def test_predict_price(self): bid = expr.randn(100) ask = expr.randn(100) res = finance.predict_price(bid, ask, 5).optimized() #print res #print res.optimized() print res.glom()
def test_slices_from_slices(self):
slices1 = (slice(0, 50), slice(0, 50))
slices2 = (slice(0, 50), slice(50, 100))
slices3 = (slice(50, 100), slice(0, 50))
slices4 = (slice(50, 100), slice(50, 100))
t1 = expr.randn(100, 100)
t2 = expr.randn(100, 100)
t3 = expr.write(t2, slices1, t1, slices1)
t4 = expr.write(t3, slices2, t1, slices2)
t5 = expr.write(t4, slices3, t1, slices3)
t6 = expr.write(t5, slices4, t1, slices4)
Assert.all_eq(t1.glom(), t6.glom())
dst = np.arange(0, 2500).reshape(50, 50)
t11 = expr.write(t1, slices1, dst, slices1)
t12 = expr.write(t11, slices2, dst, slices1)
t13 = expr.write(t12, slices3, dst, slices1)
t14 = expr.write(t13, slices4, dst, slices1)
tmp = expr.write(expr.randn(100, 100), slices4, dst, slices1)
t21 = expr.write(t2, slices1, tmp, slices4)
t22 = expr.write(t21, slices2, tmp, slices4)
t23 = expr.write(t22, slices3, tmp, slices4)
t24 = expr.write(t23, slices4, tmp, slices4)
Assert.all_eq(t14.glom(), t24.glom())
def test_predict_price(self):
bid = expr.randn(100)
ask = expr.randn(100)
res = finance.predict_price(bid, ask, 5).optimized()
#print res
#print res.optimized()
print res.glom()
def benchmark_svm(ctx, timer):
print "#worker:", ctx.num_workers
max_iter = 2
#N = 200000 * ctx.num_workers
N = 1000 * 64
D = 64
# create data
data = expr.randn(N, D, dtype=np.float64, tile_hint=(N, util.divup(D, ctx.num_workers)))
labels = expr.shuffle(data, _init_label_mapper, shape_hint=(data.shape[0], 1))
t1 = datetime.now()
w = fit(data, labels, T=max_iter).force()
t2 = datetime.now()
util.log_warn('train time per iteration:%s ms, final w:%s', millis(t1,t2)/max_iter, w.glom().T)
correct = 0
for i in range(10):
new_data = expr.randn(1, D, dtype=np.float64, tile_hint=[1, D])
new_label = predict(w, new_data)
#print 'point %s, predict %s' % (new_data.glom(), new_label)
new_data = new_data.glom()
if new_data[0,0] >= new_data[0,1] and new_label == 1.0 or new_data[0,0] < new_data[0,1] and new_label == -1.0:
correct += 1
print 'predict precision:', correct * 1.0 / 10
def test_fio_path(self):
self.create_path()
t1 = expr.randn(100, 100).evaluate()
expr.save(t1, "fiotest1", self.test_dir2, False)
expr.pickle(t1, "fiotest2", self.test_dir2, False)
Assert.all_eq(t1.glom(), expr.load("fiotest1", self.test_dir2, False).glom())
Assert.all_eq(t1.glom(), expr.unpickle("fiotest2", self.test_dir2, False).glom())
def benchmark_cholesky(ctx, timer):
print "#worker:", ctx.num_workers
#n = int(math.pow(ctx.num_workers, 1.0 / 3.0))
n = int(math.sqrt(ctx.num_workers))
#ARRAY_SIZE = 1600 * 4
ARRAY_SIZE = 1600 * n
util.log_warn('prepare data!')
#A = np.random.randn(ARRAY_SIZE, ARRAY_SIZE)
#A = np.dot(A, A.T)
#A = expr.force(from_numpy(A, tile_hint=(ARRAY_SIZE/n, ARRAY_SIZE/n)))
#A = expr.randn(ARRAY_SIZE, ARRAY_SIZE, tile_hint=(ARRAY_SIZE/n, ARRAY_SIZE/n))
A = expr.randn(ARRAY_SIZE, ARRAY_SIZE)
# FIXME: Ideally we should be able to get rid of tile_hint.
# However, current extent.change_partition_axis relies on the
# information of one-dimentional size to change tiling to grid tiling.
# It assumes that every extent should be partitioned in the same size.
# Trace extent.pyx to think about how to fix it!
A = expr.dot(A,
expr.transpose(A),
tile_hint=(ARRAY_SIZE, ARRAY_SIZE / ctx.num_workers)).force()
util.log_warn('begin cholesky!')
t1 = datetime.now()
L = cholesky(A).glom()
t2 = datetime.now()
assert np.all(np.isclose(A.glom(), np.dot(L, L.T.conj())))
cost_time = millis(t1, t2)
print "total cost time:%s ms, per iter cost time:%s ms" % (cost_time,
cost_time / n)
def test_pca(self):
ctx = blob_ctx.get()
A = expr.randn(*DIM, tile_hint=(int(DIM[0]/ctx.num_workers), DIM[1])).force()
m = PCA(N_COMPONENTS)
m2 = SK_PCA(N_COMPONENTS)
m.fit(A)
m2.fit(A.glom())
assert np.allclose(absolute(m.components_), absolute(m2.components_))
def test_qr(self): M = 100 N = 10 Y = expr.randn(M, N, tile_hint=(M/4, N)).force() Q1, R1 = qr(Y) Q1 = Q1.glom() Q2, R2 = linalg.qr(Y.glom(), mode="economic") assert np.allclose(np.absolute(Q1), np.absolute(Q2)) assert np.allclose(np.absolute(R1), np.absolute(R2))
def test_qr(self):
M = 100
N = 10
Y = expr.randn(M, N, tile_hint=(M / 4, N)).force()
Q1, R1 = qr(Y)
Q1 = Q1.glom()
Q2, R2 = linalg.qr(Y.glom(), mode="economic")
assert np.allclose(np.absolute(Q1), np.absolute(Q2))
assert np.allclose(np.absolute(R1), np.absolute(R2))
def test_ssvd(self):
ctx = blob_ctx.get()
# Create a sparse matrix.
A = expr.randn(*DIM, tile_hint = (int(DIM[0]/ctx.num_workers), DIM[1]),
dtype=np.float64)
U,S,VT = svd(A)
U2,S2,VT2 = linalg.svd(A.glom(), full_matrices=0)
assert np.allclose(absolute(U.glom()), absolute(U2))
assert np.allclose(absolute(S), absolute(S2))
assert np.allclose(absolute(VT), absolute(VT2))
def test_slices_from_np(self): npa = np.random.random(10000).reshape(100, 100) slices1 = (slice(0, 50), slice(0, 50)) slices2 = (slice(0, 50), slice(50, 100)) slices3 = (slice(50, 100), slice(0, 50)) slices4 = (slice(50, 100), slice(50, 100)) t1 = expr.randn(100, 100) t2 = expr.write(t1, slices1, npa, slices1) t3 = expr.write(t2, slices2, npa, slices2) t4 = expr.write(t3, slices3, npa, slices3) t5 = expr.write(t4, slices4, npa, slices4) Assert.all_eq(t5.glom(), npa)
def test_ssvd(self):
ctx = blob_ctx.get()
# Create a sparse matrix.
A = expr.randn(*DIM,
tile_hint=(int(DIM[0] / ctx.num_workers), DIM[1]),
dtype=np.float64)
U, S, VT = svd(A)
U2, S2, VT2 = linalg.svd(A.glom(), full_matrices=0)
assert np.allclose(absolute(U.glom()), absolute(U2))
assert np.allclose(absolute(S), absolute(S2))
assert np.allclose(absolute(VT), absolute(VT2))
def svd(A, k=None):
"""
Stochastic SVD.
Parameters
----------
A : spartan matrix
Array to compute the SVD on, of shape (M, N)
k : int, optional
Number of singular values and vectors to compute.
The operations include matrix multiplication and QR decomposition.
We parallelize both of them.
Returns
--------
U : Spartan array of shape (M, k)
S : numpy array of shape (k,)
V : numpy array of shape (k, k)
"""
if k is None:
k = A.shape[1]
ctx = blob_ctx.get()
Omega = expr.randn(A.shape[1], k, tile_hint=(A.shape[1]/ctx.num_workers, k))
r = A.shape[0] / ctx.num_workers
Y = expr.dot(A, Omega, tile_hint=(r, k)).force()
Q, R = qr(Y)
B = expr.dot(expr.transpose(Q), A)
BTB = expr.dot(B, expr.transpose(B)).glom()
S, U_ = np.linalg.eig(BTB)
S = np.sqrt(S)
# Sort by eigen values from large to small
si = np.argsort(S)[::-1]
S = S[si]
U_ = U_[:, si]
U = expr.dot(Q, U_).force()
V = np.dot(np.dot(expr.transpose(B).glom(), U_), np.diag(np.ones(S.shape[0]) / S))
return U, S, V.T
def svd(A, k=None):
"""
Stochastic SVD.
Parameters
----------
A : spartan matrix
Array to compute the SVD on, of shape (M, N)
k : int, optional
Number of singular values and vectors to compute.
The operations include matrix multiplication and QR decomposition.
We parallelize both of them.
Returns
--------
U : Spartan array of shape (M, k)
S : numpy array of shape (k,)
V : numpy array of shape (k, k)
"""
if k is None: k = A.shape[1]
Omega = expr.randn(A.shape[1], k)
Y = expr.dot(A, Omega)
Q, R = qr(Y)
B = expr.dot(expr.transpose(Q), A)
BTB = expr.dot(B, expr.transpose(B)).optimized().glom()
S, U_ = np.linalg.eig(BTB)
S = np.sqrt(S)
# Sort by eigen values from large to small
si = np.argsort(S)[::-1]
S = S[si]
U_ = U_[:, si]
U = expr.dot(Q, U_).optimized().force()
V = np.dot(np.dot(expr.transpose(B).optimized().glom(), U_),
np.diag(np.ones(S.shape[0]) / S))
return U, S, V.T
def benchmark_cholesky(ctx, timer):
print "#worker:", ctx.num_workers
#n = int(math.pow(ctx.num_workers, 1.0 / 3.0))
n = int(math.sqrt(ctx.num_workers))
#ARRAY_SIZE = 1600 * 4
ARRAY_SIZE = 900 * n
util.log_warn('prepare data!')
#A = np.random.randn(ARRAY_SIZE, ARRAY_SIZE)
#A = np.dot(A, A.T)
A = expr.randn(ARRAY_SIZE, ARRAY_SIZE)
A = expr.dot(A, expr.transpose(A))
util.log_warn('begin cholesky!')
t1 = datetime.now()
L = cholesky(A).optimized().glom()
t2 = datetime.now()
#assert np.all(np.isclose(A.glom(), np.dot(L, L.T.conj())))
cost_time = millis(t1, t2)
print "total cost time:%s ms, per iter cost time:%s ms" % (cost_time, cost_time/n)
def test_fio_partial_dense(self):
self.create_path()
t1 = expr.randn(300, 300).evaluate()
expr.save(t1, "fiotest_partial1", self.test_dir, False)
expr.pickle(t1, "fiotest_partial2", self.test_dir, False)
t2 = expr.load("fiotest_partial1", self.test_dir, False)
test_tiles = {}
for ex, v in t1.tiles.iteritems():
test_tiles[ex] = v.worker
test_tiles = expr.partial_load(test_tiles, "fiotest_partial1", self.test_dir, False)
for ex, v in test_tiles.iteritems():
t1.tiles[ex] = v
Assert.all_eq(t1.glom(), t2.glom())
test_tiles = {}
for ex, v in t1.tiles.iteritems():
test_tiles[ex] = v.worker
test_tiles = expr.partial_unpickle(test_tiles, "fiotest_partial2", self.test_dir, False)
for ex, v in test_tiles.iteritems():
t1.tiles[ex] = v
Assert.all_eq(t1.glom(), t2.glom())
def benchmark_cholesky(ctx, timer):
print "#worker:", ctx.num_workers
# n = int(math.pow(ctx.num_workers, 1.0 / 3.0))
n = int(math.sqrt(ctx.num_workers))
ARRAY_SIZE = 1600 * 4
# ARRAY_SIZE = 1600 * n
util.log_warn("prepare data!")
# A = np.random.randn(ARRAY_SIZE, ARRAY_SIZE)
# A = np.dot(A, A.T)
# A = expr.force(from_numpy(A, tile_hint=(ARRAY_SIZE/n, ARRAY_SIZE/n)))
A = expr.randn(ARRAY_SIZE, ARRAY_SIZE, tile_hint=(ARRAY_SIZE / n, ARRAY_SIZE / n))
A = expr.dot(A, expr.transpose(A)).force()
util.log_warn("begin cholesky!")
t1 = datetime.now()
L = cholesky(A).glom()
t2 = datetime.now()
assert np.all(np.isclose(A.glom(), np.dot(L, L.T.conj())))
cost_time = millis(t1, t2)
print "total cost time:%s ms, per iter cost time:%s ms" % (cost_time, cost_time / n)
def benchmark_cholesky(ctx, timer):
print "#worker:", ctx.num_workers
#n = int(math.pow(ctx.num_workers, 1.0 / 3.0))
n = int(math.sqrt(ctx.num_workers))
#ARRAY_SIZE = 1600 * 4
ARRAY_SIZE = 900 * n
util.log_warn('prepare data!')
#A = np.random.randn(ARRAY_SIZE, ARRAY_SIZE)
#A = np.dot(A, A.T)
A = expr.randn(ARRAY_SIZE, ARRAY_SIZE)
A = expr.dot(A, expr.transpose(A))
util.log_warn('begin cholesky!')
t1 = datetime.now()
L = cholesky(A).optimized().glom()
t2 = datetime.now()
#assert np.all(np.isclose(A.glom(), np.dot(L, L.T.conj())))
cost_time = millis(t1, t2)
print "total cost time:%s ms, per iter cost time:%s ms" % (cost_time,
cost_time / n)
def setUp(self):
if not hasattr(self, 'current'):
self.current = eager(expr.abs(10 + expr.randn(10)))
self.strike = eager(expr.abs(20 + expr.randn(10)))
def simulate(ts_all, te_all, lamb_all, num_paths):
"""Range over a number of independent products.
:param ts_all: DistArray
Start dates for a series of swaptions.
:param te_all: DistArray
End dates for a series of swaptions.
:param lamb_all: DistArray
Parameter values for a series of swaptions.
:param num_paths: Int
Number of paths used in random walk.
:rtype: DistArray
"""
swaptions = []
i = 0
for ts_a, te, lamb in zip(ts_all, te_all, lamb_all):
for ts in ts_a:
# start = time()
print i
time_structure = arange(None, 0, ts + DELTA, DELTA)
maturity_structure = arange(None, 0, te, DELTA)
############# MODEL ###############
# Variance reduction technique - Antithetic Variates.
eps_tmp = randn(time_structure.shape[0] - 1, num_paths)
eps = concatenate(eps_tmp, -eps_tmp, 1)
# Forward LIBOR rates for the construction of the spot measure.
f_kk = zeros((time_structure.shape[0], 2 * num_paths))
f_kk = assign(f_kk, np.s_[0, :], F_0)
# Plane kxN of simulated LIBOR rates.
f_kn = ones((maturity_structure.shape[0], 2 * num_paths)) * F_0
# Simulations of the plane f_kn for each time step.
for t in xrange(1, time_structure.shape[0]):
f_kn_new = f_kn[1:, :] * exp(
lamb * mu(f_kn, lamb) * DELTA - 0.5 * lamb * lamb * DELTA + lamb * eps[t - 1, :] * sqrt(DELTA)
)
f_kk = assign(f_kk, np.s_[t, :], f_kn_new[0])
f_kn = f_kn_new
############## PRODUCT ###############
# Value of zero coupon bonds.
zcb = ones((int((te - ts) / DELTA) + 1, 2 * num_paths))
f_kn_modified = 1 + DELTA * f_kn
for j in xrange(zcb.shape[0] - 1):
zcb = assign(zcb, np.s_[j + 1], zcb[j] / f_kn_modified[j])
# Swaption price at maturity.
last_row = zcb[zcb.shape[0] - 1, :].reshape((20,))
swap_ts = maximum(1 - last_row - THETA * DELTA * expr.sum(zcb[1:], 0), 0)
# Spot measure used for discounting.
b_ts = ones((2 * num_paths,))
tmp = 1 + DELTA * f_kk
for j in xrange(int(ts / DELTA)):
b_ts *= tmp[j].reshape((20,))
# Swaption price at time 0.
swaption = swap_ts / b_ts
# Save expected value in bps and std.
me = mean((swaption[0:num_paths] + swaption[num_paths:]) / 2) * 10000
st = std((swaption[0:num_paths] + swaption[num_paths:]) / 2) / sqrt(num_paths) * 10000
swaptions.append([me.optimized().force(), st.optimized().force()])
# print time() - start
i += 1
return swaptions
def test_trace(self):
devnull = open(os.devnull, 'w')
with RedirectStdStreams(stdout=devnull, stderr=devnull):
t1 = expr.randn(100, 100)
t2 = expr.randn(200, 100)
t3 = expr.add(t1, t2)
t4 = expr.randn(300, 100)
t5 = expr.add(t3, t4)
t6 = expr.randn(400, 100)
t7 = expr.add(t6, t5)
t8 = t7.optimized()
try:
t8.force()
except Exception as e:
pass
t1 = expr.randn(100, 100)
t2 = expr.randn(100, 100)
t3 = expr.dot(t1, t2)
t4 = expr.randn(200, 100)
t5 = expr.add(t3, t4)
t6 = t5.optimized()
try:
t6.force()
except Exception as e:
pass
t1 = expr.randn(100, 100)
t2 = expr.randn(100, 100)
t3 = expr.add(t1, t2)
t4 = expr.randn(200, 100)
t5 = expr.add(t3, t4)
t6 = expr.randn(100, 100)
t7 = expr.add(t6, t5)
t8 = expr.count_zero(t7)
t9 = t8.optimized()
try:
t9.force()
except Exception as e:
pass
def setUp(self):
if not hasattr(self, 'current'):
self.current = eager(expr.abs(10 + expr.randn(10)))
self.strike = eager(expr.abs(20 + expr.randn(10)))
def simulate(ts_all, te_all, lamb_all, num_paths):
'''Range over a number of independent products.
:param ts_all: DistArray
Start dates for a series of swaptions.
:param te_all: DistArray
End dates for a series of swaptions.
:param lamb_all: DistArray
Parameter values for a series of swaptions.
:param num_paths: Int
Number of paths used in random walk.
:rtype: DistArray
'''
swaptions = []
i = 0
for ts_a, te, lamb in zip(ts_all, te_all, lamb_all):
for ts in ts_a:
#start = time()
print i
time_structure = arange(None, 0, ts + DELTA, DELTA)
maturity_structure = arange(None, 0, te, DELTA)
############# MODEL ###############
# Variance reduction technique - Antithetic Variates.
eps_tmp = randn(time_structure.shape[0] - 1, num_paths)
eps = concatenate(eps_tmp, -eps_tmp, 1)
# Forward LIBOR rates for the construction of the spot measure.
f_kk = zeros((time_structure.shape[0], 2*num_paths))
f_kk = assign(f_kk, np.s_[0, :], F_0)
# Plane kxN of simulated LIBOR rates.
f_kn = ones((maturity_structure.shape[0], 2*num_paths))*F_0
# Simulations of the plane f_kn for each time step.
for t in xrange(1, time_structure.shape[0]):
f_kn_new = f_kn[1:, :]*exp(lamb*mu(f_kn, lamb)*DELTA-0.5*lamb*lamb *
DELTA + lamb*eps[t - 1, :]*sqrt(DELTA))
f_kk = assign(f_kk, np.s_[t, :], f_kn_new[0])
f_kn = f_kn_new
############## PRODUCT ###############
# Value of zero coupon bonds.
zcb = ones((int((te-ts)/DELTA)+1, 2*num_paths))
f_kn_modified = 1 + DELTA*f_kn
for j in xrange(zcb.shape[0] - 1):
zcb = assign(zcb, np.s_[j + 1], zcb[j] / f_kn_modified[j])
# Swaption price at maturity.
last_row = zcb[zcb.shape[0] - 1, :].reshape((20, ))
swap_ts = maximum(1 - last_row - THETA*DELTA*expr.sum(zcb[1:], 0), 0)
# Spot measure used for discounting.
b_ts = ones((2*num_paths, ))
tmp = 1 + DELTA * f_kk
for j in xrange(int(ts/DELTA)):
b_ts *= tmp[j].reshape((20, ))
# Swaption price at time 0.
swaption = swap_ts/b_ts
# Save expected value in bps and std.
me = mean((swaption[0:num_paths] + swaption[num_paths:])/2) * 10000
st = std((swaption[0:num_paths] + swaption[num_paths:])/2)/sqrt(num_paths)*10000
swaptions.append([me.optimized().force(), st.optimized().force()])
#print time() - start
i += 1
return swaptions
def test_find_change(self):
arr = expr.randn(100)
movers = finance.find_change(arr)
#util.log_info(optimize(movers))
util.log_info(movers.glom())
def test_find_change(self): arr = expr.randn(100) movers = finance.find_change(arr) #util.log_info(optimize(movers)) util.log_info(movers.glom())
def test_trace(self):
devnull = open(os.devnull, 'w')
with RedirectStdStreams(stdout = devnull, stderr = devnull):
t1 = expr.randn(100, 100)
t2 = expr.randn(200, 100)
t3 = expr.add(t1, t2)
t4 = expr.randn(300, 100)
t5 = expr.add(t3, t4)
t6 = expr.randn(400, 100)
t7 = expr.add(t6, t5)
t8 = t7.optimized()
try:
t8.force()
except Exception as e:
pass
t1 = expr.randn(100, 100)
t2 = expr.randn(100, 100)
t3 = expr.dot(t1, t2)
t4 = expr.randn(200, 100)
t5 = expr.add(t3, t4)
t6 = t5.optimized()
try:
t6.force()
except Exception as e:
pass
t1 = expr.randn(100, 100)
t2 = expr.randn(100, 100)
t3 = expr.add(t1, t2)
t4 = expr.randn(200, 100)
t5 = expr.add(t3, t4)
t6 = expr.randn(100, 100)
t7 = expr.add(t6, t5)
t8 = expr.count_zero(t7)
t9 = t8.optimized()
try:
t9.force()
except Exception as e:
pass
