def test_predict_price(self): bid = expr.randn(100) ask = expr.randn(100) res = finance.predict_price(bid, ask, 5).optimized() #print res #print res.optimized() print res.glom()
def test_slices_from_slices(self): slices1 = (slice(0, 50), slice(0, 50)) slices2 = (slice(0, 50), slice(50, 100)) slices3 = (slice(50, 100), slice(0, 50)) slices4 = (slice(50, 100), slice(50, 100)) t1 = expr.randn(100, 100) t2 = expr.randn(100, 100) t3 = expr.write(t2, slices1, t1, slices1) t4 = expr.write(t3, slices2, t1, slices2) t5 = expr.write(t4, slices3, t1, slices3) t6 = expr.write(t5, slices4, t1, slices4) Assert.all_eq(t1.glom(), t6.glom()) dst = np.arange(0, 2500).reshape(50, 50) t11 = expr.write(t1, slices1, dst, slices1) t12 = expr.write(t11, slices2, dst, slices1) t13 = expr.write(t12, slices3, dst, slices1) t14 = expr.write(t13, slices4, dst, slices1) tmp = expr.write(expr.randn(100, 100), slices4, dst, slices1) t21 = expr.write(t2, slices1, tmp, slices4) t22 = expr.write(t21, slices2, tmp, slices4) t23 = expr.write(t22, slices3, tmp, slices4) t24 = expr.write(t23, slices4, tmp, slices4) Assert.all_eq(t14.glom(), t24.glom())
def benchmark_svm(ctx, timer): print "#worker:", ctx.num_workers max_iter = 2 #N = 200000 * ctx.num_workers N = 1000 * 64 D = 64 # create data data = expr.randn(N, D, dtype=np.float64, tile_hint=(N, util.divup(D, ctx.num_workers))) labels = expr.shuffle(data, _init_label_mapper, shape_hint=(data.shape[0], 1)) t1 = datetime.now() w = fit(data, labels, T=max_iter).force() t2 = datetime.now() util.log_warn('train time per iteration:%s ms, final w:%s', millis(t1,t2)/max_iter, w.glom().T) correct = 0 for i in range(10): new_data = expr.randn(1, D, dtype=np.float64, tile_hint=[1, D]) new_label = predict(w, new_data) #print 'point %s, predict %s' % (new_data.glom(), new_label) new_data = new_data.glom() if new_data[0,0] >= new_data[0,1] and new_label == 1.0 or new_data[0,0] < new_data[0,1] and new_label == -1.0: correct += 1 print 'predict precision:', correct * 1.0 / 10
def test_fio_path(self): self.create_path() t1 = expr.randn(100, 100).evaluate() expr.save(t1, "fiotest1", self.test_dir2, False) expr.pickle(t1, "fiotest2", self.test_dir2, False) Assert.all_eq(t1.glom(), expr.load("fiotest1", self.test_dir2, False).glom()) Assert.all_eq(t1.glom(), expr.unpickle("fiotest2", self.test_dir2, False).glom())
def benchmark_cholesky(ctx, timer): print "#worker:", ctx.num_workers #n = int(math.pow(ctx.num_workers, 1.0 / 3.0)) n = int(math.sqrt(ctx.num_workers)) #ARRAY_SIZE = 1600 * 4 ARRAY_SIZE = 1600 * n util.log_warn('prepare data!') #A = np.random.randn(ARRAY_SIZE, ARRAY_SIZE) #A = np.dot(A, A.T) #A = expr.force(from_numpy(A, tile_hint=(ARRAY_SIZE/n, ARRAY_SIZE/n))) #A = expr.randn(ARRAY_SIZE, ARRAY_SIZE, tile_hint=(ARRAY_SIZE/n, ARRAY_SIZE/n)) A = expr.randn(ARRAY_SIZE, ARRAY_SIZE) # FIXME: Ideally we should be able to get rid of tile_hint. # However, current extent.change_partition_axis relies on the # information of one-dimentional size to change tiling to grid tiling. # It assumes that every extent should be partitioned in the same size. # Trace extent.pyx to think about how to fix it! A = expr.dot(A, expr.transpose(A), tile_hint=(ARRAY_SIZE, ARRAY_SIZE / ctx.num_workers)).force() util.log_warn('begin cholesky!') t1 = datetime.now() L = cholesky(A).glom() t2 = datetime.now() assert np.all(np.isclose(A.glom(), np.dot(L, L.T.conj()))) cost_time = millis(t1, t2) print "total cost time:%s ms, per iter cost time:%s ms" % (cost_time, cost_time / n)
def test_pca(self): ctx = blob_ctx.get() A = expr.randn(*DIM, tile_hint=(int(DIM[0]/ctx.num_workers), DIM[1])).force() m = PCA(N_COMPONENTS) m2 = SK_PCA(N_COMPONENTS) m.fit(A) m2.fit(A.glom()) assert np.allclose(absolute(m.components_), absolute(m2.components_))
def test_qr(self): M = 100 N = 10 Y = expr.randn(M, N, tile_hint=(M/4, N)).force() Q1, R1 = qr(Y) Q1 = Q1.glom() Q2, R2 = linalg.qr(Y.glom(), mode="economic") assert np.allclose(np.absolute(Q1), np.absolute(Q2)) assert np.allclose(np.absolute(R1), np.absolute(R2))
def test_qr(self): M = 100 N = 10 Y = expr.randn(M, N, tile_hint=(M / 4, N)).force() Q1, R1 = qr(Y) Q1 = Q1.glom() Q2, R2 = linalg.qr(Y.glom(), mode="economic") assert np.allclose(np.absolute(Q1), np.absolute(Q2)) assert np.allclose(np.absolute(R1), np.absolute(R2))
def test_ssvd(self): ctx = blob_ctx.get() # Create a sparse matrix. A = expr.randn(*DIM, tile_hint = (int(DIM[0]/ctx.num_workers), DIM[1]), dtype=np.float64) U,S,VT = svd(A) U2,S2,VT2 = linalg.svd(A.glom(), full_matrices=0) assert np.allclose(absolute(U.glom()), absolute(U2)) assert np.allclose(absolute(S), absolute(S2)) assert np.allclose(absolute(VT), absolute(VT2))
def test_slices_from_np(self): npa = np.random.random(10000).reshape(100, 100) slices1 = (slice(0, 50), slice(0, 50)) slices2 = (slice(0, 50), slice(50, 100)) slices3 = (slice(50, 100), slice(0, 50)) slices4 = (slice(50, 100), slice(50, 100)) t1 = expr.randn(100, 100) t2 = expr.write(t1, slices1, npa, slices1) t3 = expr.write(t2, slices2, npa, slices2) t4 = expr.write(t3, slices3, npa, slices3) t5 = expr.write(t4, slices4, npa, slices4) Assert.all_eq(t5.glom(), npa)
def test_ssvd(self): ctx = blob_ctx.get() # Create a sparse matrix. A = expr.randn(*DIM, tile_hint=(int(DIM[0] / ctx.num_workers), DIM[1]), dtype=np.float64) U, S, VT = svd(A) U2, S2, VT2 = linalg.svd(A.glom(), full_matrices=0) assert np.allclose(absolute(U.glom()), absolute(U2)) assert np.allclose(absolute(S), absolute(S2)) assert np.allclose(absolute(VT), absolute(VT2))
def svd(A, k=None): """ Stochastic SVD. Parameters ---------- A : spartan matrix Array to compute the SVD on, of shape (M, N) k : int, optional Number of singular values and vectors to compute. The operations include matrix multiplication and QR decomposition. We parallelize both of them. Returns -------- U : Spartan array of shape (M, k) S : numpy array of shape (k,) V : numpy array of shape (k, k) """ if k is None: k = A.shape[1] ctx = blob_ctx.get() Omega = expr.randn(A.shape[1], k, tile_hint=(A.shape[1]/ctx.num_workers, k)) r = A.shape[0] / ctx.num_workers Y = expr.dot(A, Omega, tile_hint=(r, k)).force() Q, R = qr(Y) B = expr.dot(expr.transpose(Q), A) BTB = expr.dot(B, expr.transpose(B)).glom() S, U_ = np.linalg.eig(BTB) S = np.sqrt(S) # Sort by eigen values from large to small si = np.argsort(S)[::-1] S = S[si] U_ = U_[:, si] U = expr.dot(Q, U_).force() V = np.dot(np.dot(expr.transpose(B).glom(), U_), np.diag(np.ones(S.shape[0]) / S)) return U, S, V.T
def svd(A, k=None): """ Stochastic SVD. Parameters ---------- A : spartan matrix Array to compute the SVD on, of shape (M, N) k : int, optional Number of singular values and vectors to compute. The operations include matrix multiplication and QR decomposition. We parallelize both of them. Returns -------- U : Spartan array of shape (M, k) S : numpy array of shape (k,) V : numpy array of shape (k, k) """ if k is None: k = A.shape[1] Omega = expr.randn(A.shape[1], k) Y = expr.dot(A, Omega) Q, R = qr(Y) B = expr.dot(expr.transpose(Q), A) BTB = expr.dot(B, expr.transpose(B)).optimized().glom() S, U_ = np.linalg.eig(BTB) S = np.sqrt(S) # Sort by eigen values from large to small si = np.argsort(S)[::-1] S = S[si] U_ = U_[:, si] U = expr.dot(Q, U_).optimized().force() V = np.dot(np.dot(expr.transpose(B).optimized().glom(), U_), np.diag(np.ones(S.shape[0]) / S)) return U, S, V.T
def benchmark_cholesky(ctx, timer): print "#worker:", ctx.num_workers #n = int(math.pow(ctx.num_workers, 1.0 / 3.0)) n = int(math.sqrt(ctx.num_workers)) #ARRAY_SIZE = 1600 * 4 ARRAY_SIZE = 900 * n util.log_warn('prepare data!') #A = np.random.randn(ARRAY_SIZE, ARRAY_SIZE) #A = np.dot(A, A.T) A = expr.randn(ARRAY_SIZE, ARRAY_SIZE) A = expr.dot(A, expr.transpose(A)) util.log_warn('begin cholesky!') t1 = datetime.now() L = cholesky(A).optimized().glom() t2 = datetime.now() #assert np.all(np.isclose(A.glom(), np.dot(L, L.T.conj()))) cost_time = millis(t1, t2) print "total cost time:%s ms, per iter cost time:%s ms" % (cost_time, cost_time/n)
def test_fio_partial_dense(self): self.create_path() t1 = expr.randn(300, 300).evaluate() expr.save(t1, "fiotest_partial1", self.test_dir, False) expr.pickle(t1, "fiotest_partial2", self.test_dir, False) t2 = expr.load("fiotest_partial1", self.test_dir, False) test_tiles = {} for ex, v in t1.tiles.iteritems(): test_tiles[ex] = v.worker test_tiles = expr.partial_load(test_tiles, "fiotest_partial1", self.test_dir, False) for ex, v in test_tiles.iteritems(): t1.tiles[ex] = v Assert.all_eq(t1.glom(), t2.glom()) test_tiles = {} for ex, v in t1.tiles.iteritems(): test_tiles[ex] = v.worker test_tiles = expr.partial_unpickle(test_tiles, "fiotest_partial2", self.test_dir, False) for ex, v in test_tiles.iteritems(): t1.tiles[ex] = v Assert.all_eq(t1.glom(), t2.glom())
def benchmark_cholesky(ctx, timer): print "#worker:", ctx.num_workers # n = int(math.pow(ctx.num_workers, 1.0 / 3.0)) n = int(math.sqrt(ctx.num_workers)) ARRAY_SIZE = 1600 * 4 # ARRAY_SIZE = 1600 * n util.log_warn("prepare data!") # A = np.random.randn(ARRAY_SIZE, ARRAY_SIZE) # A = np.dot(A, A.T) # A = expr.force(from_numpy(A, tile_hint=(ARRAY_SIZE/n, ARRAY_SIZE/n))) A = expr.randn(ARRAY_SIZE, ARRAY_SIZE, tile_hint=(ARRAY_SIZE / n, ARRAY_SIZE / n)) A = expr.dot(A, expr.transpose(A)).force() util.log_warn("begin cholesky!") t1 = datetime.now() L = cholesky(A).glom() t2 = datetime.now() assert np.all(np.isclose(A.glom(), np.dot(L, L.T.conj()))) cost_time = millis(t1, t2) print "total cost time:%s ms, per iter cost time:%s ms" % (cost_time, cost_time / n)
def benchmark_cholesky(ctx, timer): print "#worker:", ctx.num_workers #n = int(math.pow(ctx.num_workers, 1.0 / 3.0)) n = int(math.sqrt(ctx.num_workers)) #ARRAY_SIZE = 1600 * 4 ARRAY_SIZE = 900 * n util.log_warn('prepare data!') #A = np.random.randn(ARRAY_SIZE, ARRAY_SIZE) #A = np.dot(A, A.T) A = expr.randn(ARRAY_SIZE, ARRAY_SIZE) A = expr.dot(A, expr.transpose(A)) util.log_warn('begin cholesky!') t1 = datetime.now() L = cholesky(A).optimized().glom() t2 = datetime.now() #assert np.all(np.isclose(A.glom(), np.dot(L, L.T.conj()))) cost_time = millis(t1, t2) print "total cost time:%s ms, per iter cost time:%s ms" % (cost_time, cost_time / n)
def setUp(self): if not hasattr(self, 'current'): self.current = eager(expr.abs(10 + expr.randn(10))) self.strike = eager(expr.abs(20 + expr.randn(10)))
def simulate(ts_all, te_all, lamb_all, num_paths): """Range over a number of independent products. :param ts_all: DistArray Start dates for a series of swaptions. :param te_all: DistArray End dates for a series of swaptions. :param lamb_all: DistArray Parameter values for a series of swaptions. :param num_paths: Int Number of paths used in random walk. :rtype: DistArray """ swaptions = [] i = 0 for ts_a, te, lamb in zip(ts_all, te_all, lamb_all): for ts in ts_a: # start = time() print i time_structure = arange(None, 0, ts + DELTA, DELTA) maturity_structure = arange(None, 0, te, DELTA) ############# MODEL ############### # Variance reduction technique - Antithetic Variates. eps_tmp = randn(time_structure.shape[0] - 1, num_paths) eps = concatenate(eps_tmp, -eps_tmp, 1) # Forward LIBOR rates for the construction of the spot measure. f_kk = zeros((time_structure.shape[0], 2 * num_paths)) f_kk = assign(f_kk, np.s_[0, :], F_0) # Plane kxN of simulated LIBOR rates. f_kn = ones((maturity_structure.shape[0], 2 * num_paths)) * F_0 # Simulations of the plane f_kn for each time step. for t in xrange(1, time_structure.shape[0]): f_kn_new = f_kn[1:, :] * exp( lamb * mu(f_kn, lamb) * DELTA - 0.5 * lamb * lamb * DELTA + lamb * eps[t - 1, :] * sqrt(DELTA) ) f_kk = assign(f_kk, np.s_[t, :], f_kn_new[0]) f_kn = f_kn_new ############## PRODUCT ############### # Value of zero coupon bonds. zcb = ones((int((te - ts) / DELTA) + 1, 2 * num_paths)) f_kn_modified = 1 + DELTA * f_kn for j in xrange(zcb.shape[0] - 1): zcb = assign(zcb, np.s_[j + 1], zcb[j] / f_kn_modified[j]) # Swaption price at maturity. last_row = zcb[zcb.shape[0] - 1, :].reshape((20,)) swap_ts = maximum(1 - last_row - THETA * DELTA * expr.sum(zcb[1:], 0), 0) # Spot measure used for discounting. b_ts = ones((2 * num_paths,)) tmp = 1 + DELTA * f_kk for j in xrange(int(ts / DELTA)): b_ts *= tmp[j].reshape((20,)) # Swaption price at time 0. swaption = swap_ts / b_ts # Save expected value in bps and std. me = mean((swaption[0:num_paths] + swaption[num_paths:]) / 2) * 10000 st = std((swaption[0:num_paths] + swaption[num_paths:]) / 2) / sqrt(num_paths) * 10000 swaptions.append([me.optimized().force(), st.optimized().force()]) # print time() - start i += 1 return swaptions
def test_trace(self): devnull = open(os.devnull, 'w') with RedirectStdStreams(stdout=devnull, stderr=devnull): t1 = expr.randn(100, 100) t2 = expr.randn(200, 100) t3 = expr.add(t1, t2) t4 = expr.randn(300, 100) t5 = expr.add(t3, t4) t6 = expr.randn(400, 100) t7 = expr.add(t6, t5) t8 = t7.optimized() try: t8.force() except Exception as e: pass t1 = expr.randn(100, 100) t2 = expr.randn(100, 100) t3 = expr.dot(t1, t2) t4 = expr.randn(200, 100) t5 = expr.add(t3, t4) t6 = t5.optimized() try: t6.force() except Exception as e: pass t1 = expr.randn(100, 100) t2 = expr.randn(100, 100) t3 = expr.add(t1, t2) t4 = expr.randn(200, 100) t5 = expr.add(t3, t4) t6 = expr.randn(100, 100) t7 = expr.add(t6, t5) t8 = expr.count_zero(t7) t9 = t8.optimized() try: t9.force() except Exception as e: pass
def simulate(ts_all, te_all, lamb_all, num_paths): '''Range over a number of independent products. :param ts_all: DistArray Start dates for a series of swaptions. :param te_all: DistArray End dates for a series of swaptions. :param lamb_all: DistArray Parameter values for a series of swaptions. :param num_paths: Int Number of paths used in random walk. :rtype: DistArray ''' swaptions = [] i = 0 for ts_a, te, lamb in zip(ts_all, te_all, lamb_all): for ts in ts_a: #start = time() print i time_structure = arange(None, 0, ts + DELTA, DELTA) maturity_structure = arange(None, 0, te, DELTA) ############# MODEL ############### # Variance reduction technique - Antithetic Variates. eps_tmp = randn(time_structure.shape[0] - 1, num_paths) eps = concatenate(eps_tmp, -eps_tmp, 1) # Forward LIBOR rates for the construction of the spot measure. f_kk = zeros((time_structure.shape[0], 2*num_paths)) f_kk = assign(f_kk, np.s_[0, :], F_0) # Plane kxN of simulated LIBOR rates. f_kn = ones((maturity_structure.shape[0], 2*num_paths))*F_0 # Simulations of the plane f_kn for each time step. for t in xrange(1, time_structure.shape[0]): f_kn_new = f_kn[1:, :]*exp(lamb*mu(f_kn, lamb)*DELTA-0.5*lamb*lamb * DELTA + lamb*eps[t - 1, :]*sqrt(DELTA)) f_kk = assign(f_kk, np.s_[t, :], f_kn_new[0]) f_kn = f_kn_new ############## PRODUCT ############### # Value of zero coupon bonds. zcb = ones((int((te-ts)/DELTA)+1, 2*num_paths)) f_kn_modified = 1 + DELTA*f_kn for j in xrange(zcb.shape[0] - 1): zcb = assign(zcb, np.s_[j + 1], zcb[j] / f_kn_modified[j]) # Swaption price at maturity. last_row = zcb[zcb.shape[0] - 1, :].reshape((20, )) swap_ts = maximum(1 - last_row - THETA*DELTA*expr.sum(zcb[1:], 0), 0) # Spot measure used for discounting. b_ts = ones((2*num_paths, )) tmp = 1 + DELTA * f_kk for j in xrange(int(ts/DELTA)): b_ts *= tmp[j].reshape((20, )) # Swaption price at time 0. swaption = swap_ts/b_ts # Save expected value in bps and std. me = mean((swaption[0:num_paths] + swaption[num_paths:])/2) * 10000 st = std((swaption[0:num_paths] + swaption[num_paths:])/2)/sqrt(num_paths)*10000 swaptions.append([me.optimized().force(), st.optimized().force()]) #print time() - start i += 1 return swaptions
def test_find_change(self): arr = expr.randn(100) movers = finance.find_change(arr) #util.log_info(optimize(movers)) util.log_info(movers.glom())
def test_trace(self): devnull = open(os.devnull, 'w') with RedirectStdStreams(stdout = devnull, stderr = devnull): t1 = expr.randn(100, 100) t2 = expr.randn(200, 100) t3 = expr.add(t1, t2) t4 = expr.randn(300, 100) t5 = expr.add(t3, t4) t6 = expr.randn(400, 100) t7 = expr.add(t6, t5) t8 = t7.optimized() try: t8.force() except Exception as e: pass t1 = expr.randn(100, 100) t2 = expr.randn(100, 100) t3 = expr.dot(t1, t2) t4 = expr.randn(200, 100) t5 = expr.add(t3, t4) t6 = t5.optimized() try: t6.force() except Exception as e: pass t1 = expr.randn(100, 100) t2 = expr.randn(100, 100) t3 = expr.add(t1, t2) t4 = expr.randn(200, 100) t5 = expr.add(t3, t4) t6 = expr.randn(100, 100) t7 = expr.add(t6, t5) t8 = expr.count_zero(t7) t9 = t8.optimized() try: t9.force() except Exception as e: pass