def solve(A, b, conv_iters, max_iters, verbose): print("Solving system...") x = np.zeros(A.shape[1]) r = b - A.dot(x) p = r rsold = r.dot(r) converged = -1 # Should always converge in fewer iterations than this max_iters = (min(max_iters, b.shape[0]) if max_iters is not None else b.shape[0]) for i in range(max_iters): Ap = A.dot(p) alpha = rsold / (p.dot(Ap)) x = x + alpha * p r = r - alpha * Ap rsnew = r.dot(r) # We only do the convergence test every conv_iters or on the last # iteration if (i % conv_iters == 0 or i == (max_iters - 1)) and np.sqrt(rsnew) < 1e-10: converged = i break if verbose: print("Residual: " + str(rsnew)) beta = rsnew / rsold p = r + beta * p rsold = rsnew if converged < 0: print("Convergence FAILURE!") else: print("Converged in %d iterations" % (converged)) return x
def test(): npa = np.array([1, 4, 9]) a = lg.array(npa) assert np.array_equal(lg.sqrt(a), np.sqrt(npa)) npa = np.array([1, 4, 9], dtype=np.float) a = lg.array(npa) assert np.array_equal(lg.sqrt(a), np.sqrt(npa)) npa = np.array([1, 4, 9], dtype=np.float32) a = lg.array(npa) assert np.array_equal(lg.sqrt(a), np.sqrt(npa)) npa = np.array([1, 4, 9], dtype=np.float64) a = lg.array(npa) assert np.array_equal(lg.sqrt(a), np.sqrt(npa)) return
def test(): npa = np.array([1.0, 4.0, 9.0]) a = lg.array(npa) assert np.array_equal(lg.sqrt(a, out=a), np.sqrt(npa, out=npa)) npa = np.array([1.0, 4.0, 9.0], dtype=np.float) a = lg.array(npa) assert np.array_equal(lg.sqrt(a, out=a), np.sqrt(npa, out=npa)) npa = np.array([1.0, 4.0, 9.0], dtype=np.float32) a = lg.array(npa) assert np.array_equal(lg.sqrt(a, out=a), np.sqrt(npa, out=npa)) npa = np.array([1.0, 4.0, 9.0], dtype=np.float64) a = lg.array(npa) assert np.array_equal(lg.sqrt(a, out=a), np.sqrt(npa, out=npa)) return
def black_scholes(S, X, T, R, V): sqrt_t = np.sqrt(T) d1 = np.log(S / X) + (R + 0.5 * V * V) * T / (V * sqrt_t) d2 = d1 - V * sqrt_t cnd_d1 = cnd(d1) cnd_d2 = cnd(d2) exp_rt = np.exp(-R * T) call_result = S * cnd_d1 - X * exp_rt * cnd_d2 put_result = X * exp_rt * (1.0 - cnd_d2) - S * (1.0 - cnd_d1) return call_result, put_result
def test(): x = lg.array([ [1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16], [17, 18, 19, 20], ]) x[0, :] = 0 assert lg.array_equal(x[0, :], [0, 0, 0, 0]) y = lg.array([[20, 30, 40, 50], [70, 80, 90, 100]]) x[1:4:2] = y assert lg.array_equal(x[1, :], [20, 30, 40, 50]) assert lg.array_equal(x[3, :], [70, 80, 90, 100]) input_size = 10 hidden_size = 4 WLSTM = lg.random.randn(input_size + hidden_size + 1, 4 * hidden_size) / lg.sqrt(input_size + hidden_size) WLSTM[0, :] = 0 # initialize biases to zero assert lg.array_equal( WLSTM[0, :], [ 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ], ) lg.random.seed(10) WLSTM = lg.random.randn(15, 16) dIFOGt = lg.random.randn(3, 16) dHoutt_in = lg.random.randn(2, 3, 4) dHoutt = dHoutt_in.copy() dHint = dIFOGt.dot(WLSTM.transpose()) temp = dHoutt[0, :] + dHint[:, 11:] dHoutt[0, :] = temp assert not lg.array_equal(dHoutt[0, :], dHoutt_in[0, :]) return
def run_lstm(batch_size, hidden_size, sentence_length, word_size, timing): start = datetime.datetime.now() X = np.random.randn(sentence_length, batch_size, hidden_size) h0 = np.random.randn(1, hidden_size) WLSTM = np.random.randn( word_size + hidden_size, 4 * hidden_size ) / np.sqrt(word_size + hidden_size) xphpb = WLSTM.shape[0] d = hidden_size n = sentence_length b = batch_size Hin = np.zeros((n, b, xphpb)) Hout = np.zeros((n, b, d)) IFOG = np.zeros((n, b, d * 4)) IFOGf = np.zeros((n, b, d * 4)) C = np.zeros((n, b, d)) Ct = np.zeros((n, b, d)) for t in range(0, n): if t == 0: prev = np.tile(h0, (b, 1)) else: prev = Hout[t - 1] Hin[t, :, :word_size] = X[t] Hin[t, :, word_size:] = prev # compute all gate activations. dots: IFOG[t] = Hin[t].dot(WLSTM) # non-linearities IFOGf[t, :, : 3 * d] = 1.0 / ( 1.0 + np.exp(-IFOG[t, :, : 3 * d]) ) # sigmoids these are the gates IFOGf[t, :, 3 * d :] = np.tanh(IFOG[t, :, 3 * d :]) # tanh # compute the cell activation C[t] = IFOGf[t, :, :d] * IFOGf[t, :, 3 * d :] if t > 0: C[t] += IFOGf[t, :, d : 2 * d] * C[t - 1] Ct[t] = np.tanh(C[t]) Hout[t] = IFOGf[t, :, 2 * d : 3 * d] * Ct[t] # Do a little sum of the outputs to synchronize and check for NaNs total = np.sum(Hout) assert not math.isnan(total) stop = datetime.datetime.now() delta = stop - start total = delta.total_seconds() * 1000.0 if timing: print("Elapsed Time: " + str(total) + " ms") return total
def test(): xn = np.array( [[1 + 2j, 3 - 4j, 5 + 6j], [7 - 8j, -9 + 10j, -11 - 12j]], np.complex ) x = lg.array(xn) assert lg.all(lg.abs(lg.sin(x) - np.sin(xn)) < 1e-5) assert lg.all(lg.abs(lg.cos(x) - np.cos(xn)) < 1e-5) assert lg.all(lg.abs(lg.exp(x) - np.exp(xn)) < 1e-5) assert lg.all(lg.abs(lg.tanh(x) - np.tanh(xn)) < 1e-5) assert lg.all(lg.abs(lg.sqrt(x) - np.sqrt(xn)) < 1e-5) return
def init(input_size, hidden_size, fancy_forget_bias_init=3): """ Initialize parameters of the LSTM (both weights and biases in one matrix). One might way to have a positive fancy_forget_bias_init number (e.g. maybe even up to 5, in some papers) """ # +1 for the biases, which will be the first row of WLSTM WLSTM = np.random.randn( input_size + hidden_size + 1, 4 * hidden_size) / np.sqrt(input_size + hidden_size) WLSTM[0, :] = 0 # initialize biases to zero if fancy_forget_bias_init != 0: # forget gates get little bit negative bias initially to encourage # them to be turned off remember that due to Xavier initialization # above, the raw output activations from gates before nonlinearity # are zero mean and on order of standard deviation ~1 WLSTM[0, hidden_size:2 * hidden_size] = fancy_forget_bias_init return WLSTM
def test(): word_size = 10 hidden_size = 10 sentence_length = 2 batch_size = 3 X = np.random.randn(sentence_length, batch_size, hidden_size) h0 = np.random.randn(1, hidden_size) WLSTM = np.random.randn(word_size + hidden_size, 4 * hidden_size) / np.sqrt(word_size + hidden_size) xphpb = WLSTM.shape[0] d = hidden_size n = sentence_length b = batch_size Hin = np.zeros((n, b, xphpb)) Hout = np.zeros((n, b, d)) IFOG = np.zeros((n, b, d * 4)) IFOGf = np.zeros((n, b, d * 4)) C = np.zeros((n, b, d)) Ct = np.zeros((n, b, d)) for t in range(0, n): if t == 0: prev = np.tile(h0, (b, 1)) else: prev = Hout[t - 1] Hin[t, :, :word_size] = X[t] Hin[t, :, word_size:] = prev # compute all gate activations. dots: IFOG[t] = Hin[t].dot(WLSTM) # non-linearities IFOGf[t, :, :3 * d] = 1.0 / (1.0 + np.exp(-IFOG[t, :, :3 * d]) ) # sigmoids these are the gates IFOGf[t, :, 3 * d:] = np.tanh(IFOG[t, :, 3 * d:]) # tanh # compute the cell activation C[t] = IFOGf[t, :, :d] * IFOGf[t, :, 3 * d:] if t > 0: C[t] += IFOGf[t, :, d:2 * d] * C[t - 1] Ct[t] = np.tanh(C[t]) Hout[t] = IFOGf[t, :, 2 * d:3 * d] * Ct[t] return
def test(): pythonX = [1, 2, 3] legate_oldX = lg.array(pythonX) assert len(legate_oldX) == len(pythonX) SIZE = 100 legate_oldX = lg.random.random(SIZE) assert SIZE == len(legate_oldX) x = lg.array([1, 2, 3, 4]) y = lg.array([1, 2, 3, 4]) z = x + y print(len(z)) assert len(x) == len(y) == len(z) x = lg.array(pythonX) x = lg.sqrt(x) assert len(x) == len(pythonX) return
def test(): xn = np.array([[1, 2, 3], [4, 5, 6]]) x = lg.array(xn) # print(np.sin(xn)) # print(lg.sin(x)) assert np.allclose(np.sin(xn), lg.sin(x)) # print(np.cos(xn)) # print(lg.cos(x)) assert np.allclose(np.cos(xn), lg.cos(x)) # print(np.sqrt(xn)) # print(lg.sqrt(x)) assert np.allclose(np.sqrt(xn), lg.sqrt(x)) # print(np.exp(xn)) # print(lg.exp(x)) assert np.allclose(np.exp(xn), lg.exp(x)) # print(np.log(xn)) # print(lg.log(x)) assert np.allclose(np.log(xn), lg.log(x)) # print(np.absolute(xn)) # print(lg.absolute(x)) assert np.allclose(np.absolute(xn), lg.absolute(x)) y = lg.tanh(x) yn = np.tanh(xn) assert np.allclose(y, yn) y = lg.cos(0.5) # print(y) assert np.allclose(y, np.cos(0.5)) y = lg.sqrt(0.5) # print(y) assert np.allclose(y, np.sqrt(0.5)) y = lg.sin(0.5) # print(y) assert np.allclose(y, np.sin(0.5)) y = lg.exp(2) # print(y) assert np.allclose(y, np.exp(2)) y = lg.log(2) # print(y) assert np.allclose(y, np.log(2)) y = lg.absolute(-3) # print(y) assert y == 3 np.random.seed(42) an = np.random.randn(1, 3, 16) bn = 1.0 / (1.0 + np.exp(-an[0, :, :])) a = lg.array(an) b = 1.0 / (1.0 + lg.exp(-a[0, :, :])) assert np.allclose(b, bn) return
def testtion(): word_size = 10 hidden_size = 10 sentence_length = 5 batch_size = 3 lg.random.seed(42) WLSTM = lg.random.randn(word_size + hidden_size, 4 * hidden_size) / lg.sqrt(word_size + hidden_size) xphpb = WLSTM.shape[0] d = hidden_size n = sentence_length b = batch_size dHout = lg.random.randn(n, b, d) IFOGf = lg.random.randn(n, b, d * 4) C = lg.random.randn(n, b, d) Ct = lg.random.randn(n, b, d) Hin = lg.random.randn(n, b, xphpb) dIFOG = lg.zeros((n, b, d * 4)) dIFOGf = lg.zeros(IFOGf.shape) dHin = lg.zeros(Hin.shape) dC = lg.zeros(C.shape) dh0 = lg.zeros((1, d)) for t in reversed(range(n)): tanhCt = Ct[t] dIFOGf[t, :, 2 * d:3 * d] = tanhCt * dHout[t] # backprop tanh non-linearity first then continue backprop dC[t] += (1 - tanhCt**2) * (IFOGf[t, :, 2 * d:3 * d] * dHout[t]) if t > 0: dIFOGf[t, :, d:2 * d] = C[t - 1] * dC[t] dC[t - 1] += IFOGf[t, :, d:2 * d] * dC[t] dIFOGf[t, :, :d] = IFOGf[t, :, 3 * d:] * dC[t] dIFOGf[t, :, 3 * d:] = IFOGf[t, :, :d] * dC[t] # backprop activation functions dIFOG[t, :, 3 * d:] = (1 - IFOGf[t, :, 3 * d:]**2) * dIFOGf[t, :, 3 * d:] y = IFOGf[t, :, :3 * d] dIFOG[t, :, :3 * d] = (y * (1.0 - y)) * dIFOGf[t, :, :3 * d] # backprop matrix multiply dHin[t] = dIFOG[t].dot(WLSTM.transpose()) # backprop the identity transforms into Hin if t > 0: dHout[t - 1, :] += dHin[t, :, word_size:] else: dh0[0] += lg.sum(dHin[t, :, word_size:], 0) np.random.seed(42) WLSTM = np.random.randn(word_size + hidden_size, 4 * hidden_size) / np.sqrt(word_size + hidden_size) xphpb = WLSTM.shape[0] d = hidden_size n = sentence_length b = batch_size dHout = np.random.randn(n, b, d) IFOGf = np.random.randn(n, b, d * 4) C = np.random.randn(n, b, d) Ct = np.random.randn(n, b, d) Hin = np.random.randn(n, b, xphpb) dIFOG = np.zeros((n, b, d * 4)) dIFOGf = np.zeros(IFOGf.shape) dHin = np.zeros(Hin.shape) dC = np.zeros(C.shape) dhnp0 = np.zeros((1, d)) for t in reversed(range(n)): tanhCt = Ct[t] dIFOGf[t, :, 2 * d:3 * d] = tanhCt * dHout[t] # backprop tanh non-linearity first then continue backprop dC[t] += (1 - tanhCt**2) * (IFOGf[t, :, 2 * d:3 * d] * dHout[t]) if t > 0: dIFOGf[t, :, d:2 * d] = C[t - 1] * dC[t] dC[t - 1] += IFOGf[t, :, d:2 * d] * dC[t] dIFOGf[t, :, :d] = IFOGf[t, :, 3 * d:] * dC[t] dIFOGf[t, :, 3 * d:] = IFOGf[t, :, :d] * dC[t] # backprop activation functions dIFOG[t, :, 3 * d:] = (1 - IFOGf[t, :, 3 * d:]**2) * dIFOGf[t, :, 3 * d:] y = IFOGf[t, :, :3 * d] dIFOG[t, :, :3 * d] = (y * (1.0 - y)) * dIFOGf[t, :, :3 * d] # backprop matrix multiply dHin[t] = dIFOG[t].dot(WLSTM.transpose()) # backprop the identity transforms into Hin if t > 0: dHout[t - 1, :] += dHin[t, :, word_size:] else: dhnp0[0] += np.sum(dHin[t, :, word_size:], 0) assert np.allclose(dh0[0], dhnp0[0])
def test(): a = [1, 4, 9] for x in a: assert np.array_equal(lg.sqrt(x), np.sqrt(x)) return
def run_lstm(batch_size, hidden_size, sentence_length, word_size, timing): start = datetime.datetime.now() WLSTM = np.random.randn(word_size + hidden_size, 4 * hidden_size) / np.sqrt(word_size + hidden_size) xphpb = WLSTM.shape[0] d = hidden_size n = sentence_length b = batch_size dHout = np.random.randn(n, b, d) IFOGf = np.random.randn(n, b, d * 4) C = np.random.randn(n, b, d) Ct = np.random.randn(n, b, d) Hin = np.random.randn(n, b, xphpb) dIFOG = np.zeros((n, b, d * 4)) dIFOGf = np.zeros(IFOGf.shape) dHin = np.zeros(Hin.shape) dC = np.zeros(C.shape) dh0 = np.zeros((1, d)) for t in reversed(range(n)): tanhCt = Ct[t] dIFOGf[t, :, 2 * d:3 * d] = tanhCt * dHout[t] # backprop tanh non-linearity first then continue backprop dC[t] += (1 - tanhCt**2) * (IFOGf[t, :, 2 * d:3 * d] * dHout[t]) if t > 0: dIFOGf[t, :, d:2 * d] = C[t - 1] * dC[t] dC[t - 1] += IFOGf[t, :, d:2 * d] * dC[t] dIFOGf[t, :, :d] = IFOGf[t, :, 3 * d:] * dC[t] dIFOGf[t, :, 3 * d:] = IFOGf[t, :, :d] * dC[t] # backprop activation functions dIFOG[t, :, 3 * d:] = (1 - IFOGf[t, :, 3 * d:]**2) * dIFOGf[t, :, 3 * d:] y = IFOGf[t, :, :3 * d] dIFOG[t, :, :3 * d] = (y * (1.0 - y)) * dIFOGf[t, :, :3 * d] # backprop matrix multiply dHin[t] = dIFOG[t].dot(WLSTM.transpose()) # backprop the identity transforms into Hin if t > 0: dHout[t - 1, :] += dHin[t, :, word_size:] else: dh0[0] += np.sum(dHin[t, :, word_size:], 0) # Do a little sum to synchronize and check for NaNs total = np.sum(dh0) assert not math.isnan(total) stop = datetime.datetime.now() delta = stop - start total = delta.total_seconds() * 1000.0 if timing: print("Elapsed Time: " + str(total) + " ms") return total
def update_parameters(learning_rate, params): for p in params.all(): p.m += p.d * p.d # Calculate sum of gradients # print(learning_rate * dparam) p.v += -(learning_rate * p.d / np.sqrt(p.m + 1e-8))