def solve(A, b, conv_iters, max_iters, verbose):
print("Solving system...")
x = np.zeros(A.shape[1])
r = b - A.dot(x)
p = r
rsold = r.dot(r)
converged = -1
# Should always converge in fewer iterations than this
max_iters = (min(max_iters, b.shape[0])
if max_iters is not None else b.shape[0])
for i in range(max_iters):
Ap = A.dot(p)
alpha = rsold / (p.dot(Ap))
x = x + alpha * p
r = r - alpha * Ap
rsnew = r.dot(r)
# We only do the convergence test every conv_iters or on the last
# iteration
if (i % conv_iters == 0
or i == (max_iters - 1)) and np.sqrt(rsnew) < 1e-10:
converged = i
break
if verbose:
print("Residual: " + str(rsnew))
beta = rsnew / rsold
p = r + beta * p
rsold = rsnew
if converged < 0:
print("Convergence FAILURE!")
else:
print("Converged in %d iterations" % (converged))
return x
def test():
npa = np.array([1, 4, 9])
a = lg.array(npa)
assert np.array_equal(lg.sqrt(a), np.sqrt(npa))
npa = np.array([1, 4, 9], dtype=np.float)
a = lg.array(npa)
assert np.array_equal(lg.sqrt(a), np.sqrt(npa))
npa = np.array([1, 4, 9], dtype=np.float32)
a = lg.array(npa)
assert np.array_equal(lg.sqrt(a), np.sqrt(npa))
npa = np.array([1, 4, 9], dtype=np.float64)
a = lg.array(npa)
assert np.array_equal(lg.sqrt(a), np.sqrt(npa))
return
def test():
npa = np.array([1.0, 4.0, 9.0])
a = lg.array(npa)
assert np.array_equal(lg.sqrt(a, out=a), np.sqrt(npa, out=npa))
npa = np.array([1.0, 4.0, 9.0], dtype=np.float)
a = lg.array(npa)
assert np.array_equal(lg.sqrt(a, out=a), np.sqrt(npa, out=npa))
npa = np.array([1.0, 4.0, 9.0], dtype=np.float32)
a = lg.array(npa)
assert np.array_equal(lg.sqrt(a, out=a), np.sqrt(npa, out=npa))
npa = np.array([1.0, 4.0, 9.0], dtype=np.float64)
a = lg.array(npa)
assert np.array_equal(lg.sqrt(a, out=a), np.sqrt(npa, out=npa))
return
def black_scholes(S, X, T, R, V):
sqrt_t = np.sqrt(T)
d1 = np.log(S / X) + (R + 0.5 * V * V) * T / (V * sqrt_t)
d2 = d1 - V * sqrt_t
cnd_d1 = cnd(d1)
cnd_d2 = cnd(d2)
exp_rt = np.exp(-R * T)
call_result = S * cnd_d1 - X * exp_rt * cnd_d2
put_result = X * exp_rt * (1.0 - cnd_d2) - S * (1.0 - cnd_d1)
return call_result, put_result
def test():
x = lg.array([
[1, 2, 3, 4],
[5, 6, 7, 8],
[9, 10, 11, 12],
[13, 14, 15, 16],
[17, 18, 19, 20],
])
x[0, :] = 0
assert lg.array_equal(x[0, :], [0, 0, 0, 0])
y = lg.array([[20, 30, 40, 50], [70, 80, 90, 100]])
x[1:4:2] = y
assert lg.array_equal(x[1, :], [20, 30, 40, 50])
assert lg.array_equal(x[3, :], [70, 80, 90, 100])
input_size = 10
hidden_size = 4
WLSTM = lg.random.randn(input_size + hidden_size + 1, 4 *
hidden_size) / lg.sqrt(input_size + hidden_size)
WLSTM[0, :] = 0 # initialize biases to zero
assert lg.array_equal(
WLSTM[0, :],
[
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
],
)
lg.random.seed(10)
WLSTM = lg.random.randn(15, 16)
dIFOGt = lg.random.randn(3, 16)
dHoutt_in = lg.random.randn(2, 3, 4)
dHoutt = dHoutt_in.copy()
dHint = dIFOGt.dot(WLSTM.transpose())
temp = dHoutt[0, :] + dHint[:, 11:]
dHoutt[0, :] = temp
assert not lg.array_equal(dHoutt[0, :], dHoutt_in[0, :])
return
def run_lstm(batch_size, hidden_size, sentence_length, word_size, timing):
start = datetime.datetime.now()
X = np.random.randn(sentence_length, batch_size, hidden_size)
h0 = np.random.randn(1, hidden_size)
WLSTM = np.random.randn(
word_size + hidden_size, 4 * hidden_size
) / np.sqrt(word_size + hidden_size)
xphpb = WLSTM.shape[0]
d = hidden_size
n = sentence_length
b = batch_size
Hin = np.zeros((n, b, xphpb))
Hout = np.zeros((n, b, d))
IFOG = np.zeros((n, b, d * 4))
IFOGf = np.zeros((n, b, d * 4))
C = np.zeros((n, b, d))
Ct = np.zeros((n, b, d))
for t in range(0, n):
if t == 0:
prev = np.tile(h0, (b, 1))
else:
prev = Hout[t - 1]
Hin[t, :, :word_size] = X[t]
Hin[t, :, word_size:] = prev
# compute all gate activations. dots:
IFOG[t] = Hin[t].dot(WLSTM)
# non-linearities
IFOGf[t, :, : 3 * d] = 1.0 / (
1.0 + np.exp(-IFOG[t, :, : 3 * d])
) # sigmoids these are the gates
IFOGf[t, :, 3 * d :] = np.tanh(IFOG[t, :, 3 * d :]) # tanh
# compute the cell activation
C[t] = IFOGf[t, :, :d] * IFOGf[t, :, 3 * d :]
if t > 0:
C[t] += IFOGf[t, :, d : 2 * d] * C[t - 1]
Ct[t] = np.tanh(C[t])
Hout[t] = IFOGf[t, :, 2 * d : 3 * d] * Ct[t]
# Do a little sum of the outputs to synchronize and check for NaNs
total = np.sum(Hout)
assert not math.isnan(total)
stop = datetime.datetime.now()
delta = stop - start
total = delta.total_seconds() * 1000.0
if timing:
print("Elapsed Time: " + str(total) + " ms")
return total
def test():
xn = np.array(
[[1 + 2j, 3 - 4j, 5 + 6j], [7 - 8j, -9 + 10j, -11 - 12j]], np.complex
)
x = lg.array(xn)
assert lg.all(lg.abs(lg.sin(x) - np.sin(xn)) < 1e-5)
assert lg.all(lg.abs(lg.cos(x) - np.cos(xn)) < 1e-5)
assert lg.all(lg.abs(lg.exp(x) - np.exp(xn)) < 1e-5)
assert lg.all(lg.abs(lg.tanh(x) - np.tanh(xn)) < 1e-5)
assert lg.all(lg.abs(lg.sqrt(x) - np.sqrt(xn)) < 1e-5)
return
def init(input_size, hidden_size, fancy_forget_bias_init=3):
"""
Initialize parameters of the LSTM (both weights and biases in one
matrix). One might way to have a positive fancy_forget_bias_init number
(e.g. maybe even up to 5, in some papers)
"""
# +1 for the biases, which will be the first row of WLSTM
WLSTM = np.random.randn(
input_size + hidden_size + 1,
4 * hidden_size) / np.sqrt(input_size + hidden_size)
WLSTM[0, :] = 0 # initialize biases to zero
if fancy_forget_bias_init != 0:
# forget gates get little bit negative bias initially to encourage
# them to be turned off remember that due to Xavier initialization
# above, the raw output activations from gates before nonlinearity
# are zero mean and on order of standard deviation ~1
WLSTM[0, hidden_size:2 * hidden_size] = fancy_forget_bias_init
return WLSTM
def test():
word_size = 10
hidden_size = 10
sentence_length = 2
batch_size = 3
X = np.random.randn(sentence_length, batch_size, hidden_size)
h0 = np.random.randn(1, hidden_size)
WLSTM = np.random.randn(word_size + hidden_size,
4 * hidden_size) / np.sqrt(word_size + hidden_size)
xphpb = WLSTM.shape[0]
d = hidden_size
n = sentence_length
b = batch_size
Hin = np.zeros((n, b, xphpb))
Hout = np.zeros((n, b, d))
IFOG = np.zeros((n, b, d * 4))
IFOGf = np.zeros((n, b, d * 4))
C = np.zeros((n, b, d))
Ct = np.zeros((n, b, d))
for t in range(0, n):
if t == 0:
prev = np.tile(h0, (b, 1))
else:
prev = Hout[t - 1]
Hin[t, :, :word_size] = X[t]
Hin[t, :, word_size:] = prev
# compute all gate activations. dots:
IFOG[t] = Hin[t].dot(WLSTM)
# non-linearities
IFOGf[t, :, :3 * d] = 1.0 / (1.0 + np.exp(-IFOG[t, :, :3 * d])
) # sigmoids these are the gates
IFOGf[t, :, 3 * d:] = np.tanh(IFOG[t, :, 3 * d:]) # tanh
# compute the cell activation
C[t] = IFOGf[t, :, :d] * IFOGf[t, :, 3 * d:]
if t > 0:
C[t] += IFOGf[t, :, d:2 * d] * C[t - 1]
Ct[t] = np.tanh(C[t])
Hout[t] = IFOGf[t, :, 2 * d:3 * d] * Ct[t]
return
def test():
pythonX = [1, 2, 3]
legate_oldX = lg.array(pythonX)
assert len(legate_oldX) == len(pythonX)
SIZE = 100
legate_oldX = lg.random.random(SIZE)
assert SIZE == len(legate_oldX)
x = lg.array([1, 2, 3, 4])
y = lg.array([1, 2, 3, 4])
z = x + y
print(len(z))
assert len(x) == len(y) == len(z)
x = lg.array(pythonX)
x = lg.sqrt(x)
assert len(x) == len(pythonX)
return
def test():
xn = np.array([[1, 2, 3], [4, 5, 6]])
x = lg.array(xn)
# print(np.sin(xn))
# print(lg.sin(x))
assert np.allclose(np.sin(xn), lg.sin(x))
# print(np.cos(xn))
# print(lg.cos(x))
assert np.allclose(np.cos(xn), lg.cos(x))
# print(np.sqrt(xn))
# print(lg.sqrt(x))
assert np.allclose(np.sqrt(xn), lg.sqrt(x))
# print(np.exp(xn))
# print(lg.exp(x))
assert np.allclose(np.exp(xn), lg.exp(x))
# print(np.log(xn))
# print(lg.log(x))
assert np.allclose(np.log(xn), lg.log(x))
# print(np.absolute(xn))
# print(lg.absolute(x))
assert np.allclose(np.absolute(xn), lg.absolute(x))
y = lg.tanh(x)
yn = np.tanh(xn)
assert np.allclose(y, yn)
y = lg.cos(0.5)
# print(y)
assert np.allclose(y, np.cos(0.5))
y = lg.sqrt(0.5)
# print(y)
assert np.allclose(y, np.sqrt(0.5))
y = lg.sin(0.5)
# print(y)
assert np.allclose(y, np.sin(0.5))
y = lg.exp(2)
# print(y)
assert np.allclose(y, np.exp(2))
y = lg.log(2)
# print(y)
assert np.allclose(y, np.log(2))
y = lg.absolute(-3)
# print(y)
assert y == 3
np.random.seed(42)
an = np.random.randn(1, 3, 16)
bn = 1.0 / (1.0 + np.exp(-an[0, :, :]))
a = lg.array(an)
b = 1.0 / (1.0 + lg.exp(-a[0, :, :]))
assert np.allclose(b, bn)
return
def testtion():
word_size = 10
hidden_size = 10
sentence_length = 5
batch_size = 3
lg.random.seed(42)
WLSTM = lg.random.randn(word_size + hidden_size,
4 * hidden_size) / lg.sqrt(word_size + hidden_size)
xphpb = WLSTM.shape[0]
d = hidden_size
n = sentence_length
b = batch_size
dHout = lg.random.randn(n, b, d)
IFOGf = lg.random.randn(n, b, d * 4)
C = lg.random.randn(n, b, d)
Ct = lg.random.randn(n, b, d)
Hin = lg.random.randn(n, b, xphpb)
dIFOG = lg.zeros((n, b, d * 4))
dIFOGf = lg.zeros(IFOGf.shape)
dHin = lg.zeros(Hin.shape)
dC = lg.zeros(C.shape)
dh0 = lg.zeros((1, d))
for t in reversed(range(n)):
tanhCt = Ct[t]
dIFOGf[t, :, 2 * d:3 * d] = tanhCt * dHout[t]
# backprop tanh non-linearity first then continue backprop
dC[t] += (1 - tanhCt**2) * (IFOGf[t, :, 2 * d:3 * d] * dHout[t])
if t > 0:
dIFOGf[t, :, d:2 * d] = C[t - 1] * dC[t]
dC[t - 1] += IFOGf[t, :, d:2 * d] * dC[t]
dIFOGf[t, :, :d] = IFOGf[t, :, 3 * d:] * dC[t]
dIFOGf[t, :, 3 * d:] = IFOGf[t, :, :d] * dC[t]
# backprop activation functions
dIFOG[t, :,
3 * d:] = (1 - IFOGf[t, :, 3 * d:]**2) * dIFOGf[t, :, 3 * d:]
y = IFOGf[t, :, :3 * d]
dIFOG[t, :, :3 * d] = (y * (1.0 - y)) * dIFOGf[t, :, :3 * d]
# backprop matrix multiply
dHin[t] = dIFOG[t].dot(WLSTM.transpose())
# backprop the identity transforms into Hin
if t > 0:
dHout[t - 1, :] += dHin[t, :, word_size:]
else:
dh0[0] += lg.sum(dHin[t, :, word_size:], 0)
np.random.seed(42)
WLSTM = np.random.randn(word_size + hidden_size,
4 * hidden_size) / np.sqrt(word_size + hidden_size)
xphpb = WLSTM.shape[0]
d = hidden_size
n = sentence_length
b = batch_size
dHout = np.random.randn(n, b, d)
IFOGf = np.random.randn(n, b, d * 4)
C = np.random.randn(n, b, d)
Ct = np.random.randn(n, b, d)
Hin = np.random.randn(n, b, xphpb)
dIFOG = np.zeros((n, b, d * 4))
dIFOGf = np.zeros(IFOGf.shape)
dHin = np.zeros(Hin.shape)
dC = np.zeros(C.shape)
dhnp0 = np.zeros((1, d))
for t in reversed(range(n)):
tanhCt = Ct[t]
dIFOGf[t, :, 2 * d:3 * d] = tanhCt * dHout[t]
# backprop tanh non-linearity first then continue backprop
dC[t] += (1 - tanhCt**2) * (IFOGf[t, :, 2 * d:3 * d] * dHout[t])
if t > 0:
dIFOGf[t, :, d:2 * d] = C[t - 1] * dC[t]
dC[t - 1] += IFOGf[t, :, d:2 * d] * dC[t]
dIFOGf[t, :, :d] = IFOGf[t, :, 3 * d:] * dC[t]
dIFOGf[t, :, 3 * d:] = IFOGf[t, :, :d] * dC[t]
# backprop activation functions
dIFOG[t, :,
3 * d:] = (1 - IFOGf[t, :, 3 * d:]**2) * dIFOGf[t, :, 3 * d:]
y = IFOGf[t, :, :3 * d]
dIFOG[t, :, :3 * d] = (y * (1.0 - y)) * dIFOGf[t, :, :3 * d]
# backprop matrix multiply
dHin[t] = dIFOG[t].dot(WLSTM.transpose())
# backprop the identity transforms into Hin
if t > 0:
dHout[t - 1, :] += dHin[t, :, word_size:]
else:
dhnp0[0] += np.sum(dHin[t, :, word_size:], 0)
assert np.allclose(dh0[0], dhnp0[0])
def test():
a = [1, 4, 9]
for x in a:
assert np.array_equal(lg.sqrt(x), np.sqrt(x))
return
def run_lstm(batch_size, hidden_size, sentence_length, word_size, timing):
start = datetime.datetime.now()
WLSTM = np.random.randn(word_size + hidden_size,
4 * hidden_size) / np.sqrt(word_size + hidden_size)
xphpb = WLSTM.shape[0]
d = hidden_size
n = sentence_length
b = batch_size
dHout = np.random.randn(n, b, d)
IFOGf = np.random.randn(n, b, d * 4)
C = np.random.randn(n, b, d)
Ct = np.random.randn(n, b, d)
Hin = np.random.randn(n, b, xphpb)
dIFOG = np.zeros((n, b, d * 4))
dIFOGf = np.zeros(IFOGf.shape)
dHin = np.zeros(Hin.shape)
dC = np.zeros(C.shape)
dh0 = np.zeros((1, d))
for t in reversed(range(n)):
tanhCt = Ct[t]
dIFOGf[t, :, 2 * d:3 * d] = tanhCt * dHout[t]
# backprop tanh non-linearity first then continue backprop
dC[t] += (1 - tanhCt**2) * (IFOGf[t, :, 2 * d:3 * d] * dHout[t])
if t > 0:
dIFOGf[t, :, d:2 * d] = C[t - 1] * dC[t]
dC[t - 1] += IFOGf[t, :, d:2 * d] * dC[t]
dIFOGf[t, :, :d] = IFOGf[t, :, 3 * d:] * dC[t]
dIFOGf[t, :, 3 * d:] = IFOGf[t, :, :d] * dC[t]
# backprop activation functions
dIFOG[t, :,
3 * d:] = (1 - IFOGf[t, :, 3 * d:]**2) * dIFOGf[t, :, 3 * d:]
y = IFOGf[t, :, :3 * d]
dIFOG[t, :, :3 * d] = (y * (1.0 - y)) * dIFOGf[t, :, :3 * d]
# backprop matrix multiply
dHin[t] = dIFOG[t].dot(WLSTM.transpose())
# backprop the identity transforms into Hin
if t > 0:
dHout[t - 1, :] += dHin[t, :, word_size:]
else:
dh0[0] += np.sum(dHin[t, :, word_size:], 0)
# Do a little sum to synchronize and check for NaNs
total = np.sum(dh0)
assert not math.isnan(total)
stop = datetime.datetime.now()
delta = stop - start
total = delta.total_seconds() * 1000.0
if timing:
print("Elapsed Time: " + str(total) + " ms")
return total
def update_parameters(learning_rate, params):
for p in params.all():
p.m += p.d * p.d # Calculate sum of gradients
# print(learning_rate * dparam)
p.v += -(learning_rate * p.d / np.sqrt(p.m + 1e-8))
