def test():
npa = np.array([-1.0, 0.0, 1.0])
a = lg.array(npa)
np.exp(npa, out=npa)
lg.exp(a, out=a)
assert np.array_equal(a, npa)
return
def test():
x = lg.array([1.0, 2.0, 3.0, 4.0])
y = lg.exp(x)
# print(y)
# print(np.exp([1, 2, 3, 4]))
assert np.allclose(y, np.exp([1, 2, 3, 4]))
return
def forward(x, h_prev, C_prev, H_size, X_size, p):
assert x.shape == (X_size, 1)
assert h_prev.shape == (H_size, 1)
assert C_prev.shape == (H_size, 1)
z = np.row_stack((h_prev, x))
f = sigmoid(np.dot(p.W_f.v, z) + p.b_f.v)
i = sigmoid(np.dot(p.W_i.v, z) + p.b_i.v)
C_bar = tanh(np.dot(p.W_C.v, z) + p.b_C.v)
C = f * C_prev + i * C_bar
o = sigmoid(np.dot(p.W_o.v, z) + p.b_o.v)
h = o * tanh(C)
v = np.dot(p.W_v.v, h) + p.b_v.v
y = np.exp(v) / np.sum(np.exp(v)) # softmax
return z, f, i, C_bar, C, o, h, v, y
def forward(X, WLSTM, c0=None, h0=None):
"""
X should be of shape (n,b,input_size), where n = length of sequence, b
= batch size
"""
n, b, input_size = X.shape
d = int(WLSTM.shape[1] / 4) # hidden size
if c0 is None:
c0 = np.zeros((b, d))
if h0 is None:
h0 = np.zeros((b, d))
# Perform the LSTM forward pass with X as the input
xphpb = WLSTM.shape[0] # x plus h plus bias, lol
Hin = np.zeros(
(n, b, xphpb)) # input [1, xt, ht-1] to each tick of the LSTM
Hout = np.zeros(
(n, b,
d)) # hidden representation of the LSTM (gated cell content)
IFOG = np.zeros((n, b, d * 4)) # input, forget, output, gate (IFOG)
IFOGf = np.zeros((n, b, d * 4)) # after nonlinearity
C = np.zeros((n, b, d)) # cell content
Ct = np.zeros((n, b, d)) # tanh of cell content
for t in range(n):
# concat [x,h] as input to the LSTM
prevh = Hout[t - 1] if t > 0 else h0
Hin[t, :, 0] = 1 # bias
Hin[t, :, 1:input_size + 1] = X[t]
Hin[t, :, input_size + 1:] = prevh
# compute all gate activations. dots: (most work is this line)
IFOG[t] = Hin[t].dot(WLSTM)
# non-linearities
IFOGf[t, :, :3 * d] = 1.0 / (1.0 + np.exp(-IFOG[t, :, :3 * d])
) # sigmoids; these are the gates
IFOGf[t, :, 3 * d:] = np.tanh(IFOG[t, :, 3 * d:]) # tanh
# compute the cell activation
prevc = C[t - 1] if t > 0 else c0
C[t] = (IFOGf[t, :, :d] * IFOGf[t, :, 3 * d:] +
IFOGf[t, :, d:2 * d] * prevc)
Ct[t] = np.tanh(C[t])
Hout[t] = IFOGf[t, :, 2 * d:3 * d] * Ct[t]
cache = {}
cache["WLSTM"] = WLSTM
cache["Hout"] = Hout
cache["IFOGf"] = IFOGf
cache["IFOG"] = IFOG
cache["C"] = C
cache["Ct"] = Ct
cache["Hin"] = Hin
cache["c0"] = c0
cache["h0"] = h0
# return C[t], as well so we can continue LSTM with prev state
# init if needed
return Hout, C[t], Hout[t], cache
def black_scholes(S, X, T, R, V):
sqrt_t = np.sqrt(T)
d1 = np.log(S / X) + (R + 0.5 * V * V) * T / (V * sqrt_t)
d2 = d1 - V * sqrt_t
cnd_d1 = cnd(d1)
cnd_d2 = cnd(d2)
exp_rt = np.exp(-R * T)
call_result = S * cnd_d1 - X * exp_rt * cnd_d2
put_result = X * exp_rt * (1.0 - cnd_d2) - S * (1.0 - cnd_d1)
return call_result, put_result
def run_lstm(batch_size, hidden_size, sentence_length, word_size, timing):
start = datetime.datetime.now()
X = np.random.randn(sentence_length, batch_size, hidden_size)
h0 = np.random.randn(1, hidden_size)
WLSTM = np.random.randn(
word_size + hidden_size, 4 * hidden_size
) / np.sqrt(word_size + hidden_size)
xphpb = WLSTM.shape[0]
d = hidden_size
n = sentence_length
b = batch_size
Hin = np.zeros((n, b, xphpb))
Hout = np.zeros((n, b, d))
IFOG = np.zeros((n, b, d * 4))
IFOGf = np.zeros((n, b, d * 4))
C = np.zeros((n, b, d))
Ct = np.zeros((n, b, d))
for t in range(0, n):
if t == 0:
prev = np.tile(h0, (b, 1))
else:
prev = Hout[t - 1]
Hin[t, :, :word_size] = X[t]
Hin[t, :, word_size:] = prev
# compute all gate activations. dots:
IFOG[t] = Hin[t].dot(WLSTM)
# non-linearities
IFOGf[t, :, : 3 * d] = 1.0 / (
1.0 + np.exp(-IFOG[t, :, : 3 * d])
) # sigmoids these are the gates
IFOGf[t, :, 3 * d :] = np.tanh(IFOG[t, :, 3 * d :]) # tanh
# compute the cell activation
C[t] = IFOGf[t, :, :d] * IFOGf[t, :, 3 * d :]
if t > 0:
C[t] += IFOGf[t, :, d : 2 * d] * C[t - 1]
Ct[t] = np.tanh(C[t])
Hout[t] = IFOGf[t, :, 2 * d : 3 * d] * Ct[t]
# Do a little sum of the outputs to synchronize and check for NaNs
total = np.sum(Hout)
assert not math.isnan(total)
stop = datetime.datetime.now()
delta = stop - start
total = delta.total_seconds() * 1000.0
if timing:
print("Elapsed Time: " + str(total) + " ms")
return total
def test():
xn = np.array(
[[1 + 2j, 3 - 4j, 5 + 6j], [7 - 8j, -9 + 10j, -11 - 12j]], np.complex
)
x = lg.array(xn)
assert lg.all(lg.abs(lg.sin(x) - np.sin(xn)) < 1e-5)
assert lg.all(lg.abs(lg.cos(x) - np.cos(xn)) < 1e-5)
assert lg.all(lg.abs(lg.exp(x) - np.exp(xn)) < 1e-5)
assert lg.all(lg.abs(lg.tanh(x) - np.tanh(xn)) < 1e-5)
assert lg.all(lg.abs(lg.sqrt(x) - np.sqrt(xn)) < 1e-5)
return
def cnd(d):
A1 = 0.31938153
A2 = -0.356563782
A3 = 1.781477937
A4 = -1.821255978
A5 = 1.330274429
RSQRT2PI = 0.39894228040143267793994605993438
K = 1.0 / (1.0 + 0.2316419 * np.absolute(d))
cnd = (RSQRT2PI * np.exp(-0.5 * d * d) *
(K * (A1 + K * (A2 + K * (A3 + K * (A4 + K * A5))))))
return np.where(d > 0, 1.0 - cnd, cnd)
def test():
word_size = 10
hidden_size = 10
sentence_length = 2
batch_size = 3
X = np.random.randn(sentence_length, batch_size, hidden_size)
h0 = np.random.randn(1, hidden_size)
WLSTM = np.random.randn(word_size + hidden_size,
4 * hidden_size) / np.sqrt(word_size + hidden_size)
xphpb = WLSTM.shape[0]
d = hidden_size
n = sentence_length
b = batch_size
Hin = np.zeros((n, b, xphpb))
Hout = np.zeros((n, b, d))
IFOG = np.zeros((n, b, d * 4))
IFOGf = np.zeros((n, b, d * 4))
C = np.zeros((n, b, d))
Ct = np.zeros((n, b, d))
for t in range(0, n):
if t == 0:
prev = np.tile(h0, (b, 1))
else:
prev = Hout[t - 1]
Hin[t, :, :word_size] = X[t]
Hin[t, :, word_size:] = prev
# compute all gate activations. dots:
IFOG[t] = Hin[t].dot(WLSTM)
# non-linearities
IFOGf[t, :, :3 * d] = 1.0 / (1.0 + np.exp(-IFOG[t, :, :3 * d])
) # sigmoids these are the gates
IFOGf[t, :, 3 * d:] = np.tanh(IFOG[t, :, 3 * d:]) # tanh
# compute the cell activation
C[t] = IFOGf[t, :, :d] * IFOGf[t, :, 3 * d:]
if t > 0:
C[t] += IFOGf[t, :, d:2 * d] * C[t - 1]
Ct[t] = np.tanh(C[t])
Hout[t] = IFOGf[t, :, 2 * d:3 * d] * Ct[t]
return
def test():
xn = np.array([[1, 2, 3], [4, 5, 6]])
x = lg.array(xn)
# print(np.sin(xn))
# print(lg.sin(x))
assert np.allclose(np.sin(xn), lg.sin(x))
# print(np.cos(xn))
# print(lg.cos(x))
assert np.allclose(np.cos(xn), lg.cos(x))
# print(np.sqrt(xn))
# print(lg.sqrt(x))
assert np.allclose(np.sqrt(xn), lg.sqrt(x))
# print(np.exp(xn))
# print(lg.exp(x))
assert np.allclose(np.exp(xn), lg.exp(x))
# print(np.log(xn))
# print(lg.log(x))
assert np.allclose(np.log(xn), lg.log(x))
# print(np.absolute(xn))
# print(lg.absolute(x))
assert np.allclose(np.absolute(xn), lg.absolute(x))
y = lg.tanh(x)
yn = np.tanh(xn)
assert np.allclose(y, yn)
y = lg.cos(0.5)
# print(y)
assert np.allclose(y, np.cos(0.5))
y = lg.sqrt(0.5)
# print(y)
assert np.allclose(y, np.sqrt(0.5))
y = lg.sin(0.5)
# print(y)
assert np.allclose(y, np.sin(0.5))
y = lg.exp(2)
# print(y)
assert np.allclose(y, np.exp(2))
y = lg.log(2)
# print(y)
assert np.allclose(y, np.log(2))
y = lg.absolute(-3)
# print(y)
assert y == 3
np.random.seed(42)
an = np.random.randn(1, 3, 16)
bn = 1.0 / (1.0 + np.exp(-an[0, :, :]))
a = lg.array(an)
b = 1.0 / (1.0 + lg.exp(-a[0, :, :]))
assert np.allclose(b, bn)
return
def log_likelihood(features, target, weights):
scores = np.dot(features, weights)
return np.sum(target * scores - np.log(1.0 + np.exp(scores)))
def sigmoid(x):
return 1.0 / (1.0 + np.exp(-x))
def test():
a = [-1.0, 0.0, 1.0]
assert np.array_equal(lg.exp(a), np.exp(a))
return
def sigmoid(x):
return 1 / (1 + np.exp(-x))
def test():
a = [1, np.e, np.e**2, 0]
for x in a:
assert np.array_equal(lg.exp(x), np.exp(x))
return
