h_prev2 = np.zeros((len_of_vocab,1))
else:
input = [char_to_idx[c] for c in dataset[start_ptr:start_ptr+time_steps]]
output = [char_to_idx[c] for c in dataset[start_ptr+1:start_ptr+time_steps+1]]
lstm_layer1.zero_grad()
lstm_layer1.set_input(input=input,output=output)
lstm_layer1.forward(h_prev=h_prev1,c_prev=c_prev1)
lstm_layer2.zero_grad()
lstm_layer2.set_input(input=input[::-1],output=output[::-1])
lstm_layer2.forward(h_prev=h_prev2,c_prev=c_prev2)
loss,dout,dbout=blstm_backward(h1=lstm_layer1.hs,w1=lstm_layer1.Wout,h2=lstm_layer2.hs,w2=lstm_layer2.Wout,output=output)
do = [v for v in dout.values()]
dWi1,dWf1,dWz1,dWo1,dWout1,dRi1,dRf1,dRz1,dRo1,dPi1,dPo1,dPf1,dbi1,dbo1,dbf1,dbz1,h_prev1,c_prev1=lstm_layer1.backward(bdout=do)
dWi2,dWf2,dWz2,dWo2,dWout2,dRi2,dRf2,dRz2,dRo2,dPi2,dPo2,dPf2,dbi2,dbo2,dbf2,dbz2,h_prev2,c_prev2=lstm_layer2.backward(bdout=do)
for dparam in[dWi1,dWf1,dWz1,dWo1,dWout1,dRi1,dRf1,dRz1,dRo1,dPi1,dPo1,dPf1,dbi1,dbo1,dbf1,dbz1,\
dWi2,dWf2,dWz2,dWo2,dWout2,dRi2,dRf2,dRz2,dRo2,dPi2,dPo2,dPf2,dbi2,dbo2,dbf2,dbz2,dbout] :
np.clip(dparam,-1,1,out=dparam)
params_of_lstm = [lstm_layer1.Wi,lstm_layer1.Wf,lstm_layer1.Wz,lstm_layer1.Wo,lstm_layer1.Wout,\
lstm_layer1.Ri,lstm_layer1.Rf,lstm_layer1.Rz,lstm_layer1.Ro,lstm_layer1.Pi,lstm_layer1.Po,\
lstm_layer1.Pf,lstm_layer1.bi,lstm_layer1.bo,lstm_layer1.bf,lstm_layer1.bz,\
lstm_layer2.Wi,lstm_layer2.Wf,lstm_layer2.Wz,lstm_layer2.Wo,lstm_layer2.Wout,\
lstm_layer2.Ri,lstm_layer2.Rf,lstm_layer2.Rz,lstm_layer2.Ro,lstm_layer2.Pi,lstm_layer2.Po,\
lstm_layer2.Pf,lstm_layer2.bi,lstm_layer2.bo,lstm_layer2.bf,lstm_layer2.bz,bout]
dparams_of_lstm = [dWi1,dWf1,dWz1,dWo1,dWout1,dRi1,dRf1,dRz1,dRo1,dPi1,dPo1,dPf1,dbi1,dbo1,dbf1,dbz1,\
dWi2,dWf2,dWz2,dWo2,dWout2,dRi2,dRf2,dRz2,dRo2,dPi2,dPo2,dPf2,dbi2,dbo2,dbf2,dbz2,dbout]
for params,dparams,mparams in zip(params_of_lstm,dparams_of_lstm,mparams_of_lstm):
mparams += dparams*dparams
if start_ptr + time_steps > len_of_dataset:
start_ptr = 0
h_prev = np.zeros((len_of_vocab, 1))
else:
input = [
char_to_idx[c] for c in dataset[start_ptr:start_ptr + time_steps]
]
output = [
char_to_idx[c]
for c in dataset[start_ptr + 1:start_ptr + time_steps + 1]
]
lstm.zero_grad()
lstm.set_input(input=input, output=output)
lstm.forward(h_prev=h_prev, c_prev=c_prev)
loss, dWi, dWf, dWz, dWo, dWout, dRi, dRf, dRz, dRo, dPi, dPo, dPf, dbi, dbo, dbf, dbz, dbout, h_prev, c_prev = lstm.backward(
)
lstm.clip_grad(clip_val=1)
lstm.step()
smooth_loss = (0.999 * smooth_loss) + (0.001 * loss)
x.append(n)
y.append(smooth_loss)
if n % 1000 == 0:
print('--------------------------------------------')
print('iter:', n)
print('smooth_loss:', smooth_loss)
lstm.sample(h_prev=h_prev, c_prev=c_prev, num_char=300)
print('--------------------------------------------')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.plot(x, y, color='r')
class TestLSTM(unittest.TestCase):
def setUp(self):
Wx = np.array([[
9.72009451e-01, -4.97642862e-01, 6.45448952e-01, 8.10387855e-01,
1.13757673e+00, -5.27114694e-01, -9.08624540e-01, 1.61896844e+00,
-1.16690977e+00, 3.93476226e-01, -6.04018422e-01, 5.67830817e-01
],
[
6.68131790e-01, 6.40157016e-01, 6.90200961e-01,
-1.39750585e+00, -4.89624070e-01, 8.99789953e-01,
3.97067428e-04, 1.47459503e+00, -4.95030269e-01,
-9.22541855e-01, -1.57352198e-01, -1.67160494e+00
],
[
6.93508859e-01, -9.23177216e-01, -4.83511551e-01,
-1.18675890e+00, -7.35505045e-01, -1.61403611e+00,
-2.76067694e-02, -2.48294747e-01, 1.14474446e+00,
1.86354309e-01, -1.73018002e+00, -4.82520536e-03
]])
Wh = np.array([[
-0.88480318, 1.09509583, 0.55657863, -0.35096014, 0.18572107,
0.83823659, -0.44063768, -0.80897913, 0.35752315, 1.65812611,
1.40425671, 1.52519905
],
[
-0.22279229, 1.16363656, -0.47632291, -0.16436909,
-2.16120359, 0.28362134, 0.01817155, 0.04836914,
-0.30831619, -2.0992645, -0.07302497, -0.72868125
],
[
-1.40551611, 2.12755955, 1.76232202, 2.15703084,
-1.87387492, 1.22755896, -0.84271588, 1.07860737,
-0.35473314, -0.86293879, 1.67287773, 0.41575087
]])
b = np.array([
0.46861655, 0.15954682, 0.38782221, 1.00791178, -0.38322573,
0.83138721, 0.98675017, -0.83388618, 1.14392808, 0.37846653,
0.47617248, -1.8035631
])
self.lstm = LSTM(Wx, Wh, b)
self.x = np.array([[0.32434964, -1.25234162, -1.11446048],
[-1.00333386, -0.28908588, 0.80944727],
[-1.12554591, 0.00812256, -0.17046584],
[-0.32813507, 0.16989344, 0.53355169],
[-0.4869515, 0.1677294, -0.12831696],
[0.05832905, -0.32864825, -0.80026008],
[-0.02647765, -0.7902213, 0.15894066],
[0.6531385, 0.89562313, 0.29789637],
[2.10234369, -0.29459709, 0.10299345],
[-0.27252854, 2.00320793, 0.45730897],
[1.5004658, 1.64225418, 1.18133581],
[-0.24243905, -0.43939872, -0.13251067]])
self.h_prev = np.array([[-0.10935227, -0.44314754, 1.08169975],
[-1.21809561, -0.71525211, -1.02789959],
[-0.95964908, -0.51143773, 0.21415223],
[1.67953394, -0.35025546, -0.38320191],
[0.44725308, -1.12955008, 1.75971301],
[0.28200635, -0.0879721, 0.08046973],
[-0.57840066, -0.05648079, 0.73577611],
[-0.95991989, -0.42672155, 0.87126791],
[-0.61249768, -1.27271833, -1.17150149],
[0.62112557, 0.01982296, 1.98008052],
[-0.47977218, -0.24977873, -0.56312278],
[0.479705, 0.34781637, -0.07003237]])
self.c_prev = np.array([[1.31482411, -0.77629214, 0.58700115],
[-1.06531026, -0.87934991, 0.62284509],
[-0.60934138, -0.99160926, -0.42251352],
[-0.23641619, 0.46826475, -0.02462736],
[-0.12147368, -1.00207905, -0.12701465],
[1.07820838, 0.51889022, -1.68135737],
[2.21903061, -2.61861404, 1.16692287],
[-1.49690178, 0.29867214, -0.85135438],
[1.59958995, -0.86312955, 1.00291727],
[-0.61192179, -0.29602751, 0.78903217],
[-0.04484411, -0.13443395, -0.6548083],
[-0.9392424, -1.23116816, 1.26848987]])
def test_forward(self):
c_next, h_next = self.lstm.forward(self.x, self.h_prev, self.c_prev)
assert_almost_equal(
np.array([[0.59989084, -0.56519862, 0.98400225],
[-1.91850874, 0.02352198, -0.93321127],
[0.45227288, -0.54703237, 0.48403159],
[-0.73615581, 0.32063485, 0.7548738],
[0.42491665, -1.47394193, 0.67068475],
[1.05754984, 0.49689304, -0.61805755],
[1.49972308, -1.83582377, 1.00702007],
[-0.74679837, -0.61435311, -0.45936825],
[1.99958655, 0.77989824, 0.03898481],
[0.18246256, -1.25613678, 1.4088285],
[-0.60443486, 0.78677923, -0.91444947],
[0.27892748, -0.94084206, 1.31430749]]), c_next)
assert_almost_equal(
np.array([[4.19169498e-01, -5.03430077e-01, 5.64198599e-01],
[-6.54667825e-01, 5.95397055e-04, -1.84025417e-02],
[1.30830713e-01, -3.09895913e-01, 1.37309301e-02],
[-6.16008036e-01, 2.51996674e-01, 3.79084583e-01],
[3.33833747e-01, -8.91544720e-01, 2.74736749e-01],
[5.94326869e-01, 4.21936364e-01, -1.83699519e-01],
[3.74427572e-01, -6.48435544e-01, 2.00609507e-01],
[-1.07885596e-01, -2.15141706e-01, -1.01425325e-02],
[9.49238782e-01, 1.64936528e-02, 1.37032403e-02],
[1.78080877e-02, -8.29836071e-01, 2.47025891e-02],
[-2.55087741e-01, 8.53326914e-03, -8.05473776e-03],
[1.86996338e-01, -5.96078729e-01, 2.76307207e-01]]),
h_next)
def test_backward(self):
dc_next, dh_next = self.lstm.forward(self.x, self.h_prev, self.c_prev)
dx, dh_prev, dc_prev = self.lstm.backward(dc_next, dh_next)
assert_almost_equal(
np.array([[-6.79930933e-01, -1.97518363e-01, 4.16860765e-01],
[1.55034826e-01, -2.91419470e-01, 1.71192005e-01],
[-3.70136768e-02, 1.71824890e-02, -3.38419828e-01],
[-3.05073579e-01, 2.41424747e-01, -1.93659638e-01],
[1.54819264e-01, 2.30288841e-01, 2.25717407e-02],
[5.45401813e-01, 4.42172325e-02, -1.86619072e-01],
[-3.13126336e-04, 3.56652426e-01, -1.26116627e+00],
[-3.60309927e-02, 5.69443437e-03, 1.46531343e-02],
[1.11857111e-01, -4.33420556e-01, -3.87583328e-01],
[5.95012499e-02, -6.50019034e-03, -5.01780050e-02],
[-6.99925197e-02, -8.10853281e-02, 1.01525160e-02],
[-8.53223995e-02, -2.06294786e-01, -7.03297008e-01]]),
dx)
assert_almost_equal(
np.array([[0.34687254, 0.38344684, 0.37401971],
[0.64098696, -0.85045992, -0.5425016],
[0.26924398, 0.14994028, 0.69949726],
[0.35040516, -0.01098271, -0.15074927],
[-0.02868001, 0.07799114, 0.39807372],
[0.18752975, -0.67098506, -0.33199641],
[1.6726196, -0.19763998, 1.64359161],
[0.15319373, 0.05884558, 0.22297725],
[-0.13909709, -0.11303457, 0.49671851],
[0.05830494, -0.03137801, 0.10292926],
[0.11062537, -0.18552512, -0.17265588],
[0.94321997, 0.11497126, 1.47405578]]), dh_prev)
assert_almost_equal(
np.array([[7.84603591e-02, -7.93934519e-01, 8.02468649e-01],
[-7.07422789e-01, 1.15401664e-05, -1.38975207e-03],
[1.18023605e-01, -2.39578791e-01, 1.17225299e-02],
[-4.65581356e-01, 3.09765349e-01, 3.95637732e-01],
[4.36409729e-02, -1.13529451e+00, 4.74057958e-01],
[3.21235172e-01, 5.77017648e-01, -2.34804219e-01],
[1.85468079e-01, -4.58247852e-01, 2.10825777e-01],
[-1.53895902e-01, -2.33683523e-01, -1.75307671e-02],
[1.08007772e+00, 8.27308200e-05, 1.16063117e-02],
[6.54400485e-03, -1.16746148e+00, 3.28679438e-02],
[-4.54490114e-01, 9.55363471e-04, -8.84788951e-03],
[1.32898508e-01, -6.72948492e-01, 1.87992760e-01]]),
dc_prev)
def test_grads_diff(self):
_before_Wx_grads, _before_Wh_grads, _before_b_grads = self.lstm.grads
before_Wx_grads = copy.copy(_before_Wx_grads)
before_Wh_grads = copy.copy(_before_Wh_grads)
before_b_grads = copy.copy(_before_b_grads)
dc_next, dh_next = self.lstm.forward(self.x, self.h_prev, self.c_prev)
self.lstm.backward(dc_next, dh_next)
after_Wx_grads, after_Wh_grads, after_b_grads = self.lstm.grads
Wx_grads = before_Wx_grads == after_Wx_grads
Wh_grads = before_Wh_grads == after_Wh_grads
b_grads = before_b_grads == after_b_grads
assert_array_equal(
np.array([[
False, False, False, False, False, False, False, False, False,
False, False, False
],
[
False, False, False, False, False, False, False,
False, False, False, False, False
],
[
False, False, False, False, False, False, False,
False, False, False, False, False
]]), Wx_grads)
assert_array_equal(
np.array([[
False, False, False, False, False, False, False, False, False,
False, False, False
],
[
False, False, False, False, False, False, False,
False, False, False, False, False
],
[
False, False, False, False, False, False, False,
False, False, False, False, False
]]), Wh_grads)
assert_array_equal(
np.array([
False, False, False, False, False, False, False, False, False,
False, False, False
]), b_grads)
class Strategy:
"""
Contains the lstm, financial portfolios
"""
def __init__(self, data, dates, targets, first_ret_idx):
"""
Retrieves data and preprocess them (better for deep learning)
:param data: ndarray, all raw data
:param dates: ndarray, all dates
:param targets: ndarray, all raw targets
:param first_ret_idx: long, index of first column of returns in the data
"""
self.targets = (targets - np.mean(targets, 0)[None, :]) / np.var(targets, 0)[None, :]**(1/2)
self.data = (data - np.mean(data, 0)[None, :]) / np.var(data, 0)[None, :]**(1/2)
self.all_returns = data[:, first_ret_idx:first_ret_idx + constants.FC_OUTPUT_NEURONS]
self.dates = dates
self.bmk = None
self.ptf = None
self.lstm = None
def create_neural_network(self, act_fun_lay, act_fun_pos):
"""
Creates 2 hidden fully connected neural network with chosen last activation function
:param act_fun_lay: layer, activation function as layer
:param act_fun_pos: activation function position
:return: NNetwork, neural network created
"""
nn = NNetwork()
curt_lay = layer.FCLayer(self.data.shape[1] + self.targets.shape[1], constants.FC_1_NEURONS,
constants.EDGE_GRADIENT, True)
nn.add_layer(curt_lay, constants.FC_1_POS)
nn.add_layer(layer.ReLULayer(), constants.RELU_1_POS)
curt_lay = layer.FCLayer(constants.FC_1_NEURONS, constants.FC_2_NEURONS, constants.EDGE_GRADIENT, True)
nn.add_layer(curt_lay, constants.FC_2_POS)
nn.add_layer(layer.ReLULayer(), constants.RELU_2_POS)
curt_lay = layer.FCLayer(constants.FC_2_NEURONS, constants.FC_OUTPUT_NEURONS, constants.EDGE_GRADIENT, True)
nn.add_layer(curt_lay, constants.FC_OUTPUT_POS)
nn.add_layer(act_fun_lay, act_fun_pos)
return nn
def create_lstm(self):
"""
Creates all neural networks and the vanilla lstm
"""
nn_f = self.create_neural_network(layer.SigmoidLayer(), constants.SIG_F_POS)
nn_i = self.create_neural_network(layer.SigmoidLayer(), constants.SIG_I_POS)
nn_c = self.create_neural_network(layer.TanhLayer(), constants.TANH_C_POS)
nn_o = self.create_neural_network(layer.SigmoidLayer(), constants.SIG_O_POS)
self.lstm = LSTM(nn_f, nn_i, nn_c, nn_o, constants.TAU_QUANTILE, constants.LEARNING_RATE)
def create_portfolio(self):
"""
Creates portfolio with weights depending on lstm outputs
"""
self.ptf = Portfolio(constants.FC_OUTPUT_NEURONS, constants.TRANSACTION_FEE_RATE, constants.INITIAL_PTF_VALUE
, name='portfolio')
self.ptf.update_weights(1 / len(self.ptf.weights) * np.ones([len(self.ptf.weights), 1]))
def create_benchmark(self):
"""
Creates equal weights benchmark portfolio
"""
self.bmk = Portfolio(constants.FC_OUTPUT_NEURONS, constants.TRANSACTION_FEE_RATE, constants.INITIAL_PTF_VALUE
, name='benchmark')
self.bmk.update_weights(1/len(self.bmk.weights) * np.ones([len(self.bmk.weights), 1]))
def train(self, size_train=0.7):
"""
Trains the lstm vanilla using the data
Target is Sharpe ratios for all assets
Tau Quantile loss
Batch gradient descent
:param size_train: float, proportion of the data used for training
"""
h_prev = np.zeros((constants.FC_OUTPUT_NEURONS, 1))
c_prev = np.zeros((constants.FC_OUTPUT_NEURONS, 1))
curt_batch_size = 1
intermediate_values = []
out_data = np.zeros((constants.BATCH_SIZE, constants.FC_OUTPUT_NEURONS))
for curt_index in range(0, int(math.floor(size_train * self.data.shape[0])), 1):
in_data = self.data[curt_index, :].reshape(self.data.shape[1], 1)
h_prev, c_prev = self.lstm.forward(h_prev, c_prev, in_data)
out_data[curt_batch_size-1, :] = h_prev.reshape((constants.FC_OUTPUT_NEURONS, ))
intermediate_values.append(self.lstm.get_intermediate_values())
if curt_batch_size == constants.BATCH_SIZE:
self.lstm.backward(out_data,
intermediate_values,
self.targets[curt_index-constants.BATCH_SIZE+1:curt_index+1])
self.lstm.update_param()
curt_batch_size = 0
intermediate_values = []
out_data = np.zeros((constants.BATCH_SIZE, constants.FC_OUTPUT_NEURONS))
print(curt_index)
curt_batch_size += 1
def test(self, size_test=0.3, is_nn_to_train=False):
"""
Lstm predicts Sharpes ratios, then there is a portfolio management using those predictions with bigger weights
on better Sharpe ratios
:param size_test: float, proportion of the data used for testing
:param is_nn_to_train: boolean, true if lstm already trained
"""
if not is_nn_to_train:
h_prev = np.zeros((constants.FC_OUTPUT_NEURONS, 1))
c_prev = np.zeros((constants.FC_OUTPUT_NEURONS, 1))
curt_index = int(math.floor((1-size_test)*self.data.shape[0]))
bmk_wgts = 1 / len(self.bmk.weights) * np.ones([len(self.bmk.weights), 1])
for curt_index in range(curt_index,
self.data.shape[0]-constants.NBR_MINUTES_STEP,
constants.NBR_MINUTES_STEP):
in_data = self.data[curt_index, :].reshape(self.data.shape[1], 1)
h_prev, c_prev = self.lstm.forward(h_prev, c_prev, in_data)
temp_ret = self.all_returns[curt_index + constants.NBR_MINUTES_STEP].reshape(constants.FC_OUTPUT_NEURONS
, 1)
self.ptf.compute_return(temp_ret)
self.ptf.compute_value()
self.ptf.update_weights_inv_rdt(temp_ret)
self.ptf.update_weights_inv_val(h_prev)
self.ptf.compute_transaction_fees()
self.ptf.update_val_list()
self.ptf.update_time()
self.bmk.compute_return(temp_ret)
self.bmk.compute_value()
self.bmk.update_weights_inv_rdt(temp_ret)
self.bmk.update_weights_inv_val(bmk_wgts)
self.bmk.compute_transaction_fees()
self.bmk.update_val_list()
self.bmk.update_time()
def test_LSTM():
T = 5
batch_size = 2
nstates = 5
input_size = 4
unit = LSTM(input_size, nstates)
W = unit.get_weights()
X = np.random.randn(T, input_size, batch_size)
unit.forget()
acc_Y = unit.forward(X)
wrand = np.random.randn(*acc_Y.shape)
loss = np.sum(acc_Y * wrand)
dY = wrand
dX = unit.backward(dY)
dW = unit.get_grads()
unit.forget()
def fwd():
unit.set_weights(W)
h = unit.forward(X)
unit.forget()
return np.sum(h * wrand)
delta = 1e-4
error_threshold = 1e-3
all_values = [X, W]
backpropagated_gradients = [dX, dW]
names = ['X', 'W']
error_count = 0
for v in range(len(names)):
values = all_values[v]
dvalues = backpropagated_gradients[v]
name = names[v]
for i in range(values.size):
actual = values.flat[i]
values.flat[i] = actual + delta
loss_minus = fwd()
values.flat[i] = actual - delta
loss_plus = fwd()
values.flat[i] = actual
backpropagated_gradient = dvalues.flat[i]
numerical_gradient = (loss_minus - loss_plus) / (2 * delta)
if numerical_gradient == 0 and backpropagated_gradient == 0:
error = 0
elif abs(numerical_gradient) < 1e-7 and abs(
backpropagated_gradient) < 1e-7:
error = 0
else:
error = abs(backpropagated_gradient -
numerical_gradient) / abs(numerical_gradient +
backpropagated_gradient)
if error > error_threshold:
print 'FAILURE!!!\n'
print '\tparameter: ', name, '\tindex: ', np.unravel_index(
i, values.shape)
print '\tvalues: ', actual
print '\tbackpropagated_gradient: ', backpropagated_gradient
print '\tnumerical_gradient', numerical_gradient
print '\terror: ', error
print '\n\n'
error_count += 1
if error_count == 0:
print 'LSTM Gradient Check Passed'
else:
print 'Failed for {} parameters'.format(error_count)
def test_LSTM():
T = 5
batch_size = 2
nstates = 5
input_size = 4
unit = LSTM(input_size, nstates)
W = unit.get_weights()
X = np.random.randn(T, input_size, batch_size)
unit.forget()
acc_Y = unit.forward(X)
wrand = np.random.randn(*acc_Y.shape)
loss = np.sum(acc_Y * wrand)
dY = wrand
dX = unit.backward(dY)
dW = unit.get_grads()
unit.forget()
def fwd():
unit.set_weights(W)
h = unit.forward(X)
unit.forget()
return np.sum(h * wrand)
delta = 1e-4
error_threshold = 1e-3
all_values = [X, W]
backpropagated_gradients = [dX, dW]
names = ['X', 'W']
error_count = 0
for v in range(len(names)):
values = all_values[v]
dvalues = backpropagated_gradients[v]
name = names[v]
for i in range(values.size):
actual = values.flat[i]
values.flat[i] = actual + delta
loss_minus = fwd()
values.flat[i] = actual - delta
loss_plus = fwd()
values.flat[i] = actual
backpropagated_gradient = dvalues.flat[i]
numerical_gradient = (loss_minus - loss_plus) / (2 * delta)
if numerical_gradient == 0 and backpropagated_gradient == 0:
error = 0
elif abs(numerical_gradient) < 1e-7 and abs(backpropagated_gradient) < 1e-7:
error = 0
else:
error = abs(backpropagated_gradient - numerical_gradient) / abs(numerical_gradient + backpropagated_gradient)
if error > error_threshold:
print 'FAILURE!!!\n'
print '\tparameter: ', name, '\tindex: ', np.unravel_index(i, values.shape)
print '\tvalues: ', actual
print '\tbackpropagated_gradient: ', backpropagated_gradient
print '\tnumerical_gradient', numerical_gradient
print '\terror: ', error
print '\n\n'
error_count += 1
if error_count == 0:
print 'LSTM Gradient Check Passed'
else:
print 'Failed for {} parameters'.format(error_count)
c_prev_fg = np.copy(lstm_fg.cs[time_steps - 1])
lstm_og.zero_grad()
lstm_og.set_input(input=input, output=output)
lstm_og.forward(h_prev=h_prev_exp, c_prev=c_prev_og)
c_prev_og = np.copy(lstm_og.cs[time_steps - 1])
loss, h_prev_exp, c_prev_exp, dig_out, dim_out, dft_out, dot_out, dWout, dbout = explstm_forward(
h_prev=h_prev_exp,
c_prev=c_prev_exp,
ig=lstm_ig,
im=lstm_im,
og=lstm_og,
fg=lstm_fg,
output=output)
lstm_ig.backward(bdout=dig_out)
lstm_im.backward(bdout=dim_out)
lstm_fg.backward(bdout=dft_out)
lstm_og.backward(bdout=dot_out)
params_of_lstm = [lstm_ig.Wi,lstm_ig.Wf,lstm_ig.Wz,lstm_ig.Wo,lstm_ig.Wout,\
lstm_ig.Ri,lstm_ig.Rf,lstm_ig.Rz,lstm_ig.Ro,lstm_ig.Pi,lstm_ig.Po,\
lstm_ig.Pf,lstm_ig.bi,lstm_ig.bo,lstm_ig.bf,lstm_ig.bz,\
lstm_og.Wi,lstm_og.Wf,lstm_og.Wz,lstm_og.Wo,lstm_og.Wout,\
lstm_og.Ri,lstm_og.Rf,lstm_og.Rz,lstm_og.Ro,lstm_og.Pi,lstm_og.Po,\
lstm_og.Pf,lstm_og.bi,lstm_og.bo,lstm_og.bf,lstm_og.bz,\
lstm_fg.Wi,lstm_fg.Wf,lstm_fg.Wz,lstm_fg.Wo,lstm_fg.Wout,\
lstm_fg.Ri,lstm_fg.Rf,lstm_fg.Rz,lstm_fg.Ro,lstm_fg.Pi,lstm_fg.Po,\
lstm_fg.Pf,lstm_fg.bi,lstm_fg.bo,lstm_fg.bf,lstm_fg.bz,\
lstm_im.Wi,lstm_im.Wf,lstm_im.Wz,lstm_im.Wo,lstm_im.Wout,\
lstm_im.Ri,lstm_im.Rf,lstm_im.Rz,lstm_im.Ro,lstm_im.Pi,lstm_im.Po,\
