h_prev2 = np.zeros((len_of_vocab,1)) else: input = [char_to_idx[c] for c in dataset[start_ptr:start_ptr+time_steps]] output = [char_to_idx[c] for c in dataset[start_ptr+1:start_ptr+time_steps+1]] lstm_layer1.zero_grad() lstm_layer1.set_input(input=input,output=output) lstm_layer1.forward(h_prev=h_prev1,c_prev=c_prev1) lstm_layer2.zero_grad() lstm_layer2.set_input(input=input[::-1],output=output[::-1]) lstm_layer2.forward(h_prev=h_prev2,c_prev=c_prev2) loss,dout,dbout=blstm_backward(h1=lstm_layer1.hs,w1=lstm_layer1.Wout,h2=lstm_layer2.hs,w2=lstm_layer2.Wout,output=output) do = [v for v in dout.values()] dWi1,dWf1,dWz1,dWo1,dWout1,dRi1,dRf1,dRz1,dRo1,dPi1,dPo1,dPf1,dbi1,dbo1,dbf1,dbz1,h_prev1,c_prev1=lstm_layer1.backward(bdout=do) dWi2,dWf2,dWz2,dWo2,dWout2,dRi2,dRf2,dRz2,dRo2,dPi2,dPo2,dPf2,dbi2,dbo2,dbf2,dbz2,h_prev2,c_prev2=lstm_layer2.backward(bdout=do) for dparam in[dWi1,dWf1,dWz1,dWo1,dWout1,dRi1,dRf1,dRz1,dRo1,dPi1,dPo1,dPf1,dbi1,dbo1,dbf1,dbz1,\ dWi2,dWf2,dWz2,dWo2,dWout2,dRi2,dRf2,dRz2,dRo2,dPi2,dPo2,dPf2,dbi2,dbo2,dbf2,dbz2,dbout] : np.clip(dparam,-1,1,out=dparam) params_of_lstm = [lstm_layer1.Wi,lstm_layer1.Wf,lstm_layer1.Wz,lstm_layer1.Wo,lstm_layer1.Wout,\ lstm_layer1.Ri,lstm_layer1.Rf,lstm_layer1.Rz,lstm_layer1.Ro,lstm_layer1.Pi,lstm_layer1.Po,\ lstm_layer1.Pf,lstm_layer1.bi,lstm_layer1.bo,lstm_layer1.bf,lstm_layer1.bz,\ lstm_layer2.Wi,lstm_layer2.Wf,lstm_layer2.Wz,lstm_layer2.Wo,lstm_layer2.Wout,\ lstm_layer2.Ri,lstm_layer2.Rf,lstm_layer2.Rz,lstm_layer2.Ro,lstm_layer2.Pi,lstm_layer2.Po,\ lstm_layer2.Pf,lstm_layer2.bi,lstm_layer2.bo,lstm_layer2.bf,lstm_layer2.bz,bout] dparams_of_lstm = [dWi1,dWf1,dWz1,dWo1,dWout1,dRi1,dRf1,dRz1,dRo1,dPi1,dPo1,dPf1,dbi1,dbo1,dbf1,dbz1,\ dWi2,dWf2,dWz2,dWo2,dWout2,dRi2,dRf2,dRz2,dRo2,dPi2,dPo2,dPf2,dbi2,dbo2,dbf2,dbz2,dbout] for params,dparams,mparams in zip(params_of_lstm,dparams_of_lstm,mparams_of_lstm): mparams += dparams*dparams
if start_ptr + time_steps > len_of_dataset: start_ptr = 0 h_prev = np.zeros((len_of_vocab, 1)) else: input = [ char_to_idx[c] for c in dataset[start_ptr:start_ptr + time_steps] ] output = [ char_to_idx[c] for c in dataset[start_ptr + 1:start_ptr + time_steps + 1] ] lstm.zero_grad() lstm.set_input(input=input, output=output) lstm.forward(h_prev=h_prev, c_prev=c_prev) loss, dWi, dWf, dWz, dWo, dWout, dRi, dRf, dRz, dRo, dPi, dPo, dPf, dbi, dbo, dbf, dbz, dbout, h_prev, c_prev = lstm.backward( ) lstm.clip_grad(clip_val=1) lstm.step() smooth_loss = (0.999 * smooth_loss) + (0.001 * loss) x.append(n) y.append(smooth_loss) if n % 1000 == 0: print('--------------------------------------------') print('iter:', n) print('smooth_loss:', smooth_loss) lstm.sample(h_prev=h_prev, c_prev=c_prev, num_char=300) print('--------------------------------------------') plt.ylabel('Loss') plt.xlabel('Epoch') plt.plot(x, y, color='r')
class TestLSTM(unittest.TestCase): def setUp(self): Wx = np.array([[ 9.72009451e-01, -4.97642862e-01, 6.45448952e-01, 8.10387855e-01, 1.13757673e+00, -5.27114694e-01, -9.08624540e-01, 1.61896844e+00, -1.16690977e+00, 3.93476226e-01, -6.04018422e-01, 5.67830817e-01 ], [ 6.68131790e-01, 6.40157016e-01, 6.90200961e-01, -1.39750585e+00, -4.89624070e-01, 8.99789953e-01, 3.97067428e-04, 1.47459503e+00, -4.95030269e-01, -9.22541855e-01, -1.57352198e-01, -1.67160494e+00 ], [ 6.93508859e-01, -9.23177216e-01, -4.83511551e-01, -1.18675890e+00, -7.35505045e-01, -1.61403611e+00, -2.76067694e-02, -2.48294747e-01, 1.14474446e+00, 1.86354309e-01, -1.73018002e+00, -4.82520536e-03 ]]) Wh = np.array([[ -0.88480318, 1.09509583, 0.55657863, -0.35096014, 0.18572107, 0.83823659, -0.44063768, -0.80897913, 0.35752315, 1.65812611, 1.40425671, 1.52519905 ], [ -0.22279229, 1.16363656, -0.47632291, -0.16436909, -2.16120359, 0.28362134, 0.01817155, 0.04836914, -0.30831619, -2.0992645, -0.07302497, -0.72868125 ], [ -1.40551611, 2.12755955, 1.76232202, 2.15703084, -1.87387492, 1.22755896, -0.84271588, 1.07860737, -0.35473314, -0.86293879, 1.67287773, 0.41575087 ]]) b = np.array([ 0.46861655, 0.15954682, 0.38782221, 1.00791178, -0.38322573, 0.83138721, 0.98675017, -0.83388618, 1.14392808, 0.37846653, 0.47617248, -1.8035631 ]) self.lstm = LSTM(Wx, Wh, b) self.x = np.array([[0.32434964, -1.25234162, -1.11446048], [-1.00333386, -0.28908588, 0.80944727], [-1.12554591, 0.00812256, -0.17046584], [-0.32813507, 0.16989344, 0.53355169], [-0.4869515, 0.1677294, -0.12831696], [0.05832905, -0.32864825, -0.80026008], [-0.02647765, -0.7902213, 0.15894066], [0.6531385, 0.89562313, 0.29789637], [2.10234369, -0.29459709, 0.10299345], [-0.27252854, 2.00320793, 0.45730897], [1.5004658, 1.64225418, 1.18133581], [-0.24243905, -0.43939872, -0.13251067]]) self.h_prev = np.array([[-0.10935227, -0.44314754, 1.08169975], [-1.21809561, -0.71525211, -1.02789959], [-0.95964908, -0.51143773, 0.21415223], [1.67953394, -0.35025546, -0.38320191], [0.44725308, -1.12955008, 1.75971301], [0.28200635, -0.0879721, 0.08046973], [-0.57840066, -0.05648079, 0.73577611], [-0.95991989, -0.42672155, 0.87126791], [-0.61249768, -1.27271833, -1.17150149], [0.62112557, 0.01982296, 1.98008052], [-0.47977218, -0.24977873, -0.56312278], [0.479705, 0.34781637, -0.07003237]]) self.c_prev = np.array([[1.31482411, -0.77629214, 0.58700115], [-1.06531026, -0.87934991, 0.62284509], [-0.60934138, -0.99160926, -0.42251352], [-0.23641619, 0.46826475, -0.02462736], [-0.12147368, -1.00207905, -0.12701465], [1.07820838, 0.51889022, -1.68135737], [2.21903061, -2.61861404, 1.16692287], [-1.49690178, 0.29867214, -0.85135438], [1.59958995, -0.86312955, 1.00291727], [-0.61192179, -0.29602751, 0.78903217], [-0.04484411, -0.13443395, -0.6548083], [-0.9392424, -1.23116816, 1.26848987]]) def test_forward(self): c_next, h_next = self.lstm.forward(self.x, self.h_prev, self.c_prev) assert_almost_equal( np.array([[0.59989084, -0.56519862, 0.98400225], [-1.91850874, 0.02352198, -0.93321127], [0.45227288, -0.54703237, 0.48403159], [-0.73615581, 0.32063485, 0.7548738], [0.42491665, -1.47394193, 0.67068475], [1.05754984, 0.49689304, -0.61805755], [1.49972308, -1.83582377, 1.00702007], [-0.74679837, -0.61435311, -0.45936825], [1.99958655, 0.77989824, 0.03898481], [0.18246256, -1.25613678, 1.4088285], [-0.60443486, 0.78677923, -0.91444947], [0.27892748, -0.94084206, 1.31430749]]), c_next) assert_almost_equal( np.array([[4.19169498e-01, -5.03430077e-01, 5.64198599e-01], [-6.54667825e-01, 5.95397055e-04, -1.84025417e-02], [1.30830713e-01, -3.09895913e-01, 1.37309301e-02], [-6.16008036e-01, 2.51996674e-01, 3.79084583e-01], [3.33833747e-01, -8.91544720e-01, 2.74736749e-01], [5.94326869e-01, 4.21936364e-01, -1.83699519e-01], [3.74427572e-01, -6.48435544e-01, 2.00609507e-01], [-1.07885596e-01, -2.15141706e-01, -1.01425325e-02], [9.49238782e-01, 1.64936528e-02, 1.37032403e-02], [1.78080877e-02, -8.29836071e-01, 2.47025891e-02], [-2.55087741e-01, 8.53326914e-03, -8.05473776e-03], [1.86996338e-01, -5.96078729e-01, 2.76307207e-01]]), h_next) def test_backward(self): dc_next, dh_next = self.lstm.forward(self.x, self.h_prev, self.c_prev) dx, dh_prev, dc_prev = self.lstm.backward(dc_next, dh_next) assert_almost_equal( np.array([[-6.79930933e-01, -1.97518363e-01, 4.16860765e-01], [1.55034826e-01, -2.91419470e-01, 1.71192005e-01], [-3.70136768e-02, 1.71824890e-02, -3.38419828e-01], [-3.05073579e-01, 2.41424747e-01, -1.93659638e-01], [1.54819264e-01, 2.30288841e-01, 2.25717407e-02], [5.45401813e-01, 4.42172325e-02, -1.86619072e-01], [-3.13126336e-04, 3.56652426e-01, -1.26116627e+00], [-3.60309927e-02, 5.69443437e-03, 1.46531343e-02], [1.11857111e-01, -4.33420556e-01, -3.87583328e-01], [5.95012499e-02, -6.50019034e-03, -5.01780050e-02], [-6.99925197e-02, -8.10853281e-02, 1.01525160e-02], [-8.53223995e-02, -2.06294786e-01, -7.03297008e-01]]), dx) assert_almost_equal( np.array([[0.34687254, 0.38344684, 0.37401971], [0.64098696, -0.85045992, -0.5425016], [0.26924398, 0.14994028, 0.69949726], [0.35040516, -0.01098271, -0.15074927], [-0.02868001, 0.07799114, 0.39807372], [0.18752975, -0.67098506, -0.33199641], [1.6726196, -0.19763998, 1.64359161], [0.15319373, 0.05884558, 0.22297725], [-0.13909709, -0.11303457, 0.49671851], [0.05830494, -0.03137801, 0.10292926], [0.11062537, -0.18552512, -0.17265588], [0.94321997, 0.11497126, 1.47405578]]), dh_prev) assert_almost_equal( np.array([[7.84603591e-02, -7.93934519e-01, 8.02468649e-01], [-7.07422789e-01, 1.15401664e-05, -1.38975207e-03], [1.18023605e-01, -2.39578791e-01, 1.17225299e-02], [-4.65581356e-01, 3.09765349e-01, 3.95637732e-01], [4.36409729e-02, -1.13529451e+00, 4.74057958e-01], [3.21235172e-01, 5.77017648e-01, -2.34804219e-01], [1.85468079e-01, -4.58247852e-01, 2.10825777e-01], [-1.53895902e-01, -2.33683523e-01, -1.75307671e-02], [1.08007772e+00, 8.27308200e-05, 1.16063117e-02], [6.54400485e-03, -1.16746148e+00, 3.28679438e-02], [-4.54490114e-01, 9.55363471e-04, -8.84788951e-03], [1.32898508e-01, -6.72948492e-01, 1.87992760e-01]]), dc_prev) def test_grads_diff(self): _before_Wx_grads, _before_Wh_grads, _before_b_grads = self.lstm.grads before_Wx_grads = copy.copy(_before_Wx_grads) before_Wh_grads = copy.copy(_before_Wh_grads) before_b_grads = copy.copy(_before_b_grads) dc_next, dh_next = self.lstm.forward(self.x, self.h_prev, self.c_prev) self.lstm.backward(dc_next, dh_next) after_Wx_grads, after_Wh_grads, after_b_grads = self.lstm.grads Wx_grads = before_Wx_grads == after_Wx_grads Wh_grads = before_Wh_grads == after_Wh_grads b_grads = before_b_grads == after_b_grads assert_array_equal( np.array([[ False, False, False, False, False, False, False, False, False, False, False, False ], [ False, False, False, False, False, False, False, False, False, False, False, False ], [ False, False, False, False, False, False, False, False, False, False, False, False ]]), Wx_grads) assert_array_equal( np.array([[ False, False, False, False, False, False, False, False, False, False, False, False ], [ False, False, False, False, False, False, False, False, False, False, False, False ], [ False, False, False, False, False, False, False, False, False, False, False, False ]]), Wh_grads) assert_array_equal( np.array([ False, False, False, False, False, False, False, False, False, False, False, False ]), b_grads)
class Strategy: """ Contains the lstm, financial portfolios """ def __init__(self, data, dates, targets, first_ret_idx): """ Retrieves data and preprocess them (better for deep learning) :param data: ndarray, all raw data :param dates: ndarray, all dates :param targets: ndarray, all raw targets :param first_ret_idx: long, index of first column of returns in the data """ self.targets = (targets - np.mean(targets, 0)[None, :]) / np.var(targets, 0)[None, :]**(1/2) self.data = (data - np.mean(data, 0)[None, :]) / np.var(data, 0)[None, :]**(1/2) self.all_returns = data[:, first_ret_idx:first_ret_idx + constants.FC_OUTPUT_NEURONS] self.dates = dates self.bmk = None self.ptf = None self.lstm = None def create_neural_network(self, act_fun_lay, act_fun_pos): """ Creates 2 hidden fully connected neural network with chosen last activation function :param act_fun_lay: layer, activation function as layer :param act_fun_pos: activation function position :return: NNetwork, neural network created """ nn = NNetwork() curt_lay = layer.FCLayer(self.data.shape[1] + self.targets.shape[1], constants.FC_1_NEURONS, constants.EDGE_GRADIENT, True) nn.add_layer(curt_lay, constants.FC_1_POS) nn.add_layer(layer.ReLULayer(), constants.RELU_1_POS) curt_lay = layer.FCLayer(constants.FC_1_NEURONS, constants.FC_2_NEURONS, constants.EDGE_GRADIENT, True) nn.add_layer(curt_lay, constants.FC_2_POS) nn.add_layer(layer.ReLULayer(), constants.RELU_2_POS) curt_lay = layer.FCLayer(constants.FC_2_NEURONS, constants.FC_OUTPUT_NEURONS, constants.EDGE_GRADIENT, True) nn.add_layer(curt_lay, constants.FC_OUTPUT_POS) nn.add_layer(act_fun_lay, act_fun_pos) return nn def create_lstm(self): """ Creates all neural networks and the vanilla lstm """ nn_f = self.create_neural_network(layer.SigmoidLayer(), constants.SIG_F_POS) nn_i = self.create_neural_network(layer.SigmoidLayer(), constants.SIG_I_POS) nn_c = self.create_neural_network(layer.TanhLayer(), constants.TANH_C_POS) nn_o = self.create_neural_network(layer.SigmoidLayer(), constants.SIG_O_POS) self.lstm = LSTM(nn_f, nn_i, nn_c, nn_o, constants.TAU_QUANTILE, constants.LEARNING_RATE) def create_portfolio(self): """ Creates portfolio with weights depending on lstm outputs """ self.ptf = Portfolio(constants.FC_OUTPUT_NEURONS, constants.TRANSACTION_FEE_RATE, constants.INITIAL_PTF_VALUE , name='portfolio') self.ptf.update_weights(1 / len(self.ptf.weights) * np.ones([len(self.ptf.weights), 1])) def create_benchmark(self): """ Creates equal weights benchmark portfolio """ self.bmk = Portfolio(constants.FC_OUTPUT_NEURONS, constants.TRANSACTION_FEE_RATE, constants.INITIAL_PTF_VALUE , name='benchmark') self.bmk.update_weights(1/len(self.bmk.weights) * np.ones([len(self.bmk.weights), 1])) def train(self, size_train=0.7): """ Trains the lstm vanilla using the data Target is Sharpe ratios for all assets Tau Quantile loss Batch gradient descent :param size_train: float, proportion of the data used for training """ h_prev = np.zeros((constants.FC_OUTPUT_NEURONS, 1)) c_prev = np.zeros((constants.FC_OUTPUT_NEURONS, 1)) curt_batch_size = 1 intermediate_values = [] out_data = np.zeros((constants.BATCH_SIZE, constants.FC_OUTPUT_NEURONS)) for curt_index in range(0, int(math.floor(size_train * self.data.shape[0])), 1): in_data = self.data[curt_index, :].reshape(self.data.shape[1], 1) h_prev, c_prev = self.lstm.forward(h_prev, c_prev, in_data) out_data[curt_batch_size-1, :] = h_prev.reshape((constants.FC_OUTPUT_NEURONS, )) intermediate_values.append(self.lstm.get_intermediate_values()) if curt_batch_size == constants.BATCH_SIZE: self.lstm.backward(out_data, intermediate_values, self.targets[curt_index-constants.BATCH_SIZE+1:curt_index+1]) self.lstm.update_param() curt_batch_size = 0 intermediate_values = [] out_data = np.zeros((constants.BATCH_SIZE, constants.FC_OUTPUT_NEURONS)) print(curt_index) curt_batch_size += 1 def test(self, size_test=0.3, is_nn_to_train=False): """ Lstm predicts Sharpes ratios, then there is a portfolio management using those predictions with bigger weights on better Sharpe ratios :param size_test: float, proportion of the data used for testing :param is_nn_to_train: boolean, true if lstm already trained """ if not is_nn_to_train: h_prev = np.zeros((constants.FC_OUTPUT_NEURONS, 1)) c_prev = np.zeros((constants.FC_OUTPUT_NEURONS, 1)) curt_index = int(math.floor((1-size_test)*self.data.shape[0])) bmk_wgts = 1 / len(self.bmk.weights) * np.ones([len(self.bmk.weights), 1]) for curt_index in range(curt_index, self.data.shape[0]-constants.NBR_MINUTES_STEP, constants.NBR_MINUTES_STEP): in_data = self.data[curt_index, :].reshape(self.data.shape[1], 1) h_prev, c_prev = self.lstm.forward(h_prev, c_prev, in_data) temp_ret = self.all_returns[curt_index + constants.NBR_MINUTES_STEP].reshape(constants.FC_OUTPUT_NEURONS , 1) self.ptf.compute_return(temp_ret) self.ptf.compute_value() self.ptf.update_weights_inv_rdt(temp_ret) self.ptf.update_weights_inv_val(h_prev) self.ptf.compute_transaction_fees() self.ptf.update_val_list() self.ptf.update_time() self.bmk.compute_return(temp_ret) self.bmk.compute_value() self.bmk.update_weights_inv_rdt(temp_ret) self.bmk.update_weights_inv_val(bmk_wgts) self.bmk.compute_transaction_fees() self.bmk.update_val_list() self.bmk.update_time()
def test_LSTM(): T = 5 batch_size = 2 nstates = 5 input_size = 4 unit = LSTM(input_size, nstates) W = unit.get_weights() X = np.random.randn(T, input_size, batch_size) unit.forget() acc_Y = unit.forward(X) wrand = np.random.randn(*acc_Y.shape) loss = np.sum(acc_Y * wrand) dY = wrand dX = unit.backward(dY) dW = unit.get_grads() unit.forget() def fwd(): unit.set_weights(W) h = unit.forward(X) unit.forget() return np.sum(h * wrand) delta = 1e-4 error_threshold = 1e-3 all_values = [X, W] backpropagated_gradients = [dX, dW] names = ['X', 'W'] error_count = 0 for v in range(len(names)): values = all_values[v] dvalues = backpropagated_gradients[v] name = names[v] for i in range(values.size): actual = values.flat[i] values.flat[i] = actual + delta loss_minus = fwd() values.flat[i] = actual - delta loss_plus = fwd() values.flat[i] = actual backpropagated_gradient = dvalues.flat[i] numerical_gradient = (loss_minus - loss_plus) / (2 * delta) if numerical_gradient == 0 and backpropagated_gradient == 0: error = 0 elif abs(numerical_gradient) < 1e-7 and abs( backpropagated_gradient) < 1e-7: error = 0 else: error = abs(backpropagated_gradient - numerical_gradient) / abs(numerical_gradient + backpropagated_gradient) if error > error_threshold: print 'FAILURE!!!\n' print '\tparameter: ', name, '\tindex: ', np.unravel_index( i, values.shape) print '\tvalues: ', actual print '\tbackpropagated_gradient: ', backpropagated_gradient print '\tnumerical_gradient', numerical_gradient print '\terror: ', error print '\n\n' error_count += 1 if error_count == 0: print 'LSTM Gradient Check Passed' else: print 'Failed for {} parameters'.format(error_count)
def test_LSTM(): T = 5 batch_size = 2 nstates = 5 input_size = 4 unit = LSTM(input_size, nstates) W = unit.get_weights() X = np.random.randn(T, input_size, batch_size) unit.forget() acc_Y = unit.forward(X) wrand = np.random.randn(*acc_Y.shape) loss = np.sum(acc_Y * wrand) dY = wrand dX = unit.backward(dY) dW = unit.get_grads() unit.forget() def fwd(): unit.set_weights(W) h = unit.forward(X) unit.forget() return np.sum(h * wrand) delta = 1e-4 error_threshold = 1e-3 all_values = [X, W] backpropagated_gradients = [dX, dW] names = ['X', 'W'] error_count = 0 for v in range(len(names)): values = all_values[v] dvalues = backpropagated_gradients[v] name = names[v] for i in range(values.size): actual = values.flat[i] values.flat[i] = actual + delta loss_minus = fwd() values.flat[i] = actual - delta loss_plus = fwd() values.flat[i] = actual backpropagated_gradient = dvalues.flat[i] numerical_gradient = (loss_minus - loss_plus) / (2 * delta) if numerical_gradient == 0 and backpropagated_gradient == 0: error = 0 elif abs(numerical_gradient) < 1e-7 and abs(backpropagated_gradient) < 1e-7: error = 0 else: error = abs(backpropagated_gradient - numerical_gradient) / abs(numerical_gradient + backpropagated_gradient) if error > error_threshold: print 'FAILURE!!!\n' print '\tparameter: ', name, '\tindex: ', np.unravel_index(i, values.shape) print '\tvalues: ', actual print '\tbackpropagated_gradient: ', backpropagated_gradient print '\tnumerical_gradient', numerical_gradient print '\terror: ', error print '\n\n' error_count += 1 if error_count == 0: print 'LSTM Gradient Check Passed' else: print 'Failed for {} parameters'.format(error_count)
c_prev_fg = np.copy(lstm_fg.cs[time_steps - 1]) lstm_og.zero_grad() lstm_og.set_input(input=input, output=output) lstm_og.forward(h_prev=h_prev_exp, c_prev=c_prev_og) c_prev_og = np.copy(lstm_og.cs[time_steps - 1]) loss, h_prev_exp, c_prev_exp, dig_out, dim_out, dft_out, dot_out, dWout, dbout = explstm_forward( h_prev=h_prev_exp, c_prev=c_prev_exp, ig=lstm_ig, im=lstm_im, og=lstm_og, fg=lstm_fg, output=output) lstm_ig.backward(bdout=dig_out) lstm_im.backward(bdout=dim_out) lstm_fg.backward(bdout=dft_out) lstm_og.backward(bdout=dot_out) params_of_lstm = [lstm_ig.Wi,lstm_ig.Wf,lstm_ig.Wz,lstm_ig.Wo,lstm_ig.Wout,\ lstm_ig.Ri,lstm_ig.Rf,lstm_ig.Rz,lstm_ig.Ro,lstm_ig.Pi,lstm_ig.Po,\ lstm_ig.Pf,lstm_ig.bi,lstm_ig.bo,lstm_ig.bf,lstm_ig.bz,\ lstm_og.Wi,lstm_og.Wf,lstm_og.Wz,lstm_og.Wo,lstm_og.Wout,\ lstm_og.Ri,lstm_og.Rf,lstm_og.Rz,lstm_og.Ro,lstm_og.Pi,lstm_og.Po,\ lstm_og.Pf,lstm_og.bi,lstm_og.bo,lstm_og.bf,lstm_og.bz,\ lstm_fg.Wi,lstm_fg.Wf,lstm_fg.Wz,lstm_fg.Wo,lstm_fg.Wout,\ lstm_fg.Ri,lstm_fg.Rf,lstm_fg.Rz,lstm_fg.Ro,lstm_fg.Pi,lstm_fg.Po,\ lstm_fg.Pf,lstm_fg.bi,lstm_fg.bo,lstm_fg.bf,lstm_fg.bz,\ lstm_im.Wi,lstm_im.Wf,lstm_im.Wz,lstm_im.Wo,lstm_im.Wout,\ lstm_im.Ri,lstm_im.Rf,lstm_im.Rz,lstm_im.Ro,lstm_im.Pi,lstm_im.Po,\