def build(P, character_count, embedding_size=20, hidden_size=50):
P.V = 0.1 * np.random.randn(character_count, embedding_size)
lstm_layer_1 = lstm.build(
P,
name="recurrent_1",
input_size=embedding_size,
hidden_size=hidden_size,
)
lstm_layer_2 = lstm.build(
P,
name="recurrent_2",
input_size=hidden_size,
hidden_size=hidden_size,
)
P.W_output = np.zeros((hidden_size, character_count))
P.b_output = np.zeros((character_count, ))
def predict(X): # time x batch_size
X_rep = P.V[X] # time x batch_size x embedding_size
_, hiddens_1 = lstm_layer_1(X_rep)
_, hiddens_2 = lstm_layer_2(hiddens_1)
hiddens_ = hiddens_2.reshape(
(hiddens_2.shape[0] * hiddens_2.shape[1],
hidden_size)) # time * batch_size x hidden_size
output = T.nnet.softmax(T.dot(hiddens_, P.W_output) + P.b_output)
output = output.reshape(
(hiddens_2.shape[0], hiddens_2.shape[1],
character_count)) # time x batch_size x output_size
return output
return predict
def build(P, character_count, embedding_size=20, hidden_size=50):
P.V = 0.1 * np.random.randn(character_count, embedding_size)
lstm_layer_1 = lstm.build(P,
name="recurrent_1",
input_size=embedding_size,
hidden_size=hidden_size,
)
lstm_layer_2 = lstm.build(P,
name="recurrent_2",
input_size=hidden_size,
hidden_size=hidden_size,
)
P.W_output = np.zeros((hidden_size, character_count))
P.b_output = np.zeros((character_count,))
def predict(X): # time x batch_size
X_rep = P.V[X] # time x batch_size x embedding_size
_, hiddens_1 = lstm_layer_1(X_rep)
_, hiddens_2 = lstm_layer_2(hiddens_1)
hiddens_ = hiddens_2.reshape((hiddens_2.shape[0] * hiddens_2.shape[1], hidden_size)) # time * batch_size x hidden_size
output = T.nnet.softmax(T.dot(hiddens_, P.W_output) + P.b_output)
output = output.reshape((hiddens_2.shape[0], hiddens_2.shape[1], character_count)) # time x batch_size x output_size
return output
return predict
def build_stmt_encoder(P, name, input_size, hidden_size):
lstm_layer = lstm.build(P, name, input_size, hidden_size)
def encode_stmt(X):
cells, hiddens = lstm_layer(X)
return cells[-1], hiddens[-1]
return encode_stmt
def build_model(P, input_size, hidden_size, output_size):
lstm_layer = lstm.build(P,"lstm",input_size,hidden_size)
P.W_output = np.zeros((hidden_size,output_size))
P.b_output = np.zeros((output_size,))
def model(X):
hidden = lstm_layer(X)[1]
return T.nnet.softmax(T.dot(hidden,P.W_output) + P.b_output)
return model
def build_model(P, input_size, hidden_size, output_size):
lstm_layer = lstm.build(P, "lstm", input_size, hidden_size)
P.W_output = np.zeros((hidden_size, output_size))
P.b_output = np.zeros((output_size,))
def model(X):
hidden = lstm_layer(X)[1]
return T.nnet.softmax(T.dot(hidden, P.W_output) + P.b_output)
return model
def setup(params):
"""
If the parameters define a model to load, load it. Otherwise
build it.
"""
if 'load_model' in params and params['load_model']:
model = load_model(params)
else:
model = lstm.build(params)
print('Model built from scratch...')
# load weights into new model if its defined and not empty.
if 'load_weights' in params and params['load_weights']:
model = load_weights(model, params)
return model
def build_diag_encoder(P,stmt_size,hidden_size,output_size,encode_stmt): # P.W_stmt_diag_hidden = random_init(stmt_size,output_size) # P.W_cumstmt_diag_hidden = random_init(hidden_size,output_size) lstm_layer = lstm.build(P,"diag",stmt_size,hidden_size) def encode_diag(X,idxs): def encode_sentence(i): word_vecs = X[idxs[i]:idxs[i+1]] return encode_stmt(word_vecs)[0] stmt_vecs,_ = theano.map( encode_sentence, sequences=[T.arange(idxs.shape[0]-1)] ) cells,hiddens = lstm_layer(stmt_vecs) # output = T.dot(stmt_vecs,P.W_stmt_diag_hidden) +\ # T.dot(hiddens,P.W_cumstmt_diag_hidden) return cells,hiddens return encode_diag
def build_diag_encoder(P, stmt_size, hidden_size, output_size, encode_stmt):
# P.W_stmt_diag_hidden = random_init(stmt_size,output_size)
# P.W_cumstmt_diag_hidden = random_init(hidden_size,output_size)
lstm_layer = lstm.build(P, "diag", stmt_size, hidden_size)
def encode_diag(X, idxs):
def encode_sentence(i):
word_vecs = X[idxs[i]:idxs[i + 1]]
return encode_stmt(word_vecs)[0]
stmt_vecs, _ = theano.map(encode_sentence,
sequences=[T.arange(idxs.shape[0] - 1)])
cells, hiddens = lstm_layer(stmt_vecs)
# output = T.dot(stmt_vecs,P.W_stmt_diag_hidden) +\
# T.dot(hiddens,P.W_cumstmt_diag_hidden)
return cells, hiddens
return encode_diag
def save(model, params, prefix='', additional_epocs=0):
"""
Save the model, weights and/or the predictor version of the net.
"""
home_path = str(Path.home())
save_folder = join(join(home_path, params['project_path']),
params['networks_path'])
name_prefix = '{}_{:d}L{:d}u_{:d}e_{:02d}i_'.format(
prefix, params['lstm_numlayers'],
params['lstm_layer1'], params['lstm_num_epochs'] + additional_epocs,
len(params['columNames']))
name_suffix = '{0:%Y}{0:%m}{0:%d}_{0:%H}{0:%M}'.format(datetime.now())
base_name = name_prefix + name_suffix
# Save the model?
if param_set(params, 'save_model'):
model_name = join(save_folder, '{}.json'.format(base_name))
model_json = model.to_json()
with open(model_name, "w") as json_file:
json_file.write(model_json)
print('Model saved to {}'.format(model_name))
# Save weights?
if param_set(params, 'save_weights') is True:
net_name = join(save_folder, '{}.h5'.format(base_name))
model.save_weights(net_name)
print('Weights saved to {}'.format(net_name))
# Save predictor?
if param_set(params, 'save_predictor'):
# Build the predictor model, with batch_size = 1, and save it.
bs = params['lstm_batch_size']
params['lstm_batch_size'] = 1
pred_model = lstm.build(params)
model_name = join(save_folder, '{}_pred.json'.format(base_name))
model_json = pred_model.to_json()
with open(model_name, "w") as json_file:
json_file.write(model_json)
params['lstm_batch_size'] = bs
del pred_model
print('1-Prediction model saved to {}'.format(model_name))
%matplotlib inline
%load_ext autoreload
%autoreload 2
# Initialization of seeds
set_random_seed(2)
seed(2)
# load json and create model
params = parameters.read()
raw = data.read(params)
print('Original dataset num samples:', raw.shape)
adjusted = parameters.adjust(raw, params)
X_train, Y_train, X_test, Y_test = data.prepare(adjusted, params)
# Build the model and train it.
params['lstm_batch_size'] = 1
model = lstm.build(params)
# load weights into new model
model.load_weights("20180116_0438.h5")
print("Loaded weights from disk")
print('Actual:', params['y_scaler'].inverse_transform(Y_test[31]))
# Plot the test values for Y, and Y_hat, without scaling (inverted)
Y_hat = model.predict(X_test[31].reshape((1, 6, 8)), batch_size=params['lstm_batch_size'])
print('Prediction:', params['y_scaler'].inverse_transform(Y_hat))
def build_stmt_encoder(P,name,input_size,hidden_size): lstm_layer = lstm.build(P,name,input_size,hidden_size) def encode_stmt(X): cells,hiddens = lstm_layer(X) return cells[-1],hiddens[-1] return encode_stmt
#
adjusted = parameters.adjust(raw, params)
X, Y, Xtest, ytest = prepare(
normalize(adjusted, params), params)
#
# t r a i n i n g
#
model = setup(params)
parameters.summary(params)
model.summary()
lstm.stateless_fit(model, X, Y, Xtest, ytest,
params)
#
# r e b u i l d & p r e d i c t
#
pred = lstm.build(params, batch_size=1)
pred.set_weights(model.get_weights())
(yhat, rmse, num_errors) = lstm.range_predict(
pred, Xtest, ytest, params)
#
# w r i t e r e s u l t s
#
grid.writerow([
params['lstm_num_epochs'],
params['lstm_batch_size'],
params['lstm_timesteps'],
params['lstm_layer1'],
params['lstm_dropout1'],
params['lstm_stateful'],
params['lstm_shuffle'],
params['lstm_forget_bias'], rmse,
