def main(memory_frac, config):
tf_config = tf.ConfigProto()
tf_config.gpu_options.per_process_gpu_memory_fraction = memory_frac
set_session(tf.Session(config=tf_config))
if config is None:
config_path = constants.LSTM_CONFIG
else:
config_path = config
# process the json configuration file
config = process_config(config_path)
# create the experiments dirs
create_dirs([
config.callbacks.tensorboard_log_dir, config.callbacks.checkpoint_dir,
config.callbacks.model_dir, config.callbacks.config_dir,
config.callbacks.result_dir
])
print('Create the data generator.')
data_loader = LSTMDataLoader(config)
print('Create the model.')
model = LSTMModel(config)
copyfile(inspect.getfile(model.__class__),
os.path.join(config.callbacks.model_dir, "model.py"))
copyfile(config_path,
os.path.join(config.callbacks.config_dir, "config.json"))
print('Create the trainer')
trainer = LSTMTrainer(model.build_model(), data_loader, config)
print('Start training the model.')
trainer.train()
def lstm_model(train, val, test):
# vocab = Vocab()
# vocab.from_dataset(train+val+test)
vocab = generate_vocab()
hparams = {
"vocab": vocab,
"embedding_dim": 128,
"hidden_dim": 128,
"n_epochs": 5,
"batch_size": 32,
"lr": 0.001,
"num_layers": 1,
"dropout": 0,
}
model = LSTMModel(hparams)
model.fit(train, val)
print("lstm model perplexity:", evaluate(model, test))
progression = model.generate(10)
print("generated progression:", len(progression), progression)
# convert_to_chord_name(progression))
print("generated progression perplexity:", evaluate(model, [progression]))
print("accuracy:", accuracy(model, test))
print("vocab size:", len(vocab))
def test_train(self):
a = DataReader('F', 'google', datetime(2006, 6, 1),
datetime(2016, 6, 1))
a_returns = pd.DataFrame(np.diff(np.log(a['Close'].values)))
a_returns.index = a.index.values[1:a.index.values.shape[0]]
train_value = a_returns.values
scaler, x_train, y_train, x_test, y_test = time_step_data_convert(
original_data=train_value,
num_time_steps=5,
batch_size=10,
)
lstm_instance = LSTMModel(num_time_steps=5)
train_model_instance = TrainModel(model_object=lstm_instance,
model_name='lstm')
predicted, score, rmse = train_model_instance.train(Xtrain=x_train,
Ytrain=y_train,
Xtest=x_test,
Ytest=y_test)
score_check = False
rmse_check = False
if score < 0.001:
score_check = True
self.assertEqual(True, score_check)
if rmse < 0.031:
rmse_check = True
self.assertEqual(True, rmse_check)
color_list = ['r--', 'b:']
label_list = ['predict_x', 'real_y']
data_list = [predicted, y_train]
visualize_wave(color_list=color_list,
label_list=label_list,
data_list=data_list)
def main(args):
print(80 * "=")
print("INITIALIZING")
print(80 * "=")
embedder = Embedder(args)
embeddings, tok2idMap, learning_set = embedder.load_and_preprocess_data(
args.tiny)
# choosing train vs. test sets within our data
# TODO make a separate test set ; in order to report out-of-sample
# performance. Instead of 90% train and 10% dev, we'll do 80% train,
# 10% dev and 10% test.
train_set = learning_set[:int(len(learning_set) * args.train_fraction)]
dev_set = learning_set[int(len(learning_set) * args.train_fraction):]
if not os.path.exists('./data/weights/'):
os.makedirs('./data/weights/')
with tf.Graph().as_default() as graph:
print("Building model...", end=' ')
start = time.time()
if args.model == "naive":
config = NaiveConfig(args)
model = NaiveModel(config, embeddings, embedder)
elif args.model == "lstm":
config = LSTMConfig(args)
model = LSTMModel(config, embeddings, embedder)
elif args.model == "bilstm":
config = BiLSTMConfig(args)
model = BiLSTMModel(config, embeddings, embedder)
elif args.model == "stacked_lstm":
config = StackedLSTMConfig(args)
model = StackedLSTMModel(config, embeddings, embedder)
init_op = tf.global_variables_initializer()
saver = tf.train.Saver()
print("took {:.2f} seconds\n".format(time.time() - start))
if args.crf:
CRF = Crf([labels for sentence, labels in train_set])
model = CRFModel(model, CRF, args)
graph.finalize()
with tf.Session(graph=graph) as session:
session.run(init_op)
print(80 * "=")
print("TRAINING")
print(80 * "=")
model.fit(session, saver, train_set, dev_set)
def run_training(opts) -> None:
"""Train the model"""
# Load the data
data = load_dataset(path=opts.source)
X_train, X_test, y_train, y_test = train_test_split(data[config.FEATURES],
data[config.TARGET],
test_size=config.TEST_SIZE,
random_state=config.RANDOM_STATE)
# Fit and transform the pipeline
pipe.fit(X_train)
proc_data_x = pipe.transform(X_train)
proc_data_test = pipe.transform(X_test)
# Initialize and fit the model
model = LSTMModel(lstm_units=opts.lstm_units,
embedding_dim=opts.embedding_dim,
input_dim=opts.input_dim,
optimizer=opts.optimizer)
model.fit(proc_data_x, y_train, epochs=opts.epochs, validation_data=(proc_data_test, y_test))
params = model.get_numeric_params
avg_metrics = {}
for key, value in model.model.history.history.items():
avg_metrics[key] = np.mean(value)
log_metrics(metrics=avg_metrics, params=params, tracking_uri=config.TRACKING_URI, experiment_name='test_experiment')
# Save the trained model
model.save(opts.model_path)
# Save the pipeline
save_pipeline(path=opts.pipe_path, pipeline_to_persist=pipe)
def get_model(model_id):
"""
Returns the object associated to the model selected in the args.
"""
if args.mode in [0, 2]:
if model_id == 0:
return BaselineRNNModel(vocab_size=len(word_to_idx),
device=device,
embedding_dim=args.embedding_dim,
dropout=args.dropout).to(device)
elif model_id == 1:
return LSTMModel(vocab_size=len(word_to_idx),
device=device,
embedding_dim=args.embedding_dim,
dropout=args.dropout).to(device)
elif model_id == 2:
return AttentionLSTMModel(vocab_size=len(word_to_idx),
device=device,
embedding_dim=args.embedding_dim,
dropout=args.dropout).to(device)
else:
raise ValueError(
"Model not known or not applicable to selected mode. Available Models:\n"
"0 for BaselineRNNModel.\n"
"1 for BasicLSTMModel.\n"
"2 for LSTM with Attention model.")
else:
if model_id == 3:
return BaselineEncoderDecoderModel(len(word_to_idx),
seq_length=args.seq_length,
embedding_dim=300,
dropout=args.dropout,
device=device).to(device)
elif model_id == 4:
return AttentionEncoderDecoderModel(len(word_to_idx),
seq_length=args.seq_length,
embedding_dim=300,
dropout=args.dropout,
device=device).to(device)
else:
raise ValueError(
"Model not known or not applicable to selected mode. Available Models:\n"
"3 for BaselineEncoderDecoderModel\n"
"4 for AttentionEncoderDecoderModel.")
def callback(channel, method, properties, body):
print(f'[x] Received {body} from {properties}')
# Clear file in tmp/ folder
for f in os.listdir('/tmp'):
os.remove('tmp/' + f)
'''
LOAD DATA FROM MINIO --> CREATE - TRAIN - SAVE MODEL --> UPLOAD MODEL TO MINIO
'''
received_msg = json.loads(body)
to_path = 'tmp/' + received_msg['name'] + received_msg['type']
from_path = received_msg['file_uri']
# download data from minio
DataStore_Handler.download(from_path, to_path)
# read data from downloaded file
data = pandas.read_csv(to_path, header=None)
data = data.to_numpy()
print(data[0])
# split data to train set and test set
train_data, test_data = train_test_split(data,
test_size=0.2,
shuffle=False)
scaler_file = 'scaler.pkl'
model_file = 'model.h5'
# ==================================================================
# PREDICTION FOR THREE MONTHS
# ==================================================================
''' train models '''
model_lstm = LSTMModel(train_data, test_data)
model_lstm.compile()
model_lstm.train()
''' save the best model '''
model_lstm.save()
K.clear_session()
# upload model and necessary files to minio
files = [model_file, scaler_file] # filelist for forwarding to edge-server
filename = received_msg['name']
file_extension = '.' + model_file.split('.')[-1]
dest = filename + '/model/'
for fname in files:
if os.path.isfile('tmp/' +
fname): # some models don't have scaler.pkl, etc.
DataStore_Handler.upload('tmp/' + fname, dest + fname)
os.remove('tmp/' + fname)
else:
files.remove(fname)
# SAVE LOGS TO MONGO
logs = {
"name": filename,
"type": file_extension,
'date': time.strftime("%Y-%m-%d %H:%M:%S"),
"file_uri": dest,
'preprocessor_id': received_msg.get('preprocessor_id', '')
}
logged_info = Database_Handler.insert(config.MONGO_COLLECTION, logs)
# send notification
msg = {
"name": filename,
"type": file_extension,
'date': time.strftime("%Y-%m-%d %H:%M:%S"),
"file_uri": dest,
'files': files,
'creator_id': str(logged_info.inserted_id)
}
Message_Handler.sendMessage('from_creator', json.dumps(msg))
ASSETS = ASSET_LIST
LR = 1e-4
if __name__ == '__main__':
batch_gen = Batchifier(data_path=DATA_PATH,
bsz=BSZ,
bptt=BPTT,
idx=IDX,
asset_list=ASSETS,
randomize_train=randomize_train,
overlapping_train=overlapping_train)
model = LSTMModel(num_hid=NUM_HID,
bptt=BPTT,
num_assets=len(asset_list),
lr=LR,
clip_norm=5.0)
with tf.Session() as sess:
sess.run(model.tf_init())
losses = []
for epoch in range(1, NUM_EPOCHS + 1):
for bTrainX, bTrainY in batch_gen.load_train():
loss = model.optimize(sess, bTrainX, bTrainY)
losses.append(loss)
print("Epoch {} Average Train Loss: {}, validating...".format(
epoch,
sum(losses) / len(losses)))
losses = []
allocation_wts = []
N = opt.N
seq_len = opt.seq_len
sample_gap = opt.sample_gap
batch_size = opt.batch_size
input_size = opt.input_size
encode_size = opt.encode_size
hidden_size = opt.hidden_size
num_layers = opt.num_layers
if os.path.exists(path="./data/mydata.pickle"):
with open('./data/mydata.pickle', 'rb') as load_data:
(train_input, train_label), (dev_input, dev_label), (test_input, test_label) = pickle.load(load_data)
else:
(train_input, train_label), (dev_input, dev_label), (test_input, test_label) = DataLoader("./data/data.csv", N, seq_len, sample_gap, batch_size)
with open('./data/mydata.pickle', 'wb') as save_data:
data_list = [(train_input, train_label), (dev_input, dev_label), (test_input, test_label)]
pickle.dump(data_list, save_data)
train_model = LSTMModel(input_size, encode_size, hidden_size, num_layers, batch_size).to(opt.device)
# set loss function and optimizer
loss_function = select_loss_function(opt.loss_function)
optimizer = set_optimizer(train_model, opt)
solver = LSTMSolver(train_model, loss_function, optimizer)
print("Training starts at %s" % (time.asctime(time.localtime(time.time()))))
solver.train_and_decode(train_input, train_label, dev_input, dev_label,
test_input, test_label, opt, max_epoch=opt.max_epoch)