def run_service():
stocks = StockHelper.get_stock_symbol_mapping()
for stock, symbol in stocks.items():
logger.info(f"Starting training for {stock} [{symbol}] at {ctime()}")
models = {
"SVM": SVM(symbol, scaler=StandardScaler),
"ARIMA": ARIMA(symbol, scaler=LogScaler),
"LSTM": LSTM(symbol, scaler=MinMaxScaler, is_keras=True),
}
for model_name, model in models.items():
logger.info(f"\tTraining {model_name} for {stock}")
start_time = time()
train_data = model.train_data
n_days = 300
test_data = Series(index=get_next_n_trading_days(n_days))
predictions = model.fit_predict(n_days)
predictions = Series(data=predictions, index=test_data.index[: len(predictions)])
save_predictions(predictions, type(model).__name__, symbol, train_data.index.max().to_pydatetime())
logger.info(f"\tTrained {model_name} for {stock} in {time() - start_time:.3f} seconds")
logger.info(f"Finished training for {stock} [{symbol}] at {ctime()}")
def predict(img_shape, model):
prediction_file = 'predictions.csv'
predict_folder = './data/test_stg1/*.jpg'
image_paths = list(glob(predict_folder))
features, _ = preprocess(img_shape, image_paths)
predictions = model.predict(features)
save_predictions(
prediction_file, image_paths,
['ALB', 'BET', 'DOL', 'LAG', 'NoF', 'OTHER', 'SHARK', 'YFT'],
predictions)
def run():
model_path = '/tmp/output_graph.pb'
prediction_file = 'predictions.csv'
image_paths = list(glob('./data/test_stg1/*.jpg'))
image_predictions = []
with open('/tmp/output_labels.txt', 'rb') as labels_file:
labels = [
line.decode("utf-8").upper()
for line in labels_file.read().splitlines()
]
with tf.Session() as sess:
create_graph(model_path)
for image_path in tqdm(image_paths):
image_data = tf.gfile.FastGFile(image_path, 'rb').read()
softmax_tensor = sess.graph.get_tensor_by_name('final_result:0')
predictions = sess.run(softmax_tensor,
{'DecodeJpeg/contents:0': image_data})
predictions = np.squeeze(predictions)
image_predictions.append(predictions)
save_predictions(prediction_file, image_paths, labels, image_predictions)
ax.plot(train_data, c="red", label="Training Data")
ax.plot(test_data, c="black", label="Testing Data")
ax.legend()
plt.show()
# %%
predictions = model.fit_predict(n_days)
predictions
# %%
predictions = Series(data=predictions,
index=test_data.index[:len(predictions)])
# %%
if to_save_predictions:
save_predictions(predictions,
type(model).__name__, symbol,
train_data.index.max().to_pydatetime())
# %%
fig, ax = plt.subplots(figsize=(10, 6))
ax.plot(train_data, c="red", label="Training Data")
ax.plot(test_data, c="black", label="Testing Data")
ax.plot(predictions, c="green", label="Predicted Price")
ax.grid(True)
ax.legend()
plt.show()
# %%
if testing:
mae_score = mean_absolute_error(test_data, predictions)
print(f"MAE: {mae_score}")
def evaluate_forecasting(model, loader, args, isTest=False, getLoss=False):
with torch.no_grad():
model.eval()
for name, param in model.named_parameters():
if "transition_model" in name and "weight" in name and "stage" in name:
print(name)
#print(param)
print(torch.norm(param[:, :40], p=2, keepdim=True))
print(torch.norm(param[:, 40:], p=2, keepdim=True))
#sys.exit()
state_predictions = []
state_groundTruth = []
loss_total = 0
objwise_mse = 0
total_len = 0
pred_states = []
next_states = []
input_space = False
for batch_idx, data_batch in enumerate(loader):
data_batch = [tensor.to(device) for tensor in data_batch]
statePast = data_batch[0].float()
contPast = data_batch[1].float()
action = data_batch[2].float()
nextState = data_batch[3].float()
# multi step
loss = 0
message_loss = 0
per_obj_loss = 0
l1_term = 0
state_pred = []
state_gt = []
for i in range(action.shape[2]):
if input_space:
state_encoding, cont_encoding, action_encoding = model.getEncodings(
torch.cat([
statePast[:, :, i:], nextState[:, :, :i] if i == 0
else predicted[:, :,
max(0, i - args.window_size):i]
],
dim=-1),
torch.cat([
contPast[:, :, i:],
action[:, :, max(0, i - args.window_size):i]
],
dim=-1), action[:, :, i].unsqueeze(2))
pred = model.getTransition(
state_encoding, cont_encoding, action_encoding
) if args.full else state_encoding + model.getTransition(
state_encoding if i == 0 else pred, cont_encoding,
action_encoding)
if (i == 0):
predicted = model.decode(pred)
else:
predicted = torch.cat(
[predicted, model.decode(pred)], dim=-1)
#print(predicted.size())
else:
state_encoding, cont_encoding, action_encoding = model.getEncodings(
torch.cat([statePast[:, :, i:], nextState[:, :, :i]],
dim=-1),
torch.cat([
contPast[:, :, i:],
action[:, :, max(0, i - args.window_size):i]
],
dim=-1), action[:, :, i].unsqueeze(2))
pred = model.getTransition(
state_encoding if i == 0 else pred, cont_encoding,
action_encoding
) if args.full else state_encoding + model.getTransition(
state_encoding if i == 0 else pred, cont_encoding,
action_encoding)
if (args.message_pass):
message_loss += model.get_l1_Message()
mse_loss = MSEloss(model.decode(pred),
nextState[:, :, i].unsqueeze(-1))
objwise = helper.per_obj_mse(model.decode(pred),
nextState[:, :, i].unsqueeze(-1))
loss += mse_loss
per_obj_loss += objwise
if (args.save_predictions):
state_gt.append(nextState[:, :, i].unsqueeze(-1))
state_pred.append(model.decode(pred))
if (i == action.shape[2] - 1):
state_groundTruth.append(torch.cat(state_gt, dim=-1))
state_predictions.append(torch.cat(state_pred, dim=-1))
#helper.save_predictions(nextState[:,:,i].unsqueeze(2), pred, i)
# helper.save_predictions(save_folder, used_params, nextState[:,:,i].unsqueeze(-1), model.decode(pred), i)
if (args.hierarchical_ls and args.layer_l1):
l1_term = args.l1 * helper.getStages_norm(model)
elif (args.per_node_MLP and args.layer_l1):
l1_term = args.l1 * helper.getTM_norm(model)
if (args.soft_decoder_l1):
decoder_params = [
x.view(-1) for x in model.decoder.parameters()
][0]
l1_term = args.decoder_l1 * torch.norm(
decoder_params, 1) / decoder_params.size()[0]
if (args.message_pass and getLoss):
message_loss = args.message_l1 * message_loss
loss += message_loss
if (getLoss):
loss_total += ((loss.item()) * len(pred) + l1_term)
else:
loss_total += (loss.item()) * len(pred)
objwise_mse += per_obj_loss * len(pred)
total_len += len(pred)
if getLoss:
model.train()
return loss_total / float(total_len)
if (args.save_predictions):
state_groundTruth = torch.cat(state_groundTruth, dim=0)
state_predictions = torch.cat(state_predictions, dim=0)
helper.save_predictions(save_folder, used_params,
state_groundTruth, state_predictions, 0)
objwise_mse_list = (objwise_mse / float(total_len)).tolist()
dump_object_wise = [
'{}'.format(per_object) for per_object in objwise_mse_list
]
save_name = 'result_M3.txt' if not args.sepCTRL else 'results_M4.txt'
re_loss = loss_total / float(total_len)
objwise_mse_list.append(re_loss)
results = np.expand_dims(np.array(objwise_mse_list), axis=0)
print('Reconstruction Loss {}'.format(loss_total / float(total_len)))
print('per_obj_mse: ' + str(dump_object_wise))
return results[0]