def testNormalize(self):
df = dt.load_data(params.global_params['db_path'],
'C',
index_col='date',
from_date=20100101,
to_date=20100405,
limit=30)
df = df['close']
look_back = 20
look_ahead = 1
coeff = 3.0
print(df.tail())
data = df.values
print('data.shape', data.shape)
_, y_data = dt.normalize(data,
look_back=look_back,
look_ahead=look_ahead,
alpha=coeff)
tmp = dt.denormalize(y_data, data, look_back, look_ahead, coeff)
print('denorma.shape', tmp.shape)
plt.plot(data[look_back:], label='actual')
plt.plot(tmp, label='denorm')
plt.legend(loc='upper left')
plt.show()
def predict_single(self, observation):
""" Predict the action of a single observation
Args:
observation: (num_stocks + 1, window_length)
Returns: a single action array with shape (num_stocks + 1,)
"""
obsX = observation[:, -self.window_length:, 3:4] / observation[:, -self.window_length:, 0:1]
obsX = normalize(obsX)
obsX = np.expand_dims(obsX, axis=0)
with self.graph.as_default():
return np.squeeze(self.model.predict(obsX), axis=0)
def obs_normalizer(observation):
""" Preprocess observation obtained by environment
Args:
observation: (nb_classes, window_length, num_features) or with info
Returns: normalized
"""
if isinstance(observation, tuple):
observation = observation[0]
# directly use close/open ratio as feature
observation = observation[:, :, 3:4] / observation[:, :, 0:1]
observation = normalize(observation)
return observation
def obs_normalizer(observation, feature_number):
""" Preprocess observation obtained by environment
Args:
observation: (nb_classes, window_length, num_features) or with info
Returns: normalized
"""
if isinstance(observation, tuple):
observation = observation[0]
if feature_number == 1: # directly use close/open ratio as feature
observation = observation[:, :, 3:4] / observation[:, :, 0:1]
observation = normalize(observation)
if feature_number == 2: # use close/open ratio and volumn / volumn_last ratio
observation_price = observation[:,:, 3:4] / observation[:,:, 0:1]
observation_price = normalize(observation_price)
observation_volume = stats.zscore(observation[:,:,5:6].astype('float64'), axis = 1)
observation_volume = np.nan_to_num(observation_volume)
observation = np.concatenate((observation_price, observation_volume), axis = 2)
if feature_number == 4: # use open,low,high,close/close
observation = observation[:,:, 0:4]/observation[:,-1:,3:4]
observation = normalize(observation)
if feature_number == 5: # use open,low,high,close/close and volumn / volumn_last ratio
observation_price = observation[:,:, 0:4]/ observation[:,-1:,3:4]
observation_price = normalize(observation_price)
observation_volume = stats.zscore(observation[:,:,5:6].astype('float64'), axis = 1)
observation_volume = np.nan_to_num(observation_volume)
observation = np.concatenate((observation_price, observation_volume), axis = 2)
return observation
def predict_single(self, observation):
""" Predict the action of a single observation
Args:
observation: (num_stocks + 1, window_length)
Returns: a single action array with shape (num_stocks + 1,)
"""
action = np.zeros((self.num_classes, ))
obsX = observation[:, -self.window_length:,
3] / observation[:, -self.window_length:, 0]
obsX = normalize(obsX)
obsX = np.expand_dims(obsX, axis=0)
current_action_index = self.model.predict_classes(obsX, verbose=False)
action[current_action_index] = 1.0
return action
def obs_normalizer(observation):
""" Preprocess observation obtained by environment
Args:
observation: (nb_classes, window_length, num_features) or with info
Returns: normalized
"""
# print("data before normalization={}".format(observation))
if isinstance(observation, tuple):
observation = observation[0]
# print("data in first element used={}".format(observation))
# directly use close/open ratio as feature
# print("shape before normalization={}".format(observation.shape))
observation = observation[:, :, 3:4] / observation[:, :, 0:1]
observation = normalize(observation)
# print("shape after normalization={}".format(observation.shape))
return observation
def get_dataframe():
data = get_data()
headers = data.pop(0)
df = pd.DataFrame(data, columns=headers)
origin = df['DateTime']
df['DateTime'] = pd.to_datetime(origin,
format='%d.%m.%Y %H:%M:%S',
errors='coerce')
mask = df['DateTime'].isnull()
df.loc[mask, 'DateTime'] = pd.to_datetime(origin[mask],
format='%d.%m.%Y %H:%M',
errors='coerce')
mask = df['DateTime'].isnull()
df.loc[mask, 'DateTime'] = pd.to_datetime(origin[mask],
format='%Y-%m-%d %H:%M:%S',
errors='coerce')
mask = df['DateTime'].isnull()
df.loc[mask, 'DateTime'] = pd.to_datetime(origin[mask],
format='%Y-%m-%d %H:%M',
errors='coerce')
return normalize(df.sort_values(by='DateTime', ascending=True))
def main(args):
# Init tensorboard
writer = SummaryWriter('./runs/' + args.runname + str(args.trialnumber))
model_name = 'VanillaDMM'
# Set evaluation log file
evaluation_logpath = './logs/{}/evaluation_result.log'.format(
model_name.lower())
log_evaluation(evaluation_logpath,
'Evaluation Trial - {}\n'.format(args.trialnumber))
# Constants
time_length = 30
input_length_for_pred = 20
pred_length = time_length - input_length_for_pred
train_batch_size = 16
valid_batch_size = 1
# For model
input_channels = 1
z_channels = 50
emission_channels = [64, 32]
transition_channels = 64
encoder_channels = [32, 64]
rnn_input_dim = 256
rnn_channels = 128
kernel_size = 3
pred_length = 0
# Device checking
use_cuda = torch.cuda.is_available()
device = torch.device("cuda:0" if use_cuda else "cpu")
# Make dataset
logging.info("Generate data")
train_datapath = args.datapath / 'train'
valid_datapath = args.datapath / 'valid'
train_dataset = DiffusionDataset(train_datapath)
valid_dataset = DiffusionDataset(valid_datapath)
# Create data loaders from pickle data
logging.info("Generate data loaders")
train_dataloader = DataLoader(
train_dataset, batch_size=train_batch_size, shuffle=True, num_workers=8)
valid_dataloader = DataLoader(
valid_dataset, batch_size=valid_batch_size, num_workers=4)
# Training parameters
width = 100
height = 100
input_dim = width * height
# Create model
logging.warning("Generate model")
logging.warning(input_dim)
pred_input_dim = 10
dmm = DMM(input_channels=input_channels, z_channels=z_channels, emission_channels=emission_channels,
transition_channels=transition_channels, encoder_channels=encoder_channels, rnn_input_dim=rnn_input_dim, rnn_channels=rnn_channels, kernel_size=kernel_size, height=height, width=width, pred_input_dim=pred_input_dim, num_layers=1, rnn_dropout_rate=0.0,
num_iafs=0, iaf_dim=50, use_cuda=use_cuda)
# Initialize model
logging.info("Initialize model")
epochs = args.endepoch
learning_rate = 0.0001
beta1 = 0.9
beta2 = 0.999
clip_norm = 10.0
lr_decay = 1.0
weight_decay = 0
adam_params = {"lr": learning_rate, "betas": (beta1, beta2),
"clip_norm": clip_norm, "lrd": lr_decay,
"weight_decay": weight_decay}
adam = ClippedAdam(adam_params)
elbo = Trace_ELBO()
svi = SVI(dmm.model, dmm.guide, adam, loss=elbo)
# saves the model and optimizer states to disk
save_model = Path('./checkpoints/' + model_name)
def save_checkpoint(epoch):
save_dir = save_model / '{}.model'.format(epoch)
save_opt_dir = save_model / '{}.opt'.format(epoch)
logging.info("saving model to %s..." % save_dir)
torch.save(dmm.state_dict(), save_dir)
logging.info("saving optimizer states to %s..." % save_opt_dir)
adam.save(save_opt_dir)
logging.info("done saving model and optimizer checkpoints to disk.")
# Starting epoch
start_epoch = args.startepoch
# loads the model and optimizer states from disk
if start_epoch != 0:
load_opt = './checkpoints/' + model_name + \
'/e{}-i188-opt-tn{}.opt'.format(start_epoch - 1, args.trialnumber)
load_model = './checkpoints/' + model_name + \
'/e{}-i188-tn{}.pt'.format(start_epoch - 1, args.trialnumber)
def load_checkpoint():
# assert exists(load_opt) and exists(load_model), \
# "--load-model and/or --load-opt misspecified"
logging.info("loading model from %s..." % load_model)
dmm.load_state_dict(torch.load(load_model, map_location=device))
# logging.info("loading optimizer states from %s..." % load_opt)
# adam.load(load_opt)
# logging.info("done loading model and optimizer states.")
if load_model != '':
logging.info('Load checkpoint')
load_checkpoint()
# Validation only?
validation_only = args.validonly
# Train the model
if not validation_only:
logging.info("Training model")
annealing_epochs = 1000
minimum_annealing_factor = 0.2
N_train_size = 3000
N_mini_batches = int(N_train_size / train_batch_size +
int(N_train_size % train_batch_size > 0))
for epoch in tqdm(range(start_epoch, epochs), desc='Epoch', leave=True):
r_loss_train = 0
dmm.train(True)
idx = 0
mov_avg_loss = 0
mov_data_len = 0
for which_mini_batch, data in enumerate(tqdm(train_dataloader, desc='Train', leave=True)):
if annealing_epochs > 0 and epoch < annealing_epochs:
# compute the KL annealing factor approriate for the current mini-batch in the current epoch
min_af = minimum_annealing_factor
annealing_factor = min_af + (1.0 - min_af) * \
(float(which_mini_batch + epoch * N_mini_batches + 1) /
float(annealing_epochs * N_mini_batches))
else:
# by default the KL annealing factor is unity
annealing_factor = 1.0
data['observation'] = normalize(
data['observation'].unsqueeze(2).to(device))
batch_size, length, _, w, h = data['observation'].shape
data_reversed = reverse_sequences(data['observation'])
data_mask = torch.ones(
batch_size, length, input_channels, w, h).cuda()
loss = svi.step(data['observation'],
data_reversed, data_mask, annealing_factor)
# Running losses
mov_avg_loss += loss
mov_data_len += batch_size
r_loss_train += loss
idx += 1
# Average losses
train_loss_avg = r_loss_train / (len(train_dataset) * time_length)
writer.add_scalar('Loss/train', train_loss_avg, epoch)
logging.info("Epoch: %d, Training loss: %1.5f",
epoch, train_loss_avg)
# # Time to time evaluation
if epoch == epochs - 1:
for temp_pred_length in [20]:
r_loss_valid = 0
r_loss_loc_valid = 0
r_loss_scale_valid = 0
r_loss_latent_valid = 0
dmm.train(False)
val_pred_length = temp_pred_length
val_pred_input_length = 10
with torch.no_grad():
for i, data in enumerate(tqdm(valid_dataloader, desc='Eval', leave=True)):
data['observation'] = normalize(
data['observation'].unsqueeze(2).to(device))
batch_size, length, _, w, h = data['observation'].shape
data_reversed = reverse_sequences(
data['observation'])
data_mask = torch.ones(
batch_size, length, input_channels, w, h).cuda()
pred_tensor = data['observation'][:,
:input_length_for_pred, :, :, :]
pred_tensor_reversed = reverse_sequences(
pred_tensor)
pred_tensor_mask = torch.ones(
batch_size, input_length_for_pred, input_channels, w, h).cuda()
ground_truth = data['observation'][:,
input_length_for_pred:, :, :, :]
val_nll = svi.evaluate_loss(
data['observation'], data_reversed, data_mask)
preds, _, loss_loc, loss_scale = do_prediction_rep_inference(
dmm, pred_tensor_mask, val_pred_length, val_pred_input_length, data['observation'])
ground_truth = denormalize(
data['observation'].squeeze().cpu().detach()
)
pred_with_input = denormalize(
torch.cat(
[data['observation'][:, :-val_pred_length, :, :, :].squeeze(),
preds.squeeze()], dim=0
).cpu().detach()
)
# Running losses
r_loss_valid += val_nll
r_loss_loc_valid += loss_loc
r_loss_scale_valid += loss_scale
# Average losses
valid_loss_avg = r_loss_valid / \
(len(valid_dataset) * time_length)
valid_loss_loc_avg = r_loss_loc_valid / \
(len(valid_dataset) * val_pred_length * width * height)
valid_loss_scale_avg = r_loss_scale_valid / \
(len(valid_dataset) * val_pred_length * width * height)
writer.add_scalar('Loss/test', valid_loss_avg, epoch)
writer.add_scalar(
'Loss/test_obs', valid_loss_loc_avg, epoch)
writer.add_scalar('Loss/test_scale',
valid_loss_scale_avg, epoch)
logging.info("Validation loss: %1.5f", valid_loss_avg)
logging.info("Validation obs loss: %1.5f",
valid_loss_loc_avg)
logging.info("Validation scale loss: %1.5f",
valid_loss_scale_avg)
log_evaluation(evaluation_logpath, "Validation obs loss for {}s pred {}: {}\n".format(
val_pred_length, args.trialnumber, valid_loss_loc_avg))
log_evaluation(evaluation_logpath, "Validation scale loss for {}s pred {}: {}\n".format(
val_pred_length, args.trialnumber, valid_loss_scale_avg))
# Save model
if epoch % 50 == 0 or epoch == epochs - 1:
torch.save(dmm.state_dict(), args.modelsavepath / model_name /
'e{}-i{}-tn{}.pt'.format(epoch, idx, args.trialnumber))
adam.save(args.modelsavepath / model_name /
'e{}-i{}-opt-tn{}.opt'.format(epoch, idx, args.trialnumber))
# Last validation after training
test_samples_indices = range(100)
total_n = 0
if validation_only:
r_loss_loc_valid = 0
r_loss_scale_valid = 0
r_loss_latent_valid = 0
dmm.train(False)
val_pred_length = args.validpredlength
val_pred_input_length = 10
with torch.no_grad():
for i in tqdm(test_samples_indices, desc='Valid', leave=True):
# Data processing
data = valid_dataset[i]
if torch.isnan(torch.sum(data['observation'])):
print("Skip {}".format(i))
continue
else:
total_n += 1
data['observation'] = normalize(
data['observation'].unsqueeze(0).unsqueeze(2).to(device))
batch_size, length, _, w, h = data['observation'].shape
data_reversed = reverse_sequences(data['observation'])
data_mask = torch.ones(
batch_size, length, input_channels, w, h).to(device)
# Prediction
pred_tensor_mask = torch.ones(
batch_size, input_length_for_pred, input_channels, w, h).to(device)
preds, _, loss_loc, loss_scale = do_prediction_rep_inference(
dmm, pred_tensor_mask, val_pred_length, val_pred_input_length, data['observation'])
ground_truth = denormalize(
data['observation'].squeeze().cpu().detach()
)
pred_with_input = denormalize(
torch.cat(
[data['observation'][:, :-val_pred_length, :, :, :].squeeze(),
preds.squeeze()], dim=0
).cpu().detach()
)
# Save samples
if i < 5:
save_dir_samples = Path('./samples/more_variance_long')
with open(save_dir_samples / '{}-gt-test.pkl'.format(i), 'wb') as fout:
pickle.dump(ground_truth, fout)
with open(save_dir_samples / '{}-vanilladmm-pred-test.pkl'.format(i), 'wb') as fout:
pickle.dump(pred_with_input, fout)
# Running losses
r_loss_loc_valid += loss_loc
r_loss_scale_valid += loss_scale
r_loss_latent_valid += np.sum((preds.squeeze().detach().cpu().numpy(
) - data['latent'][time_length - val_pred_length:, :, :].detach().cpu().numpy()) ** 2)
# Average losses
test_samples_indices = range(total_n)
print(total_n)
valid_loss_loc_avg = r_loss_loc_valid / \
(total_n * val_pred_length * width * height)
valid_loss_scale_avg = r_loss_scale_valid / \
(total_n * val_pred_length * width * height)
valid_loss_latent_avg = r_loss_latent_valid / \
(total_n * val_pred_length * width * height)
logging.info("Validation obs loss for %ds pred VanillaDMM: %f",
val_pred_length, valid_loss_loc_avg)
logging.info("Validation latent loss: %f", valid_loss_latent_avg)
with open('VanillaDMMResult.log', 'a+') as fout:
validation_log = 'Pred {}s VanillaDMM: {}\n'.format(
val_pred_length, valid_loss_loc_avg)
fout.write(validation_log)
batch_size = 32
alpha = 3.0
''' Loading data '''
df = dt.load_data(params.global_params['db_path'], symbol, index_col='date')
df = df[['open', 'high', 'low', 'close']]
''' Preparing data - Inline data as input parameters '''
data = df.values
train_rows = int(data.shape[0] * train_size)
train_data = data[:train_rows]
test_data = data[train_rows:]
test_dates = df.index.values[train_rows + look_back:]
x_close_train, y_train = dt.normalize(train_data[:, 3],
look_back=look_back,
look_ahead=look_ahead,
alpha=alpha)
x_close_test, y_test = dt.normalize(test_data[:, 3],
look_back=look_back,
look_ahead=look_ahead,
alpha=alpha)
x_open_train, _ = dt.normalize(train_data[:, 0],
ref_data=train_data[:, 3],
look_back=look_back,
look_ahead=look_ahead,
alpha=alpha)
x_open_test, _ = dt.normalize(test_data[:, 0],
ref_data=test_data[:, 3],
look_back=look_back,
look_ahead=look_ahead,
def validate_dataset(dataloader, name='test'):
for temp_pred_length in validation_pred_lengths:
r_loss_valid = 0
r_loss_loc_valid = 0
r_loss_scale_valid = 0
val_pred_length = temp_pred_length
val_pred_input_length = 10
dmm.train(False)
dmm.convlstm.eval()
with torch.no_grad():
for i, data in enumerate(
tqdm(dataloader, desc=name, leave=True)):
data['observation'] = normalize(
data['observation'].unsqueeze(2).to(
device), data_min_val, data_max_val)
batch_size, length, _, w, h = data[
'observation'].shape
data_reversed = reverse_sequences(
data['observation'])
data_mask = torch.ones(batch_size, length,
input_channels, w,
h).cuda()
pred_tensor = data[
'observation'][:, :
input_length_for_pred, :, :, :]
pred_tensor_reversed = reverse_sequences(
pred_tensor)
pred_tensor_mask = torch.ones(
batch_size, input_length_for_pred,
input_channels, w, h).cuda()
ground_truth = data[
'observation'][:,
input_length_for_pred:, :, :, :]
val_nll = svi.evaluate_loss(
data['observation'], data_reversed,
data_mask)
preds, _, loss_loc, loss_scale = do_prediction_rep_inference(
dmm, pred_tensor_mask, val_pred_length,
val_pred_input_length, data['observation'])
ground_truth = denormalize(
data['observation'].squeeze().cpu().detach(
), data_min_val, data_max_val)
pred_with_input = denormalize(
torch.cat([
data['observation']
[:, :-val_pred_length, :, :, :].
squeeze(),
preds.squeeze()
],
dim=0).cpu().detach(),
data_min_val, data_max_val)
# Running losses
r_loss_valid += val_nll
r_loss_loc_valid += loss_loc
r_loss_scale_valid += loss_scale
# Average losses
valid_loss_avg = r_loss_valid / \
(len(valid_dataset) * time_length)
valid_loss_loc_avg = r_loss_loc_valid / \
(len(valid_dataset) *
val_pred_length * width * height)
writer.add_scalar('Loss/{}'.format(name),
valid_loss_avg, epoch)
writer.add_scalar('Loss/{}_obs'.format(name),
valid_loss_loc_avg, epoch)
logging.info("Validation loss: %1.5f",
valid_loss_avg)
logging.info("Validation obs loss: %1.5f",
valid_loss_loc_avg)
log_evaluation(
evaluation_logpath,
"Validation obs loss for {}s pred {} ConvDMM: {}\n"
.format(val_pred_length, args.trialnumber,
valid_loss_loc_avg))
def main(args):
# Init tensorboard
writer = SummaryWriter('./runs/' + args.runname + str(args.trialnumber))
# Set evaluation log file
evaluation_logpath = './logs/convlstm/evaluation_result.log'
log_evaluation(evaluation_logpath,
'Evaluation Trial - {}\n'.format(args.trialnumber))
# Constants
time_length = 30
input_length = 20
pred_length = time_length - input_length
validation_pred_lengths = [5, 10, 15, 20]
train_batch_size = 16
valid_batch_size = 1
min_val, max_val = 0, 1000
# Device checking
use_cuda = torch.cuda.is_available()
device = torch.device("cuda:0" if use_cuda else "cpu")
# Make dataset
logging.info("Generate data")
train_datapath = args.datapath / 'train'
valid_datapath = args.datapath / 'valid'
train_dataset = DiffusionDataset(train_datapath)
valid_dataset = DiffusionDataset(valid_datapath)
# Create data loaders from pickle data
logging.info("Generate data loaders")
train_dataloader = DataLoader(train_dataset,
batch_size=train_batch_size,
shuffle=True,
num_workers=4)
valid_dataloader = DataLoader(valid_dataset,
batch_size=valid_batch_size,
num_workers=4,
shuffle=False)
# Training parameters
row_dim = 100
col_dim = 100
input_dim = row_dim * col_dim
# Create model
logging.warning("Generate model")
logging.warning(input_dim)
input_channels = 1
hidden_channels = [64]
kernel_size = 3
pred_input_dim = 10
model = ConvLSTM(input_channels, hidden_channels, kernel_size,
pred_input_dim, device)
# Initialize model
logging.info("Initialize model")
epochs = args.endepoch
learning_rate = 0.001
criterion = torch.nn.MSELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
# Starting epoch
start_epoch = args.startepoch
# Load model
if start_epoch != 0:
load_model = Path('./checkpoints/ConvLSTM/e{}-i188-tn{}.pt'.format(
start_epoch - 1, args.trialnumber))
if load_model is not None:
logging.info("loading model from %s..." % load_model)
model.load_state_dict(torch.load(load_model, map_location=device))
# Validation only?
validation_only = args.validonly
# Train the model
if not validation_only:
logging.info("Training model")
train_losses = []
valid_losses = []
for epoch in tqdm(range(start_epoch, epochs), desc='Epoch',
leave=True):
r_loss_train = 0
model.train(True)
idx = 0
mov_avg_loss = 0
for data in tqdm(train_dataloader, desc='Train', leave=True):
optimizer.zero_grad()
data['observation'] = normalize(
data['observation'].unsqueeze(2).to(device), min_val,
max_val)
batch_size, length, _, w, h = data['observation'].shape
preds, _ = model(
data['observation'][:, :-pred_length, :, :, :],
pred_length)
loss = criterion(
preds, data['observation'][:, input_length:, :, :, :])
loss.backward()
optimizer.step()
# Running losses
mov_avg_loss += loss.item() * data['observation'].size(0)
if idx % 3 == 2:
with open('training.log', 'a+') as fout:
temp_avg = mov_avg_loss / 3
training_log = 'Epoch {}: Iter {}: {}\n'.format(
epoch, idx, temp_avg)
fout.write(training_log)
mov_avg_loss = 0
r_loss_train += loss.item() * data['observation'].size(0)
idx += 1
# Average losses
train_loss_avg = r_loss_train / len(train_dataset)
writer.add_scalar('Loss/train', train_loss_avg, epoch)
logging.info("Epoch: %d, Training loss: %1.5f", epoch,
train_loss_avg)
# Time to time evaluation
if epoch == epochs - 1:
for temp_pred_length in validation_pred_lengths:
r_loss_valid = 0
r_loss_loc_valid = 0
val_input_length = time_length - temp_pred_length
model.train(False)
with torch.no_grad():
for i, data in enumerate(
tqdm(valid_dataloader,
desc='Valid',
leave=True)):
data['observation'] = normalize(
data['observation'].unsqueeze(2).to(device),
min_val, max_val)
batch_size, length, _, w, h = data[
'observation'].shape
preds = model.forward_with_gt(
data['observation'], temp_pred_length)
loss = criterion(
preds, data['observation']
[:, val_input_length:, :, :, :])
ground_truth = denormalize(
data['observation'].squeeze().cpu().detach(),
min_val, max_val)
pred_with_input = denormalize(
torch.cat([
data['observation']
[:, :-temp_pred_length, :, :, :].squeeze(),
preds.squeeze()
],
dim=0).cpu().detach(), min_val,
max_val)
# Running losses
r_loss_valid += loss.item() * \
data['observation'].size(0)
r_loss_loc_valid += np.sum((
preds.detach().cpu().numpy() -
data['observation'][:,
val_input_length:, :, :, :]
.detach().cpu().numpy())**2)
# Average losses
valid_loss_avg = r_loss_valid / len(valid_dataset)
valid_loss_loc_avg = r_loss_loc_valid / \
(len(valid_dataset) * temp_pred_length * col_dim * row_dim)
writer.add_scalar('Loss/test_obs', valid_loss_loc_avg,
epoch)
writer.add_scalar('Loss/test', valid_loss_avg, epoch)
logging.info("Validation loss: %1.5f", valid_loss_avg)
logging.info("Validation obs loss: %1.5f",
valid_loss_loc_avg)
log_evaluation(
evaluation_logpath,
"Validation obs loss for {}s pred {}: {}\n".format(
temp_pred_length, args.trialnumber,
valid_loss_loc_avg))
# Save model
torch.save(
model.state_dict(), args.modelsavepath /
'e{}-i{}-tn{}.pt'.format(epoch, idx, args.trialnumber))
# Last validation after training
test_samples_indices = range(len(valid_dataset))
total_n = len(valid_dataset)
if validation_only:
r_loss_valid = 0
r_loss_loc_valid = 0
r_loss_latent_valid = 0
model.train(False)
pred_length = args.validpredlength
input_length = time_length - pred_length
with torch.no_grad():
for i in tqdm(test_samples_indices, desc='Valid', leave=True):
# Data processing
data = valid_dataset[i]
data['observation'] = normalize(
data['observation'].unsqueeze(0).unsqueeze(2).to(device),
min_val, max_val)
batch_size, length, _, w, h = data['observation'].shape
preds = model.forward_with_gt(data['observation'], pred_length)
loss = criterion(
preds, data['observation'][:, input_length:, :, :, :])
ground_truth = denormalize(
data['observation'].squeeze().cpu().detach(), min_val,
max_val)
pred_with_input = denormalize(
torch.cat([
data['observation']
[:, :-pred_length, :, :, :].squeeze(),
preds.squeeze()
],
dim=0).cpu().detach(), min_val, max_val)
# Running losses
r_loss_valid += loss.item() * data['observation'].size(0)
r_loss_loc_valid += np.sum(
(preds.detach().cpu().numpy() - data['observation']
[:, input_length:, :, :, :].detach().cpu().numpy())**2)
r_loss_latent_valid += np.sum((
preds.squeeze().detach().cpu().numpy() -
data['latent'][input_length:, :, :].detach().cpu().numpy()
)**2)
# Average losses
print(total_n)
valid_loss_avg = r_loss_valid / total_n
valid_loss_loc_avg = r_loss_loc_valid / \
(total_n * pred_length * col_dim * row_dim)
valid_loss_latent_avg = r_loss_latent_valid / \
(total_n * pred_length * col_dim * row_dim)
logging.info("Validation loss: %f", valid_loss_avg)
logging.info("Validation obs loss: %f", valid_loss_loc_avg)
logging.info("Validation latent loss: %f", valid_loss_latent_avg)
def data_for_prediction(symbol,
date,
next_date,
look_back,
look_ahead,
alpha,
hmm_model,
db_path=params.global_params['db_path']):
limit = look_back * 2
x_df = dt.load_data(db_path,
symbol,
to_date=date,
limit=limit,
index_col='date')
x_df = x_df[['open', 'high', 'low', 'close', 'volume']]
y_df = dt.load_data(db_path,
symbol,
to_date=next_date,
limit=limit,
index_col='date')
y_data = y_df[['close']].values
x_test_dates = x_df.index.values
''' Preparing x_data - Inline x_data as input parameters '''
_log_returns = x_df['close'].values
_log_returns = np.diff(np.log(_log_returns), n=look_ahead)
_log_returns = np.insert(_log_returns, 0, np.zeros((look_ahead)), axis=0)
x_data = x_df.values
x_data = np.insert(x_data, x_data.shape[1], _log_returns, axis=1) #5
x_data[:, 4] = dt.min_max_normalize(x_data[:, 4], method='tanh')
hmm_input_test = np.column_stack([x_data[:, 5]])
hmm_test = hmm_model.predict_proba(hmm_input_test)
# print('hmm_test.shape', hmm_test.shape)
# print('x_data.shape', x_data.shape)
# normalize and stack OHLC x_data for NN
x_close_test = dt.normalize_for_future(x_data[:, 3],
look_back=look_back,
look_ahead=look_ahead,
alpha=alpha)
_, y_test = dt.normalize(y_data,
look_back=look_back,
look_ahead=look_ahead,
alpha=alpha)
# print('x_close_test.shape', x_close_test.shape)
# print('y_test.shape', y_test.shape)
x_open_test = dt.normalize_for_future(x_data[:, 0],
ref_data=x_data[:, 3],
look_back=look_back,
look_ahead=look_ahead,
alpha=alpha)
x_high_test = dt.normalize_for_future(x_data[:, 1],
ref_data=x_data[:, 3],
look_back=look_back,
look_ahead=look_ahead,
alpha=alpha)
x_low_test = dt.normalize_for_future(x_data[:, 2],
ref_data=x_data[:, 3],
look_back=look_back,
look_ahead=look_ahead,
alpha=alpha)
x_volume_test = dt.normalize_for_future(x_data[:, 4],
ref_data=x_data[:, 4],
look_back=look_back,
look_ahead=look_ahead,
alpha=alpha)
x_volume_test[np.isnan(x_volume_test)] = 0.0
x_returns_test = dt.stack(x_data[:, 5],
look_back=look_back,
look_ahead=look_ahead)
hmm_hidden0_test = dt.stack(hmm_test[:, 0],
look_back=look_back,
look_ahead=look_ahead)
hmm_hidden1_test = dt.stack(hmm_test[:, 1],
look_back=look_back,
look_ahead=look_ahead)
hmm_hidden2_test = dt.stack(hmm_test[:, 2],
look_back=look_back,
look_ahead=look_ahead)
''' Reshape x_data for model '''
x_close_test = np.reshape(x_close_test,
(x_close_test.shape[0], x_close_test.shape[1]))
x_open_test = np.reshape(x_open_test,
(x_open_test.shape[0], x_open_test.shape[1]))
x_high_test = np.reshape(x_high_test,
(x_high_test.shape[0], x_high_test.shape[1]))
x_low_test = np.reshape(x_low_test,
(x_low_test.shape[0], x_low_test.shape[1]))
x_volume_test = np.reshape(
x_volume_test, (x_volume_test.shape[0], x_volume_test.shape[1]))
x_returns_test = np.reshape(
x_returns_test, (x_returns_test.shape[0], x_returns_test.shape[1]))
hmm_hidden0_test = np.reshape(
hmm_hidden0_test,
(hmm_hidden0_test.shape[0], hmm_hidden0_test.shape[1])) * 2 - 1
hmm_hidden1_test = np.reshape(
hmm_hidden1_test,
(hmm_hidden1_test.shape[0], hmm_hidden1_test.shape[1])) * 2 - 1
hmm_hidden2_test = np.reshape(
hmm_hidden2_test,
(hmm_hidden2_test.shape[0], hmm_hidden2_test.shape[1])) * 2 - 1
# print('hmm_hidden0_test.shape', hmm_hidden0_test.shape)
x_test = np.hstack((x_open_test, x_high_test, x_low_test, x_returns_test,
hmm_hidden0_test, hmm_hidden1_test, hmm_hidden2_test))
dimensions = int(x_test.shape[1] / x_open_test.shape[1])
return x_test, x_test_dates[-look_back:], y_test, dimensions
batch_size = 32
alpha = 3.0
''' Loading data '''
df = dt.load_data(params.global_params['db_path'], symbol, index_col='date')
df = df['close']
''' Preparing data - Inline data as input parameters '''
data = df.values
train_rows = int(data.shape[0] * train_size)
train_data = data[:train_rows]
test_data = data[train_rows:]
test_dates = df.index.values[train_rows + look_back:]
x_close_train, y_train = dt.normalize(train_data,
look_back=look_back,
look_ahead=look_ahead,
alpha=alpha)
x_close_test, y_test = dt.normalize(test_data,
look_back=look_back,
look_ahead=look_ahead,
alpha=alpha)
''' Randomize train data '''
shuffled = list(np.random.permutation(x_close_train.shape[0]))
x_close_train = x_close_train[shuffled]
y_train = y_train[shuffled]
''' Reshape data for model '''
x_close_train = np.reshape(x_close_train,
(x_close_train.shape[0], x_close_train.shape[1]))
x_close_test = np.reshape(x_close_test,
(x_close_test.shape[0], x_close_test.shape[1]))
y_train = np.reshape(y_train, (y_train.shape[0], 1))
