def format_data(ae_1_obj, ae_2_obj, type='test'):
_, X_val, X_test, _, Y_val, Y_test, _ = data_feeder.get_data_val(True)
if type == 'val':
X = X_val
Y = Y_val
elif type == 'test':
X = X_test
Y = Y_test
# Test
xformed_exog_test = ae_1_obj.model.predict(X)
_x1 = get_windowed_data(data=xformed_exog_test, window_size=exog_lag)
_x2 = get_windowed_data(data=Y, window_size=endo_lag + 1)
_x2 = _x2[:, 0:endo_lag]
_y = _x2[:, -1:]
num_samples = min(_x1.shape[0], _x2.shape[0])
_x1 = _x1[-num_samples:]
_x2 = _x2[-num_samples:]
_y = _y[-num_samples:, :]
ae2_inp = np.concatenate([_x1, _x2], axis=-1)
_x = ae_2_obj.model.predict(ae2_inp)
return _x, _y
def test_model(self):
_, _, exog_test, _, _, end_test, _ = data_feeder.get_data_val(True)
x1, x2, y = self.format_data_xy(exog_test, end_test)
score = self.model.evaluate([x1, x2], y)
test_mse = score[0]
return test_mse
def FTDNN():
global target_window
batch_size = 128
global epochs
X_train, X_val, X_test, Y_train, Y_val, Y_test, _ = data_feeder.get_data_val(
True)
train_windowed_data = get_windowed_data_ex(Y_train, target_window + 1)
test_windowed_data = get_windowed_data_ex(Y_test, target_window + 1)
val_windowed_data = get_windowed_data_ex(Y_val, target_window + 1)
# Set up training input and output
x_train = []
y_train = []
for i in range(train_windowed_data.shape[0]):
x_train.append(train_windowed_data[i, 0:target_window])
y_train.append(train_windowed_data[i, -1:])
x_train = np.asarray(x_train)
y_train = np.asarray(y_train)
x_train = np.asarray(x_train)
x_train = np.reshape(x_train, [x_train.shape[0], x_train.shape[1]])
y_train = np.asarray(y_train)
y_train = np.reshape(y_train, [y_train.shape[0], y_train.shape[1]])
x_val = []
y_val = []
for i in range(val_windowed_data.shape[0]):
x_val.append(val_windowed_data[i, 0:target_window])
y_val.append(val_windowed_data[i, -1:])
x_val = np.asarray(x_val)
x_val = np.reshape(x_val, [x_val.shape[0], x_val.shape[1]])
y_val = np.asarray(y_val)
y_val = np.reshape(y_val, [y_val.shape[0], y_val.shape[1]])
x_test = []
y_test = []
for i in range(test_windowed_data.shape[0]):
x_test.append(test_windowed_data[i, 0:target_window])
y_test.append(test_windowed_data[i, -1:])
x_test = np.asarray(x_test)
x_test = np.reshape(x_test, [x_test.shape[0], x_test.shape[1]])
y_test = np.asarray(y_test)
y_test = np.reshape(y_test, [y_test.shape[0], y_test.shape[1]])
model = FTDNN_model(target_window)
history = model.fit(x_train, y_train, epochs=epochs, batch_size=batch_size)
train_loss = np.mean(history.history['loss'])
val_mse = model.evaluate(x_val, y_val, batch_size=1)
test_mse = model.evaluate(x_test, y_test, batch_size=1)
return train_loss, val_mse, test_mse
def train_model(self):
exog_train, _, _, end_train, _, _, _ = data_feeder.get_data_val(True)
x1, x2, y = self.format_data_xy(exog_train, end_train)
hist = self.model.fit([x1, x2], y, epochs=self.epochs)
trian_loss = hist.history['loss']
return np.mean(trian_loss)
def get_data(time_window, type):
# window of training data = time_window
X_train, X_val, X_test, Y_train, Y_val, Y_test, _ = data_feeder.get_data_val(
std=True)
if type == 'test':
exg_x = X_test
end_x = Y_test
elif type == 'val':
exg_x = X_val
end_x = Y_val
else:
exg_x = X_train
end_x = Y_train
exg_x = utils.get_windowed_data_md(exg_x, time_window)
# Reshape exg_shape to have same number of channels as exogenous dimensions
dim = exg_x.shape[-1]
exg_x = np.reshape(exg_x, [exg_x.shape[0], time_window, dim])
# print 'exg_x shape', exg_x.shape
y = np.asanyarray(end_x)
y = np.reshape(y, [-1, 1])
res = utils.get_windowed_data(y, time_window + 1)
end_x = res[:, 0:time_window]
y = res[:, -1:]
end_x = np.reshape(end_x, [end_x.shape[0], time_window, 1])
# print 'end_x shape', end_x.shape
y = np.reshape(y, [y.shape[0], 1])
num_samples = y.shape[0]
exg_x = exg_x[0:num_samples]
print exg_x.shape, end_x.shape, y.shape
return exg_x, end_x, y
def ctdnn_get_data(window_size, type='trian'):
exog_train, exog_val, exog_test, end_train, end_val, end_test, scaler_array = data_feeder.get_data_val(
True)
if type == 'train':
train_x_exog, train_x_end, train_y = ctdnn_get_data_aux(
exog_train, end_train, window_size)
return train_x_exog, train_x_end, train_y
elif type == 'val':
val_x_exog, val_x_end, val_y = ctdnn_get_data_aux(
exog_val, end_val, window_size)
return val_x_exog, val_x_end, val_y
else:
test_x_exog, test_x_end, test_y = ctdnn_get_data_aux(
exog_test, end_test, window_size)
return test_x_exog, test_x_end, test_y
def get_val_data():
global ae_obj
global window_size
_, X_val, _, _, Y_val, _, _ = data_feeder.get_data_val(True)
# set up test data the model
val_y_inp, val_y_op = get_windowed_y(Y_val, window_size=window_size)
X_val_ae = ae_obj.reduce(X_val)
val_exog_inp = X_val_ae[window_size:, :]
# concatenate train_exog_inp and train_y_op
val_data_x = np.concatenate([val_y_inp, val_exog_inp], axis=1)
print 'Validation Data'
print val_data_x.shape
print val_y_op.shape
return val_data_x, val_y_op
def get_test_data():
global ae_obj
global window_size
_, _, X_test, _, _, Y_test, _ = data_feeder.get_data_val(True)
# set up test data the model
test_y_inp, test_y_op = get_windowed_y(Y_test, window_size=window_size)
X_test_ae = ae_obj.reduce(X_test)
test_exog_inp = X_test_ae[window_size:, :]
# concatenate train_exog_inp and train_y_op
test_data_x = np.concatenate([test_y_inp, test_exog_inp], axis=1)
print 'Test Data'
print test_data_x.shape
print test_y_op.shape
return test_data_x, test_y_op
def get_data(time_window, type):
# window of training data = time_window
X_train, _, X_test, Y_train, Y_val, Y_test, _ = data_feeder.get_data_val(
std=True)
if type == 'test':
y = Y_test
elif type == 'val':
y = Y_val
else:
y = Y_train
y = np.asanyarray(y)
y = np.reshape(y, [-1, 1])
res = utils.get_windowed_data(y, time_window + 1)
x = res[:, 0:time_window]
y = res[:, -1:]
x = np.reshape(x, [x.shape[0], time_window, 1])
y = np.reshape(y, [y.shape[0], 1])
return x, y
def val_test_model(val=False):
# -------- #
global lstm_time_step
global batch_size
global epochs
global window_size
# --------- #
ae_model = load_model('ae_model.h5')
lstm_model = load_model('model_1.h5')
if val == True:
_, _, X_test, _, _, Y_test, _ = data_feeder.get_data_val(True)
else:
_, X_test, _, Y_test, _ = data_feeder.get_data(True)
# create windows
inp, op = get_windowed_inp(Y_test, window_size)
num_samples = inp.shape[0]
print 'num_samples', num_samples
X_test = X_test[-num_samples:, :]
print 'X_test', X_test.shape
# concatenate X_train and input passed through ae
ae_op = ae_model.predict(X_test)
# print 'ae_op', ae_op.shape
inp = np.concatenate([inp, ae_op], axis=-1)
print 'inp shape', inp.shape
# Reshape the data
pad_len = lstm_time_step - (inp.shape[0] % lstm_time_step)
# add padding
_inp_pad = np.zeros(shape=[pad_len, inp.shape[-1]])
_op_pad = np.zeros(shape=[pad_len, op.shape[-1]])
inp = np.concatenate([_inp_pad, inp], axis=0)
op = np.concatenate([_op_pad, op], axis=0)
inp_dim = inp.shape[-1]
op_dim = op.shape[-1]
inp = np.reshape(inp, [-1, lstm_time_step, inp_dim])
op = np.reshape(op, [-1, lstm_time_step, op_dim])
num_samples = (inp.shape[0] // batch_size) * batch_size
inp = inp[-num_samples:, :, :]
op = op[-num_samples:, :, :]
# -------- #
print 'Shape input dimension ', inp.shape
print 'Shape output dimension ', op.shape
test_x = inp
test_y = op
score = lstm_model.evaluate(x=test_x, y=test_y, batch_size=batch_size)
print 'Mean Square Error', score[0]
return score[0]
def val_model(self):
_, exog_val, _, _, end_val, _, _ = data_feeder.get_data_val(True)
x1, x2, y = self.format_data_xy(exog_val, end_val)
score = self.model.evaluate([x1, x2], y)
val_mse = score[0]
return val_mse