class TestZouwuModelMTNetForecaster(TestCase):
def setUp(self):
tf.keras.backend.clear_session()
self.ft = TimeSequenceFeatureTransformer()
self.create_data()
def tearDown(self):
pass
def create_data(self):
def gen_train_sample(data, past_seq_len, future_seq_len):
data = pd.DataFrame(data)
x, y = self.ft._roll_train(data,
past_seq_len=past_seq_len,
future_seq_len=future_seq_len)
return x, y
def gen_test_sample(data, past_seq_len):
test_data = pd.DataFrame(data)
x = self.ft._roll_test(test_data, past_seq_len=past_seq_len)
return x
self.long_num = 6
self.time_step = 2
look_back = (self.long_num + 1) * self.time_step
look_forward = 1
self.x_train, self.y_train = gen_train_sample(
data=np.random.randn(64, 4),
past_seq_len=look_back,
future_seq_len=look_forward)
self.x_val, self.y_val = gen_train_sample(data=np.random.randn(16, 4),
past_seq_len=look_back,
future_seq_len=look_forward)
self.x_test = gen_test_sample(data=np.random.randn(16, 4),
past_seq_len=look_back)
def test_forecast_mtnet(self):
# TODO hacking to fix a bug
target_dim = 1
model = MTNetForecaster(target_dim=target_dim,
feature_dim=self.x_train.shape[-1],
long_series_num=self.long_num,
series_length=self.time_step)
x_train_long, x_train_short = model.preprocess_input(self.x_train)
x_val_long, x_val_short = model.preprocess_input(self.x_val)
x_test_long, x_test_short = model.preprocess_input(self.x_test)
model.fit([x_train_long, x_train_short],
self.y_train,
validation_data=([x_val_long, x_val_short], self.y_val),
batch_size=32,
distributed=False)
assert model.evaluate([x_val_long, x_val_short], self.y_val)
predict_result = model.predict([x_test_long, x_test_short])
assert predict_result.shape == (self.x_test.shape[0], target_dim)
class TestSeq2Seq(ZooTestCase):
def setup_method(self, method):
# super().setup_method(method)
self.train_data = pd.DataFrame(data=np.random.randn(64, 4))
self.val_data = pd.DataFrame(data=np.random.randn(16, 4))
self.test_data = pd.DataFrame(data=np.random.randn(16, 4))
self.past_seq_len = 6
self.future_seq_len_1 = 1
self.future_seq_len_2 = 2
# use roll method in time_sequence
self.feat = TimeSequenceFeatureTransformer()
self.config = {'batch_size': 32, 'epochs': 1}
self.model_1 = LSTMSeq2Seq(check_optional_config=False,
future_seq_len=self.future_seq_len_1)
self.model_2 = LSTMSeq2Seq(check_optional_config=False,
future_seq_len=self.future_seq_len_2)
self.fitted = False
self.predict_1 = None
self.predict_2 = None
def teardown_method(self, method):
pass
def test_fit_eval_1(self):
x_train_1, y_train_1 = self.feat._roll_train(
self.train_data,
past_seq_len=self.past_seq_len,
future_seq_len=self.future_seq_len_1)
print("fit_eval_future_seq_len_1:",
self.model_1.fit_eval(x_train_1, y_train_1, **self.config))
assert self.model_1.past_seq_len == 6
assert self.model_1.feature_num == 4
assert self.model_1.future_seq_len == 1
assert self.model_1.target_col_num == 1
def test_fit_eval(self):
past_seq_len = 6
future_seq_len = 2
input_dim = 5
output_dim = 4
x_train = np.random.rand(100, past_seq_len, input_dim)
y_train = np.random.rand(100, future_seq_len, output_dim)
x_test = np.random.rand(100, past_seq_len, input_dim)
y_test = np.random.rand(100, future_seq_len, output_dim)
model = LSTMSeq2Seq(check_optional_config=False,
future_seq_len=future_seq_len)
model_config = {
'batch_size': 32,
'epochs': 1,
'latent_dim': 128,
'dropout': 0.2
}
model.fit_eval(x_train, y_train, **model_config)
y_pred = model.predict(x_test)
rmse, smape = model.evaluate(x=x_test,
y=y_test,
metric=["rmse", "smape"])
assert rmse.shape == smape.shape
assert rmse.shape == (future_seq_len, output_dim)
assert model.past_seq_len == past_seq_len
assert model.future_seq_len == future_seq_len
assert model.feature_num == input_dim
assert model.target_col_num == output_dim
assert y_pred.shape == y_test.shape
def test_fit_eval_2(self):
x_train_2, y_train_2 = self.feat._roll_train(
self.train_data,
past_seq_len=self.past_seq_len,
future_seq_len=self.future_seq_len_2)
print("fit_eval_future_seq_len_2:",
self.model_2.fit_eval(x_train_2, y_train_2, **self.config))
assert self.model_2.future_seq_len == 2
self.fitted = True
def test_evaluate_1(self):
x_train_1, y_train_1 = self.feat._roll_train(
self.train_data,
past_seq_len=self.past_seq_len,
future_seq_len=self.future_seq_len_1)
x_val_1, y_val_1 = self.feat._roll_train(
self.val_data,
past_seq_len=self.past_seq_len,
future_seq_len=self.future_seq_len_1)
self.model_1.fit_eval(x_train_1, y_train_1, **self.config)
print("evaluate_future_seq_len_1:",
self.model_1.evaluate(x_val_1, y_val_1, metric=['mse', 'r2']))
def test_evaluate_2(self):
x_train_2, y_train_2 = self.feat._roll_train(
self.train_data,
past_seq_len=self.past_seq_len,
future_seq_len=self.future_seq_len_2)
x_val_2, y_val_2 = self.feat._roll_train(
self.val_data,
past_seq_len=self.past_seq_len,
future_seq_len=self.future_seq_len_2)
self.model_2.fit_eval(x_train_2, y_train_2, **self.config)
print("evaluate_future_seq_len_2:",
self.model_2.evaluate(x_val_2, y_val_2, metric=['mse', 'r2']))
def test_predict_1(self):
x_train_1, y_train_1 = self.feat._roll_train(
self.train_data,
past_seq_len=self.past_seq_len,
future_seq_len=self.future_seq_len_1)
x_test_1 = self.feat._roll_test(self.test_data,
past_seq_len=self.past_seq_len)
self.model_1.fit_eval(x_train_1, y_train_1, **self.config)
predict_1 = self.model_1.predict(x_test_1)
assert predict_1.shape == (x_test_1.shape[0], self.future_seq_len_1)
def test_predict_2(self):
x_train_2, y_train_2 = self.feat._roll_train(
self.train_data,
past_seq_len=self.past_seq_len,
future_seq_len=self.future_seq_len_2)
x_test_2 = self.feat._roll_test(self.test_data,
past_seq_len=self.past_seq_len)
self.model_2.fit_eval(x_train_2, y_train_2, **self.config)
predict_2 = self.model_2.predict(x_test_2)
assert predict_2.shape == (x_test_2.shape[0], self.future_seq_len_2)
def test_save_restore_1(self):
x_train_1, y_train_1 = self.feat._roll_train(
self.train_data,
past_seq_len=self.past_seq_len,
future_seq_len=self.future_seq_len_1)
x_test_1 = self.feat._roll_test(self.test_data,
past_seq_len=self.past_seq_len)
self.model_1.fit_eval(x_train_1, y_train_1, **self.config)
predict_1_before = self.model_1.predict(x_test_1)
new_model_1 = LSTMSeq2Seq(check_optional_config=False)
dirname = tempfile.mkdtemp(prefix="automl_test_feature")
try:
save(dirname, model=self.model_1)
restore(dirname, model=new_model_1, config=self.config)
predict_1_after = new_model_1.predict(x_test_1)
assert_array_almost_equal(predict_1_before, predict_1_after, decimal=2), \
"Prediction values are not the same after restore: " \
"predict before is {}, and predict after is {}".format(predict_1_before,
predict_1_after)
new_config = {'epochs': 1}
new_model_1.fit_eval(x_train_1, y_train_1, **new_config)
finally:
shutil.rmtree(dirname)
def test_save_restore_2(self):
x_train_2, y_train_2 = self.feat._roll_train(
self.train_data,
past_seq_len=self.past_seq_len,
future_seq_len=self.future_seq_len_2)
x_test_2 = self.feat._roll_test(self.test_data,
past_seq_len=self.past_seq_len)
self.model_2.fit_eval(x_train_2, y_train_2, **self.config)
predict_2_before = self.model_2.predict(x_test_2)
new_model_2 = LSTMSeq2Seq(check_optional_config=False)
dirname = tempfile.mkdtemp(prefix="automl_test_feature")
try:
save(dirname, model=self.model_2)
restore(dirname, model=new_model_2, config=self.config)
predict_2_after = new_model_2.predict(x_test_2)
assert_array_almost_equal(predict_2_before, predict_2_after, decimal=2), \
"Prediction values are not the same after restore: " \
"predict before is {}, and predict after is {}".format(predict_2_before,
predict_2_after)
new_config = {'epochs': 2}
new_model_2.fit_eval(x_train_2, y_train_2, **new_config)
finally:
shutil.rmtree(dirname)
def test_predict_with_uncertainty(self, ):
x_train_2, y_train_2 = self.feat._roll_train(
self.train_data,
past_seq_len=self.past_seq_len,
future_seq_len=self.future_seq_len_2)
x_test_2 = self.feat._roll_test(self.test_data,
past_seq_len=self.past_seq_len)
self.model_2.fit_eval(x_train_2, y_train_2, mc=True, **self.config)
prediction, uncertainty = self.model_2.predict_with_uncertainty(
x_test_2, n_iter=2)
assert prediction.shape == (x_test_2.shape[0], self.future_seq_len_2)
assert uncertainty.shape == (x_test_2.shape[0], self.future_seq_len_2)
assert np.any(uncertainty)
new_model_2 = LSTMSeq2Seq(check_optional_config=False)
dirname = tempfile.mkdtemp(prefix="automl_test_feature")
try:
save(dirname, model=self.model_2)
restore(dirname, model=new_model_2, config=self.config)
prediction, uncertainty = new_model_2.predict_with_uncertainty(
x_test_2, n_iter=2)
assert prediction.shape == (x_test_2.shape[0],
self.future_seq_len_2)
assert uncertainty.shape == (x_test_2.shape[0],
self.future_seq_len_2)
assert np.any(uncertainty)
finally:
shutil.rmtree(dirname)
class TestMTNetKeras(ZooTestCase):
def setup_method(self, method):
tf.keras.backend.clear_session()
self.ft = TimeSequenceFeatureTransformer()
self.create_data()
self.model = MTNetKeras()
self.config = {
"long_num": self.long_num,
"time_step": self.time_step,
"ar_window": np.random.randint(1, 3),
"cnn_height": np.random.randint(1, 3),
"epochs": 1
}
def teardown_method(self, method):
pass
def create_data(self):
def gen_train_sample(data, past_seq_len, future_seq_len):
data = pd.DataFrame(data)
x, y = self.ft._roll_train(data,
past_seq_len=past_seq_len,
future_seq_len=future_seq_len)
return x, y
def gen_test_sample(data, past_seq_len):
test_data = pd.DataFrame(data)
x = self.ft._roll_test(test_data, past_seq_len=past_seq_len)
return x
self.long_num = 6
self.time_step = 2
look_back = (self.long_num + 1) * self.time_step
look_forward = 1
self.x_train, self.y_train = gen_train_sample(
data=np.random.randn(64, 4),
past_seq_len=look_back,
future_seq_len=look_forward)
self.x_val, self.y_val = gen_train_sample(data=np.random.randn(16, 4),
past_seq_len=look_back,
future_seq_len=look_forward)
self.x_test = gen_test_sample(data=np.random.randn(16, 4),
past_seq_len=look_back)
def test_fit_evaluate(self):
self.model.fit_eval(data=(self.x_train, self.y_train),
validation_data=(self.x_val, self.y_val),
**self.config)
self.model.evaluate(self.x_val, self.y_val)
def test_save_restore(self):
self.model.fit_eval(data=(self.x_train, self.y_train),
validation_data=(self.x_val, self.y_val),
**self.config)
y_pred = self.model.predict(self.x_test)
assert y_pred.shape == (self.x_test.shape[0], self.y_train.shape[1])
dirname = "tmp"
restored_model = MTNetKeras()
try:
save(dirname, model=self.model)
restore(dirname, model=restored_model, config=self.config)
predict_after = restored_model.predict(self.x_test)
assert_array_almost_equal(y_pred, predict_after, decimal=2), \
"Prediction values are not the same after restore: " \
"predict before is {}, and predict after is {}".format(y_pred, predict_after)
restored_model.fit_eval((self.x_train, self.y_train), epochs=1)
restored_model.evaluate(self.x_val, self.y_val)
finally:
shutil.rmtree("tmp")
def test_predict_with_uncertainty(self):
self.model.fit_eval(data=(self.x_train, self.y_train),
validation_data=(self.x_val, self.y_val),
mc=True,
**self.config)
pred, uncertainty = self.model.predict_with_uncertainty(self.x_test,
n_iter=2)
assert pred.shape == (self.x_test.shape[0], self.y_train.shape[1])
assert uncertainty.shape == pred.shape
assert np.any(uncertainty)