def _test_initialization(self, warmup_iterations, batch_size):
stub_model = StubTimeSeriesModel()
data = self._make_test_data(length=20,
cut_start=None,
cut_end=None,
offset=0.)
if batch_size == -1:
input_fn = test_utils.AllWindowInputFn(
input_pipeline.NumpyReader(data), window_size=10)
else:
input_fn = input_pipeline.RandomWindowInputFn(
input_pipeline.NumpyReader(data),
window_size=10,
batch_size=batch_size)
chainer = state_management.ChainingStateManager(
state_saving_interval=1)
features, _ = input_fn()
stub_model.initialize_graph()
chainer.initialize_graph(model=stub_model)
model_outputs = chainer.define_loss(model=stub_model,
features=features,
mode=estimator_lib.ModeKeys.TRAIN)
with self.cached_session() as session:
variables.global_variables_initializer().run()
coordinator = coordinator_lib.Coordinator()
queue_runner_impl.start_queue_runners(session, coord=coordinator)
for _ in range(warmup_iterations):
# Warm up saved state
model_outputs.loss.eval()
outputs = model_outputs.loss.eval()
coordinator.request_stop()
coordinator.join()
return outputs
def _test_missing_values(self, cut_start, cut_end, offset):
stub_model = StubTimeSeriesModel()
data = self._make_test_data(length=100,
cut_start=cut_start,
cut_end=cut_end,
offset=offset)
input_fn = test_utils.AllWindowInputFn(
input_pipeline.NumpyReader(data), window_size=10)
chainer = state_management.ChainingStateManager(
state_saving_interval=1)
features, _ = input_fn()
stub_model.initialize_graph()
chainer.initialize_graph(model=stub_model)
model_outputs = chainer.define_loss(model=stub_model,
features=features,
mode=estimator_lib.ModeKeys.TRAIN)
with self.cached_session() as session:
variables.global_variables_initializer().run()
coordinator = coordinator_lib.Coordinator()
queue_runner_impl.start_queue_runners(session, coord=coordinator)
for _ in range(10):
model_outputs.loss.eval()
returned_loss = model_outputs.loss.eval()
coordinator.request_stop()
coordinator.join()
return returned_loss
def chained_model_outputs(original_model, data):
input_fn = test_utils.AllWindowInputFn(
input_pipeline.NumpyReader(data), window_size=chunk_size)
state_manager = state_management.ChainingStateManager(
state_saving_interval=1)
features, _ = input_fn()
state_manager.initialize_graph(original_model)
model_outputs = state_manager.define_loss(
model=original_model,
features=features,
mode=estimator_lib.ModeKeys.TRAIN)
def _eval_outputs(session):
for _ in range(50):
# Warm up saved state
model_outputs.loss.eval()
(posterior_mean, posterior_var,
priors_from_time) = model_outputs.end_state
posteriors = ((posterior_mean, ), (posterior_var, ),
priors_from_time)
outputs = (model_outputs.loss, posteriors,
model_outputs.predictions)
chunked_outputs_evaled = session.run(outputs)
return chunked_outputs_evaled
return _eval_outputs
def _all_window_input_fn_test_template(
self, time_series_reader, num_samples, window_size,
original_numpy_features=None):
input_fn = test_utils.AllWindowInputFn(
time_series_reader=time_series_reader,
window_size=window_size)
features, _ = input_fn()
init_op = variables.local_variables_initializer()
with self.test_session() as session:
coordinator = coordinator_lib.Coordinator()
queue_runner_impl.start_queue_runners(session, coord=coordinator)
session.run(init_op)
chunked_times, chunked_values = session.run(
[features[TrainEvalFeatures.TIMES],
features[TrainEvalFeatures.VALUES]])
coordinator.request_stop()
coordinator.join()
self.assertAllEqual([num_samples - window_size + 1, window_size],
chunked_times.shape)
if original_numpy_features is not None:
original_times = original_numpy_features[TrainEvalFeatures.TIMES]
original_values = original_numpy_features[TrainEvalFeatures.VALUES]
self.assertAllEqual(original_times, numpy.unique(chunked_times))
self.assertAllEqual(original_values[chunked_times],
chunked_values)
def test_long_eval(self):
g = ops.Graph()
with g.as_default():
model = ar_model.ARModel(periodicities=2,
num_features=1,
num_time_buckets=10,
input_window_size=2,
output_window_size=1)
raw_features = {
TrainEvalFeatures.TIMES: [[1, 3, 5, 7, 11]],
TrainEvalFeatures.VALUES: [[[1.], [2.], [3.], [4.], [5.]]]
}
chunked_features, _ = test_utils.AllWindowInputFn(
time_series_reader=input_pipeline.NumpyReader(raw_features),
window_size=3)()
model.initialize_graph()
with variable_scope.variable_scope("armodel") as scope:
raw_evaluation = model.define_loss(
raw_features, mode=estimator_lib.ModeKeys.EVAL)
with variable_scope.variable_scope(scope, reuse=True):
chunked_evaluation = model.define_loss(
chunked_features, mode=estimator_lib.ModeKeys.EVAL)
with session.Session() as sess:
coordinator = coordinator_lib.Coordinator()
queue_runner_impl.start_queue_runners(sess, coord=coordinator)
variables.global_variables_initializer().run()
raw_evaluation_evaled, chunked_evaluation_evaled = sess.run(
[raw_evaluation, chunked_evaluation])
self.assertAllClose(chunked_evaluation_evaled.loss,
raw_evaluation_evaled.loss)
last_chunk_evaluation_state = [
state[-1, None]
for state in chunked_evaluation_evaled.end_state
]
for last_chunk_state_member, raw_state_member in zip(
last_chunk_evaluation_state,
raw_evaluation_evaled.end_state):
self.assertAllClose(last_chunk_state_member,
raw_state_member)
self.assertAllEqual([[5, 7, 11]],
raw_evaluation_evaled.prediction_times)
for feature_name in raw_evaluation.predictions:
self.assertAllEqual(
[
1, 3, 1
], # batch, window, num_features. The window size has 2
# cut off for the first input_window.
raw_evaluation_evaled.predictions[feature_name].shape)
self.assertAllClose(
np.reshape(
chunked_evaluation_evaled.
predictions[feature_name], [-1]),
np.reshape(
raw_evaluation_evaled.predictions[feature_name],
[-1]))
coordinator.request_stop()
coordinator.join()
def test_chained_exact_posterior_recovery_no_transition_noise(self):
with self.test_session() as session:
stub_model, data, true_params = self._get_single_model()
chunk_size = 10
input_fn = test_utils.AllWindowInputFn(
input_pipeline.NumpyReader(data), window_size=chunk_size)
features, _ = input_fn()
state_manager = state_management.ChainingStateManager(
state_saving_interval=1)
state_manager.initialize_graph(stub_model)
model_outputs = state_manager.define_loss(
model=stub_model,
features=features,
mode=estimator_lib.ModeKeys.TRAIN)
variables.global_variables_initializer().run()
coordinator = coordinator_lib.Coordinator()
queue_runner_impl.start_queue_runners(session, coord=coordinator)
for _ in range(
data[feature_keys.TrainEvalFeatures.TIMES].shape[1] //
chunk_size):
model_outputs.loss.eval()
posterior_mean, posterior_var, posterior_times = session.run(
model_outputs.end_state, feed_dict=true_params)
coordinator.request_stop()
coordinator.join()
self.assertAllClose(numpy.zeros([1, 4, 4]),
posterior_var,
atol=1e-2)
self.assertAllClose(numpy.dot(
numpy.linalg.matrix_power(
stub_model.transition,
data[feature_keys.TrainEvalFeatures.TIMES].shape[1]),
true_params[stub_model.prior_state_mean]),
posterior_mean[0],
rtol=1e-1)
self.assertAllClose(
data[feature_keys.TrainEvalFeatures.TIMES][:, -1],
posterior_times)
def train_helper(self, input_window_size, loss,
max_loss=None, train_steps=200,
anomaly_prob=0.01,
anomaly_distribution=None,
multiple_periods=False):
np.random.seed(3)
data_noise_stddev = 0.2
if max_loss is None:
if loss == ARModel.NORMAL_LIKELIHOOD_LOSS:
max_loss = 1.0
else:
max_loss = 0.05 / (data_noise_stddev ** 2)
train_data, test_data = self.create_data(
noise_stddev=data_noise_stddev,
anomaly_prob=anomaly_prob,
multiple_periods=multiple_periods)
output_window_size = 10
window_size = input_window_size + output_window_size
class _RunConfig(estimator_lib.RunConfig):
@property
def tf_random_seed(self):
return 3
estimator = ARRegressor(
periodicities=self.period,
anomaly_prior_probability=0.01 if anomaly_distribution else None,
anomaly_distribution=anomaly_distribution,
num_features=2,
output_window_size=output_window_size,
num_time_buckets=20,
input_window_size=input_window_size,
hidden_layer_sizes=[16],
loss=loss,
config=_RunConfig())
train_input_fn = input_pipeline.RandomWindowInputFn(
time_series_reader=input_pipeline.NumpyReader(train_data),
window_size=window_size,
batch_size=64,
num_threads=1,
shuffle_seed=2)
test_input_fn = test_utils.AllWindowInputFn(
time_series_reader=input_pipeline.NumpyReader(test_data),
window_size=window_size)
# Test training
estimator.train(
input_fn=train_input_fn,
steps=train_steps)
test_evaluation = estimator.evaluate(input_fn=test_input_fn, steps=1)
test_loss = test_evaluation["loss"]
logging.info("Final test loss: %f", test_loss)
self.assertLess(test_loss, max_loss)
if loss == ARModel.SQUARED_LOSS:
# Test that the evaluation loss is reported without input scaling.
self.assertAllClose(
test_loss,
np.mean((test_evaluation["mean"] - test_evaluation["observed"]) ** 2))
# Test predict
train_data_times = train_data[TrainEvalFeatures.TIMES]
train_data_values = train_data[TrainEvalFeatures.VALUES]
test_data_times = test_data[TrainEvalFeatures.TIMES]
test_data_values = test_data[TrainEvalFeatures.VALUES]
predict_times = np.expand_dims(np.concatenate(
[train_data_times[input_window_size:], test_data_times]), 0)
predict_true_values = np.expand_dims(np.concatenate(
[train_data_values[input_window_size:], test_data_values]), 0)
state_times = np.expand_dims(train_data_times[:input_window_size], 0)
state_values = np.expand_dims(
train_data_values[:input_window_size, :], 0)
state_exogenous = state_times[:, :, None][:, :, :0]
def prediction_input_fn():
return ({
PredictionFeatures.TIMES: training.limit_epochs(
predict_times, num_epochs=1),
PredictionFeatures.STATE_TUPLE: (state_times,
state_values,
state_exogenous)
}, {})
(predictions,) = tuple(estimator.predict(input_fn=prediction_input_fn))
predicted_mean = predictions["mean"][:, 0]
true_values = predict_true_values[0, :, 0]
if loss == ARModel.NORMAL_LIKELIHOOD_LOSS:
variances = predictions["covariance"][:, 0]
standard_deviations = np.sqrt(variances)
# Note that we may get tighter bounds with more training steps.
errors = np.abs(predicted_mean - true_values) > 4 * standard_deviations
fraction_errors = np.mean(errors)
logging.info("Fraction errors: %f", fraction_errors)
