def _all_window_input_fn_test_template(self, time_series_reader, num_samples, window_size, original_numpy_features=None): input_fn = test_utils.AllWindowInputFn( time_series_reader=time_series_reader, window_size=window_size) features, _ = input_fn() init_op = variables.local_variables_initializer() with self.cached_session() as session: coordinator = coordinator_lib.Coordinator() queue_runner_impl.start_queue_runners(session, coord=coordinator) session.run(init_op) chunked_times, chunked_values = session.run([ features[TrainEvalFeatures.TIMES], features[TrainEvalFeatures.VALUES] ]) coordinator.request_stop() coordinator.join() self.assertAllEqual([num_samples - window_size + 1, window_size], chunked_times.shape) if original_numpy_features is not None: original_times = original_numpy_features[TrainEvalFeatures.TIMES] original_values = original_numpy_features[TrainEvalFeatures.VALUES] self.assertAllEqual(original_times, numpy.unique(chunked_times)) self.assertAllEqual(original_values[chunked_times], chunked_values)
def _test_missing_values(self, cut_start, cut_end, offset): stub_model = StubTimeSeriesModel() data = self._make_test_data(length=100, cut_start=cut_start, cut_end=cut_end, offset=offset) input_fn = test_utils.AllWindowInputFn( input_pipeline.NumpyReader(data), window_size=10) chainer = state_management.ChainingStateManager( state_saving_interval=1) features, _ = input_fn() stub_model.initialize_graph() chainer.initialize_graph(model=stub_model) model_outputs = chainer.define_loss(model=stub_model, features=features, mode=estimator_lib.ModeKeys.TRAIN) with self.cached_session() as session: variables.global_variables_initializer().run() coordinator = coordinator_lib.Coordinator() queue_runner_impl.start_queue_runners(session, coord=coordinator) for _ in range(10): model_outputs.loss.eval() returned_loss = model_outputs.loss.eval() coordinator.request_stop() coordinator.join() return returned_loss
def _test_initialization(self, warmup_iterations, batch_size): stub_model = StubTimeSeriesModel() data = self._make_test_data(length=20, cut_start=None, cut_end=None, offset=0.) if batch_size == -1: input_fn = test_utils.AllWindowInputFn( input_pipeline.NumpyReader(data), window_size=10) else: input_fn = input_pipeline.RandomWindowInputFn( input_pipeline.NumpyReader(data), window_size=10, batch_size=batch_size) chainer = state_management.ChainingStateManager( state_saving_interval=1) features, _ = input_fn() stub_model.initialize_graph() chainer.initialize_graph(model=stub_model) model_outputs = chainer.define_loss(model=stub_model, features=features, mode=estimator_lib.ModeKeys.TRAIN) with self.cached_session() as session: variables.global_variables_initializer().run() coordinator = coordinator_lib.Coordinator() queue_runner_impl.start_queue_runners(session, coord=coordinator) for _ in range(warmup_iterations): # Warm up saved state model_outputs.loss.eval() outputs = model_outputs.loss.eval() coordinator.request_stop() coordinator.join() return outputs
def chained_model_outputs(original_model, data): input_fn = test_utils.AllWindowInputFn( input_pipeline.NumpyReader(data), window_size=chunk_size) state_manager = state_management.ChainingStateManager( state_saving_interval=1) features, _ = input_fn() state_manager.initialize_graph(original_model) model_outputs = state_manager.define_loss( model=original_model, features=features, mode=estimator_lib.ModeKeys.TRAIN) def _eval_outputs(session): for _ in range(50): # Warm up saved state model_outputs.loss.eval() (posterior_mean, posterior_var, priors_from_time) = model_outputs.end_state posteriors = ((posterior_mean, ), (posterior_var, ), priors_from_time) outputs = (model_outputs.loss, posteriors, model_outputs.predictions) chunked_outputs_evaled = session.run(outputs) return chunked_outputs_evaled return _eval_outputs
def test_long_eval(self): g = ops.Graph() with g.as_default(): model = ar_model.ARModel(periodicities=2, num_features=1, num_time_buckets=10, input_window_size=2, output_window_size=1) raw_features = { TrainEvalFeatures.TIMES: [[1, 3, 5, 7, 11]], TrainEvalFeatures.VALUES: [[[1.], [2.], [3.], [4.], [5.]]] } chunked_features, _ = test_utils.AllWindowInputFn( time_series_reader=input_pipeline.NumpyReader(raw_features), window_size=3)() model.initialize_graph() with variable_scope.variable_scope("armodel") as scope: raw_evaluation = model.define_loss( raw_features, mode=estimator_lib.ModeKeys.EVAL) with variable_scope.variable_scope(scope, reuse=True): chunked_evaluation = model.define_loss( chunked_features, mode=estimator_lib.ModeKeys.EVAL) with session.Session() as sess: coordinator = coordinator_lib.Coordinator() queue_runner_impl.start_queue_runners(sess, coord=coordinator) variables.global_variables_initializer().run() raw_evaluation_evaled, chunked_evaluation_evaled = sess.run( [raw_evaluation, chunked_evaluation]) self.assertAllClose(chunked_evaluation_evaled.loss, raw_evaluation_evaled.loss) last_chunk_evaluation_state = [ state[-1, None] for state in chunked_evaluation_evaled.end_state ] for last_chunk_state_member, raw_state_member in zip( last_chunk_evaluation_state, raw_evaluation_evaled.end_state): self.assertAllClose(last_chunk_state_member, raw_state_member) self.assertAllEqual([[5, 7, 11]], raw_evaluation_evaled.prediction_times) for feature_name in raw_evaluation.predictions: self.assertAllEqual( [ 1, 3, 1 ], # batch, window, num_features. The window size has 2 # cut off for the first input_window. raw_evaluation_evaled.predictions[feature_name].shape) self.assertAllClose( np.reshape( chunked_evaluation_evaled. predictions[feature_name], [-1]), np.reshape( raw_evaluation_evaled.predictions[feature_name], [-1])) coordinator.request_stop() coordinator.join()
def test_chained_exact_posterior_recovery_no_transition_noise(self): with self.cached_session() as session: stub_model, data, true_params = self._get_single_model() chunk_size = 10 input_fn = test_utils.AllWindowInputFn( input_pipeline.NumpyReader(data), window_size=chunk_size) features, _ = input_fn() state_manager = state_management.ChainingStateManager( state_saving_interval=1) state_manager.initialize_graph(stub_model) model_outputs = state_manager.define_loss( model=stub_model, features=features, mode=estimator_lib.ModeKeys.TRAIN) variables.global_variables_initializer().run() coordinator = coordinator_lib.Coordinator() queue_runner_impl.start_queue_runners(session, coord=coordinator) for _ in range( data[feature_keys.TrainEvalFeatures.TIMES].shape[1] // chunk_size): model_outputs.loss.eval() posterior_mean, posterior_var, posterior_times = session.run( model_outputs.end_state, feed_dict=true_params) coordinator.request_stop() coordinator.join() self.assertAllClose(numpy.zeros([1, 4, 4]), posterior_var, atol=1e-2) self.assertAllClose(numpy.dot( numpy.linalg.matrix_power( stub_model.transition, data[feature_keys.TrainEvalFeatures.TIMES].shape[1]), true_params[stub_model.prior_state_mean]), posterior_mean[0], rtol=1e-1) self.assertAllClose( data[feature_keys.TrainEvalFeatures.TIMES][:, -1], posterior_times)
def train_helper(self, input_window_size, loss, max_loss=None, train_steps=200, anomaly_prob=0.01, anomaly_distribution=None, multiple_periods=False): np.random.seed(3) data_noise_stddev = 0.2 if max_loss is None: if loss == ar_model.ARModel.NORMAL_LIKELIHOOD_LOSS: max_loss = 1.0 else: max_loss = 0.05 / (data_noise_stddev**2) train_data, test_data = self.create_data( noise_stddev=data_noise_stddev, anomaly_prob=anomaly_prob, multiple_periods=multiple_periods) output_window_size = 10 window_size = input_window_size + output_window_size class _RunConfig(estimator_lib.RunConfig): @property def tf_random_seed(self): return 3 estimator = ARRegressor( periodicities=self.period, anomaly_prior_probability=0.01 if anomaly_distribution else None, anomaly_distribution=anomaly_distribution, num_features=2, output_window_size=output_window_size, num_time_buckets=20, input_window_size=input_window_size, hidden_layer_sizes=[16], loss=loss, config=_RunConfig()) train_input_fn = input_pipeline.RandomWindowInputFn( time_series_reader=input_pipeline.NumpyReader(train_data), window_size=window_size, batch_size=64, num_threads=1, shuffle_seed=2) test_input_fn = test_utils.AllWindowInputFn( time_series_reader=input_pipeline.NumpyReader(test_data), window_size=window_size) # Test training estimator.train(input_fn=train_input_fn, steps=train_steps) test_evaluation = estimator.evaluate(input_fn=test_input_fn, steps=1) test_loss = test_evaluation["loss"] logging.info("Final test loss: %f", test_loss) self.assertLess(test_loss, max_loss) if loss == ar_model.ARModel.SQUARED_LOSS: # Test that the evaluation loss is reported without input scaling. self.assertAllClose( test_loss, np.mean((test_evaluation["mean"] - test_evaluation["observed"])**2)) # Test predict train_data_times = train_data[TrainEvalFeatures.TIMES] train_data_values = train_data[TrainEvalFeatures.VALUES] test_data_times = test_data[TrainEvalFeatures.TIMES] test_data_values = test_data[TrainEvalFeatures.VALUES] predict_times = np.expand_dims( np.concatenate( [train_data_times[input_window_size:], test_data_times]), 0) predict_true_values = np.expand_dims( np.concatenate( [train_data_values[input_window_size:], test_data_values]), 0) state_times = np.expand_dims(train_data_times[:input_window_size], 0) state_values = np.expand_dims(train_data_values[:input_window_size, :], 0) state_exogenous = state_times[:, :, None][:, :, :0] def prediction_input_fn(): return ({ PredictionFeatures.TIMES: training.limit_epochs(predict_times, num_epochs=1), PredictionFeatures.STATE_TUPLE: (state_times, state_values, state_exogenous) }, {}) (predictions, ) = tuple( estimator.predict(input_fn=prediction_input_fn)) predicted_mean = predictions["mean"][:, 0] true_values = predict_true_values[0, :, 0] if loss == ar_model.ARModel.NORMAL_LIKELIHOOD_LOSS: variances = predictions["covariance"][:, 0] standard_deviations = np.sqrt(variances) # Note that we may get tighter bounds with more training steps. errors = np.abs(predicted_mean - true_values) > 4 * standard_deviations fraction_errors = np.mean(errors) logging.info("Fraction errors: %f", fraction_errors)