def test_forward(self): helper = Helper() x, x2, x3, z, z2 = helper.get_xz_variables() with self.test_session() as sess: ensure_variables_initialized() # test single sample vae = helper.vae() z_s = vae(x) np.testing.assert_equal(tf.shape(z_s).eval(), [BATCH_SIZE, Z_DIMS]) q_net_zs = helper.zs_variational(x, observed={'z': z_s}) np.testing.assert_allclose(*sess.run([ z_s.log_prob(), q_net_zs.local_log_prob('z') ])) # test multiple samples vae = helper.vae() z_s = vae(x, n_z=N_Z) np.testing.assert_equal( tf.shape(z_s).eval(), [N_Z, BATCH_SIZE, Z_DIMS]) q_net_zs = helper.zs_variational(x, observed={'z': z_s}) np.testing.assert_allclose(*sess.run([ z_s.log_prob(), q_net_zs.local_log_prob('z') ]))
def test_variational(self): helper = Helper() x, x2, x3, z, z2 = helper.get_xz_variables() with self.test_session() as sess: ensure_variables_initialized() # test log-prob of one z observation vae = helper.vae() q_net = vae.variational(x, z=z) q_net_zs = helper.zs_variational(x, observed={'z': z}) np.testing.assert_allclose(q_net['z'].eval(), z.eval()) np.testing.assert_allclose(*sess.run( [q_net.local_log_prob('z'), q_net_zs.local_log_prob('z')])) # test log-prob of multiple z observations vae = helper.vae() q_net = vae.variational(x, z=z2, n_z=N_Z) q_net_zs = helper.zs_variational(x, observed={'z': z2}, n_z=N_Z) np.testing.assert_allclose(q_net['z'].eval(), z2.eval()) np.testing.assert_allclose(*sess.run( [q_net.local_log_prob('z'), q_net_zs.local_log_prob('z')])) # test the shape of z samples and their log-probs vae = helper.vae() q_net = vae.variational(x, n_z=N_Z) q_net_zs = helper.zs_variational(x, observed={'z': q_net['z']}) np.testing.assert_equal( tf.shape(q_net['z']).eval(), [N_Z, BATCH_SIZE, Z_DIMS]) np.testing.assert_allclose(*sess.run( [q_net.local_log_prob('z'), q_net_zs.local_log_prob('z')]))
def test_transform(self): def transform(x, u, w, b): wxb = np.dot(x, w.T) + b y = x + u * np.tanh(wxb) # shape: [?, n_units] tanh_wxb = np.tanh(wxb) phi = (1 - tanh_wxb**2) * w # shape: [?, n_units] log_det = np.log(np.abs(1 + np.dot(phi, u.T))) # shape: [?, 1] return y, np.squeeze(log_det, -1) n_units = 5 tf.set_random_seed(1234) flow = PlanarNormalizingFlow() flow.apply(tf.random_normal(shape=[1, n_units], dtype=tf.float64)) x = np.arange(30, dtype=np.float64).reshape([2, 3, n_units]) with self.test_session() as sess: ensure_variables_initialized() # compute the ground-truth y and log_det y = x w, b, u_hat = flow._w, flow._b, flow._u_hat u_hat, w, b = sess.run([u_hat, w, b]) y, log_det = transform(y, u_hat, w, b) # check the flow-derived y and log_det y2, log_det2 = sess.run( flow.transform(tf.constant(x, dtype=tf.float64))) np.testing.assert_allclose(y2, y, rtol=1e-5) np.testing.assert_allclose(log_det2, log_det, rtol=1e-5)
def test_group_ndims(self): helper = Helper() x, x2, x3, z, z2 = helper.get_xz_variables() with self.test_session() as sess: ensure_variables_initialized() vae = helper.vae(z_group_ndims=0, x_group_ndims=2) q_net = vae.variational(x, z=z) q_net_zs = helper.zs_variational(x, observed={'z': z}, z_group_ndims=0) np.testing.assert_allclose(*sess.run( [q_net.local_log_prob('z'), q_net_zs.local_log_prob('z')])) p_net = vae.model(z=z, x=x) p_net_zs = helper.zs_model(observed={ 'z': z, 'x': x }, z_group_ndims=0, x_group_ndims=2) np.testing.assert_allclose(*sess.run( [p_net.local_log_prob('z'), p_net_zs.local_log_prob('z')])) np.testing.assert_allclose(*sess.run( [p_net.local_log_prob('x'), p_net_zs.local_log_prob('x')]))
def test_reconstruct(self): class Capture(object): def __init__(self, vae): self._variational_func = vae.variational self._model_func = vae.model self.q_net = None self.p_net = None def variational(*args, **kwargs): self.q_net = self._variational_func(*args, **kwargs) return self.q_net def model(*args, **kwargs): self.p_net = self._model_func(*args, **kwargs) return self.p_net vae.variational = variational vae.model = model helper = Helper() x, x2, x3, z, z2 = helper.get_xz_variables() with self.test_session() as sess: ensure_variables_initialized() # test single sample vae = helper.vae() capture = Capture(vae) x_r = vae.reconstruct(x) np.testing.assert_equal(tf.shape(x_r).eval(), [BATCH_SIZE, X_DIMS]) p_net_zs = helper.zs_model( observed={'z': capture.q_net['z'], 'x': x_r}) np.testing.assert_allclose(*sess.run([x_r, capture.p_net['x']])) np.testing.assert_allclose(*sess.run([ capture.p_net['z'].log_prob(), p_net_zs.local_log_prob('z') ])) np.testing.assert_allclose(*sess.run([ x_r.log_prob(), p_net_zs.local_log_prob('x') ])) # test multiple samples vae = helper.vae() capture = Capture(vae) x_r = vae.reconstruct(x, n_z=N_Z, n_x=N_X) np.testing.assert_equal( tf.shape(x_r).eval(), [N_X, N_Z, BATCH_SIZE, X_DIMS]) p_net_zs = helper.zs_model( observed={'z': capture.q_net['z'], 'x': x_r}, n_z=N_Z, n_x=N_X) np.testing.assert_allclose(*sess.run([x_r, capture.p_net['x']])) np.testing.assert_allclose(*sess.run([ capture.p_net['z'].log_prob(), p_net_zs.local_log_prob('z') ])) np.testing.assert_allclose(*sess.run([ x_r.log_prob(), p_net_zs.local_log_prob('x') ]))
def test_basic_logging(self): v = ScheduledVariable('v', 1.) logger = MetricLogger() self.assertEqual(logger.format_logs(), '') with self.test_session() as sess: ensure_variables_initialized() logger.collect_metrics(dict(loss=v)) logger.collect_metrics(dict(loss=2., valid_loss=3., valid_timer=0.1)) logger.collect_metrics(dict(loss=4., valid_acc=5., train_time=0.2)) logger.collect_metrics(dict(loss=6., valid_acc=7., train_time=0.3)) logger.collect_metrics(dict(other_metric=5.)) self.assertEqual( logger.format_logs(), 'train time: 0.25s (±0.05s); ' 'valid timer: 0.1s; ' 'other metric: 5; ' 'loss: 3.25 (±1.92029); ' 'valid loss: 3; ' 'valid acc: 6 (±1)' ) logger.clear() self.assertEqual(logger.format_logs(), '') logger.collect_metrics({'loss': 1.}) self.assertEqual(logger.format_logs(), 'loss: 1')
def test_ScheduledVariable(self): v = ScheduledVariable('v', 123., dtype=tf.int32, model_var=True, collections=['my_variables']) assert_variables(['v'], trainable=False, collections=['my_variables']) with TemporaryDirectory() as tmpdir: saver = tf.train.Saver(var_list=[v.variable]) save_path = os.path.join(tmpdir, 'saved_var') with self.test_session() as sess: ensure_variables_initialized() self.assertEqual(v.get(), 123) self.assertEqual(sess.run(v), 123) self.assertEqual(v.set(456), 456) self.assertEqual(v.get(), 456) saver.save(sess, save_path) with self.test_session() as sess: saver.restore(sess, save_path) self.assertEqual(v.get(), 456) sess.run(v.assign_op, feed_dict={v.assign_ph: 789}) self.assertEqual(v.get(), 789)
def run(self): """Run training loop.""" if self._is_fitting: raise RuntimeError('`run()` is not re-entrant.') self._is_fitting = True try: # initialize global training status session = get_default_session_or_error() ensure_variables_initialized() self.loop.print_training_summary() # initialize internal status for hook_list in self.hook_lists: hook_list.reset() for epoch in self.loop.iter_epochs(): # run before epoch hook self.before_epochs.call_hooks() # run steps of this epoch for payload in self._iter_steps(): # run before step hook self.before_steps.call_hooks() # run the step self._run_step(session, payload) # run after step hook self.after_steps.call_hooks() # run after epoch hook self.after_epochs.call_hooks() finally: self._is_fitting = False
def test_ensure_variables_initialized_using_dict(self): a = tf.get_variable('a', dtype=tf.int32, initializer=1) b = tf.get_variable('b', dtype=tf.int32, initializer=2) # test using dict with self.test_session(): ensure_variables_initialized({'a': a}) self.assertEqual(get_uninitialized_variables([a, b]), [b])
def fit(self, train_iterator, summary_dir=None): """ Train the :class:`OmniAnomaly` model with given data. Args: values (np.ndarray): 1-D `float32` array, the standardized KPI observations. summary_dir (str): Optional summary directory for :class:`tf.summary.FileWriter`. (default :obj:`None`, summary is disabled) """ sess = get_default_session_or_error() # initialize the variables of the trainer, and the model sess.run(self._trainer_initializer) ensure_variables_initialized(self._train_params) # training loop lr = self._initial_lr with TrainLoop( param_vars=self._train_params, summary_dir=summary_dir, max_epoch=self._max_epoch, max_step=self._max_step, ) as loop: # type: TrainLoop # loop.print_training_summary() train_batch_time = [] valid_batch_time = [] time_train_start = time.time() for epoch in loop.iter_epochs(): start_time = time.time() for step, idx in loop.iter_steps(range(len(train_iterator))): # run a training step batch_x = train_iterator[idx] start_batch_time = time.time() feed_dict = dict(six.iteritems(self._feed_dict)) feed_dict[self._learning_rate] = lr feed_dict[self._input_x] = batch_x loss, _ = sess.run([self._loss, self._train_op], feed_dict=feed_dict) loop.collect_metrics({"loss": loss}) train_batch_time.append(time.time() - start_batch_time) # anneal the learning rate if self._lr_anneal_epochs and epoch % self._lr_anneal_epochs == 0: lr *= self._lr_anneal_factor loop.println("Learning rate decreased to {}".format(lr), with_tag=True) time_train_end = time.time() return { # "best_valid_loss": float(loop.best_valid_metric), "train_time": np.sum(train_batch_time), "total_train_time": time_train_end - time_train_start, }
def test_model(self): helper = Helper() x, x2, x3, z, z2 = helper.get_xz_variables() with self.test_session() as sess: ensure_variables_initialized() # test log-prob of one z observation vae = helper.vae() p_net = vae.model(z=z, x=x) p_net_zs = helper.zs_model(observed={'z': z, 'x': x}) np.testing.assert_allclose(p_net['z'].eval(), z.eval()) np.testing.assert_allclose(p_net['x'].eval(), x.eval()) np.testing.assert_allclose(*sess.run( [p_net.local_log_prob('z'), p_net_zs.local_log_prob('z')])) np.testing.assert_allclose(*sess.run( [p_net.local_log_prob('x'), p_net_zs.local_log_prob('x')])) # test log-prob of multiple x & z observations vae = helper.vae() p_net = vae.model(z=z2, x=x3, n_z=N_Z, n_x=N_X) p_net_zs = helper.zs_model(observed={ 'z': z2, 'x': x3 }, n_z=N_Z, n_x=N_X) np.testing.assert_allclose(p_net['z'].eval(), z2.eval()) np.testing.assert_allclose(p_net['x'].eval(), x3.eval()) np.testing.assert_allclose(*sess.run( [p_net.local_log_prob('z'), p_net_zs.local_log_prob('z')])) np.testing.assert_allclose(*sess.run( [p_net.local_log_prob('x'), p_net_zs.local_log_prob('x')])) # test the shape of samples and their log-probs vae = helper.vae() p_net = vae.model(n_x=N_X, n_z=N_Z) p_net_zs = helper.zs_model(observed={ 'z': p_net['z'], 'x': p_net['x'] }, n_z=N_Z, n_x=N_X) np.testing.assert_equal( tf.shape(p_net['z']).eval(), [N_Z, BATCH_SIZE, Z_DIMS]) np.testing.assert_equal( tf.shape(p_net['x']).eval(), [N_X, N_Z, BATCH_SIZE, X_DIMS]) np.testing.assert_allclose(*sess.run( [p_net.local_log_prob('z'), p_net_zs.local_log_prob('z')])) np.testing.assert_allclose(*sess.run( [p_net.local_log_prob('x'), p_net_zs.local_log_prob('x')]))
def test_inplace(self): with self.test_session(): d = { 'a': 12, 'b': ScheduledVariable('b', 34), 'c': lambda: 56, } ensure_variables_initialized() self.assertIs(d, resolve_feed_dict(d, inplace=True)) self.assertDictEqual({'a': 12, 'b': 34, 'c': 56}, d)
def ensure_variables_initialized(self): """Initialize all uninitialized variables within model's scope. If the `global_step` is created by the model itself, then it will also be initialized. Otherwise the `global_step` must be manually initialized. """ self.build() var_list = list(six.itervalues(self.get_variables())) ensure_variables_initialized(var_list)
def test_tensor_arguments(self): with self.test_session(): a = tf.get_variable('a', initializer=0, dtype=tf.int32) ensure_variables_initialized() with TrainLoop([a], early_stopping=True, max_epoch=tf.constant(6), max_step=tf.constant(7)) as loop: self.assertEqual(loop.max_epoch, 6) self.assertEqual(loop.max_step, 7)
def test_copy(self): with self.test_session(): d = { 'a': 12, 'b': ScheduledVariable('b', 34), 'c': lambda: 56, } ensure_variables_initialized() d2 = resolve_feed_dict(d) self.assertIsNot(d2, d) self.assertDictEqual({'a': 12, 'b': 34, 'c': 56}, d2) self.assertIsInstance(d['b'], ScheduledVariable)
def test_tensor_arguments(self): with self.test_session(): a = tf.get_variable('a', initializer=0, dtype=tf.int32) ensure_variables_initialized() with TrainLoop([a], early_stopping=True, initial_valid_metric=tf.constant(1.23), initial_epoch=tf.constant(4), initial_step=tf.constant(5), max_epoch=tf.constant(6), max_step=tf.constant(7)) as loop: self.assertAlmostEqual(loop._early_stopping._best_metric, 1.23) self.assertEqual(loop.epoch, 4) self.assertEqual(loop.step, 5) self.assertEqual(loop.max_epoch, 6) self.assertEqual(loop.max_step, 7)
def test_run(self): ph = tf.placeholder(tf.int32, [5]) var = tf.get_variable('var', shape=[5], dtype=tf.int32, initializer=tf.zeros_initializer()) train_op = tf.assign(var, ph) df = DataFlow.arrays([np.arange(10, 15, dtype=np.int32)], batch_size=5) with self.test_session() as session, \ TrainLoop([var], max_epoch=1, early_stopping=False) as loop: loop.collect_metrics = Mock(wraps=loop.collect_metrics) t = LossTrainer(loop, tf.reduce_sum(ph), train_op, [ph], df, metric_name='loss_x') ensure_variables_initialized() t.run() self.assertEqual( {'loss_x': 60}, loop.collect_metrics.call_args_list[0][0][0]) np.testing.assert_equal([10, 11, 12, 13, 14], session.run(var))
def test_run(self): ph = tf.placeholder(tf.int32, [5]) var = tf.get_variable('var', shape=[5], dtype=tf.int32, initializer=tf.zeros_initializer()) summary = tf.summary.histogram(var.name, var) train_op = tf.assign(var, ph) df = DataFlow.arrays([np.arange(10, 15, dtype=np.int32)], batch_size=5) with TemporaryDirectory() as tmpdir: with self.test_session() as session, \ TrainLoop([var], max_epoch=1, early_stopping=False, summary_dir=tmpdir) as loop: loop.collect_metrics = Mock(wraps=loop.collect_metrics) loop.add_summary = Mock(wraps=loop.add_summary) t = Trainer(loop, train_op, [ph], df, metrics={'loss_x': tf.reduce_sum(ph)}, summaries=[summary]) ensure_variables_initialized() t.run() self.assertEqual( {'loss_x': 60}, loop.collect_metrics.call_args_list[0][0][0] ) self.assertTrue(loop.add_summary.called) np.testing.assert_equal([10, 11, 12, 13, 14], session.run(var)) # test to specify summaries, but without loop.summary_dir, so no # summary should run with self.test_session() as session, \ TrainLoop([var], max_epoch=1, early_stopping=False) as loop: loop.collect_metrics = Mock(wraps=loop.collect_metrics) loop.add_summary = Mock(wraps=loop.add_summary) t = Trainer(loop, train_op, [ph], df, metrics={'loss_x': tf.reduce_sum(ph)}, summaries=[summary]) ensure_variables_initialized() t.run() self.assertEqual( {'loss_x': 60}, loop.collect_metrics.call_args_list[0][0][0] ) self.assertFalse(loop.add_summary.called)
def run(self): """Run training loop.""" if self._is_fitting: raise RuntimeError('`run()` is not re-entrant.') self._is_fitting = True try: # trigger the before execution event self.events.fire(EventKeys.BEFORE_EXECUTION, self) # initialize global training status session = get_default_session_or_error() if self._ensure_variables_initialized: ensure_variables_initialized() self.loop.print_training_summary() for _ in self.loop.iter_epochs(): # trigger before epoch event self.events.fire(EventKeys.BEFORE_EPOCH, self) # run steps of this epoch for payload in self._iter_steps(): # trigger before step event self.events.fire(EventKeys.BEFORE_STEP, self) # run the step self._run_step(session, payload) # trigger after step events self.events.fire(EventKeys.STEP_EVALUATION, self) self.events.fire(EventKeys.STEP_ANNEALING, self) self.events.fire(EventKeys.STEP_LOGGING, self) self.events.reverse_fire(EventKeys.AFTER_STEP, self) # trigger after epoch events self.events.fire(EventKeys.EPOCH_EVALUATION, self) self.events.fire(EventKeys.EPOCH_ANNEALING, self) self.events.fire(EventKeys.EPOCH_LOGGING, self) self.events.reverse_fire(EventKeys.AFTER_EPOCH, self) # trigger the after execution event self.events.reverse_fire(EventKeys.AFTER_EXECUTION, self) finally: self._is_fitting = False
def test_ensure_variables_initialized(self): a = tf.get_variable('a', dtype=tf.int32, initializer=1) b = tf.get_variable('b', dtype=tf.int32, initializer=2) c = tf.get_variable('c', dtype=tf.int32, initializer=3, collections=[tf.GraphKeys.MODEL_VARIABLES]) d = tf.get_variable('d', dtype=tf.int32, initializer=4, collections=[tf.GraphKeys.MODEL_VARIABLES]) # test using list with self.test_session(): self.assertEqual(get_uninitialized_variables([a, b, c, d]), [a, b, c, d]) ensure_variables_initialized() self.assertEqual(get_uninitialized_variables([a, b, c, d]), [c, d]) ensure_variables_initialized([a, b, c, d]) self.assertEqual(get_uninitialized_variables([a, b, c, d]), [])
def test_nvil(self): assert_allclose = functools.partial(np.testing.assert_allclose, rtol=1e-5, atol=1e-6) with self.test_session() as sess: log_p = tf.random_normal(shape=[5, 7]) log_q1 = tf.random_normal(shape=[1, 3, 5, 7]) log_q2 = tf.random_normal(shape=[4, 1, 5, 7]) baseline = tf.random_normal(shape=[5, 7]) # test without sampling axis vi = VariationalInference(log_p, [log_q1, log_q2]) output = vi.training.nvil(baseline=None, center_by_moving_average=True) a, b = nvil_estimator( values=log_p - (log_q1 + log_q2), latent_log_joint=log_q1 + log_q2, baseline=None, center_by_moving_average=True, decay=0.8, ) answer = -a ensure_variables_initialized() assert_allclose(*sess.run([output, answer])) # test with sampling axis vi = VariationalInference(log_p, [log_q1, log_q2], axis=[0, 1]) output = vi.training.nvil(baseline=baseline, center_by_moving_average=False) a, b = nvil_estimator( values=log_p - (log_q1 + log_q2), latent_log_joint=log_q1 + log_q2, axis=[0, 1], baseline=baseline, center_by_moving_average=False, decay=0.8, ) answer = b - a ensure_variables_initialized() assert_allclose(*sess.run([output, answer]))
def test_AnnealingDynamicValue(self): with self.test_session() as sess: # test without min_value v = AnnealingVariable('v', 1, 2) ensure_variables_initialized() self.assertEqual(v.get(), 1) self.assertEqual(v.anneal(), 2) self.assertEqual(v.get(), 2) self.assertEqual(v.anneal(), 4) self.assertEqual(v.get(), 4) self.assertEqual(v.set(2), 2) self.assertEqual(v.get(), 2) self.assertEqual(v.anneal(), 4) self.assertEqual(v.get(), 4) # test with min_value v = AnnealingVariable('v2', 1, .5, 2) ensure_variables_initialized() self.assertEqual(v.get(), 2) v = AnnealingVariable('v3', 1, .5, .5) ensure_variables_initialized() self.assertEqual(v.get(), 1) self.assertEqual(v.anneal(), .5) self.assertEqual(v.get(), .5) self.assertEqual(v.anneal(), .5) self.assertEqual(v.get(), .5)
def test_u_hat(self): n_units = 5 tf.set_random_seed(1234) flow = PlanarNormalizingFlow(name='planar_nf') flow.apply(tf.random_normal(shape=[1, n_units])) # ensure these parameters exist w, b, u, u_hat = flow._w, flow._b, flow._u, flow._u_hat for v in [w, u, b, u_hat]: self.assertIn('planar_nf/', v.name) # ensure these parameters have expected shapes self.assertEqual(get_static_shape(w), (1, n_units)) self.assertEqual(get_static_shape(u), (1, n_units)) self.assertEqual(get_static_shape(b), (1, )) self.assertEqual(get_static_shape(u_hat), (1, n_units)) with self.test_session() as sess: ensure_variables_initialized() w, b, u, u_hat = sess.run([w, b, u, u_hat]) m = lambda a: -1 + np.log(1 + np.exp(a)) wu = np.dot(w, u.T) # shape: [1] np.testing.assert_allclose(u + w * (m(wu) - wu) / np.sum(w**2), u_hat)
def test_pixelcnn_conv2d_resnet_pixel_dep(self): tf.set_random_seed(1234) with mock.patch('tensorflow.nn.conv2d', patched_conv2d), \ self.test_session() as sess: ###################### # test one layer dep # ###################### H, W = 11, 12 x = np.eye(H * W, dtype=np.float32).reshape([H * W, H, W, 1]) * \ np.ones([1, 1, 1, 2], dtype=np.float32) vertical_predicate = lambda i, j, h, w: ( (h - 3 <= i <= h - 1) and (w - 2 <= j <= w + 2)) horizontal_predicate = lambda i, j, h, w: ( # value from vertical stack ((h - 4 <= i <= h - 1) and (w - 3 <= j <= w + 2)) or # value from horizontal stack ((h - 2 <= i <= h) and (w - 3 <= j <= w - 1))) # NHWC y = pixelcnn_2d_input(x, channels_last=True, auxiliary_channel=False) o = pixelcnn_conv2d_resnet( y, out_channels=3, vertical_kernel_size=(2, 3), horizontal_kernel_size=(2, 2), strides=(1, 1), channels_last=True, ) ensure_variables_initialized() vertical, horizontal = sess.run([o.vertical, o.horizontal]) check_pixel_deps( self, H, W, vertical, horizontal, vertical_predicate, horizontal_predicate, ) # NCHW y = pixelcnn_2d_input(np.transpose(x, [0, 3, 1, 2]), channels_last=False, auxiliary_channel=False) o = pixelcnn_conv2d_resnet(y, out_channels=3, vertical_kernel_size=(2, 3), horizontal_kernel_size=(2, 2), strides=(1, 1), channels_last=False, activation_fn=tf.nn.leaky_relu) ensure_variables_initialized() vertical, horizontal = sess.run([o.vertical, o.horizontal]) vertical = np.transpose(vertical, [0, 2, 3, 1]) horizontal = np.transpose(horizontal, [0, 2, 3, 1]) check_pixel_deps( self, H, W, vertical, horizontal, vertical_predicate, horizontal_predicate, ) ######################## # test multi layer dep # ######################## H, W = 6, 7 x = np.eye(H * W, dtype=np.float32).reshape([H * W, H, W, 1]) * \ np.ones([1, 1, 1, 2], dtype=np.float32) n_layers = 3 vertical_predicate = lambda i, j, h, w: i <= h - 1 horizontal_predicate = lambda i, j, h, w: ( # value from vertical stack i <= h - 1 or # value from horizontal stack (j <= w - 1 and i <= h)) y = pixelcnn_2d_input(x, channels_last=True, auxiliary_channel=False) for i in range(n_layers): y = pixelcnn_conv2d_resnet( y, out_channels=3, vertical_kernel_size=(2, 3), horizontal_kernel_size=(2, 2), strides=(1, 1), channels_last=True, ) ensure_variables_initialized() vertical, horizontal = sess.run([y.vertical, y.horizontal]) check_pixel_deps( self, H, W, vertical, horizontal, vertical_predicate, horizontal_predicate, )
def fit(self, values, valid_portion=0.01, summary_dir=None): """ Train the :class:`OmniAnomaly` model with given data. Args: values (np.ndarray): 1-D `float32` array, the standardized KPI observations. valid_portion (float): Ratio of validation data out of all the specified training data. (default 0.3) summary_dir (str): Optional summary directory for :class:`tf.summary.FileWriter`. (default :obj:`None`, summary is disabled) """ sess = get_default_session_or_error() # split the training & validation set values = np.asarray(values, dtype=np.float32) if len(values.shape) != 2: raise ValueError("`values` must be a 2-D array") n = int(len(values) * valid_portion) train_values, v_x = values[:-n], values[-n:] train_sliding_window = BatchSlidingWindow( array_size=len(train_values), window_size=self.model.window_length, batch_size=self._batch_size, shuffle=True, ignore_incomplete_batch=True, ) valid_sliding_window = BatchSlidingWindow( array_size=len(v_x), window_size=self.model.window_length, batch_size=self._valid_batch_size, ) # initialize the variables of the trainer, and the model sess.run(self._trainer_initializer) ensure_variables_initialized(self._train_params) # training loop lr = self._initial_lr with TrainLoop( param_vars=self._train_params, early_stopping=True, summary_dir=summary_dir, max_epoch=self._max_epoch, max_step=self._max_step, ) as loop: # type: TrainLoop loop.print_training_summary() train_batch_time = [] valid_batch_time = [] time_train_start = time.time() for epoch in loop.iter_epochs(): print("train_values:", train_values.shape) train_iterator = train_sliding_window.get_iterator([train_values]) start_time = time.time() for step, (batch_x,) in loop.iter_steps(train_iterator): # run a training step start_batch_time = time.time() feed_dict = dict(six.iteritems(self._feed_dict)) feed_dict[self._learning_rate] = lr feed_dict[self._input_x] = batch_x loss, _ = sess.run( [self._loss, self._train_op], feed_dict=feed_dict ) loop.collect_metrics({"loss": loss}) train_batch_time.append(time.time() - start_batch_time) # if step % self._valid_step_freq == 0: # train_duration = time.time() - start_time # loop.collect_metrics({"train_time": train_duration}) # # collect variable summaries # if summary_dir is not None: # loop.add_summary(sess.run(self._summary_op)) # # do validation in batches # with loop.timeit("valid_time"), loop.metric_collector( # "valid_loss" # ) as mc: # v_it = valid_sliding_window.get_iterator([v_x]) # for (b_v_x,) in v_it: # start_batch_time = time.time() # feed_dict = dict(six.iteritems(self._valid_feed_dict)) # feed_dict[self._input_x] = b_v_x # loss = sess.run(self._loss, feed_dict=feed_dict) # valid_batch_time.append(time.time() - start_batch_time) # mc.collect(loss, weight=len(b_v_x)) # # print the logs of recent steps # loop.print_logs() # start_time = time.time() # anneal the learning rate if self._lr_anneal_epochs and epoch % self._lr_anneal_epochs == 0: lr *= self._lr_anneal_factor loop.println( "Learning rate decreased to {}".format(lr), with_tag=True ) time_train_end = time.time() return { # "best_valid_loss": float(loop.best_valid_metric), "train_time": np.sum(train_batch_time), "valid_time": 0, "total_train_time": time_train_end - time_train_start, }
def test_nvil(self): assert_allclose = functools.partial(np.testing.assert_allclose, rtol=1e-5, atol=1e-6) with self.test_session() as sess: x, y, z, f, log_f, log_q = \ prepare_test_payload(is_reparameterized=False) baseline = 3.14 * tf.cos(y) alt_f = tf.exp(2 * y * z) # baseline is None, center by moving average, no sampling cost, baseline_cost = nvil_estimator(values=f, latent_log_joint=log_q) self.assertIsNone(baseline_cost) var_count = len(tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)) moving_mean = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)[0] self.assertEqual(moving_mean.name, 'nvil_estimator/moving_mean:0') self.assertEqual(get_static_shape(moving_mean), (1, 1)) ensure_variables_initialized() sess.run(tf.assign(moving_mean, [[6.]])) cost_shape = cost.get_shape().as_list() moving_mean = 4.8 + .2 * tf.reduce_mean(f) assert_allclose(*sess.run([ tf.gradients([cost], [y])[0], tf.reduce_sum(z * f + (f - moving_mean) * (3 * (x**2 - 1) * (y**2)), axis=0) ])) # baseline is given, no center by moving average cost, baseline_cost = nvil_estimator( values=f, latent_log_joint=log_q, baseline=baseline, center_by_moving_average=False, ) self.assertEqual( len(tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)), var_count) self.assertListEqual(cost.get_shape().as_list(), cost_shape) assert_allclose(*sess.run([ tf.gradients([cost], [y])[0], tf.reduce_sum(z * f + (f - 3.14 * tf.cos(y)) * (3 * (x**2 - 1) * (y**2)), axis=0) ])) assert_allclose(*sess.run([ tf.gradients([baseline_cost], [y])[0], # -2 * (f(x,z) - C(x)) * C'(x) tf.reduce_sum(-2 * (f - baseline) * (-3.14 * tf.sin(y)), axis=0) ])) # baseline is given, no center by moving average, axis = [0] cost, baseline_cost = nvil_estimator( values=f, latent_log_joint=log_q, baseline=baseline, center_by_moving_average=False, axis=[0]) self.assertEqual( len(tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)), var_count) self.assertListEqual(cost.get_shape().as_list(), cost_shape[1:]) assert_allclose(*sess.run([ tf.gradients([cost], [y])[0], tf.reduce_sum(z * f + (f - 3.14 * tf.cos(y)) * (3 * (x**2 - 1) * (y**2)), axis=0) / 7 ])) assert_allclose(*sess.run([ tf.gradients([baseline_cost], [y])[0], # -2 * (f(x,z) - C(x)) * C'(x) tf.reduce_sum(-2 * (f - baseline) * (-3.14 * tf.sin(y)), axis=0) / 7 ]))
def fit(self, values, labels, mean, std, excludes=None, valid_portion=0.3, summary_dir=None): """ Train the :class:`Donut` model with given data. Args: values (np.ndarray): 1-D `float32` array, the standardized KPI observations. labels (np.ndarray): 1-D `int32` array, the anomaly labels. missing (np.ndarray): 1-D `int32` array, the indicator of missing points. mean (float): The mean of KPI observations before standardization. std (float): The standard deviation of KPI observations before standardization. excludes (np.ndarray): 1-D `bool` array, indicators of whether or not to totally exclude a point. If a point is excluded, any window which contains that point is excluded. (default :obj:`None`, no point is totally excluded) valid_portion (float): Ratio of validation data out of all the specified training data. (default 0.3) summary_dir (str): Optional summary directory for :class:`tf.summary.FileWriter`. (default :obj:`None`, summary is disabled) """ sess = get_default_session_or_error() # split the training & validation set values = np.asarray(values, dtype=np.float32) labels = np.asarray(labels, dtype=np.int32) if len(values.shape) != 1: raise ValueError('`values` must be a 1-D array') if labels.shape != values.shape: raise ValueError('The shape of `labels` does not agree with ' 'the shape of `values` ({} vs {})'.format( labels.shape, values.shape)) n = int(len(values) * valid_portion) train_values, v_x = values[:-n], values[-n:] train_labels, valid_labels = labels[:-n], labels[-n:] v_y = valid_labels.astype(np.int32) if excludes is None: train_excludes, valid_excludes = None, None else: train_excludes, valid_excludes = excludes[:-n], excludes[-n:] # data augmentation object and the sliding window iterator aug = MissingDataInjection(mean, std, self._missing_data_injection_rate) train_sliding_window = BatchSlidingWindow( array_size=len(train_values), window_size=self.model.x_dims, batch_size=self._batch_size, excludes=train_excludes, shuffle=True, ignore_incomplete_batch=True, ) valid_sliding_window = BatchSlidingWindow( array_size=len(v_x), window_size=self.model.x_dims, batch_size=self._valid_batch_size, excludes=valid_excludes, ) # initialize the variables of the trainer, and the model sess.run(self._trainer_initializer) ensure_variables_initialized(self._train_params) # training loop lr = self._initial_lr with TrainLoop(param_vars=self._train_params, early_stopping=True, summary_dir=summary_dir, max_epoch=self._max_epoch, max_step=self._max_step) as loop: # type: TrainLoop loop.print_training_summary() for epoch in loop.iter_epochs(): x, y1 = aug.augment(train_values, train_labels) y = y1.astype(np.int32) train_iterator = train_sliding_window.get_iterator([x, y]) for step, (batch_x, batch_y) in loop.iter_steps(train_iterator): # run a training step feed_dict = dict(six.iteritems(self._feed_dict)) feed_dict[self._learning_rate] = lr feed_dict[self._input_x] = batch_x feed_dict[self._input_y] = batch_y loss, _ = sess.run([self._loss, self._train_op], feed_dict=feed_dict) loop.collect_metrics({'loss': loss}) if step % self._valid_step_freq == 0: # collect variable summaries if summary_dir is not None: loop.add_summary(sess.run(self._summary_op)) # do validation in batches with loop.timeit('valid_time'), \ loop.metric_collector('valid_loss') as mc: v_it = valid_sliding_window.get_iterator( [v_x, v_y]) for b_v_x, b_v_y in v_it: feed_dict = dict( six.iteritems(self._valid_feed_dict)) feed_dict[self._input_x] = b_v_x feed_dict[self._input_y] = b_v_y loss = sess.run(self._loss, feed_dict=feed_dict) mc.collect(loss, weight=len(b_v_x)) # print the logs of recent steps loop.print_logs() # anneal the learning rate if self._lr_anneal_epochs and \ epoch % self._lr_anneal_epochs == 0: lr *= self._lr_anneal_factor loop.println('Learning rate decreased to {}'.format(lr), with_tag=True)
def test_training_loss(self): class Capture(object): def __init__(self, vae): self._chain_func = vae.chain self._solvers = {} self.called_solver = None self.chain = None vae.chain = self def _call_solver(self, solver_name, *args, **kwargs): self.called_solver = solver_name return self._solvers[solver_name](*args, **kwargs) def __call__(self, *args, **kwargs): chain = self._chain_func(*args, **kwargs) self.chain = chain for k in ['iwae', 'vimco', 'sgvb', 'reinforce']: self._solvers[k] = getattr(chain.vi.training, k) setattr(chain.vi.training, k, functools.partial(self._call_solver, solver_name=k)) return chain @property def q_net(self): return self.chain.variational helper = Helper() x, x2, x3, z, z2 = helper.get_xz_variables() M = tf.reduce_mean with self.test_session() as sess: ensure_variables_initialized() # test error for n_z vae = helper.vae() with pytest.raises( TypeError, match='Cannot choose the variational solver ' 'automatically for dynamic `n_z`'): vae.get_training_loss(x, n_z=tf.constant(1)) # test sgvb vae = helper.vae() capture = Capture(vae) loss = vae.get_training_loss(x) self.assertEqual(capture.called_solver, 'sgvb') zs_obj = helper.zs_objective( zs.variational.elbo, observed={'x': x}, latent={'z': capture.q_net.query(['z'])[0]} ) np.testing.assert_allclose(*sess.run([loss, M(zs_obj.sgvb())])) # test sgvb with explicit n_z vae = helper.vae() capture = Capture(vae) loss = vae.get_training_loss(x, n_z=1) self.assertEqual(capture.called_solver, 'sgvb') zs_obj = helper.zs_objective( zs.variational.elbo, observed={'x': x}, latent={'z': capture.q_net.query(['z'])[0]}, axis=0, ) np.testing.assert_allclose(*sess.run([loss, M(zs_obj.sgvb())])) # test iwae vae = helper.vae() capture = Capture(vae) loss = vae.get_training_loss(x, n_z=N_Z) self.assertEqual(capture.called_solver, 'iwae') zs_obj = helper.zs_objective( zs.variational.importance_weighted_objective, observed={'x': x}, latent={'z': capture.q_net.query(['z'])[0]}, axis=0, ) np.testing.assert_allclose(*sess.run([loss, M(zs_obj.sgvb())])) # test reinforce vae = helper.vae(is_reparameterized=False) capture = Capture(vae) loss = vae.get_training_loss(x) self.assertEqual(capture.called_solver, 'reinforce') # TODO: The output of reinforce mismatches on some platform # # REINFORCE requires additional moving average variable, causing # it very hard to ensure two calls should have identical outputs. # So we disable such tests for the time being. # zs_obj = helper.zs_objective( # zs.variational.elbo, # observed={'x': x}, # latent={'z': capture.q_net.query(['z'])[0]} # ) # with tf.variable_scope(None, default_name='reinforce'): # zs_reinforce = zs_obj.reinforce() # ensure_variables_initialized() # np.testing.assert_allclose(*sess.run([loss, zs_reinforce])) # test reinforce with explicit n_z vae = helper.vae(is_reparameterized=False) capture = Capture(vae) loss = vae.get_training_loss(x, n_z=1) self.assertEqual(capture.called_solver, 'reinforce') # TODO: The output of reinforce mismatches on some platform # zs_obj = helper.zs_objective( # zs.variational.elbo, # observed={'x': x}, # latent={'z': capture.q_net.query(['z'])[0]}, # axis=0, # ) # with tf.variable_scope(None, default_name='reinforce'): # zs_reinforce = zs_obj.reinforce() # ensure_variables_initialized() # np.testing.assert_allclose(*sess.run([loss, zs_reinforce])) # test vimco vae = helper.vae(is_reparameterized=False) capture = Capture(vae) loss = vae.get_training_loss(x, n_z=N_Z) self.assertEqual(capture.called_solver, 'vimco') zs_obj = helper.zs_objective( zs.variational.importance_weighted_objective, observed={'x': x}, latent={'z': capture.q_net.query(['z'])[0]}, axis=0, ) np.testing.assert_allclose(*sess.run([loss, M(zs_obj.vimco())]))
def fit(self, values_list, sample_ratio=1.0, valid_portion=0.3, summary_dir=None): """ Train the :class:`Omni` model with given data. Args: values_list (list of np.ndarray): 1-D `float32` array, the standardized KPI observations. valid_portion (float): Ratio of validation data out of all the specified training data. (default 0.3) summary_dir (str): Optional summary directory for :class:`tf.summary.FileWriter`. (default :obj:`None`, summary is disabled) """ sess = get_default_session_or_error() np.random.seed(2020) # split the training & validation set train_values_list, v_x_list, train_excludes_list, v_excludes_list = [], [], [], [] for values in values_list: values = np.asarray(values, dtype=np.float32) if len(values.shape) != 2: raise ValueError('`values` must be a 2-D array') N_sample = int(sample_ratio * len(values)) N_start = int(np.random.random() * (len(values) - N_sample)) values = values[N_start:int(N_start + N_sample), :] n = int(len(values) * valid_portion) train_values, v_x = values[:-n], values[-n:] train_values_list.append(train_values) v_x_list.append(v_x) train_excludes_list.append( np.hstack([[1], np.zeros([len(train_values) - 1])])) v_excludes_list.append(np.hstack([[1], np.zeros([len(v_x) - 1])])) train_values, v_x = np.vstack(train_values_list), np.vstack(v_x_list) train_excludes, v_excludes = np.hstack(train_excludes_list), np.hstack( v_excludes_list) train_sliding_window = BatchSlidingWindow( array_size=len(train_values), window_size=self.model.window_length, batch_size=self._batch_size, excludes=train_excludes, shuffle=True, ignore_incomplete_batch=True, ) valid_sliding_window = BatchSlidingWindow( array_size=len(v_x), window_size=self.model.window_length, batch_size=self._valid_batch_size, excludes=v_excludes, ) # initialize the variables of the trainer, and the model sess.run(self._trainer_initializer) ensure_variables_initialized(self._train_params) # training loop lr = self._initial_lr with TrainLoop(param_vars=self._train_params, early_stopping=True, summary_dir=summary_dir, max_epoch=self._max_epoch, max_step=self._max_step) as loop: # type: TrainLoop loop.print_training_summary() train_batch_time = [] valid_batch_time = [] for epoch in loop.iter_epochs(): print('train_values:', train_values.shape) train_iterator = train_sliding_window.get_iterator( [train_values]) start_time = time.time() for step, (batch_x, ) in loop.iter_steps(train_iterator): # run a training step start_batch_time = time.time() feed_dict = dict(six.iteritems(self._feed_dict)) feed_dict[self._learning_rate] = lr feed_dict[self._input_x] = batch_x loss, _ = sess.run([self._loss, self._train_op], feed_dict=feed_dict) loop.collect_metrics({'loss': loss}) train_batch_time.append(time.time() - start_batch_time) if step % self._valid_step_freq == 0: train_duration = time.time() - start_time loop.collect_metrics({'train_time': train_duration}) # collect variable summaries if summary_dir is not None: loop.add_summary(sess.run(self._summary_op)) # do validation in batches with loop.timeit('valid_time'), \ loop.metric_collector('valid_loss') as mc: v_it = valid_sliding_window.get_iterator([v_x]) for (b_v_x, ) in v_it: start_batch_time = time.time() feed_dict = dict( six.iteritems(self._valid_feed_dict)) feed_dict[self._input_x] = b_v_x loss = sess.run(self._loss, feed_dict=feed_dict) valid_batch_time.append(time.time() - start_batch_time) mc.collect(loss, weight=len(b_v_x)) # print the logs of recent steps loop.print_logs() start_time = time.time() # anneal the learning rate if self._lr_anneal_epochs and \ epoch % self._lr_anneal_epochs == 0: lr *= self._lr_anneal_factor loop.println('Learning rate decreased to {}'.format(lr), with_tag=True) try: best_valid_loss = float(loop.best_valid_metric) except: best_valid_loss = np.nan return { 'best_valid_loss': best_valid_loss, 'train_batch_time': np.mean(train_batch_time), 'valid_batch_time': np.mean(valid_batch_time), 'train_all_time': np.sum(train_batch_time), 'valid_all_time': np.sum(valid_batch_time), }
def test_elbo(self): # test no sampling axis with self.test_session() as sess: prepared = self.prepare_model(zs.variational.elbo, axis=None, n_z=None) vi = prepared.vi zs_obj = prepared.zs_obj # test :meth:`VariationalInference.zs_objective` vi_obj = prepared.vi.zs_objective(zs.variational.elbo) self.assertIsInstance(vi_obj, zs.variational.EvidenceLowerBoundObjective) np.testing.assert_allclose(*sess.run([zs_obj, vi_obj])) # test :meth:`VariationalInference.zs_elbo` vi_obj = prepared.vi.zs_elbo() self.assertIsInstance(vi_obj, zs.variational.EvidenceLowerBoundObjective) np.testing.assert_allclose(*sess.run([zs_obj, vi_obj])) # test :meth:`VariationalLowerBounds.elbo` np.testing.assert_allclose( *sess.run([zs_obj, vi.lower_bound.elbo()])) # test :meth:`VariationalTrainingObjectives.sgvb` np.testing.assert_allclose( *sess.run([zs_obj.sgvb(), vi.training.sgvb()])) # test :meth:`VariationalTrainingObjectives.reinforce` # TODO: The output of reinforce mismatches on some platform # if variance_reduction == True # REINFORCE requires additional moving average variable, causing # it very hard to ensure two calls should have identical outputs. # So we disable such tests for the time being. with tf.variable_scope(None, default_name='reinforce'): # reinforce requires extra variables, but ZhuSuan does not # obtain a dedicated variable scope. so we open one here. zs_reinforce = zs_obj.reinforce(variance_reduction=False) vi_reinforce = vi.training.reinforce(variance_reduction=False) ensure_variables_initialized() np.testing.assert_allclose(*sess.run([zs_reinforce, vi_reinforce])) # test with sampling axis with self.test_session() as sess: prepared = self.prepare_model(zs.variational.elbo, axis=0, n_z=7) vi = prepared.vi zs_obj = prepared.zs_obj # test :meth:`VariationalInference.zs_objective` vi_obj = prepared.vi.zs_objective(zs.variational.elbo) self.assertIsInstance(vi_obj, zs.variational.EvidenceLowerBoundObjective) np.testing.assert_allclose(*sess.run([zs_obj, vi_obj])) # test :meth:`VariationalInference.zs_elbo` vi_obj = prepared.vi.zs_elbo() self.assertIsInstance(vi_obj, zs.variational.EvidenceLowerBoundObjective) np.testing.assert_allclose(*sess.run([zs_obj, vi_obj])) # test :meth:`VariationalLowerBounds.elbo` np.testing.assert_allclose( *sess.run([zs_obj, vi.lower_bound.elbo()])) # test :meth:`VariationalTrainingObjectives.sgvb` np.testing.assert_allclose( *sess.run([zs_obj.sgvb(), vi.training.sgvb()])) # test :meth:`VariationalTrainingObjectives.reinforce` # TODO: The output of reinforce mismatches on some platform # if variance_reduction == True # REINFORCE requires additional moving average variable, causing # it very hard to ensure two calls should have identical outputs. # So we disable such tests for the time being. with tf.variable_scope(None, default_name='reinforce'): # reinforce requires extra variables, but ZhuSuan does not # obtain a dedicated variable scope. so we open one here. zs_reinforce = zs_obj.reinforce(variance_reduction=False) vi_reinforce = vi.training.reinforce(variance_reduction=False) ensure_variables_initialized() np.testing.assert_allclose(*sess.run([zs_reinforce, vi_reinforce]))