def test_forward(self):
helper = Helper()
x, x2, x3, z, z2 = helper.get_xz_variables()
with self.test_session() as sess:
ensure_variables_initialized()
# test single sample
vae = helper.vae()
z_s = vae(x)
np.testing.assert_equal(tf.shape(z_s).eval(), [BATCH_SIZE, Z_DIMS])
q_net_zs = helper.zs_variational(x, observed={'z': z_s})
np.testing.assert_allclose(*sess.run([
z_s.log_prob(),
q_net_zs.local_log_prob('z')
]))
# test multiple samples
vae = helper.vae()
z_s = vae(x, n_z=N_Z)
np.testing.assert_equal(
tf.shape(z_s).eval(), [N_Z, BATCH_SIZE, Z_DIMS])
q_net_zs = helper.zs_variational(x, observed={'z': z_s})
np.testing.assert_allclose(*sess.run([
z_s.log_prob(),
q_net_zs.local_log_prob('z')
]))
def test_variational(self):
helper = Helper()
x, x2, x3, z, z2 = helper.get_xz_variables()
with self.test_session() as sess:
ensure_variables_initialized()
# test log-prob of one z observation
vae = helper.vae()
q_net = vae.variational(x, z=z)
q_net_zs = helper.zs_variational(x, observed={'z': z})
np.testing.assert_allclose(q_net['z'].eval(), z.eval())
np.testing.assert_allclose(*sess.run(
[q_net.local_log_prob('z'), q_net_zs.local_log_prob('z')]))
# test log-prob of multiple z observations
vae = helper.vae()
q_net = vae.variational(x, z=z2, n_z=N_Z)
q_net_zs = helper.zs_variational(x, observed={'z': z2}, n_z=N_Z)
np.testing.assert_allclose(q_net['z'].eval(), z2.eval())
np.testing.assert_allclose(*sess.run(
[q_net.local_log_prob('z'), q_net_zs.local_log_prob('z')]))
# test the shape of z samples and their log-probs
vae = helper.vae()
q_net = vae.variational(x, n_z=N_Z)
q_net_zs = helper.zs_variational(x, observed={'z': q_net['z']})
np.testing.assert_equal(
tf.shape(q_net['z']).eval(), [N_Z, BATCH_SIZE, Z_DIMS])
np.testing.assert_allclose(*sess.run(
[q_net.local_log_prob('z'), q_net_zs.local_log_prob('z')]))
def test_transform(self):
def transform(x, u, w, b):
wxb = np.dot(x, w.T) + b
y = x + u * np.tanh(wxb) # shape: [?, n_units]
tanh_wxb = np.tanh(wxb)
phi = (1 - tanh_wxb**2) * w # shape: [?, n_units]
log_det = np.log(np.abs(1 + np.dot(phi, u.T))) # shape: [?, 1]
return y, np.squeeze(log_det, -1)
n_units = 5
tf.set_random_seed(1234)
flow = PlanarNormalizingFlow()
flow.apply(tf.random_normal(shape=[1, n_units], dtype=tf.float64))
x = np.arange(30, dtype=np.float64).reshape([2, 3, n_units])
with self.test_session() as sess:
ensure_variables_initialized()
# compute the ground-truth y and log_det
y = x
w, b, u_hat = flow._w, flow._b, flow._u_hat
u_hat, w, b = sess.run([u_hat, w, b])
y, log_det = transform(y, u_hat, w, b)
# check the flow-derived y and log_det
y2, log_det2 = sess.run(
flow.transform(tf.constant(x, dtype=tf.float64)))
np.testing.assert_allclose(y2, y, rtol=1e-5)
np.testing.assert_allclose(log_det2, log_det, rtol=1e-5)
def test_group_ndims(self):
helper = Helper()
x, x2, x3, z, z2 = helper.get_xz_variables()
with self.test_session() as sess:
ensure_variables_initialized()
vae = helper.vae(z_group_ndims=0, x_group_ndims=2)
q_net = vae.variational(x, z=z)
q_net_zs = helper.zs_variational(x,
observed={'z': z},
z_group_ndims=0)
np.testing.assert_allclose(*sess.run(
[q_net.local_log_prob('z'),
q_net_zs.local_log_prob('z')]))
p_net = vae.model(z=z, x=x)
p_net_zs = helper.zs_model(observed={
'z': z,
'x': x
},
z_group_ndims=0,
x_group_ndims=2)
np.testing.assert_allclose(*sess.run(
[p_net.local_log_prob('z'),
p_net_zs.local_log_prob('z')]))
np.testing.assert_allclose(*sess.run(
[p_net.local_log_prob('x'),
p_net_zs.local_log_prob('x')]))
def test_reconstruct(self):
class Capture(object):
def __init__(self, vae):
self._variational_func = vae.variational
self._model_func = vae.model
self.q_net = None
self.p_net = None
def variational(*args, **kwargs):
self.q_net = self._variational_func(*args, **kwargs)
return self.q_net
def model(*args, **kwargs):
self.p_net = self._model_func(*args, **kwargs)
return self.p_net
vae.variational = variational
vae.model = model
helper = Helper()
x, x2, x3, z, z2 = helper.get_xz_variables()
with self.test_session() as sess:
ensure_variables_initialized()
# test single sample
vae = helper.vae()
capture = Capture(vae)
x_r = vae.reconstruct(x)
np.testing.assert_equal(tf.shape(x_r).eval(), [BATCH_SIZE, X_DIMS])
p_net_zs = helper.zs_model(
observed={'z': capture.q_net['z'], 'x': x_r})
np.testing.assert_allclose(*sess.run([x_r, capture.p_net['x']]))
np.testing.assert_allclose(*sess.run([
capture.p_net['z'].log_prob(),
p_net_zs.local_log_prob('z')
]))
np.testing.assert_allclose(*sess.run([
x_r.log_prob(),
p_net_zs.local_log_prob('x')
]))
# test multiple samples
vae = helper.vae()
capture = Capture(vae)
x_r = vae.reconstruct(x, n_z=N_Z, n_x=N_X)
np.testing.assert_equal(
tf.shape(x_r).eval(), [N_X, N_Z, BATCH_SIZE, X_DIMS])
p_net_zs = helper.zs_model(
observed={'z': capture.q_net['z'], 'x': x_r}, n_z=N_Z, n_x=N_X)
np.testing.assert_allclose(*sess.run([x_r, capture.p_net['x']]))
np.testing.assert_allclose(*sess.run([
capture.p_net['z'].log_prob(),
p_net_zs.local_log_prob('z')
]))
np.testing.assert_allclose(*sess.run([
x_r.log_prob(),
p_net_zs.local_log_prob('x')
]))
def test_basic_logging(self):
v = ScheduledVariable('v', 1.)
logger = MetricLogger()
self.assertEqual(logger.format_logs(), '')
with self.test_session() as sess:
ensure_variables_initialized()
logger.collect_metrics(dict(loss=v))
logger.collect_metrics(dict(loss=2., valid_loss=3., valid_timer=0.1))
logger.collect_metrics(dict(loss=4., valid_acc=5., train_time=0.2))
logger.collect_metrics(dict(loss=6., valid_acc=7., train_time=0.3))
logger.collect_metrics(dict(other_metric=5.))
self.assertEqual(
logger.format_logs(),
'train time: 0.25s (±0.05s); '
'valid timer: 0.1s; '
'other metric: 5; '
'loss: 3.25 (±1.92029); '
'valid loss: 3; '
'valid acc: 6 (±1)'
)
logger.clear()
self.assertEqual(logger.format_logs(), '')
logger.collect_metrics({'loss': 1.})
self.assertEqual(logger.format_logs(), 'loss: 1')
def test_ScheduledVariable(self):
v = ScheduledVariable('v', 123., dtype=tf.int32, model_var=True,
collections=['my_variables'])
assert_variables(['v'], trainable=False, collections=['my_variables'])
with TemporaryDirectory() as tmpdir:
saver = tf.train.Saver(var_list=[v.variable])
save_path = os.path.join(tmpdir, 'saved_var')
with self.test_session() as sess:
ensure_variables_initialized()
self.assertEqual(v.get(), 123)
self.assertEqual(sess.run(v), 123)
self.assertEqual(v.set(456), 456)
self.assertEqual(v.get(), 456)
saver.save(sess, save_path)
with self.test_session() as sess:
saver.restore(sess, save_path)
self.assertEqual(v.get(), 456)
sess.run(v.assign_op, feed_dict={v.assign_ph: 789})
self.assertEqual(v.get(), 789)
def run(self):
"""Run training loop."""
if self._is_fitting:
raise RuntimeError('`run()` is not re-entrant.')
self._is_fitting = True
try:
# initialize global training status
session = get_default_session_or_error()
ensure_variables_initialized()
self.loop.print_training_summary()
# initialize internal status
for hook_list in self.hook_lists:
hook_list.reset()
for epoch in self.loop.iter_epochs():
# run before epoch hook
self.before_epochs.call_hooks()
# run steps of this epoch
for payload in self._iter_steps():
# run before step hook
self.before_steps.call_hooks()
# run the step
self._run_step(session, payload)
# run after step hook
self.after_steps.call_hooks()
# run after epoch hook
self.after_epochs.call_hooks()
finally:
self._is_fitting = False
def test_ensure_variables_initialized_using_dict(self):
a = tf.get_variable('a', dtype=tf.int32, initializer=1)
b = tf.get_variable('b', dtype=tf.int32, initializer=2)
# test using dict
with self.test_session():
ensure_variables_initialized({'a': a})
self.assertEqual(get_uninitialized_variables([a, b]), [b])
def fit(self, train_iterator, summary_dir=None):
"""
Train the :class:`OmniAnomaly` model with given data.
Args:
values (np.ndarray): 1-D `float32` array, the standardized
KPI observations.
summary_dir (str): Optional summary directory for
:class:`tf.summary.FileWriter`. (default :obj:`None`,
summary is disabled)
"""
sess = get_default_session_or_error()
# initialize the variables of the trainer, and the model
sess.run(self._trainer_initializer)
ensure_variables_initialized(self._train_params)
# training loop
lr = self._initial_lr
with TrainLoop(
param_vars=self._train_params,
summary_dir=summary_dir,
max_epoch=self._max_epoch,
max_step=self._max_step,
) as loop: # type: TrainLoop
# loop.print_training_summary()
train_batch_time = []
valid_batch_time = []
time_train_start = time.time()
for epoch in loop.iter_epochs():
start_time = time.time()
for step, idx in loop.iter_steps(range(len(train_iterator))):
# run a training step
batch_x = train_iterator[idx]
start_batch_time = time.time()
feed_dict = dict(six.iteritems(self._feed_dict))
feed_dict[self._learning_rate] = lr
feed_dict[self._input_x] = batch_x
loss, _ = sess.run([self._loss, self._train_op],
feed_dict=feed_dict)
loop.collect_metrics({"loss": loss})
train_batch_time.append(time.time() - start_batch_time)
# anneal the learning rate
if self._lr_anneal_epochs and epoch % self._lr_anneal_epochs == 0:
lr *= self._lr_anneal_factor
loop.println("Learning rate decreased to {}".format(lr),
with_tag=True)
time_train_end = time.time()
return {
# "best_valid_loss": float(loop.best_valid_metric),
"train_time": np.sum(train_batch_time),
"total_train_time": time_train_end - time_train_start,
}
def test_model(self):
helper = Helper()
x, x2, x3, z, z2 = helper.get_xz_variables()
with self.test_session() as sess:
ensure_variables_initialized()
# test log-prob of one z observation
vae = helper.vae()
p_net = vae.model(z=z, x=x)
p_net_zs = helper.zs_model(observed={'z': z, 'x': x})
np.testing.assert_allclose(p_net['z'].eval(), z.eval())
np.testing.assert_allclose(p_net['x'].eval(), x.eval())
np.testing.assert_allclose(*sess.run(
[p_net.local_log_prob('z'),
p_net_zs.local_log_prob('z')]))
np.testing.assert_allclose(*sess.run(
[p_net.local_log_prob('x'),
p_net_zs.local_log_prob('x')]))
# test log-prob of multiple x & z observations
vae = helper.vae()
p_net = vae.model(z=z2, x=x3, n_z=N_Z, n_x=N_X)
p_net_zs = helper.zs_model(observed={
'z': z2,
'x': x3
},
n_z=N_Z,
n_x=N_X)
np.testing.assert_allclose(p_net['z'].eval(), z2.eval())
np.testing.assert_allclose(p_net['x'].eval(), x3.eval())
np.testing.assert_allclose(*sess.run(
[p_net.local_log_prob('z'),
p_net_zs.local_log_prob('z')]))
np.testing.assert_allclose(*sess.run(
[p_net.local_log_prob('x'),
p_net_zs.local_log_prob('x')]))
# test the shape of samples and their log-probs
vae = helper.vae()
p_net = vae.model(n_x=N_X, n_z=N_Z)
p_net_zs = helper.zs_model(observed={
'z': p_net['z'],
'x': p_net['x']
},
n_z=N_Z,
n_x=N_X)
np.testing.assert_equal(
tf.shape(p_net['z']).eval(), [N_Z, BATCH_SIZE, Z_DIMS])
np.testing.assert_equal(
tf.shape(p_net['x']).eval(), [N_X, N_Z, BATCH_SIZE, X_DIMS])
np.testing.assert_allclose(*sess.run(
[p_net.local_log_prob('z'),
p_net_zs.local_log_prob('z')]))
np.testing.assert_allclose(*sess.run(
[p_net.local_log_prob('x'),
p_net_zs.local_log_prob('x')]))
def test_inplace(self):
with self.test_session():
d = {
'a': 12,
'b': ScheduledVariable('b', 34),
'c': lambda: 56,
}
ensure_variables_initialized()
self.assertIs(d, resolve_feed_dict(d, inplace=True))
self.assertDictEqual({'a': 12, 'b': 34, 'c': 56}, d)
def ensure_variables_initialized(self):
"""Initialize all uninitialized variables within model's scope.
If the `global_step` is created by the model itself, then it will
also be initialized. Otherwise the `global_step` must be manually
initialized.
"""
self.build()
var_list = list(six.itervalues(self.get_variables()))
ensure_variables_initialized(var_list)
def test_tensor_arguments(self):
with self.test_session():
a = tf.get_variable('a', initializer=0, dtype=tf.int32)
ensure_variables_initialized()
with TrainLoop([a],
early_stopping=True,
max_epoch=tf.constant(6),
max_step=tf.constant(7)) as loop:
self.assertEqual(loop.max_epoch, 6)
self.assertEqual(loop.max_step, 7)
def test_copy(self):
with self.test_session():
d = {
'a': 12,
'b': ScheduledVariable('b', 34),
'c': lambda: 56,
}
ensure_variables_initialized()
d2 = resolve_feed_dict(d)
self.assertIsNot(d2, d)
self.assertDictEqual({'a': 12, 'b': 34, 'c': 56}, d2)
self.assertIsInstance(d['b'], ScheduledVariable)
def test_tensor_arguments(self):
with self.test_session():
a = tf.get_variable('a', initializer=0, dtype=tf.int32)
ensure_variables_initialized()
with TrainLoop([a],
early_stopping=True,
initial_valid_metric=tf.constant(1.23),
initial_epoch=tf.constant(4),
initial_step=tf.constant(5),
max_epoch=tf.constant(6),
max_step=tf.constant(7)) as loop:
self.assertAlmostEqual(loop._early_stopping._best_metric, 1.23)
self.assertEqual(loop.epoch, 4)
self.assertEqual(loop.step, 5)
self.assertEqual(loop.max_epoch, 6)
self.assertEqual(loop.max_step, 7)
def test_run(self):
ph = tf.placeholder(tf.int32, [5])
var = tf.get_variable('var', shape=[5], dtype=tf.int32,
initializer=tf.zeros_initializer())
train_op = tf.assign(var, ph)
df = DataFlow.arrays([np.arange(10, 15, dtype=np.int32)], batch_size=5)
with self.test_session() as session, \
TrainLoop([var], max_epoch=1, early_stopping=False) as loop:
loop.collect_metrics = Mock(wraps=loop.collect_metrics)
t = LossTrainer(loop, tf.reduce_sum(ph), train_op, [ph], df,
metric_name='loss_x')
ensure_variables_initialized()
t.run()
self.assertEqual(
{'loss_x': 60}, loop.collect_metrics.call_args_list[0][0][0])
np.testing.assert_equal([10, 11, 12, 13, 14], session.run(var))
def test_run(self):
ph = tf.placeholder(tf.int32, [5])
var = tf.get_variable('var', shape=[5], dtype=tf.int32,
initializer=tf.zeros_initializer())
summary = tf.summary.histogram(var.name, var)
train_op = tf.assign(var, ph)
df = DataFlow.arrays([np.arange(10, 15, dtype=np.int32)], batch_size=5)
with TemporaryDirectory() as tmpdir:
with self.test_session() as session, \
TrainLoop([var], max_epoch=1, early_stopping=False,
summary_dir=tmpdir) as loop:
loop.collect_metrics = Mock(wraps=loop.collect_metrics)
loop.add_summary = Mock(wraps=loop.add_summary)
t = Trainer(loop, train_op, [ph], df,
metrics={'loss_x': tf.reduce_sum(ph)},
summaries=[summary])
ensure_variables_initialized()
t.run()
self.assertEqual(
{'loss_x': 60},
loop.collect_metrics.call_args_list[0][0][0]
)
self.assertTrue(loop.add_summary.called)
np.testing.assert_equal([10, 11, 12, 13, 14], session.run(var))
# test to specify summaries, but without loop.summary_dir, so no
# summary should run
with self.test_session() as session, \
TrainLoop([var], max_epoch=1, early_stopping=False) as loop:
loop.collect_metrics = Mock(wraps=loop.collect_metrics)
loop.add_summary = Mock(wraps=loop.add_summary)
t = Trainer(loop, train_op, [ph], df,
metrics={'loss_x': tf.reduce_sum(ph)},
summaries=[summary])
ensure_variables_initialized()
t.run()
self.assertEqual(
{'loss_x': 60},
loop.collect_metrics.call_args_list[0][0][0]
)
self.assertFalse(loop.add_summary.called)
def run(self):
"""Run training loop."""
if self._is_fitting:
raise RuntimeError('`run()` is not re-entrant.')
self._is_fitting = True
try:
# trigger the before execution event
self.events.fire(EventKeys.BEFORE_EXECUTION, self)
# initialize global training status
session = get_default_session_or_error()
if self._ensure_variables_initialized:
ensure_variables_initialized()
self.loop.print_training_summary()
for _ in self.loop.iter_epochs():
# trigger before epoch event
self.events.fire(EventKeys.BEFORE_EPOCH, self)
# run steps of this epoch
for payload in self._iter_steps():
# trigger before step event
self.events.fire(EventKeys.BEFORE_STEP, self)
# run the step
self._run_step(session, payload)
# trigger after step events
self.events.fire(EventKeys.STEP_EVALUATION, self)
self.events.fire(EventKeys.STEP_ANNEALING, self)
self.events.fire(EventKeys.STEP_LOGGING, self)
self.events.reverse_fire(EventKeys.AFTER_STEP, self)
# trigger after epoch events
self.events.fire(EventKeys.EPOCH_EVALUATION, self)
self.events.fire(EventKeys.EPOCH_ANNEALING, self)
self.events.fire(EventKeys.EPOCH_LOGGING, self)
self.events.reverse_fire(EventKeys.AFTER_EPOCH, self)
# trigger the after execution event
self.events.reverse_fire(EventKeys.AFTER_EXECUTION, self)
finally:
self._is_fitting = False
def test_ensure_variables_initialized(self):
a = tf.get_variable('a', dtype=tf.int32, initializer=1)
b = tf.get_variable('b', dtype=tf.int32, initializer=2)
c = tf.get_variable('c',
dtype=tf.int32,
initializer=3,
collections=[tf.GraphKeys.MODEL_VARIABLES])
d = tf.get_variable('d',
dtype=tf.int32,
initializer=4,
collections=[tf.GraphKeys.MODEL_VARIABLES])
# test using list
with self.test_session():
self.assertEqual(get_uninitialized_variables([a, b, c, d]),
[a, b, c, d])
ensure_variables_initialized()
self.assertEqual(get_uninitialized_variables([a, b, c, d]), [c, d])
ensure_variables_initialized([a, b, c, d])
self.assertEqual(get_uninitialized_variables([a, b, c, d]), [])
def test_nvil(self):
assert_allclose = functools.partial(np.testing.assert_allclose,
rtol=1e-5,
atol=1e-6)
with self.test_session() as sess:
log_p = tf.random_normal(shape=[5, 7])
log_q1 = tf.random_normal(shape=[1, 3, 5, 7])
log_q2 = tf.random_normal(shape=[4, 1, 5, 7])
baseline = tf.random_normal(shape=[5, 7])
# test without sampling axis
vi = VariationalInference(log_p, [log_q1, log_q2])
output = vi.training.nvil(baseline=None,
center_by_moving_average=True)
a, b = nvil_estimator(
values=log_p - (log_q1 + log_q2),
latent_log_joint=log_q1 + log_q2,
baseline=None,
center_by_moving_average=True,
decay=0.8,
)
answer = -a
ensure_variables_initialized()
assert_allclose(*sess.run([output, answer]))
# test with sampling axis
vi = VariationalInference(log_p, [log_q1, log_q2], axis=[0, 1])
output = vi.training.nvil(baseline=baseline,
center_by_moving_average=False)
a, b = nvil_estimator(
values=log_p - (log_q1 + log_q2),
latent_log_joint=log_q1 + log_q2,
axis=[0, 1],
baseline=baseline,
center_by_moving_average=False,
decay=0.8,
)
answer = b - a
ensure_variables_initialized()
assert_allclose(*sess.run([output, answer]))
def test_AnnealingDynamicValue(self):
with self.test_session() as sess:
# test without min_value
v = AnnealingVariable('v', 1, 2)
ensure_variables_initialized()
self.assertEqual(v.get(), 1)
self.assertEqual(v.anneal(), 2)
self.assertEqual(v.get(), 2)
self.assertEqual(v.anneal(), 4)
self.assertEqual(v.get(), 4)
self.assertEqual(v.set(2), 2)
self.assertEqual(v.get(), 2)
self.assertEqual(v.anneal(), 4)
self.assertEqual(v.get(), 4)
# test with min_value
v = AnnealingVariable('v2', 1, .5, 2)
ensure_variables_initialized()
self.assertEqual(v.get(), 2)
v = AnnealingVariable('v3', 1, .5, .5)
ensure_variables_initialized()
self.assertEqual(v.get(), 1)
self.assertEqual(v.anneal(), .5)
self.assertEqual(v.get(), .5)
self.assertEqual(v.anneal(), .5)
self.assertEqual(v.get(), .5)
def test_u_hat(self):
n_units = 5
tf.set_random_seed(1234)
flow = PlanarNormalizingFlow(name='planar_nf')
flow.apply(tf.random_normal(shape=[1, n_units]))
# ensure these parameters exist
w, b, u, u_hat = flow._w, flow._b, flow._u, flow._u_hat
for v in [w, u, b, u_hat]:
self.assertIn('planar_nf/', v.name)
# ensure these parameters have expected shapes
self.assertEqual(get_static_shape(w), (1, n_units))
self.assertEqual(get_static_shape(u), (1, n_units))
self.assertEqual(get_static_shape(b), (1, ))
self.assertEqual(get_static_shape(u_hat), (1, n_units))
with self.test_session() as sess:
ensure_variables_initialized()
w, b, u, u_hat = sess.run([w, b, u, u_hat])
m = lambda a: -1 + np.log(1 + np.exp(a))
wu = np.dot(w, u.T) # shape: [1]
np.testing.assert_allclose(u + w * (m(wu) - wu) / np.sum(w**2),
u_hat)
def test_pixelcnn_conv2d_resnet_pixel_dep(self):
tf.set_random_seed(1234)
with mock.patch('tensorflow.nn.conv2d', patched_conv2d), \
self.test_session() as sess:
######################
# test one layer dep #
######################
H, W = 11, 12
x = np.eye(H * W, dtype=np.float32).reshape([H * W, H, W, 1]) * \
np.ones([1, 1, 1, 2], dtype=np.float32)
vertical_predicate = lambda i, j, h, w: (
(h - 3 <= i <= h - 1) and (w - 2 <= j <= w + 2))
horizontal_predicate = lambda i, j, h, w: (
# value from vertical stack
((h - 4 <= i <= h - 1) and (w - 3 <= j <= w + 2)) or
# value from horizontal stack
((h - 2 <= i <= h) and (w - 3 <= j <= w - 1)))
# NHWC
y = pixelcnn_2d_input(x,
channels_last=True,
auxiliary_channel=False)
o = pixelcnn_conv2d_resnet(
y,
out_channels=3,
vertical_kernel_size=(2, 3),
horizontal_kernel_size=(2, 2),
strides=(1, 1),
channels_last=True,
)
ensure_variables_initialized()
vertical, horizontal = sess.run([o.vertical, o.horizontal])
check_pixel_deps(
self,
H,
W,
vertical,
horizontal,
vertical_predicate,
horizontal_predicate,
)
# NCHW
y = pixelcnn_2d_input(np.transpose(x, [0, 3, 1, 2]),
channels_last=False,
auxiliary_channel=False)
o = pixelcnn_conv2d_resnet(y,
out_channels=3,
vertical_kernel_size=(2, 3),
horizontal_kernel_size=(2, 2),
strides=(1, 1),
channels_last=False,
activation_fn=tf.nn.leaky_relu)
ensure_variables_initialized()
vertical, horizontal = sess.run([o.vertical, o.horizontal])
vertical = np.transpose(vertical, [0, 2, 3, 1])
horizontal = np.transpose(horizontal, [0, 2, 3, 1])
check_pixel_deps(
self,
H,
W,
vertical,
horizontal,
vertical_predicate,
horizontal_predicate,
)
########################
# test multi layer dep #
########################
H, W = 6, 7
x = np.eye(H * W, dtype=np.float32).reshape([H * W, H, W, 1]) * \
np.ones([1, 1, 1, 2], dtype=np.float32)
n_layers = 3
vertical_predicate = lambda i, j, h, w: i <= h - 1
horizontal_predicate = lambda i, j, h, w: (
# value from vertical stack
i <= h - 1 or
# value from horizontal stack
(j <= w - 1 and i <= h))
y = pixelcnn_2d_input(x,
channels_last=True,
auxiliary_channel=False)
for i in range(n_layers):
y = pixelcnn_conv2d_resnet(
y,
out_channels=3,
vertical_kernel_size=(2, 3),
horizontal_kernel_size=(2, 2),
strides=(1, 1),
channels_last=True,
)
ensure_variables_initialized()
vertical, horizontal = sess.run([y.vertical, y.horizontal])
check_pixel_deps(
self,
H,
W,
vertical,
horizontal,
vertical_predicate,
horizontal_predicate,
)
def fit(self, values, valid_portion=0.01, summary_dir=None):
"""
Train the :class:`OmniAnomaly` model with given data.
Args:
values (np.ndarray): 1-D `float32` array, the standardized
KPI observations.
valid_portion (float): Ratio of validation data out of all the
specified training data. (default 0.3)
summary_dir (str): Optional summary directory for
:class:`tf.summary.FileWriter`. (default :obj:`None`,
summary is disabled)
"""
sess = get_default_session_or_error()
# split the training & validation set
values = np.asarray(values, dtype=np.float32)
if len(values.shape) != 2:
raise ValueError("`values` must be a 2-D array")
n = int(len(values) * valid_portion)
train_values, v_x = values[:-n], values[-n:]
train_sliding_window = BatchSlidingWindow(
array_size=len(train_values),
window_size=self.model.window_length,
batch_size=self._batch_size,
shuffle=True,
ignore_incomplete_batch=True,
)
valid_sliding_window = BatchSlidingWindow(
array_size=len(v_x),
window_size=self.model.window_length,
batch_size=self._valid_batch_size,
)
# initialize the variables of the trainer, and the model
sess.run(self._trainer_initializer)
ensure_variables_initialized(self._train_params)
# training loop
lr = self._initial_lr
with TrainLoop(
param_vars=self._train_params,
early_stopping=True,
summary_dir=summary_dir,
max_epoch=self._max_epoch,
max_step=self._max_step,
) as loop: # type: TrainLoop
loop.print_training_summary()
train_batch_time = []
valid_batch_time = []
time_train_start = time.time()
for epoch in loop.iter_epochs():
print("train_values:", train_values.shape)
train_iterator = train_sliding_window.get_iterator([train_values])
start_time = time.time()
for step, (batch_x,) in loop.iter_steps(train_iterator):
# run a training step
start_batch_time = time.time()
feed_dict = dict(six.iteritems(self._feed_dict))
feed_dict[self._learning_rate] = lr
feed_dict[self._input_x] = batch_x
loss, _ = sess.run(
[self._loss, self._train_op], feed_dict=feed_dict
)
loop.collect_metrics({"loss": loss})
train_batch_time.append(time.time() - start_batch_time)
# if step % self._valid_step_freq == 0:
# train_duration = time.time() - start_time
# loop.collect_metrics({"train_time": train_duration})
# # collect variable summaries
# if summary_dir is not None:
# loop.add_summary(sess.run(self._summary_op))
# # do validation in batches
# with loop.timeit("valid_time"), loop.metric_collector(
# "valid_loss"
# ) as mc:
# v_it = valid_sliding_window.get_iterator([v_x])
# for (b_v_x,) in v_it:
# start_batch_time = time.time()
# feed_dict = dict(six.iteritems(self._valid_feed_dict))
# feed_dict[self._input_x] = b_v_x
# loss = sess.run(self._loss, feed_dict=feed_dict)
# valid_batch_time.append(time.time() - start_batch_time)
# mc.collect(loss, weight=len(b_v_x))
# # print the logs of recent steps
# loop.print_logs()
# start_time = time.time()
# anneal the learning rate
if self._lr_anneal_epochs and epoch % self._lr_anneal_epochs == 0:
lr *= self._lr_anneal_factor
loop.println(
"Learning rate decreased to {}".format(lr), with_tag=True
)
time_train_end = time.time()
return {
# "best_valid_loss": float(loop.best_valid_metric),
"train_time": np.sum(train_batch_time),
"valid_time": 0,
"total_train_time": time_train_end - time_train_start,
}
def test_nvil(self):
assert_allclose = functools.partial(np.testing.assert_allclose,
rtol=1e-5,
atol=1e-6)
with self.test_session() as sess:
x, y, z, f, log_f, log_q = \
prepare_test_payload(is_reparameterized=False)
baseline = 3.14 * tf.cos(y)
alt_f = tf.exp(2 * y * z)
# baseline is None, center by moving average, no sampling
cost, baseline_cost = nvil_estimator(values=f,
latent_log_joint=log_q)
self.assertIsNone(baseline_cost)
var_count = len(tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES))
moving_mean = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)[0]
self.assertEqual(moving_mean.name, 'nvil_estimator/moving_mean:0')
self.assertEqual(get_static_shape(moving_mean), (1, 1))
ensure_variables_initialized()
sess.run(tf.assign(moving_mean, [[6.]]))
cost_shape = cost.get_shape().as_list()
moving_mean = 4.8 + .2 * tf.reduce_mean(f)
assert_allclose(*sess.run([
tf.gradients([cost], [y])[0],
tf.reduce_sum(z * f + (f - moving_mean) * (3 * (x**2 - 1) *
(y**2)),
axis=0)
]))
# baseline is given, no center by moving average
cost, baseline_cost = nvil_estimator(
values=f,
latent_log_joint=log_q,
baseline=baseline,
center_by_moving_average=False,
)
self.assertEqual(
len(tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)),
var_count)
self.assertListEqual(cost.get_shape().as_list(), cost_shape)
assert_allclose(*sess.run([
tf.gradients([cost], [y])[0],
tf.reduce_sum(z * f + (f - 3.14 * tf.cos(y)) *
(3 * (x**2 - 1) * (y**2)),
axis=0)
]))
assert_allclose(*sess.run([
tf.gradients([baseline_cost], [y])[0],
# -2 * (f(x,z) - C(x)) * C'(x)
tf.reduce_sum(-2 * (f - baseline) * (-3.14 * tf.sin(y)),
axis=0)
]))
# baseline is given, no center by moving average, axis = [0]
cost, baseline_cost = nvil_estimator(
values=f,
latent_log_joint=log_q,
baseline=baseline,
center_by_moving_average=False,
axis=[0])
self.assertEqual(
len(tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)),
var_count)
self.assertListEqual(cost.get_shape().as_list(), cost_shape[1:])
assert_allclose(*sess.run([
tf.gradients([cost], [y])[0],
tf.reduce_sum(z * f + (f - 3.14 * tf.cos(y)) *
(3 * (x**2 - 1) * (y**2)),
axis=0) / 7
]))
assert_allclose(*sess.run([
tf.gradients([baseline_cost], [y])[0],
# -2 * (f(x,z) - C(x)) * C'(x)
tf.reduce_sum(-2 * (f - baseline) * (-3.14 * tf.sin(y)),
axis=0) / 7
]))
def fit(self,
values,
labels,
mean,
std,
excludes=None,
valid_portion=0.3,
summary_dir=None):
"""
Train the :class:`Donut` model with given data.
Args:
values (np.ndarray): 1-D `float32` array, the standardized
KPI observations.
labels (np.ndarray): 1-D `int32` array, the anomaly labels.
missing (np.ndarray): 1-D `int32` array, the indicator of
missing points.
mean (float): The mean of KPI observations before standardization.
std (float): The standard deviation of KPI observations before
standardization.
excludes (np.ndarray): 1-D `bool` array, indicators of whether
or not to totally exclude a point. If a point is excluded,
any window which contains that point is excluded.
(default :obj:`None`, no point is totally excluded)
valid_portion (float): Ratio of validation data out of all the
specified training data. (default 0.3)
summary_dir (str): Optional summary directory for
:class:`tf.summary.FileWriter`. (default :obj:`None`,
summary is disabled)
"""
sess = get_default_session_or_error()
# split the training & validation set
values = np.asarray(values, dtype=np.float32)
labels = np.asarray(labels, dtype=np.int32)
if len(values.shape) != 1:
raise ValueError('`values` must be a 1-D array')
if labels.shape != values.shape:
raise ValueError('The shape of `labels` does not agree with '
'the shape of `values` ({} vs {})'.format(
labels.shape, values.shape))
n = int(len(values) * valid_portion)
train_values, v_x = values[:-n], values[-n:]
train_labels, valid_labels = labels[:-n], labels[-n:]
v_y = valid_labels.astype(np.int32)
if excludes is None:
train_excludes, valid_excludes = None, None
else:
train_excludes, valid_excludes = excludes[:-n], excludes[-n:]
# data augmentation object and the sliding window iterator
aug = MissingDataInjection(mean, std,
self._missing_data_injection_rate)
train_sliding_window = BatchSlidingWindow(
array_size=len(train_values),
window_size=self.model.x_dims,
batch_size=self._batch_size,
excludes=train_excludes,
shuffle=True,
ignore_incomplete_batch=True,
)
valid_sliding_window = BatchSlidingWindow(
array_size=len(v_x),
window_size=self.model.x_dims,
batch_size=self._valid_batch_size,
excludes=valid_excludes,
)
# initialize the variables of the trainer, and the model
sess.run(self._trainer_initializer)
ensure_variables_initialized(self._train_params)
# training loop
lr = self._initial_lr
with TrainLoop(param_vars=self._train_params,
early_stopping=True,
summary_dir=summary_dir,
max_epoch=self._max_epoch,
max_step=self._max_step) as loop: # type: TrainLoop
loop.print_training_summary()
for epoch in loop.iter_epochs():
x, y1 = aug.augment(train_values, train_labels)
y = y1.astype(np.int32)
train_iterator = train_sliding_window.get_iterator([x, y])
for step, (batch_x,
batch_y) in loop.iter_steps(train_iterator):
# run a training step
feed_dict = dict(six.iteritems(self._feed_dict))
feed_dict[self._learning_rate] = lr
feed_dict[self._input_x] = batch_x
feed_dict[self._input_y] = batch_y
loss, _ = sess.run([self._loss, self._train_op],
feed_dict=feed_dict)
loop.collect_metrics({'loss': loss})
if step % self._valid_step_freq == 0:
# collect variable summaries
if summary_dir is not None:
loop.add_summary(sess.run(self._summary_op))
# do validation in batches
with loop.timeit('valid_time'), \
loop.metric_collector('valid_loss') as mc:
v_it = valid_sliding_window.get_iterator(
[v_x, v_y])
for b_v_x, b_v_y in v_it:
feed_dict = dict(
six.iteritems(self._valid_feed_dict))
feed_dict[self._input_x] = b_v_x
feed_dict[self._input_y] = b_v_y
loss = sess.run(self._loss,
feed_dict=feed_dict)
mc.collect(loss, weight=len(b_v_x))
# print the logs of recent steps
loop.print_logs()
# anneal the learning rate
if self._lr_anneal_epochs and \
epoch % self._lr_anneal_epochs == 0:
lr *= self._lr_anneal_factor
loop.println('Learning rate decreased to {}'.format(lr),
with_tag=True)
def test_training_loss(self):
class Capture(object):
def __init__(self, vae):
self._chain_func = vae.chain
self._solvers = {}
self.called_solver = None
self.chain = None
vae.chain = self
def _call_solver(self, solver_name, *args, **kwargs):
self.called_solver = solver_name
return self._solvers[solver_name](*args, **kwargs)
def __call__(self, *args, **kwargs):
chain = self._chain_func(*args, **kwargs)
self.chain = chain
for k in ['iwae', 'vimco', 'sgvb', 'reinforce']:
self._solvers[k] = getattr(chain.vi.training, k)
setattr(chain.vi.training, k,
functools.partial(self._call_solver, solver_name=k))
return chain
@property
def q_net(self):
return self.chain.variational
helper = Helper()
x, x2, x3, z, z2 = helper.get_xz_variables()
M = tf.reduce_mean
with self.test_session() as sess:
ensure_variables_initialized()
# test error for n_z
vae = helper.vae()
with pytest.raises(
TypeError, match='Cannot choose the variational solver '
'automatically for dynamic `n_z`'):
vae.get_training_loss(x, n_z=tf.constant(1))
# test sgvb
vae = helper.vae()
capture = Capture(vae)
loss = vae.get_training_loss(x)
self.assertEqual(capture.called_solver, 'sgvb')
zs_obj = helper.zs_objective(
zs.variational.elbo,
observed={'x': x},
latent={'z': capture.q_net.query(['z'])[0]}
)
np.testing.assert_allclose(*sess.run([loss, M(zs_obj.sgvb())]))
# test sgvb with explicit n_z
vae = helper.vae()
capture = Capture(vae)
loss = vae.get_training_loss(x, n_z=1)
self.assertEqual(capture.called_solver, 'sgvb')
zs_obj = helper.zs_objective(
zs.variational.elbo,
observed={'x': x},
latent={'z': capture.q_net.query(['z'])[0]},
axis=0,
)
np.testing.assert_allclose(*sess.run([loss, M(zs_obj.sgvb())]))
# test iwae
vae = helper.vae()
capture = Capture(vae)
loss = vae.get_training_loss(x, n_z=N_Z)
self.assertEqual(capture.called_solver, 'iwae')
zs_obj = helper.zs_objective(
zs.variational.importance_weighted_objective,
observed={'x': x},
latent={'z': capture.q_net.query(['z'])[0]},
axis=0,
)
np.testing.assert_allclose(*sess.run([loss, M(zs_obj.sgvb())]))
# test reinforce
vae = helper.vae(is_reparameterized=False)
capture = Capture(vae)
loss = vae.get_training_loss(x)
self.assertEqual(capture.called_solver, 'reinforce')
# TODO: The output of reinforce mismatches on some platform
#
# REINFORCE requires additional moving average variable, causing
# it very hard to ensure two calls should have identical outputs.
# So we disable such tests for the time being.
# zs_obj = helper.zs_objective(
# zs.variational.elbo,
# observed={'x': x},
# latent={'z': capture.q_net.query(['z'])[0]}
# )
# with tf.variable_scope(None, default_name='reinforce'):
# zs_reinforce = zs_obj.reinforce()
# ensure_variables_initialized()
# np.testing.assert_allclose(*sess.run([loss, zs_reinforce]))
# test reinforce with explicit n_z
vae = helper.vae(is_reparameterized=False)
capture = Capture(vae)
loss = vae.get_training_loss(x, n_z=1)
self.assertEqual(capture.called_solver, 'reinforce')
# TODO: The output of reinforce mismatches on some platform
# zs_obj = helper.zs_objective(
# zs.variational.elbo,
# observed={'x': x},
# latent={'z': capture.q_net.query(['z'])[0]},
# axis=0,
# )
# with tf.variable_scope(None, default_name='reinforce'):
# zs_reinforce = zs_obj.reinforce()
# ensure_variables_initialized()
# np.testing.assert_allclose(*sess.run([loss, zs_reinforce]))
# test vimco
vae = helper.vae(is_reparameterized=False)
capture = Capture(vae)
loss = vae.get_training_loss(x, n_z=N_Z)
self.assertEqual(capture.called_solver, 'vimco')
zs_obj = helper.zs_objective(
zs.variational.importance_weighted_objective,
observed={'x': x},
latent={'z': capture.q_net.query(['z'])[0]},
axis=0,
)
np.testing.assert_allclose(*sess.run([loss, M(zs_obj.vimco())]))
def fit(self,
values_list,
sample_ratio=1.0,
valid_portion=0.3,
summary_dir=None):
"""
Train the :class:`Omni` model with given data.
Args:
values_list (list of np.ndarray): 1-D `float32` array, the standardized
KPI observations.
valid_portion (float): Ratio of validation data out of all the
specified training data. (default 0.3)
summary_dir (str): Optional summary directory for
:class:`tf.summary.FileWriter`. (default :obj:`None`,
summary is disabled)
"""
sess = get_default_session_or_error()
np.random.seed(2020)
# split the training & validation set
train_values_list, v_x_list, train_excludes_list, v_excludes_list = [], [], [], []
for values in values_list:
values = np.asarray(values, dtype=np.float32)
if len(values.shape) != 2:
raise ValueError('`values` must be a 2-D array')
N_sample = int(sample_ratio * len(values))
N_start = int(np.random.random() * (len(values) - N_sample))
values = values[N_start:int(N_start + N_sample), :]
n = int(len(values) * valid_portion)
train_values, v_x = values[:-n], values[-n:]
train_values_list.append(train_values)
v_x_list.append(v_x)
train_excludes_list.append(
np.hstack([[1], np.zeros([len(train_values) - 1])]))
v_excludes_list.append(np.hstack([[1], np.zeros([len(v_x) - 1])]))
train_values, v_x = np.vstack(train_values_list), np.vstack(v_x_list)
train_excludes, v_excludes = np.hstack(train_excludes_list), np.hstack(
v_excludes_list)
train_sliding_window = BatchSlidingWindow(
array_size=len(train_values),
window_size=self.model.window_length,
batch_size=self._batch_size,
excludes=train_excludes,
shuffle=True,
ignore_incomplete_batch=True,
)
valid_sliding_window = BatchSlidingWindow(
array_size=len(v_x),
window_size=self.model.window_length,
batch_size=self._valid_batch_size,
excludes=v_excludes,
)
# initialize the variables of the trainer, and the model
sess.run(self._trainer_initializer)
ensure_variables_initialized(self._train_params)
# training loop
lr = self._initial_lr
with TrainLoop(param_vars=self._train_params,
early_stopping=True,
summary_dir=summary_dir,
max_epoch=self._max_epoch,
max_step=self._max_step) as loop: # type: TrainLoop
loop.print_training_summary()
train_batch_time = []
valid_batch_time = []
for epoch in loop.iter_epochs():
print('train_values:', train_values.shape)
train_iterator = train_sliding_window.get_iterator(
[train_values])
start_time = time.time()
for step, (batch_x, ) in loop.iter_steps(train_iterator):
# run a training step
start_batch_time = time.time()
feed_dict = dict(six.iteritems(self._feed_dict))
feed_dict[self._learning_rate] = lr
feed_dict[self._input_x] = batch_x
loss, _ = sess.run([self._loss, self._train_op],
feed_dict=feed_dict)
loop.collect_metrics({'loss': loss})
train_batch_time.append(time.time() - start_batch_time)
if step % self._valid_step_freq == 0:
train_duration = time.time() - start_time
loop.collect_metrics({'train_time': train_duration})
# collect variable summaries
if summary_dir is not None:
loop.add_summary(sess.run(self._summary_op))
# do validation in batches
with loop.timeit('valid_time'), \
loop.metric_collector('valid_loss') as mc:
v_it = valid_sliding_window.get_iterator([v_x])
for (b_v_x, ) in v_it:
start_batch_time = time.time()
feed_dict = dict(
six.iteritems(self._valid_feed_dict))
feed_dict[self._input_x] = b_v_x
loss = sess.run(self._loss,
feed_dict=feed_dict)
valid_batch_time.append(time.time() -
start_batch_time)
mc.collect(loss, weight=len(b_v_x))
# print the logs of recent steps
loop.print_logs()
start_time = time.time()
# anneal the learning rate
if self._lr_anneal_epochs and \
epoch % self._lr_anneal_epochs == 0:
lr *= self._lr_anneal_factor
loop.println('Learning rate decreased to {}'.format(lr),
with_tag=True)
try:
best_valid_loss = float(loop.best_valid_metric)
except:
best_valid_loss = np.nan
return {
'best_valid_loss': best_valid_loss,
'train_batch_time': np.mean(train_batch_time),
'valid_batch_time': np.mean(valid_batch_time),
'train_all_time': np.sum(train_batch_time),
'valid_all_time': np.sum(valid_batch_time),
}
def test_elbo(self):
# test no sampling axis
with self.test_session() as sess:
prepared = self.prepare_model(zs.variational.elbo,
axis=None,
n_z=None)
vi = prepared.vi
zs_obj = prepared.zs_obj
# test :meth:`VariationalInference.zs_objective`
vi_obj = prepared.vi.zs_objective(zs.variational.elbo)
self.assertIsInstance(vi_obj,
zs.variational.EvidenceLowerBoundObjective)
np.testing.assert_allclose(*sess.run([zs_obj, vi_obj]))
# test :meth:`VariationalInference.zs_elbo`
vi_obj = prepared.vi.zs_elbo()
self.assertIsInstance(vi_obj,
zs.variational.EvidenceLowerBoundObjective)
np.testing.assert_allclose(*sess.run([zs_obj, vi_obj]))
# test :meth:`VariationalLowerBounds.elbo`
np.testing.assert_allclose(
*sess.run([zs_obj, vi.lower_bound.elbo()]))
# test :meth:`VariationalTrainingObjectives.sgvb`
np.testing.assert_allclose(
*sess.run([zs_obj.sgvb(), vi.training.sgvb()]))
# test :meth:`VariationalTrainingObjectives.reinforce`
# TODO: The output of reinforce mismatches on some platform
# if variance_reduction == True
# REINFORCE requires additional moving average variable, causing
# it very hard to ensure two calls should have identical outputs.
# So we disable such tests for the time being.
with tf.variable_scope(None, default_name='reinforce'):
# reinforce requires extra variables, but ZhuSuan does not
# obtain a dedicated variable scope. so we open one here.
zs_reinforce = zs_obj.reinforce(variance_reduction=False)
vi_reinforce = vi.training.reinforce(variance_reduction=False)
ensure_variables_initialized()
np.testing.assert_allclose(*sess.run([zs_reinforce, vi_reinforce]))
# test with sampling axis
with self.test_session() as sess:
prepared = self.prepare_model(zs.variational.elbo, axis=0, n_z=7)
vi = prepared.vi
zs_obj = prepared.zs_obj
# test :meth:`VariationalInference.zs_objective`
vi_obj = prepared.vi.zs_objective(zs.variational.elbo)
self.assertIsInstance(vi_obj,
zs.variational.EvidenceLowerBoundObjective)
np.testing.assert_allclose(*sess.run([zs_obj, vi_obj]))
# test :meth:`VariationalInference.zs_elbo`
vi_obj = prepared.vi.zs_elbo()
self.assertIsInstance(vi_obj,
zs.variational.EvidenceLowerBoundObjective)
np.testing.assert_allclose(*sess.run([zs_obj, vi_obj]))
# test :meth:`VariationalLowerBounds.elbo`
np.testing.assert_allclose(
*sess.run([zs_obj, vi.lower_bound.elbo()]))
# test :meth:`VariationalTrainingObjectives.sgvb`
np.testing.assert_allclose(
*sess.run([zs_obj.sgvb(), vi.training.sgvb()]))
# test :meth:`VariationalTrainingObjectives.reinforce`
# TODO: The output of reinforce mismatches on some platform
# if variance_reduction == True
# REINFORCE requires additional moving average variable, causing
# it very hard to ensure two calls should have identical outputs.
# So we disable such tests for the time being.
with tf.variable_scope(None, default_name='reinforce'):
# reinforce requires extra variables, but ZhuSuan does not
# obtain a dedicated variable scope. so we open one here.
zs_reinforce = zs_obj.reinforce(variance_reduction=False)
vi_reinforce = vi.training.reinforce(variance_reduction=False)
ensure_variables_initialized()
np.testing.assert_allclose(*sess.run([zs_reinforce, vi_reinforce]))
