def test_training_summary(self):
a = tf.get_variable('a', dtype=tf.float32, shape=(2, 3))
b = tf.get_variable('b', dtype=tf.float32, shape=(4, ))
c = tf.get_variable('c', dtype=tf.float32, shape=())
# test param variables in list
logs = []
with train_loop([a, b], print_function=logs.append) as loop:
self.assertEqual(loop.param_vars, [a, b])
loop.print_training_summary()
self.assertEqual('\n'.join(logs),
('Trainable Parameters (10 in total)\n'
'----------------------------------\n'
'a (2, 3) 6\n'
'b (4,) 4\n'))
# test param variables in dict
logs = []
with train_loop({
'aa': a,
'bb': b
}, print_function=logs.append) as loop:
self.assertEqual(loop.param_vars, {'aa': a, 'bb': b})
loop.print_training_summary()
self.assertEqual('\n'.join(logs),
('Trainable Parameters (10 in total)\n'
'----------------------------------\n'
'aa (2, 3) 6\n'
'bb (4,) 4\n'))
def test_logs(self):
logs = []
with train_loop([], max_step=6, print_function=logs.append) as loop:
for epoch in loop.iter_epochs():
for step, x in loop.iter_steps(np.arange(4)):
time.sleep(0.01)
loop.add_metrics(x=x)
if step % 2 == 0:
loop.print_logs()
loop.add_metrics(y=epoch)
loop.print_logs()
self.assertMatches(
'\n'.join(logs),
re.compile(
r'^'
r'\[Epoch 1, Step 2/6\] step time: 0\.01\d* sec \(±[^ ]+ sec\); '
r'x: 0\.5 \(±0\.5\)\n'
r'\[Epoch 1, Step 4/6\] step time: 0\.01\d* sec \(±[^ ]+ sec\); '
r'x: 2\.5 \(±0\.5\)\n'
r'\[Epoch 1\] epoch time: 0\.0[456]\d* sec; '
r'step time: 0\.01\d* sec \(±[^ ]+ sec\); x: 1\.5 \(±1\.11803\); '
r'y: 1\n'
r'\[Epoch 2, Step 6/6\] step time: 0\.01\d* sec \(±[^ ]+ sec\); '
r'x: 0\.5 \(±0\.5\)\n'
r'\[Epoch 2\] epoch time: 0\.0[23]\d* sec; '
r'step time: 0\.01\d* sec \(±[^ ]+ sec\); x: 0\.5 \(±0\.5\); y: 2'
r'$'))
def test_counter_attributes(self):
with train_loop([]) as loop:
self.assertEqual(loop.epoch, 0)
self.assertEqual(loop.step, 0)
self.assertIsNone(loop.max_epoch)
self.assertIsNone(loop.max_step)
with train_loop([],
initial_epoch=1,
initial_step=3,
max_epoch=2,
max_step=10) as loop:
self.assertEqual(loop.epoch, 1)
self.assertEqual(loop.step, 3)
self.assertEqual(loop.max_epoch, 2)
self.assertEqual(loop.max_step, 10)
loop.max_epoch = 20
loop.max_step = 100
self.assertEqual(loop.max_epoch, 20)
self.assertEqual(loop.max_step, 100)
def test_accumulator(self):
logs = []
with train_loop([], max_epoch=1, print_function=logs.append) as loop:
for _ in loop.iter_epochs():
with loop.accumulator('x') as acc:
acc.add(2)
acc.add(3, weight=3)
loop.print_logs()
self.assertMatches(
'\n'.join(logs),
re.compile(r'^'
r'\[Epoch 1/1\] epoch time: [^ ]+ sec; x: 2\.75'
r'$'))
def test_early_stopping(self):
with self.get_session():
a = tf.get_variable('a', shape=(), dtype=tf.int32)
b = tf.get_variable('b', shape=(), dtype=tf.int32)
# test early-stopping with no valid metric committed
set_variable_values([a, b], [1, 2])
self.assertEqual(get_variable_values([a, b]), [1, 2])
with train_loop([a], early_stopping=True):
set_variable_values([a, b], [10, 20])
self.assertEqual(get_variable_values([a, b]), [10, 20])
# test early-stopping with smaller-better metric
set_variable_values([a, b], [1, 2])
self.assertEqual(get_variable_values([a, b]), [1, 2])
with train_loop([a], max_epoch=1, early_stopping=True) as loop:
for _ in loop.iter_epochs():
for step, valid_loss in loop.iter_steps([0.7, 0.6, 0.8]):
set_variable_values([a, b], [10 + step, 20 + step])
loop.add_metrics(valid_loss=valid_loss)
self.assertAlmostEqual(loop.best_valid_metric, 0.6)
self.assertEqual(get_variable_values([a, b]), [12, 23])
# test early-stopping with larger-better metric
set_variable_values([a, b], [1, 2])
self.assertEqual(get_variable_values([a, b]), [1, 2])
with train_loop([a],
max_epoch=1,
valid_metric=('y', False),
early_stopping=True) as loop:
for _ in loop.iter_epochs():
for step, y in loop.iter_steps([0.7, 0.6, 0.8]):
set_variable_values([a, b], [10 + step, 20 + step])
loop.add_metrics(y=y)
self.assertAlmostEqual(loop.best_valid_metric, 0.8)
self.assertEqual(get_variable_values([a, b]), [13, 23])
def test_timeit(self):
logs = []
with train_loop([], max_epoch=1, print_function=logs.append) as loop:
for _ in loop.iter_epochs():
with loop.timeit('x_timer'):
time.sleep(0.01)
with loop.timeit('y_time'):
time.sleep(0.02)
loop.print_logs()
self.assertMatches(
'\n'.join(logs),
re.compile(r'^'
r'\[Epoch 1/1\] epoch time: 0\.0[345]\d* sec; '
r'x timer: 0\.01\d* sec; y time: 0\.0[23]\d* sec'
r'$'))
def test_tensor_arguments(self):
with self.get_session():
a = tf.get_variable('a', initializer=0, dtype=tf.int32)
ensure_variables_initialized()
with train_loop([a],
early_stopping=True,
initial_valid_metric=tf.constant(1.23),
initial_epoch=tf.constant(4),
initial_step=tf.constant(5),
max_epoch=tf.constant(6),
max_step=tf.constant(7)) as loop:
self.assertAlmostEqual(loop._early_stopping._best_metric, 1.23)
self.assertEqual(loop.epoch, 4)
self.assertEqual(loop.step, 5)
self.assertEqual(loop.max_epoch, 6)
self.assertEqual(loop.max_step, 7)
def test_errors(self):
with self.assertRaisesRegex(RuntimeError,
'Another epoch loop has been opened.'):
with train_loop([], max_epoch=10) as loop:
for _ in loop.iter_epochs():
for _ in loop.iter_epochs():
pass
with self.assertRaisesRegex(
RuntimeError, 'Step loop must be opened within active epoch '
'loop.'):
with train_loop([], max_step=10) as loop:
for _ in loop.iter_steps():
pass
with self.assertRaisesRegex(RuntimeError,
'Another step loop has been opened.'):
with train_loop([], max_epoch=10, max_step=10) as loop:
for _ in loop.iter_epochs():
for _ in loop.iter_steps():
for _ in loop.iter_steps():
pass
with self.assertRaisesRegex(
RuntimeError, 'Neither an epoch nor a step loop has been '
'opened.'):
with train_loop([]) as loop:
with loop.timeit('timer'):
pass
with self.assertRaisesRegex(
RuntimeError, 'Neither an epoch nor a step loop has been '
'opened.'):
with train_loop([]) as loop:
with loop.accumulator('metric'):
pass
with self.assertRaisesRegex(
RuntimeError, 'Neither an epoch nor a step loop has been '
'opened.'):
with train_loop([]) as loop:
loop.add_metrics(loss=1.)
with self.assertRaisesRegex(
RuntimeError, 'Neither an epoch nor a step loop has been '
'opened.'):
with train_loop([]) as loop:
loop.print_logs()
with self.assertRaisesRegex(
RuntimeError, '`data_generator` is required when `max_step` '
'is not configured, so as to prevent an '
'unstoppable step loop.'):
with train_loop([], max_epoch=10) as loop:
for _ in loop.iter_epochs():
for _ in loop.iter_steps():
pass
with self.assertRaisesRegex(TypeError, '`metrics` should be a dict.'):
with train_loop([], max_epoch=10) as loop:
for _ in loop.iter_epochs():
loop.add_metrics(())
with self.assertRaisesRegex(
TypeError, '`summary_writer` is expected to be '
'`SummaryWriter`, but got .*'):
with train_loop([], max_epoch=1, summary_writer=()):
pass
def test_counters(self):
# test loop with configured `max_epoch`
with train_loop([], max_epoch=2) as loop:
epoch_counter = 0
step_counter = 0
for epoch in loop.iter_epochs():
epoch_counter += 1
self.assertEqual(epoch, epoch_counter)
x_ans = 0
for step, x in loop.iter_steps(np.arange(4)):
self.assertEqual(step, loop.step)
self.assertEqual(epoch, loop.epoch)
self.assertEqual(x, x_ans)
x_ans += 1
step_counter += 1
self.assertEqual(step, step_counter)
self.assertEqual(step_counter, loop.step)
self.assertEqual(epoch, loop.epoch)
self.assertEqual(epoch_counter, 2)
self.assertEqual(step_counter, 8)
# test loop with configured `max_step`
with train_loop([], max_step=10) as loop:
epoch_counter = 0
step_counter = 0
for epoch in loop.iter_epochs():
epoch_counter += 1
self.assertEqual(epoch, epoch_counter)
for step in loop.iter_steps():
step_counter += 1
self.assertEqual(step, step_counter)
self.assertEqual(epoch_counter, 1)
self.assertEqual(step_counter, 10)
# test loop with configured `max_step` with payload
with train_loop([], max_step=10) as loop:
epoch_counter = 0
step_counter = 0
for epoch in loop.iter_epochs():
epoch_counter += 1
self.assertEqual(epoch, epoch_counter)
x_ans = 0
for step, x in loop.iter_steps(np.arange(4)):
self.assertEqual(x, x_ans)
x_ans += 1
step_counter += 1
self.assertEqual(step, step_counter)
self.assertEqual(epoch_counter, 3)
self.assertEqual(step_counter, 10)
# test loop with configured `max_step` and `max_epoch`,
# while `max_epoch` finishes first
with train_loop([], max_step=10, max_epoch=2) as loop:
epoch_counter = 0
step_counter = 0
for epoch in loop.iter_epochs():
epoch_counter += 1
self.assertEqual(epoch, epoch_counter)
for step, _ in loop.iter_steps(np.arange(4)):
step_counter += 1
self.assertEqual(step, step_counter)
self.assertEqual(epoch_counter, 2)
self.assertEqual(step_counter, 8)
# test loop with configured `max_step` and `max_epoch`,
# while `max_step` finishes first
with train_loop([], max_step=10, max_epoch=3) as loop:
epoch_counter = 0
step_counter = 0
for epoch in loop.iter_epochs():
epoch_counter += 1
self.assertEqual(epoch, epoch_counter)
for step, _ in loop.iter_steps(np.arange(4)):
step_counter += 1
self.assertEqual(step, step_counter)
self.assertEqual(epoch_counter, 3)
self.assertEqual(step_counter, 10)
def test_summary_writer(self):
def read_summary(summary_dir):
# read the metric summary
loss_steps = []
loss_values = []
valid_loss_steps = []
valid_loss_values = []
x_steps = []
x_values = []
tags = set()
event_file_path = os.path.join(summary_dir,
os.listdir(summary_dir)[0])
for e in tf.train.summary_iterator(event_file_path):
for v in e.summary.value:
tags.add(v.tag)
if v.tag == 'loss':
loss_steps.append(e.step)
loss_values.append(v.simple_value)
elif v.tag == 'valid_loss':
valid_loss_steps.append(e.step)
valid_loss_values.append(v.simple_value)
elif v.tag == 'x':
x_steps.append(e.step)
x_values.append(v.simple_value)
return (tags, loss_steps, loss_values, valid_loss_steps,
valid_loss_values, x_steps, x_values)
# test enable summary with `summary_dir`
with TemporaryDirectory() as tempdir:
with train_loop([], max_epoch=2, summary_dir=tempdir) as loop:
self.assertIsNone(loop._early_stopping)
for epoch in loop.iter_epochs():
for _, loss in loop.iter_steps([0.7, 0.6, 0.8]):
loop.add_metrics(loss=epoch + loss)
loop.add_metrics(valid_loss=epoch)
with self.get_session():
summary_op = tf.summary.scalar('x', tf.constant(1.23))
loop.add_summary(summary_op.eval())
obj = read_summary(tempdir)
self.assertEqual(
sorted(obj[0]),
['epoch_time', 'loss', 'step_time', 'valid_loss', 'x'])
np.testing.assert_equal(obj[1], [1, 2, 3, 4, 5, 6])
np.testing.assert_almost_equal(obj[2],
[1.7, 1.6, 1.8, 2.7, 2.6, 2.8])
np.testing.assert_equal(obj[3], [3, 6])
np.testing.assert_almost_equal(obj[4], [1, 2])
np.testing.assert_equal(obj[5], [6])
np.testing.assert_almost_equal(obj[6], [1.23])
# test enable summary with `summary_writer`
with TemporaryDirectory() as tempdir:
sw = tf.summary.FileWriter(tempdir)
with train_loop([], max_epoch=2, summary_writer=sw) as loop:
self.assertIsNone(loop._early_stopping)
self.assertIs(loop._summary_writer._writer, sw)
for epoch in loop.iter_epochs():
for _, loss in loop.iter_steps([0.7, 0.6, 0.8]):
loop.add_metrics(loss=epoch + loss)
loop.add_metrics(valid_loss=epoch)
sw.close()
self.assertEqual(sorted(read_summary(tempdir)[0]),
['epoch_time', 'loss', 'step_time', 'valid_loss'])
with TemporaryDirectory() as tempdir:
sw = SummaryWriter(tf.summary.FileWriter(tempdir))
with train_loop([], max_epoch=2, summary_writer=sw) as loop:
self.assertIsNone(loop._early_stopping)
self.assertIs(loop._summary_writer, sw)
for epoch in loop.iter_epochs():
for _, loss in loop.iter_steps([0.7, 0.6, 0.8]):
loop.add_metrics(loss=epoch + loss)
loop.add_metrics(valid_loss=epoch)
sw.close()
self.assertEqual(sorted(read_summary(tempdir)[0]),
['epoch_time', 'loss', 'step_time', 'valid_loss'])
def test_valid_metric(self):
# test default "valid_loss"
logs = []
with train_loop([], print_function=logs.append) as loop:
self.assertEqual(loop._valid_metric, 'valid_loss')
self.assertTrue(loop._valid_metric_smaller_is_better)
for _ in loop.iter_epochs():
best_metric = 1.
for _, valid_loss in loop.iter_steps([0.8, 0.6, 0.7]):
loop.add_metrics(valid_loss=valid_loss)
best_metric = min(best_metric, valid_loss)
self.assertAlmostEqual(loop.best_valid_metric, best_metric)
loop.print_logs()
loop.print_logs()
break
self.assertAlmostEqual(loop.best_valid_metric, 0.6)
self.assertMatches(
'\n'.join(logs),
re.compile(
r'^'
r'\[Epoch 1, Step 1\] step time: [^ ]+ sec; '
r'valid loss: 0\.8 \(\*\)\n'
r'\[Epoch 1, Step 2\] step time: [^ ]+ sec; '
r'valid loss: 0\.6 \(\*\)\n'
r'\[Epoch 1, Step 3\] step time: [^ ]+ sec; '
r'valid loss: 0\.7\n'
r'\[Epoch 1\] epoch time: [^ ]+ sec; step time: [^ ]+ sec '
r'\(±[^ ]+ sec\); valid loss: 0\.7 \(±0\.0816497\)'
r'$'))
# test customized valid setting
logs = []
with train_loop([],
print_function=logs.append,
valid_metric=('y', False)) as loop:
self.assertEqual(loop._valid_metric, 'y')
self.assertFalse(loop._valid_metric_smaller_is_better)
for _ in loop.iter_epochs():
best_metric = 0.
for _, y in loop.iter_steps([0.7, 0.6, 0.8]):
loop.add_metrics(y=y)
best_metric = max(best_metric, y)
self.assertAlmostEqual(loop.best_valid_metric, best_metric)
loop.print_logs()
loop.print_logs()
break
self.assertAlmostEqual(loop.best_valid_metric, 0.8)
self.assertMatches(
'\n'.join(logs),
re.compile(
r'^'
r'\[Epoch 1, Step 1\] step time: [^ ]+ sec; '
r'y: 0\.7 \(\*\)\n'
r'\[Epoch 1, Step 2\] step time: [^ ]+ sec; '
r'y: 0\.6\n'
r'\[Epoch 1, Step 3\] step time: [^ ]+ sec; '
r'y: 0\.8 \(\*\)\n'
r'\[Epoch 1\] epoch time: [^ ]+ sec; step time: [^ ]+ sec '
r'\(±[^ ]+ sec\); y: 0\.7 \(±0\.0816497\)'
r'$'))
# test the setting of "valid_acc"
with train_loop([],
print_function=logs.append,
valid_metric='valid_acc') as loop:
self.assertEqual(loop._valid_metric, 'valid_acc')
self.assertFalse(loop._valid_metric_smaller_is_better)
# test the setting of "y"
with train_loop([], print_function=logs.append,
valid_metric='y') as loop:
self.assertEqual(loop._valid_metric, 'y')
self.assertTrue(loop._valid_metric_smaller_is_better)
def fit(self,
values,
labels,
missing,
mean,
std,
excludes=None,
valid_portion=0.3,
summary_dir=None):
"""
Train the :class:`Donut` model with given data.
Args:
values (np.ndarray): 1-D `float32` array, the standardized
KPI observations.
labels (np.ndarray): 1-D `int32` array, the anomaly labels.
missing (np.ndarray): 1-D `int32` array, the indicator of
missing points.
mean (float): The mean of KPI observations before standardization.
std (float): The standard deviation of KPI observations before
standardization.
excludes (np.ndarray): 1-D `bool` array, indicators of whether
or not to totally exclude a point. If a point is excluded,
any window which contains that point is excluded.
(default :obj:`None`, no point is totally excluded)
valid_portion (float): Ratio of validation data out of all the
specified training data. (default 0.3)
summary_dir (str): Optional summary directory for
:class:`tf.summary.FileWriter`. (default :obj:`None`,
summary is disabled)
"""
sess = get_default_session_or_error()
# split the training & validation set
values = np.asarray(values, dtype=np.float32)
labels = np.asarray(labels, dtype=np.int32)
missing = np.asarray(missing, dtype=np.int32)
if len(values.shape) != 1:
raise ValueError('`values` must be a 1-D array')
if labels.shape != values.shape:
raise ValueError('The shape of `labels` does not agree with '
'the shape of `values` ({} vs {})'.format(
labels.shape, values.shape))
if missing.shape != values.shape:
raise ValueError('The shape of `missing` does not agree with '
'the shape of `values` ({} vs {})'.format(
missing.shape, values.shape))
n = int(len(values) * valid_portion)
train_values, v_x = values[:-n], values[-n:]
train_labels, valid_labels = labels[:-n], labels[-n:]
train_missing, valid_missing = missing[:-n], missing[-n:]
v_y = np.logical_or(valid_labels, valid_missing).astype(np.int32)
if excludes is None:
train_excludes, valid_excludes = None, None
else:
train_excludes, valid_excludes = excludes[:-n], excludes[-n:]
# data augmentation object and the sliding window iterator
aug = MissingDataInjection(mean, std,
self._missing_data_injection_rate)
train_sliding_window = BatchSlidingWindow(
array_size=len(train_values),
window_size=self.model.x_dims,
batch_size=self._batch_size,
excludes=train_excludes,
shuffle=True,
ignore_incomplete_batch=True,
)
valid_sliding_window = BatchSlidingWindow(
array_size=len(v_x),
window_size=self.model.x_dims,
batch_size=self._valid_batch_size,
excludes=valid_excludes,
)
# initialize the variables of the trainer, and the model
sess.run(self._trainer_initializer)
ensure_variables_initialized(self._train_params)
# training loop
lr = self._initial_lr
with train_loop(
param_vars=self._train_params,
early_stopping=True,
summary_dir=summary_dir,
max_epoch=self._max_epoch,
max_step=self._max_step) as loop: # type: TrainLoopContext
loop.print_training_summary()
for epoch in loop.iter_epochs():
x, y1, y2 = aug.augment(train_values, train_labels,
train_missing)
y = np.logical_or(y1, y2).astype(np.int32)
train_iterator = train_sliding_window.get_iterator([x, y])
for step, (batch_x,
batch_y) in loop.iter_steps(train_iterator):
# run a training step
feed_dict = dict(six.iteritems(self._feed_dict))
feed_dict[self._learning_rate] = lr
feed_dict[self._input_x] = batch_x
feed_dict[self._input_y] = batch_y
loss, _ = sess.run([self._loss, self._train_op],
feed_dict=feed_dict)
loop.collect_metrics({'loss': loss})
if step % self._valid_step_freq == 0:
# collect variable summaries
if summary_dir is not None:
loop.add_summary(sess.run(self._summary_op))
# do validation in batches
with loop.timeit('valid_time'), \
loop.metric_collector('valid_loss') as mc:
v_it = valid_sliding_window.get_iterator(
[v_x, v_y])
for b_v_x, b_v_y in v_it:
feed_dict = dict(
six.iteritems(self._valid_feed_dict))
feed_dict[self._input_x] = b_v_x
feed_dict[self._input_y] = b_v_y
loss = sess.run(self._loss,
feed_dict=feed_dict)
mc.collect(loss, weight=len(b_v_x))
# print the logs of recent steps
loop.print_logs()
# anneal the learning rate
if self._lr_anneal_epochs and \
epoch % self._lr_anneal_epochs == 0:
lr *= self._lr_anneal_factor
loop.println('Learning rate decreased to {}'.format(lr),
with_tag=True)