def test_error_construction(self):
with pytest.raises(ValueError,
match='`x_dims` must be a positive integer'):
_ = Donut(lambda x: x, lambda x: x, x_dims=-1, z_dims=1)
with pytest.raises(ValueError,
match='`x_dims` must be a positive integer'):
_ = Donut(lambda x: x, lambda x: x, x_dims=object(), z_dims=1)
with pytest.raises(ValueError,
match='`z_dims` must be a positive integer'):
_ = Donut(lambda x: x, lambda x: x, x_dims=1, z_dims=0)
with pytest.raises(ValueError,
match='`z_dims` must be a positive integer'):
_ = Donut(lambda x: x, lambda x: x, x_dims=1, z_dims=object())
def get_donut():
return Donut(
h_for_p_x=lambda x: x,
h_for_q_z=lambda x: x,
x_dims=5,
z_dims=3,
)
def test_fit_args(self):
values, labels, missing, excludes = self._payload()
tf.set_random_seed(1234)
donut = Donut(
h_for_p_x=lambda x: x, h_for_q_z=lambda x: x, x_dims=5,
z_dims=3
)
trainer = DonutTrainer(donut, max_epoch=1)
with self.test_session():
# test no exclude
trainer.fit(values=values, labels=labels, missing=missing,
mean=1., std=2.)
# test shape error
with pytest.raises(
ValueError, match='`values` must be a 1-D array'):
trainer.fit(values=np.array([[1.]]), labels=labels,
missing=missing, mean=1., std=2.)
with pytest.raises(
ValueError, match='The shape of `labels` does not agree '
'with the shape of `values`'):
trainer.fit(values=values, labels=labels[:-1],
missing=missing, mean=1., std=2.)
with pytest.raises(
ValueError, match='The shape of `missing` does not agree '
'with the shape of `values`'):
trainer.fit(values=values, labels=labels,
missing=missing[:-1], mean=1., std=2.)
def test_props(self):
donut = Donut(
h_for_p_x=Mock(wraps=lambda x: x),
h_for_q_z=Mock(wraps=lambda x: x),
x_dims=5,
z_dims=3,
std_epsilon=0.125,
)
self.assertEqual(donut.x_dims, 5)
self.assertEqual(donut.z_dims, 3)
self.assertIsInstance(donut.vae, VAE)
def test_construction_args(self):
values, labels, missing, excludes = self._payload()
tf.set_random_seed(1234)
donut = Donut(h_for_p_x=lambda x: x,
h_for_q_z=lambda x: x,
x_dims=5,
z_dims=3)
# test feed_dict
is_training = tf.placeholder(tf.bool, ())
trainer = DonutTrainer(donut,
max_epoch=1,
feed_dict={is_training: True})
with self.test_session():
trainer.fit(values=values,
labels=labels,
missing=missing,
mean=1.,
std=2.,
excludes=excludes)
# test valid_feed_dict
trainer = DonutTrainer(donut,
max_epoch=1,
valid_feed_dict={is_training: True})
with self.test_session():
trainer.fit(values=values,
labels=labels,
missing=missing,
mean=1.,
std=2.,
excludes=excludes)
# test max_epoch is None and max_step is None
with pytest.raises(ValueError,
match='At least one of `max_epoch` and `max_step` '
'should be specified'):
_ = DonutTrainer(donut, max_epoch=None, max_step=None)
# test optimizer and optimizer_params
trainer = DonutTrainer(donut,
max_epoch=1,
optimizer=tf.train.MomentumOptimizer,
optimizer_params={'momentum': 0.01})
with self.test_session():
trainer.fit(values=values,
labels=labels,
missing=missing,
mean=1.,
std=2.,
excludes=excludes)
def test_fit(self):
values, labels, missing, excludes = self._payload()
with TemporaryDirectory() as tmpdir:
tf.set_random_seed(1234)
donut = Donut(
h_for_p_x=lambda x: x, h_for_q_z=lambda x: x, x_dims=5,
z_dims=3
)
trainer = DonutTrainer(
donut, max_epoch=3, batch_size=7, valid_step_freq=50,
lr_anneal_epochs=2
)
with self.test_session():
trainer.fit(
values=values, labels=labels, missing=missing, mean=1.,
std=2., excludes=excludes, summary_dir=tmpdir
)
def check_status(self, order_id):
order_donuts = []
for donut_data in loads(
get('http://127.0.0.1:8082/kitchen/get_donuts').content):
donut = Donut(donut_data['flavor'], donut_data['order_id'],
donut_data['status'])
if donut.order_id == order_id:
order_donuts.append(donut)
estimated_time = 0
status = READY
for order_donut in order_donuts:
status = order_donut.status
if status == NEW_ORDER:
estimated_time = estimated_time + 3
elif status == RECEIVED:
estimated_time = estimated_time + 2
elif status == COOKING:
estimated_time = estimated_time + 1
return dumps({
'order_id': order_id,
'estimated_delivery_time': estimated_time,
'state': status
})
def test_training_loss(self):
class Capture(object):
def __init__(self, vae):
self._vae = vae
self._vae_variational = vae.variational
vae.variational = self._variational
self.q_net = None
def _variational(self, x, z=None, n_z=None):
assert (z is None)
if n_z is None:
z = tf.reshape(tf.range(12, dtype=tf.float32), [4, 3])
else:
z = tf.reshape(tf.range(n_z * 12, dtype=tf.float32),
[n_z, 4, 3])
self.q_net = self._vae_variational(x, z=z, n_z=n_z)
return self.q_net
# payloads
x = tf.reshape(tf.range(20, dtype=tf.float32), [4, 5])
y = tf.constant([[1, 0, 0, 0, 0], [0, 1, 0, 0, 0], [0, 0, 1, 0, 0],
[0, 0, 0, 1, 0]],
dtype=tf.int32)
alpha = tf.cast(1 - y, dtype=tf.float32)
beta = tf.reduce_mean(alpha, axis=-1)
# create and patch donut model
tf.set_random_seed(1234)
donut = Donut(
h_for_p_x=Mock(wraps=lambda x: x),
h_for_q_z=Mock(wraps=lambda x: x),
x_dims=5,
z_dims=3,
std_epsilon=0.125,
)
capture = Capture(donut.vae)
_ = donut.get_training_loss(x, y) # ensure model is built
# training loss with n_z is None
with self.test_session() as sess:
ensure_variables_initialized()
loss = donut.get_training_loss(x, y)
np.testing.assert_equal(capture.q_net['z'].eval(),
np.arange(12).reshape([4, 3]))
p_net = donut.vae.model(z=capture.q_net['z'], x=x)
sgvb = (tf.reduce_sum(p_net['x'].log_prob(group_ndims=0) * alpha,
axis=-1) + p_net['z'].log_prob() * beta -
capture.q_net['z'].log_prob())
self.assertEqual(sgvb.get_shape(), tf.TensorShape([4]))
loss2 = -tf.reduce_mean(sgvb)
np.testing.assert_allclose(*sess.run([loss, loss2]))
# training loss with n_z > 1
with self.test_session() as sess:
ensure_variables_initialized()
loss = donut.get_training_loss(x, y, n_z=7)
np.testing.assert_equal(capture.q_net['z'].eval(),
np.arange(84).reshape([7, 4, 3]))
p_net = donut.vae.model(z=capture.q_net['z'], x=x, n_z=7)
sgvb = (tf.reduce_sum(p_net['x'].log_prob(group_ndims=0) * alpha,
axis=-1) + p_net['z'].log_prob() * beta -
capture.q_net['z'].log_prob())
self.assertEqual(sgvb.get_shape(), tf.TensorShape([7, 4]))
loss2 = -tf.reduce_mean(sgvb)
np.testing.assert_allclose(*sess.run([loss, loss2]))
def test_step2(input2,steps2):
donut = Donut(input2)
assert donut.shortest_path(True) == steps2
def test_step2(input2, steps2):
donut = Donut(join(dirname(realpath(__file__)), input2))
assert donut.shortest_path(True) == steps2
def test_step1(input1,steps1):
donut = Donut(input1)
assert donut.shortest_path() == steps1
def test_get_score(self):
class Capture(object):
def __init__(self, vae):
self._vae = vae
self._vae_variational = vae.variational
vae.variational = self._variational
self.q_net = None
def _variational(self, x, z=None, n_z=None):
self.q_net = self._vae_variational(x, z=z, n_z=n_z)
return self.q_net
tf.set_random_seed(1234)
donut = Donut(
h_for_p_x=lambda x: x,
h_for_q_z=lambda x: x,
x_dims=5,
z_dims=3,
)
capture = Capture(donut.vae)
donut.vae.reconstruct = Mock(
wraps=lambda x: x + tf.reduce_sum(x) # only called by MCMC
)
x = tf.reshape(tf.range(20, dtype=tf.float32), [4, 5])
y = tf.constant([[1, 0, 0, 0, 0], [0, 1, 0, 0, 0], [0, 0, 1, 0, 0],
[0, 0, 0, 1, 0]],
dtype=tf.int32)
_ = donut.get_score(x, y) # ensure variables created
def r_prob(x, z, n_z=None, x_in=None):
if x_in is None:
x_in = x
q_net = donut.vae.variational(x_in, z=z, n_z=n_z)
p_net = donut.vae.model(z=q_net['z'], x=x, n_z=n_z)
p = p_net['x'].log_prob(group_ndims=0)
if n_z is not None:
p = tf.reduce_mean(p, axis=0)
return p
with self.test_session() as sess:
ensure_variables_initialized()
# test y is None
donut.vae.reconstruct.reset_mock()
np.testing.assert_allclose(*sess.run([
donut.get_score(
x, y=None, mcmc_iteration=1, last_point_only=False),
r_prob(x, z=capture.q_net['z'])
]))
self.assertEqual(donut.vae.reconstruct.call_count, 0)
# test mcmc_iteration is None
donut.vae.reconstruct.reset_mock()
np.testing.assert_allclose(*sess.run([
donut.get_score(
x, y=y, mcmc_iteration=None, last_point_only=False),
r_prob(x, z=capture.q_net['z'])
]))
self.assertEqual(donut.vae.reconstruct.call_count, 0)
# test mcmc once
x2 = tf.where(
tf.cast(y, dtype=tf.bool),
x,
x + tf.reduce_sum(x),
)
donut.vae.reconstruct.reset_mock()
np.testing.assert_allclose(*sess.run([
donut.get_score(
x, y=y, mcmc_iteration=1, last_point_only=False),
r_prob(x, z=capture.q_net['z'], x_in=x2)
]))
self.assertEqual(donut.vae.reconstruct.call_count, 1)
# test mcmc with n_z > 1
donut.vae.reconstruct.reset_mock()
np.testing.assert_allclose(*sess.run([
donut.get_score(
x, y=y, n_z=7, mcmc_iteration=1, last_point_only=False),
r_prob(x, z=capture.q_net['z'], x_in=x2, n_z=7)
]))
self.assertEqual(capture.q_net['z'].get_shape(),
tf.TensorShape([7, 4, 3]))
self.assertEqual(donut.vae.reconstruct.call_count, 1)
def test_step1(input1, steps1):
donut = Donut(join(dirname(realpath(__file__)), input1))
assert donut.shortest_path() == steps1
)
# We build the entire model within the scope of `model_vs`,
# it should hold exactly all the variables of `model`, including
# the variables created by Keras layers.
with tf.variable_scope('model') as model_vs:
model = Donut(
h_for_p_x=Sequential([
K.layers.Dense(100,
kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
K.layers.Dense(100,
kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
]),
h_for_q_z=Sequential([
K.layers.Dense(100,
kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
K.layers.Dense(100,
kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
]),
x_dims=120,
z_dims=5,
)
trainer = DonutTrainer(model=model, model_vs=model_vs, max_epoch=300)
predictor = DonutPredictor(model)
with tf.Session().as_default():
trainer.fit(train_values, train_labels, train_missing, mean, std)
from donut import Donut
import time
if __name__ == "__main__":
t0 = time.time()
donut = Donut('../data/input.txt')
part_one = donut.shortest_path()
time_part_one = round((time.time() - t0) * 1e3)
print("Solution to part one: %s (time taken %s[ms])" %
(part_one, time_part_one))
t0 = time.time()
part_two = donut.shortest_path(True)
time_part_two = round((time.time() - t0) * 1e3)
print("Solution to part two: %s (time taken %s[ms])" %
(part_two, time_part_two))
def generate_score(number):
# Read the raw data.
data_dir_path = 'C:/Users/Administrator/Downloads/research/donut-master/SMD/data_concat/data-' + number + '.csv'
data = np.array(pd.read_csv(data_dir_path, header=None), dtype=np.float64)
tag_dir_path = './SMD/test_label/machine-' + number + '.csv'
tag = np.array(pd.read_csv(tag_dir_path, header=None), dtype=np.int)
labels = np.append(np.zeros(int(len(data) / 2)), tag)
# pick one colume
values = data[:, 1]
timestamp = np.arange(len(data)) + 1
# If there is no label, simply use all zeros.
# labels = np.zeros_like(values, dtype=np.int32)
# Complete the timestamp, and obtain the missing point indicators.
timestamp, (values, labels) = \
complete_timestamp(timestamp, (values, labels))
# Split the training and testing data.
test_portion = 0.5
test_n = int(len(values) * test_portion)
train_values = values[:-test_n]
test_values = values[-len(train_values):]
train_labels, test_labels = labels[:-test_n], labels[-test_n:]
# print(len(test_values), len(test_labels))
# Standardize the training and testing data.
train_values, mean, std = standardize_kpi(train_values,
excludes=train_labels)
test_values, _, _ = standardize_kpi(test_values, mean=mean, std=std)
import tensorflow as tf
from donut import Donut
from tensorflow import keras as K
from tfsnippet.modules import Sequential
# We build the entire model within the scope of `model_vs`,
# it should hold exactly all the variables of `model`, including
# the variables created by Keras layers.
with tf.variable_scope('model') as model_vs:
model = Donut(
h_for_p_x=Sequential([
K.layers.Dense(100,
kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
K.layers.Dense(100,
kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
]),
h_for_q_z=Sequential([
K.layers.Dense(100,
kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
K.layers.Dense(100,
kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
]),
x_dims=120,
z_dims=5,
)
from donut import DonutTrainer, DonutPredictor
trainer = DonutTrainer(model=model, model_vs=model_vs)
predictor = DonutPredictor(model)
with tf.Session().as_default():
trainer.fit(train_values, train_labels, mean, std)
test_score = predictor.get_score(test_values)
if not os.path.exists('./score'):
os.makedirs('./score')
np.save('./score/' + number + '.npy', test_score)
from os.path import dirname
from os.path import realpath
from os.path import join
from donut import Donut
import time
if __name__ == "__main__":
dir_path = dirname(realpath(__file__))
file_location = join(dir_path, "../data/input.txt")
t0 = time.time()
donut = Donut(file_location)
part_one = donut.shortest_path()
time_part_one = round((time.time() - t0) * 1e3)
print(
"Solution to part one: %s (time taken %s[ms])"
% (part_one, time_part_one)
)
t0 = time.time()
part_two = donut.shortest_path(True)
time_part_two = round((time.time() - t0) * 1e3)
print(
"Solution to part two: %s (time taken %s[ms])"
% (part_two, time_part_two)
)
def donut_test(src_dir, output_dir, file, batch):
if os.path.exists(output_dir + "performance-donut-" + str(batch) + ".csv"):
perform = pd.read_csv(output_dir + "performance-donut-" + str(batch) +
".csv")
else:
perform = pd.DataFrame({
"file": [],
"storage": [],
"train-time": [],
"codisp-time": [],
"test-time": [],
"precision": [],
"recall": [],
"best-F1": [],
"best-threshold": []
})
perform = perform.append([{
'file': file,
"storage": 0.0,
"train-time": 0.0,
"codisp-time": 0.0,
"test-time": 0.0,
"precision": 0.0,
"recall": 0.0,
"best-F1": 0.0,
"best-threshold": 0.0
}],
ignore_index=True)
perform.index = perform["file"]
data = pd.read_csv(src_dir + file)
timestamp, value, labels = data["timestamp"], data["value"], data[
"anomaly"]
missing = np.zeros(len(timestamp))
test_portion = 0.5
test_n = int(len(value) * test_portion)
train_values, test_values = value[:-test_n], value[-test_n:]
train_labels, test_labels = labels[:-test_n], labels[-test_n:]
train_time, test_time = timestamp[:-test_n], timestamp[-test_n:]
train_missing, test_missing = missing[:-test_n], missing[-test_n:]
train_values, mean, std = standardize_kpi(train_values,
excludes=np.logical_or(
train_labels, train_missing))
test_values, _, _ = standardize_kpi(test_values, mean=mean, std=std)
with tf.variable_scope('model') as model_vs:
model = Donut(
h_for_p_x=Sequential([
K.layers.Dense(100,
kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
K.layers.Dense(100,
kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
]),
h_for_q_z=Sequential([
K.layers.Dense(100,
kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
K.layers.Dense(100,
kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
]),
x_dims=120,
z_dims=5,
)
trainer = DonutTrainer(model=model, model_vs=model_vs)
predictor = DonutPredictor(model)
with tf.Session().as_default():
start = time.time()
trainer.fit(train_values, train_labels, train_missing, mean, std)
end = time.time()
perform.loc[file, "train-time"] = end - start
start = time.time()
test_score = predictor.get_score(test_values, test_missing)
end = time.time()
perform.loc[file, "test-time"] = end - start
storage = get_size(trainer) + get_size(predictor)
perform.loc[file, "storage"] = storage
pd.DataFrame({
"timestamp": test_time[-len(test_score):],
"score": test_score
}).to_csv(output_dir + "test-donut" + file, index=False)
best_F1, best_threshold, precision, recall = compute_best_F1(
src_dir + file,
output_dir + "test-donut" + file,
reverse=True,
mean_start=False)
perform.loc[file, "best-F1"] = best_F1
perform.loc[file, "best-threshold"] = best_threshold
perform.loc[file, "precision"] = precision
perform.loc[file, "recall"] = recall
perform.to_csv(output_dir + "performance-donut-" + str(batch) + ".csv",
index=False)
def test_prediction(self):
np.random.seed(1234)
tf.set_random_seed(1234)
# test last_point_only == True
donut = Donut(h_for_p_x=lambda x: x,
h_for_q_z=lambda x: x,
x_dims=5,
z_dims=3)
_ = donut.get_score(tf.zeros([4, 5], dtype=tf.float32),
tf.zeros(
[4, 5],
dtype=tf.int32)) # ensure variables created
pred = DonutPredictor(donut, n_z=2, batch_size=4)
self.assertIs(pred.model, donut)
with self.test_session():
ensure_variables_initialized()
# test without missing
res = pred.get_score(values=np.arange(5, dtype=np.float32))
self.assertEqual(res.shape, (1, ))
res = pred.get_score(values=np.arange(8, dtype=np.float32))
self.assertEqual(res.shape, (4, ))
res = pred.get_score(values=np.arange(10, dtype=np.float32))
self.assertEqual(res.shape, (6, ))
# test with missing
res = pred.get_score(values=np.arange(10, dtype=np.float32),
missing=np.random.binomial(1, .5, size=10))
self.assertEqual(res.shape, (6, ))
# test errors
with self.test_session():
with pytest.raises(ValueError,
match='`values` must be a 1-D array'):
_ = pred.get_score(
np.arange(10, dtype=np.float32).reshape([-1, 1]))
with pytest.raises(ValueError,
match='The shape of `missing` does not agree '
'with the shape of `values`'):
_ = pred.get_score(np.arange(10, dtype=np.float32),
np.arange(9, dtype=np.int32))
# test last_point_only == False
pred = DonutPredictor(donut,
n_z=2,
batch_size=4,
last_point_only=False)
with self.test_session():
ensure_variables_initialized()
# test without missing
res = pred.get_score(values=np.arange(10, dtype=np.float32))
self.assertEqual(res.shape, (6, 5))
# test with missing
res = pred.get_score(values=np.arange(10, dtype=np.float32),
missing=np.random.binomial(1, .5, size=10))
self.assertEqual(res.shape, (6, 5))
# test set feed_dict
is_training = tf.placeholder(tf.bool, shape=())
pred = DonutPredictor(donut,
n_z=2,
batch_size=4,
feed_dict={is_training: False})
with self.test_session():
ensure_variables_initialized()
_ = pred.get_score(values=np.arange(10, dtype=np.float32))
def add_donut(*args, **kwargs):
kitchen_service.add_donut(
Donut(request.form['flavor'], request.form['order_id'], NEW_ORDER))
return '200'
def vae_donut(ts_obj,
window_size,
mcmc_iteration,
latent_dim,
gaussian_window_size,
step_size,
plot_reconstruction=False,
plot_anomaly_score=False):
# authors use window_size = 120
# mcmc_iteration = 10
# https://github.com/kratzert/finetune_alexnet_with_tensorflow/issues/8
tf.reset_default_graph()
start = time.time()
# if there are missing time steps, we DO NOT fill them with NaNs because donut will replace them with 0s
# using complete_timestamp
# see line 6 in https://github.com/NetManAIOps/donut/blob/master/donut/preprocessing.py
timestamp, values, labels = ts_obj.dataframe[
"timestamp"].values, ts_obj.dataframe["value"].values, np.zeros_like(
ts_obj.dataframe["value"].values, dtype=np.int32)
# print(len(timestamp))
# print(len(values))
# print(len(labels))
# Complete the timestamp, and obtain the missing point indicators
# replaces missing with 0s.
# donut cannot handle this date format for some reason
if ts_obj.dateformat == "%Y-%m":
rng = pd.date_range('2000-01-01', periods=len(values), freq='T')
timestamp, missing, (values, labels) = complete_timestamp(
rng, (values, labels))
else:
timestamp, missing, (values, labels) = complete_timestamp(
timestamp, (values, labels))
# print(len(timestamp))
# print(len(values))
# print(len(labels))
# print(sum(missing))
# Standardize the training and testing data.
values, mean, std = standardize_kpi(values,
excludes=np.logical_or(
labels, missing))
with tf.variable_scope('model') as model_vs:
model = Donut(
h_for_p_x=Sequential([
K.layers.Dense(100,
kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
K.layers.Dense(100,
kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
]),
h_for_q_z=Sequential([
K.layers.Dense(100,
kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
K.layers.Dense(100,
kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
]),
x_dims=window_size,
z_dims=latent_dim,
)
trainer = DonutTrainer(model=model, model_vs=model_vs)
predictor = DonutPredictor(model)
with tf.Session().as_default():
trainer.fit(values, labels, missing, mean, std)
score = predictor.get_score(values, missing)
# if time series is [1,2,3,4...] and ts_length is 3
# this gives us [[1,2,3],[2,3,4]...]
ts_strided = ah.as_sliding_window(values, window_size)
ts_strided = my_func_float(np.array(ts_strided, dtype=np.float32))
missing_strided = ah.as_sliding_window(missing, window_size)
missing_strided = my_func_int(
np.array(missing_strided, dtype=np.int32))
# print(ts_strided)
# print(missing_strided)
x = model.vae.reconstruct(
iterative_masked_reconstruct(reconstruct=model.vae.reconstruct,
x=ts_strided,
mask=missing_strided,
iter_count=mcmc_iteration,
back_prop=False))
# `x` is a :class:`tfsnippet.stochastic.StochasticTensor`, from which
# you may derive many useful outputs, for example:
# print(x.tensor.eval()) # the `x` samples
# print(x.log_prob(group_ndims=0).eval()) # element-wise log p(x|z) of sampled x
# print(x.distribution.log_prob(ts_strided).eval()) # the reconstruction probability
# print(x.distribution.mean.eval(), x.distribution.std.eval()) # mean and std of p(x|z)
tensor_reconstruction_probabilities = x.distribution.log_prob(
ts_strided).eval()
# because of the way strided works, we use the first 120 anomaly scores in the first slide
# and then for remaining slides, we use the last point/score
reconstruction_probabilities = list(
tensor_reconstruction_probabilities[0])
for i in range(len(tensor_reconstruction_probabilities)):
if i != 0:
slide = tensor_reconstruction_probabilities[i]
reconstruction_probabilities.append(slide[-1])
# print(len(reconstruction_probabilities))
# print(len(ts_obj.dataframe))
if ts_obj.miss:
ref_date_range = ch.get_ref_date_range(ts_obj.dataframe,
ts_obj.dateformat,
ts_obj.timestep)
gaps = ref_date_range[~ref_date_range.isin(ts_obj.
dataframe["timestamp"])]
filled_df = ch.fill_df(ts_obj.dataframe, ts_obj.timestep,
ref_date_range, "fill_nan")
# print("NaNs exist?: ",filled_df['value'].isnull().values.any())
filled_df[
"reconstruction_probabilities"] = reconstruction_probabilities
# remove nans
filled_df = filled_df.dropna()
reconstruction_probabilities = list(
filled_df["reconstruction_probabilities"].values)
# print(len(reconstruction_probabilities))
# print(len(ts_obj.dataframe))
reconstruction_probabilities = [
abs(item) for item in reconstruction_probabilities
]
anomaly_scores = anomaly_scores = ah.determine_anomaly_scores_error(
reconstruction_probabilities,
np.zeros_like(reconstruction_probabilities), ts_obj.get_length(),
gaussian_window_size, step_size)
end = time.time()
if plot_reconstruction:
plt.subplot(211)
# see lines 98 to 100 of https://github.com/NetManAIOps/donut/blob/master/donut/prediction.py
plt.title("Negative of Reconstruction Probabilities")
plt.plot(reconstruction_probabilities)
# plt.ylim([.99,1])
plt.subplot(212)
plt.title("Time Series")
plt.plot(ts_obj.dataframe["value"].values)
plt.axvline(ts_obj.get_probationary_index(),
color="black",
label="probationary line")
plt.tight_layout()
plt.show()
if plot_anomaly_score:
plt.subplot(211)
plt.title("Anomaly Scores")
plt.plot(anomaly_scores)
plt.ylim([.998, 1])
plt.subplot(212)
plt.title("Time Series")
plt.plot(ts_obj.dataframe["value"].values)
plt.axvline(ts_obj.get_probationary_index(),
color="black",
label="probationary line")
plt.tight_layout()
plt.show()
return {
"Anomaly Scores": anomaly_scores,
"Time": end - start,
"Reconstruction Probabilities": reconstruction_probabilities
}