def __init__(self):
super(Helper, self).__init__()
tf.set_random_seed(1234)
self.x = tf.get_variable('x',
shape=[BATCH_SIZE, X_DIMS],
initializer=tf.random_normal_initializer())
self.x2 = tf.get_variable('x2',
shape=[N_X, BATCH_SIZE, X_DIMS],
initializer=tf.random_normal_initializer())
self.x3 = tf.get_variable('x3',
shape=[N_X, N_Z, BATCH_SIZE, X_DIMS],
initializer=tf.random_normal_initializer())
self.z = tf.get_variable('z',
shape=[BATCH_SIZE, Z_DIMS],
initializer=tf.random_normal_initializer())
self.z2 = tf.get_variable('z2',
shape=[N_Z, BATCH_SIZE, Z_DIMS],
initializer=tf.random_normal_initializer())
self.h_for_p_x = Sequential([
K.layers.Dense(100, activation=tf.nn.relu),
DictMapper({'mean': K.layers.Dense(X_DIMS),
'logstd': K.layers.Dense(X_DIMS)})
])
self.h_for_q_z = Sequential([
K.layers.Dense(100, activation=tf.nn.relu),
DictMapper({'mean': K.layers.Dense(Z_DIMS),
'logstd': K.layers.Dense(Z_DIMS)})
])
# ensure variables created
_ = self.h_for_p_x(self.z)
_ = self.h_for_q_z(self.x)
def test_scopes(self):
def f(inputs):
return tf.get_variable('v', shape=()) * inputs
class C(Module):
def __init__(self, flag):
self.flag = flag
super(C, self).__init__()
def _forward(self, inputs, **kwargs):
v = tf.get_variable('a' if self.flag else 'b', shape=())
return v * inputs
c = C(flag=True)
seq = Sequential([
f,
c,
tf.nn.relu,
f,
C(flag=False),
c,
])
self.assertEqual(tf.global_variables(), [])
seq(tf.constant(1.))
self.assertEqual(
sorted(v.name for v in tf.global_variables()),
['c/a:0', 'c_1/b:0', 'sequential/_0/v:0', 'sequential/_3/v:0'])
seq(tf.constant(2.))
self.assertEqual(
sorted(v.name for v in tf.global_variables()),
['c/a:0', 'c_1/b:0', 'sequential/_0/v:0', 'sequential/_3/v:0'])
def __init__(self,
h_for_p_x,
h_for_q_z,
x_dims,
z_dims,
std_epsilon=1e-4,
name=None,
scope=None):
if not is_integer(x_dims) or x_dims <= 0:
raise ValueError('`x_dims` must be a positive integer')
if not is_integer(z_dims) or z_dims <= 0:
raise ValueError('`z_dims` must be a positive integer')
super(Donut, self).__init__(name=name, scope=scope)
with reopen_variable_scope(self.variable_scope):
self._vae = VAE(
p_z=Normal(mean=tf.zeros([z_dims]), std=tf.ones([z_dims])),
p_x_given_z=Normal,
q_z_given_x=Normal,
h_for_p_x=Sequential([
h_for_p_x,
DictMapper(
{
'mean':
K.layers.Dense(x_dims),
'std':
lambda x: (std_epsilon + K.layers.Dense(
x_dims, activation=tf.nn.softplus)(x))
},
name='p_x_given_z')
]),
h_for_q_z=Sequential([
h_for_q_z,
DictMapper(
{
'mean':
K.layers.Dense(z_dims),
'std':
lambda z: (std_epsilon + K.layers.Dense(
z_dims, activation=tf.nn.softplus)(z))
},
name='q_z_given_x')
]),
)
self._x_dims = x_dims
self._z_dims = z_dims
def fit(self, X: pd.DataFrame):
with self.device:
# Reset all results from last run to avoid reusing variables
self.means, self.stds, self.tf_sessions, self.models = [], [], [], []
for col_idx in trange(len(X.columns)):
col = X.columns[col_idx]
tf_session = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
timestamps = X.index
features = X.loc[:, col].interpolate().bfill().values
labels = pd.Series(0, X.index)
timestamps, _, (features, labels) = complete_timestamp(timestamps, (features, labels))
missing = np.isnan(X.loc[:, col].values)
_, mean, std = standardize_kpi(features, excludes=np.logical_or(labels, missing))
with tf.variable_scope('model') as model_vs:
model = DonutModel(
h_for_p_x=Sequential([
K.layers.Dense(100, kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
K.layers.Dense(100, kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
]),
h_for_q_z=Sequential([
K.layers.Dense(100, kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
K.layers.Dense(100, kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
]),
x_dims=self.x_dims,
z_dims=5,
)
trainer = QuietDonutTrainer(model=model, model_vs=model_vs, max_epoch=self.max_epoch,
batch_size=self.batch_size, valid_batch_size=self.batch_size,
missing_data_injection_rate=0.0, lr_anneal_factor=1.0)
with tf_session.as_default():
trainer.fit(features, labels, missing, mean, std, valid_portion=0.25)
self.means.append(mean)
self.stds.append(std)
self.tf_sessions.append(tf_session)
self.models.append(model)
test_values, _, _ = standardize_kpi(test_values, mean=mean, std=std)
return train_values, train_labels, train_missing, mean, std, test_values, test_labels, test_missing
train_values, train_labels, train_missing, mean, std, test_values, test_labels, test_missing = get_data(
)
# We build the entire model within the scope of `model_vs`,
# it should hold exactly all the variables of `model`, including
# the variables created by Keras layers.
with tf.variable_scope('model') as model_vs:
model = Donut(
h_for_p_x=Sequential([
K.layers.Dense(100,
kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
K.layers.Dense(100,
kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
]),
h_for_q_z=Sequential([
K.layers.Dense(100,
kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
K.layers.Dense(100,
kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
]),
x_dims=120,
z_dims=5,
)
def main():
# load mnist data
(train_x, train_y), (test_x, test_y) = datasets.load_mnist()
# the parameters of this experiment
x_dim = train_x.shape[1]
z_dim = 2
max_epoch = 10
batch_size = 256
valid_portion = 0.2
# construct the graph
with tf.Graph().as_default(), tf.Session().as_default() as session:
input_x = tf.placeholder(dtype=tf.float32,
shape=(None, x_dim),
name='input_x')
x_binarized = tf.stop_gradient(sample_input_x(input_x))
batch_size_tensor = tf.shape(input_x)[0]
# derive the VAE
z_shape = tf.stack([batch_size_tensor, z_dim])
vae = VAE(p_z=Normal(mean=tf.zeros(z_shape), std=tf.ones(z_shape)),
p_x_given_z=Bernoulli,
q_z_given_x=Normal,
h_for_p_x=Sequential([
K.layers.Dense(100, activation=tf.nn.relu),
K.layers.Dense(100, activation=tf.nn.relu),
DictMapper(
{'logits': K.layers.Dense(x_dim, name='x_logits')})
]),
h_for_q_z=Sequential([
tf.to_float,
K.layers.Dense(100, activation=tf.nn.relu),
K.layers.Dense(100, activation=tf.nn.relu),
DictMapper({
'mean':
K.layers.Dense(z_dim, name='z_mean'),
'logstd':
K.layers.Dense(z_dim, name='z_logstd'),
})
]))
# train the network
train(vae.get_training_loss(x_binarized), input_x, train_x, max_epoch,
batch_size, valid_portion)
# plot the latent space
q_net = vae.variational(x_binarized)
z_posterior = q_net['z']
z_predict = []
for [batch_x] in DataFlow.arrays([test_x], batch_size=batch_size):
z_predict.append(
session.run(z_posterior, feed_dict={input_x: batch_x}))
z_predict = np.concatenate(z_predict, axis=0)
plt.figure(figsize=(8, 6))
plt.scatter(z_predict[:, 0], z_predict[:, 1], c=test_y)
plt.colorbar()
plt.grid()
plt.show()
def donut_test(src_dir, output_dir, file, batch):
if os.path.exists(output_dir + "performance-donut-" + str(batch) + ".csv"):
perform = pd.read_csv(output_dir + "performance-donut-" + str(batch) +
".csv")
else:
perform = pd.DataFrame({
"file": [],
"storage": [],
"train-time": [],
"codisp-time": [],
"test-time": [],
"precision": [],
"recall": [],
"best-F1": [],
"best-threshold": []
})
perform = perform.append([{
'file': file,
"storage": 0.0,
"train-time": 0.0,
"codisp-time": 0.0,
"test-time": 0.0,
"precision": 0.0,
"recall": 0.0,
"best-F1": 0.0,
"best-threshold": 0.0
}],
ignore_index=True)
perform.index = perform["file"]
data = pd.read_csv(src_dir + file)
timestamp, value, labels = data["timestamp"], data["value"], data[
"anomaly"]
missing = np.zeros(len(timestamp))
test_portion = 0.5
test_n = int(len(value) * test_portion)
train_values, test_values = value[:-test_n], value[-test_n:]
train_labels, test_labels = labels[:-test_n], labels[-test_n:]
train_time, test_time = timestamp[:-test_n], timestamp[-test_n:]
train_missing, test_missing = missing[:-test_n], missing[-test_n:]
train_values, mean, std = standardize_kpi(train_values,
excludes=np.logical_or(
train_labels, train_missing))
test_values, _, _ = standardize_kpi(test_values, mean=mean, std=std)
with tf.variable_scope('model') as model_vs:
model = Donut(
h_for_p_x=Sequential([
K.layers.Dense(100,
kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
K.layers.Dense(100,
kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
]),
h_for_q_z=Sequential([
K.layers.Dense(100,
kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
K.layers.Dense(100,
kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
]),
x_dims=120,
z_dims=5,
)
trainer = DonutTrainer(model=model, model_vs=model_vs)
predictor = DonutPredictor(model)
with tf.Session().as_default():
start = time.time()
trainer.fit(train_values, train_labels, train_missing, mean, std)
end = time.time()
perform.loc[file, "train-time"] = end - start
start = time.time()
test_score = predictor.get_score(test_values, test_missing)
end = time.time()
perform.loc[file, "test-time"] = end - start
storage = get_size(trainer) + get_size(predictor)
perform.loc[file, "storage"] = storage
pd.DataFrame({
"timestamp": test_time[-len(test_score):],
"score": test_score
}).to_csv(output_dir + "test-donut" + file, index=False)
best_F1, best_threshold, precision, recall = compute_best_F1(
src_dir + file,
output_dir + "test-donut" + file,
reverse=True,
mean_start=False)
perform.loc[file, "best-F1"] = best_F1
perform.loc[file, "best-threshold"] = best_threshold
perform.loc[file, "precision"] = precision
perform.loc[file, "recall"] = recall
perform.to_csv(output_dir + "performance-donut-" + str(batch) + ".csv",
index=False)
def generate_score(number):
# Read the raw data.
data_dir_path = 'C:/Users/Administrator/Downloads/research/donut-master/SMD/data_concat/data-' + number + '.csv'
data = np.array(pd.read_csv(data_dir_path, header=None), dtype=np.float64)
tag_dir_path = './SMD/test_label/machine-' + number + '.csv'
tag = np.array(pd.read_csv(tag_dir_path, header=None), dtype=np.int)
labels = np.append(np.zeros(int(len(data) / 2)), tag)
# pick one colume
values = data[:, 1]
timestamp = np.arange(len(data)) + 1
# If there is no label, simply use all zeros.
# labels = np.zeros_like(values, dtype=np.int32)
# Complete the timestamp, and obtain the missing point indicators.
timestamp, (values, labels) = \
complete_timestamp(timestamp, (values, labels))
# Split the training and testing data.
test_portion = 0.5
test_n = int(len(values) * test_portion)
train_values = values[:-test_n]
test_values = values[-len(train_values):]
train_labels, test_labels = labels[:-test_n], labels[-test_n:]
# print(len(test_values), len(test_labels))
# Standardize the training and testing data.
train_values, mean, std = standardize_kpi(train_values,
excludes=train_labels)
test_values, _, _ = standardize_kpi(test_values, mean=mean, std=std)
import tensorflow as tf
from donut import Donut
from tensorflow import keras as K
from tfsnippet.modules import Sequential
# We build the entire model within the scope of `model_vs`,
# it should hold exactly all the variables of `model`, including
# the variables created by Keras layers.
with tf.variable_scope('model') as model_vs:
model = Donut(
h_for_p_x=Sequential([
K.layers.Dense(100,
kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
K.layers.Dense(100,
kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
]),
h_for_q_z=Sequential([
K.layers.Dense(100,
kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
K.layers.Dense(100,
kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
]),
x_dims=120,
z_dims=5,
)
from donut import DonutTrainer, DonutPredictor
trainer = DonutTrainer(model=model, model_vs=model_vs)
predictor = DonutPredictor(model)
with tf.Session().as_default():
trainer.fit(train_values, train_labels, mean, std)
test_score = predictor.get_score(test_values)
if not os.path.exists('./score'):
os.makedirs('./score')
np.save('./score/' + number + '.npy', test_score)
def test_errors(self):
with pytest.raises(ValueError, match='`components` must not be empty'):
Sequential([])
def vae_donut(ts_obj,
window_size,
mcmc_iteration,
latent_dim,
gaussian_window_size,
step_size,
plot_reconstruction=False,
plot_anomaly_score=False):
# authors use window_size = 120
# mcmc_iteration = 10
# https://github.com/kratzert/finetune_alexnet_with_tensorflow/issues/8
tf.reset_default_graph()
start = time.time()
# if there are missing time steps, we DO NOT fill them with NaNs because donut will replace them with 0s
# using complete_timestamp
# see line 6 in https://github.com/NetManAIOps/donut/blob/master/donut/preprocessing.py
timestamp, values, labels = ts_obj.dataframe[
"timestamp"].values, ts_obj.dataframe["value"].values, np.zeros_like(
ts_obj.dataframe["value"].values, dtype=np.int32)
# print(len(timestamp))
# print(len(values))
# print(len(labels))
# Complete the timestamp, and obtain the missing point indicators
# replaces missing with 0s.
# donut cannot handle this date format for some reason
if ts_obj.dateformat == "%Y-%m":
rng = pd.date_range('2000-01-01', periods=len(values), freq='T')
timestamp, missing, (values, labels) = complete_timestamp(
rng, (values, labels))
else:
timestamp, missing, (values, labels) = complete_timestamp(
timestamp, (values, labels))
# print(len(timestamp))
# print(len(values))
# print(len(labels))
# print(sum(missing))
# Standardize the training and testing data.
values, mean, std = standardize_kpi(values,
excludes=np.logical_or(
labels, missing))
with tf.variable_scope('model') as model_vs:
model = Donut(
h_for_p_x=Sequential([
K.layers.Dense(100,
kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
K.layers.Dense(100,
kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
]),
h_for_q_z=Sequential([
K.layers.Dense(100,
kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
K.layers.Dense(100,
kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
]),
x_dims=window_size,
z_dims=latent_dim,
)
trainer = DonutTrainer(model=model, model_vs=model_vs)
predictor = DonutPredictor(model)
with tf.Session().as_default():
trainer.fit(values, labels, missing, mean, std)
score = predictor.get_score(values, missing)
# if time series is [1,2,3,4...] and ts_length is 3
# this gives us [[1,2,3],[2,3,4]...]
ts_strided = ah.as_sliding_window(values, window_size)
ts_strided = my_func_float(np.array(ts_strided, dtype=np.float32))
missing_strided = ah.as_sliding_window(missing, window_size)
missing_strided = my_func_int(
np.array(missing_strided, dtype=np.int32))
# print(ts_strided)
# print(missing_strided)
x = model.vae.reconstruct(
iterative_masked_reconstruct(reconstruct=model.vae.reconstruct,
x=ts_strided,
mask=missing_strided,
iter_count=mcmc_iteration,
back_prop=False))
# `x` is a :class:`tfsnippet.stochastic.StochasticTensor`, from which
# you may derive many useful outputs, for example:
# print(x.tensor.eval()) # the `x` samples
# print(x.log_prob(group_ndims=0).eval()) # element-wise log p(x|z) of sampled x
# print(x.distribution.log_prob(ts_strided).eval()) # the reconstruction probability
# print(x.distribution.mean.eval(), x.distribution.std.eval()) # mean and std of p(x|z)
tensor_reconstruction_probabilities = x.distribution.log_prob(
ts_strided).eval()
# because of the way strided works, we use the first 120 anomaly scores in the first slide
# and then for remaining slides, we use the last point/score
reconstruction_probabilities = list(
tensor_reconstruction_probabilities[0])
for i in range(len(tensor_reconstruction_probabilities)):
if i != 0:
slide = tensor_reconstruction_probabilities[i]
reconstruction_probabilities.append(slide[-1])
# print(len(reconstruction_probabilities))
# print(len(ts_obj.dataframe))
if ts_obj.miss:
ref_date_range = ch.get_ref_date_range(ts_obj.dataframe,
ts_obj.dateformat,
ts_obj.timestep)
gaps = ref_date_range[~ref_date_range.isin(ts_obj.
dataframe["timestamp"])]
filled_df = ch.fill_df(ts_obj.dataframe, ts_obj.timestep,
ref_date_range, "fill_nan")
# print("NaNs exist?: ",filled_df['value'].isnull().values.any())
filled_df[
"reconstruction_probabilities"] = reconstruction_probabilities
# remove nans
filled_df = filled_df.dropna()
reconstruction_probabilities = list(
filled_df["reconstruction_probabilities"].values)
# print(len(reconstruction_probabilities))
# print(len(ts_obj.dataframe))
reconstruction_probabilities = [
abs(item) for item in reconstruction_probabilities
]
anomaly_scores = anomaly_scores = ah.determine_anomaly_scores_error(
reconstruction_probabilities,
np.zeros_like(reconstruction_probabilities), ts_obj.get_length(),
gaussian_window_size, step_size)
end = time.time()
if plot_reconstruction:
plt.subplot(211)
# see lines 98 to 100 of https://github.com/NetManAIOps/donut/blob/master/donut/prediction.py
plt.title("Negative of Reconstruction Probabilities")
plt.plot(reconstruction_probabilities)
# plt.ylim([.99,1])
plt.subplot(212)
plt.title("Time Series")
plt.plot(ts_obj.dataframe["value"].values)
plt.axvline(ts_obj.get_probationary_index(),
color="black",
label="probationary line")
plt.tight_layout()
plt.show()
if plot_anomaly_score:
plt.subplot(211)
plt.title("Anomaly Scores")
plt.plot(anomaly_scores)
plt.ylim([.998, 1])
plt.subplot(212)
plt.title("Time Series")
plt.plot(ts_obj.dataframe["value"].values)
plt.axvline(ts_obj.get_probationary_index(),
color="black",
label="probationary line")
plt.tight_layout()
plt.show()
return {
"Anomaly Scores": anomaly_scores,
"Time": end - start,
"Reconstruction Probabilities": reconstruction_probabilities
}
def train_prediction_v1(use_plt, train_values, train_labels, train_missing, test_values, test_missing, test_labels,
train_mean, train_std, valid_num):
"""
训练与预测
Args:
test_labels: 测试数据异常标签
valid_num: 测试数据数量
use_plt: 使用plt输出
train_values: 训练数据值
train_labels: 训练数据异常标签
train_missing: 训练数据缺失点
test_values: 测试数据
test_missing: 测试数据缺失点
train_mean: 平均值
train_std: 标准差
Returns:
refactor_probability: 重构概率
epoch_list: 遍数列表
lr_list: 学习率变化列表
epoch_time: 遍数
model_time: 构建Donut模型时间
trainer_time: 构建训练器时间
predictor_time: 构建预测期时间
fit_time: 训练时间
probability_time: 获得重构概率时间
"""
# 1.构造模型
tc = TimeCounter()
tc.start()
with tf.variable_scope('model') as model_vs:
model = Donut(
# 构建`p(x|z)`的隐藏网络
hidden_net_p_x_z=Sequential([
# units：该层的输出维度。
# kernel_regularizer：施加在权重上的正则项。L2正则化 使权重尽可能小 惩罚力度不大
# activation：激活函数 ReLU
K.layers.Dense(100, kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
K.layers.Dense(100, kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
]),
hidden_net_q_z_x=Sequential([
K.layers.Dense(100, kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
K.layers.Dense(100, kernel_regularizer=K.regularizers.l2(0.001),
activation=tf.nn.relu),
]),
# x维的数量
x_dims=120,
# z维的数量
z_dims=5,
)
tc.end()
model_time = tc.get_s() + "秒"
print_info(use_plt, "5.构建Donut模型【共用时{}】".format(model_time))
# 2.构造训练器
tc.start()
trainer = DonutTrainer(model=model, model_vs=model_vs)
tc.end()
trainer_time = tc.get_s() + "秒"
print_info(use_plt, "6.构造训练器【共用时{}】".format(trainer_time))
# 3.构造预测器
tc.start()
predictor = DonutPredictor(model)
tc.end()
predictor_time = tc.get_s() + "秒"
print_info(use_plt, "7.构造预测器【共用时{}】".format(predictor_time))
with tf.Session().as_default():
# 4.训练器训练模型
tc.start()
epoch_list, lr_list, epoch_time, train_message = \
trainer.fit(use_plt, train_values, train_labels, train_missing, test_values, test_labels, test_missing,
train_mean, train_std, valid_num)
tc.end()
fit_time = tc.get_s() + "秒"
print_info(use_plt, "8.训练器训练模型【共用时{}】".format(fit_time))
print_text(use_plt, "所有epoch【共用时：{}】".format(epoch_time))
print_text(use_plt, "退火学习率 学习率随epoch变化")
show_line_chart(use_plt, epoch_list, lr_list, 'annealing learning rate')
# 5.预测器获取重构概率
tc.start()
refactor_probability = predictor.get_refactor_probability(test_values, test_missing)
tc.end()
probability_time = tc.get_s() + "秒"
print_info(use_plt, "9.预测器获取重构概率【共用时{}】".format(probability_time))
return refactor_probability, epoch_list, lr_list, epoch_time, model_time, trainer_time, predictor_time, fit_time, probability_time, train_message
