def __init__(self): super(Helper, self).__init__() tf.set_random_seed(1234) self.x = tf.get_variable('x', shape=[BATCH_SIZE, X_DIMS], initializer=tf.random_normal_initializer()) self.x2 = tf.get_variable('x2', shape=[N_X, BATCH_SIZE, X_DIMS], initializer=tf.random_normal_initializer()) self.x3 = tf.get_variable('x3', shape=[N_X, N_Z, BATCH_SIZE, X_DIMS], initializer=tf.random_normal_initializer()) self.z = tf.get_variable('z', shape=[BATCH_SIZE, Z_DIMS], initializer=tf.random_normal_initializer()) self.z2 = tf.get_variable('z2', shape=[N_Z, BATCH_SIZE, Z_DIMS], initializer=tf.random_normal_initializer()) self.h_for_p_x = Sequential([ K.layers.Dense(100, activation=tf.nn.relu), DictMapper({'mean': K.layers.Dense(X_DIMS), 'logstd': K.layers.Dense(X_DIMS)}) ]) self.h_for_q_z = Sequential([ K.layers.Dense(100, activation=tf.nn.relu), DictMapper({'mean': K.layers.Dense(Z_DIMS), 'logstd': K.layers.Dense(Z_DIMS)}) ]) # ensure variables created _ = self.h_for_p_x(self.z) _ = self.h_for_q_z(self.x)
def test_scopes(self): def f(inputs): return tf.get_variable('v', shape=()) * inputs class C(Module): def __init__(self, flag): self.flag = flag super(C, self).__init__() def _forward(self, inputs, **kwargs): v = tf.get_variable('a' if self.flag else 'b', shape=()) return v * inputs c = C(flag=True) seq = Sequential([ f, c, tf.nn.relu, f, C(flag=False), c, ]) self.assertEqual(tf.global_variables(), []) seq(tf.constant(1.)) self.assertEqual( sorted(v.name for v in tf.global_variables()), ['c/a:0', 'c_1/b:0', 'sequential/_0/v:0', 'sequential/_3/v:0']) seq(tf.constant(2.)) self.assertEqual( sorted(v.name for v in tf.global_variables()), ['c/a:0', 'c_1/b:0', 'sequential/_0/v:0', 'sequential/_3/v:0'])
def __init__(self, h_for_p_x, h_for_q_z, x_dims, z_dims, std_epsilon=1e-4, name=None, scope=None): if not is_integer(x_dims) or x_dims <= 0: raise ValueError('`x_dims` must be a positive integer') if not is_integer(z_dims) or z_dims <= 0: raise ValueError('`z_dims` must be a positive integer') super(Donut, self).__init__(name=name, scope=scope) with reopen_variable_scope(self.variable_scope): self._vae = VAE( p_z=Normal(mean=tf.zeros([z_dims]), std=tf.ones([z_dims])), p_x_given_z=Normal, q_z_given_x=Normal, h_for_p_x=Sequential([ h_for_p_x, DictMapper( { 'mean': K.layers.Dense(x_dims), 'std': lambda x: (std_epsilon + K.layers.Dense( x_dims, activation=tf.nn.softplus)(x)) }, name='p_x_given_z') ]), h_for_q_z=Sequential([ h_for_q_z, DictMapper( { 'mean': K.layers.Dense(z_dims), 'std': lambda z: (std_epsilon + K.layers.Dense( z_dims, activation=tf.nn.softplus)(z)) }, name='q_z_given_x') ]), ) self._x_dims = x_dims self._z_dims = z_dims
def fit(self, X: pd.DataFrame): with self.device: # Reset all results from last run to avoid reusing variables self.means, self.stds, self.tf_sessions, self.models = [], [], [], [] for col_idx in trange(len(X.columns)): col = X.columns[col_idx] tf_session = tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) timestamps = X.index features = X.loc[:, col].interpolate().bfill().values labels = pd.Series(0, X.index) timestamps, _, (features, labels) = complete_timestamp(timestamps, (features, labels)) missing = np.isnan(X.loc[:, col].values) _, mean, std = standardize_kpi(features, excludes=np.logical_or(labels, missing)) with tf.variable_scope('model') as model_vs: model = DonutModel( h_for_p_x=Sequential([ K.layers.Dense(100, kernel_regularizer=K.regularizers.l2(0.001), activation=tf.nn.relu), K.layers.Dense(100, kernel_regularizer=K.regularizers.l2(0.001), activation=tf.nn.relu), ]), h_for_q_z=Sequential([ K.layers.Dense(100, kernel_regularizer=K.regularizers.l2(0.001), activation=tf.nn.relu), K.layers.Dense(100, kernel_regularizer=K.regularizers.l2(0.001), activation=tf.nn.relu), ]), x_dims=self.x_dims, z_dims=5, ) trainer = QuietDonutTrainer(model=model, model_vs=model_vs, max_epoch=self.max_epoch, batch_size=self.batch_size, valid_batch_size=self.batch_size, missing_data_injection_rate=0.0, lr_anneal_factor=1.0) with tf_session.as_default(): trainer.fit(features, labels, missing, mean, std, valid_portion=0.25) self.means.append(mean) self.stds.append(std) self.tf_sessions.append(tf_session) self.models.append(model)
test_values, _, _ = standardize_kpi(test_values, mean=mean, std=std) return train_values, train_labels, train_missing, mean, std, test_values, test_labels, test_missing train_values, train_labels, train_missing, mean, std, test_values, test_labels, test_missing = get_data( ) # We build the entire model within the scope of `model_vs`, # it should hold exactly all the variables of `model`, including # the variables created by Keras layers. with tf.variable_scope('model') as model_vs: model = Donut( h_for_p_x=Sequential([ K.layers.Dense(100, kernel_regularizer=K.regularizers.l2(0.001), activation=tf.nn.relu), K.layers.Dense(100, kernel_regularizer=K.regularizers.l2(0.001), activation=tf.nn.relu), ]), h_for_q_z=Sequential([ K.layers.Dense(100, kernel_regularizer=K.regularizers.l2(0.001), activation=tf.nn.relu), K.layers.Dense(100, kernel_regularizer=K.regularizers.l2(0.001), activation=tf.nn.relu), ]), x_dims=120, z_dims=5, )
def main(): # load mnist data (train_x, train_y), (test_x, test_y) = datasets.load_mnist() # the parameters of this experiment x_dim = train_x.shape[1] z_dim = 2 max_epoch = 10 batch_size = 256 valid_portion = 0.2 # construct the graph with tf.Graph().as_default(), tf.Session().as_default() as session: input_x = tf.placeholder(dtype=tf.float32, shape=(None, x_dim), name='input_x') x_binarized = tf.stop_gradient(sample_input_x(input_x)) batch_size_tensor = tf.shape(input_x)[0] # derive the VAE z_shape = tf.stack([batch_size_tensor, z_dim]) vae = VAE(p_z=Normal(mean=tf.zeros(z_shape), std=tf.ones(z_shape)), p_x_given_z=Bernoulli, q_z_given_x=Normal, h_for_p_x=Sequential([ K.layers.Dense(100, activation=tf.nn.relu), K.layers.Dense(100, activation=tf.nn.relu), DictMapper( {'logits': K.layers.Dense(x_dim, name='x_logits')}) ]), h_for_q_z=Sequential([ tf.to_float, K.layers.Dense(100, activation=tf.nn.relu), K.layers.Dense(100, activation=tf.nn.relu), DictMapper({ 'mean': K.layers.Dense(z_dim, name='z_mean'), 'logstd': K.layers.Dense(z_dim, name='z_logstd'), }) ])) # train the network train(vae.get_training_loss(x_binarized), input_x, train_x, max_epoch, batch_size, valid_portion) # plot the latent space q_net = vae.variational(x_binarized) z_posterior = q_net['z'] z_predict = [] for [batch_x] in DataFlow.arrays([test_x], batch_size=batch_size): z_predict.append( session.run(z_posterior, feed_dict={input_x: batch_x})) z_predict = np.concatenate(z_predict, axis=0) plt.figure(figsize=(8, 6)) plt.scatter(z_predict[:, 0], z_predict[:, 1], c=test_y) plt.colorbar() plt.grid() plt.show()
def donut_test(src_dir, output_dir, file, batch): if os.path.exists(output_dir + "performance-donut-" + str(batch) + ".csv"): perform = pd.read_csv(output_dir + "performance-donut-" + str(batch) + ".csv") else: perform = pd.DataFrame({ "file": [], "storage": [], "train-time": [], "codisp-time": [], "test-time": [], "precision": [], "recall": [], "best-F1": [], "best-threshold": [] }) perform = perform.append([{ 'file': file, "storage": 0.0, "train-time": 0.0, "codisp-time": 0.0, "test-time": 0.0, "precision": 0.0, "recall": 0.0, "best-F1": 0.0, "best-threshold": 0.0 }], ignore_index=True) perform.index = perform["file"] data = pd.read_csv(src_dir + file) timestamp, value, labels = data["timestamp"], data["value"], data[ "anomaly"] missing = np.zeros(len(timestamp)) test_portion = 0.5 test_n = int(len(value) * test_portion) train_values, test_values = value[:-test_n], value[-test_n:] train_labels, test_labels = labels[:-test_n], labels[-test_n:] train_time, test_time = timestamp[:-test_n], timestamp[-test_n:] train_missing, test_missing = missing[:-test_n], missing[-test_n:] train_values, mean, std = standardize_kpi(train_values, excludes=np.logical_or( train_labels, train_missing)) test_values, _, _ = standardize_kpi(test_values, mean=mean, std=std) with tf.variable_scope('model') as model_vs: model = Donut( h_for_p_x=Sequential([ K.layers.Dense(100, kernel_regularizer=K.regularizers.l2(0.001), activation=tf.nn.relu), K.layers.Dense(100, kernel_regularizer=K.regularizers.l2(0.001), activation=tf.nn.relu), ]), h_for_q_z=Sequential([ K.layers.Dense(100, kernel_regularizer=K.regularizers.l2(0.001), activation=tf.nn.relu), K.layers.Dense(100, kernel_regularizer=K.regularizers.l2(0.001), activation=tf.nn.relu), ]), x_dims=120, z_dims=5, ) trainer = DonutTrainer(model=model, model_vs=model_vs) predictor = DonutPredictor(model) with tf.Session().as_default(): start = time.time() trainer.fit(train_values, train_labels, train_missing, mean, std) end = time.time() perform.loc[file, "train-time"] = end - start start = time.time() test_score = predictor.get_score(test_values, test_missing) end = time.time() perform.loc[file, "test-time"] = end - start storage = get_size(trainer) + get_size(predictor) perform.loc[file, "storage"] = storage pd.DataFrame({ "timestamp": test_time[-len(test_score):], "score": test_score }).to_csv(output_dir + "test-donut" + file, index=False) best_F1, best_threshold, precision, recall = compute_best_F1( src_dir + file, output_dir + "test-donut" + file, reverse=True, mean_start=False) perform.loc[file, "best-F1"] = best_F1 perform.loc[file, "best-threshold"] = best_threshold perform.loc[file, "precision"] = precision perform.loc[file, "recall"] = recall perform.to_csv(output_dir + "performance-donut-" + str(batch) + ".csv", index=False)
def generate_score(number): # Read the raw data. data_dir_path = 'C:/Users/Administrator/Downloads/research/donut-master/SMD/data_concat/data-' + number + '.csv' data = np.array(pd.read_csv(data_dir_path, header=None), dtype=np.float64) tag_dir_path = './SMD/test_label/machine-' + number + '.csv' tag = np.array(pd.read_csv(tag_dir_path, header=None), dtype=np.int) labels = np.append(np.zeros(int(len(data) / 2)), tag) # pick one colume values = data[:, 1] timestamp = np.arange(len(data)) + 1 # If there is no label, simply use all zeros. # labels = np.zeros_like(values, dtype=np.int32) # Complete the timestamp, and obtain the missing point indicators. timestamp, (values, labels) = \ complete_timestamp(timestamp, (values, labels)) # Split the training and testing data. test_portion = 0.5 test_n = int(len(values) * test_portion) train_values = values[:-test_n] test_values = values[-len(train_values):] train_labels, test_labels = labels[:-test_n], labels[-test_n:] # print(len(test_values), len(test_labels)) # Standardize the training and testing data. train_values, mean, std = standardize_kpi(train_values, excludes=train_labels) test_values, _, _ = standardize_kpi(test_values, mean=mean, std=std) import tensorflow as tf from donut import Donut from tensorflow import keras as K from tfsnippet.modules import Sequential # We build the entire model within the scope of `model_vs`, # it should hold exactly all the variables of `model`, including # the variables created by Keras layers. with tf.variable_scope('model') as model_vs: model = Donut( h_for_p_x=Sequential([ K.layers.Dense(100, kernel_regularizer=K.regularizers.l2(0.001), activation=tf.nn.relu), K.layers.Dense(100, kernel_regularizer=K.regularizers.l2(0.001), activation=tf.nn.relu), ]), h_for_q_z=Sequential([ K.layers.Dense(100, kernel_regularizer=K.regularizers.l2(0.001), activation=tf.nn.relu), K.layers.Dense(100, kernel_regularizer=K.regularizers.l2(0.001), activation=tf.nn.relu), ]), x_dims=120, z_dims=5, ) from donut import DonutTrainer, DonutPredictor trainer = DonutTrainer(model=model, model_vs=model_vs) predictor = DonutPredictor(model) with tf.Session().as_default(): trainer.fit(train_values, train_labels, mean, std) test_score = predictor.get_score(test_values) if not os.path.exists('./score'): os.makedirs('./score') np.save('./score/' + number + '.npy', test_score)
def test_errors(self): with pytest.raises(ValueError, match='`components` must not be empty'): Sequential([])
def vae_donut(ts_obj, window_size, mcmc_iteration, latent_dim, gaussian_window_size, step_size, plot_reconstruction=False, plot_anomaly_score=False): # authors use window_size = 120 # mcmc_iteration = 10 # https://github.com/kratzert/finetune_alexnet_with_tensorflow/issues/8 tf.reset_default_graph() start = time.time() # if there are missing time steps, we DO NOT fill them with NaNs because donut will replace them with 0s # using complete_timestamp # see line 6 in https://github.com/NetManAIOps/donut/blob/master/donut/preprocessing.py timestamp, values, labels = ts_obj.dataframe[ "timestamp"].values, ts_obj.dataframe["value"].values, np.zeros_like( ts_obj.dataframe["value"].values, dtype=np.int32) # print(len(timestamp)) # print(len(values)) # print(len(labels)) # Complete the timestamp, and obtain the missing point indicators # replaces missing with 0s. # donut cannot handle this date format for some reason if ts_obj.dateformat == "%Y-%m": rng = pd.date_range('2000-01-01', periods=len(values), freq='T') timestamp, missing, (values, labels) = complete_timestamp( rng, (values, labels)) else: timestamp, missing, (values, labels) = complete_timestamp( timestamp, (values, labels)) # print(len(timestamp)) # print(len(values)) # print(len(labels)) # print(sum(missing)) # Standardize the training and testing data. values, mean, std = standardize_kpi(values, excludes=np.logical_or( labels, missing)) with tf.variable_scope('model') as model_vs: model = Donut( h_for_p_x=Sequential([ K.layers.Dense(100, kernel_regularizer=K.regularizers.l2(0.001), activation=tf.nn.relu), K.layers.Dense(100, kernel_regularizer=K.regularizers.l2(0.001), activation=tf.nn.relu), ]), h_for_q_z=Sequential([ K.layers.Dense(100, kernel_regularizer=K.regularizers.l2(0.001), activation=tf.nn.relu), K.layers.Dense(100, kernel_regularizer=K.regularizers.l2(0.001), activation=tf.nn.relu), ]), x_dims=window_size, z_dims=latent_dim, ) trainer = DonutTrainer(model=model, model_vs=model_vs) predictor = DonutPredictor(model) with tf.Session().as_default(): trainer.fit(values, labels, missing, mean, std) score = predictor.get_score(values, missing) # if time series is [1,2,3,4...] and ts_length is 3 # this gives us [[1,2,3],[2,3,4]...] ts_strided = ah.as_sliding_window(values, window_size) ts_strided = my_func_float(np.array(ts_strided, dtype=np.float32)) missing_strided = ah.as_sliding_window(missing, window_size) missing_strided = my_func_int( np.array(missing_strided, dtype=np.int32)) # print(ts_strided) # print(missing_strided) x = model.vae.reconstruct( iterative_masked_reconstruct(reconstruct=model.vae.reconstruct, x=ts_strided, mask=missing_strided, iter_count=mcmc_iteration, back_prop=False)) # `x` is a :class:`tfsnippet.stochastic.StochasticTensor`, from which # you may derive many useful outputs, for example: # print(x.tensor.eval()) # the `x` samples # print(x.log_prob(group_ndims=0).eval()) # element-wise log p(x|z) of sampled x # print(x.distribution.log_prob(ts_strided).eval()) # the reconstruction probability # print(x.distribution.mean.eval(), x.distribution.std.eval()) # mean and std of p(x|z) tensor_reconstruction_probabilities = x.distribution.log_prob( ts_strided).eval() # because of the way strided works, we use the first 120 anomaly scores in the first slide # and then for remaining slides, we use the last point/score reconstruction_probabilities = list( tensor_reconstruction_probabilities[0]) for i in range(len(tensor_reconstruction_probabilities)): if i != 0: slide = tensor_reconstruction_probabilities[i] reconstruction_probabilities.append(slide[-1]) # print(len(reconstruction_probabilities)) # print(len(ts_obj.dataframe)) if ts_obj.miss: ref_date_range = ch.get_ref_date_range(ts_obj.dataframe, ts_obj.dateformat, ts_obj.timestep) gaps = ref_date_range[~ref_date_range.isin(ts_obj. dataframe["timestamp"])] filled_df = ch.fill_df(ts_obj.dataframe, ts_obj.timestep, ref_date_range, "fill_nan") # print("NaNs exist?: ",filled_df['value'].isnull().values.any()) filled_df[ "reconstruction_probabilities"] = reconstruction_probabilities # remove nans filled_df = filled_df.dropna() reconstruction_probabilities = list( filled_df["reconstruction_probabilities"].values) # print(len(reconstruction_probabilities)) # print(len(ts_obj.dataframe)) reconstruction_probabilities = [ abs(item) for item in reconstruction_probabilities ] anomaly_scores = anomaly_scores = ah.determine_anomaly_scores_error( reconstruction_probabilities, np.zeros_like(reconstruction_probabilities), ts_obj.get_length(), gaussian_window_size, step_size) end = time.time() if plot_reconstruction: plt.subplot(211) # see lines 98 to 100 of https://github.com/NetManAIOps/donut/blob/master/donut/prediction.py plt.title("Negative of Reconstruction Probabilities") plt.plot(reconstruction_probabilities) # plt.ylim([.99,1]) plt.subplot(212) plt.title("Time Series") plt.plot(ts_obj.dataframe["value"].values) plt.axvline(ts_obj.get_probationary_index(), color="black", label="probationary line") plt.tight_layout() plt.show() if plot_anomaly_score: plt.subplot(211) plt.title("Anomaly Scores") plt.plot(anomaly_scores) plt.ylim([.998, 1]) plt.subplot(212) plt.title("Time Series") plt.plot(ts_obj.dataframe["value"].values) plt.axvline(ts_obj.get_probationary_index(), color="black", label="probationary line") plt.tight_layout() plt.show() return { "Anomaly Scores": anomaly_scores, "Time": end - start, "Reconstruction Probabilities": reconstruction_probabilities }
def train_prediction_v1(use_plt, train_values, train_labels, train_missing, test_values, test_missing, test_labels, train_mean, train_std, valid_num): """ 训练与预测 Args: test_labels: 测试数据异常标签 valid_num: 测试数据数量 use_plt: 使用plt输出 train_values: 训练数据值 train_labels: 训练数据异常标签 train_missing: 训练数据缺失点 test_values: 测试数据 test_missing: 测试数据缺失点 train_mean: 平均值 train_std: 标准差 Returns: refactor_probability: 重构概率 epoch_list: 遍数列表 lr_list: 学习率变化列表 epoch_time: 遍数 model_time: 构建Donut模型时间 trainer_time: 构建训练器时间 predictor_time: 构建预测期时间 fit_time: 训练时间 probability_time: 获得重构概率时间 """ # 1.构造模型 tc = TimeCounter() tc.start() with tf.variable_scope('model') as model_vs: model = Donut( # 构建`p(x|z)`的隐藏网络 hidden_net_p_x_z=Sequential([ # units:该层的输出维度。 # kernel_regularizer:施加在权重上的正则项。L2正则化 使权重尽可能小 惩罚力度不大 # activation:激活函数 ReLU K.layers.Dense(100, kernel_regularizer=K.regularizers.l2(0.001), activation=tf.nn.relu), K.layers.Dense(100, kernel_regularizer=K.regularizers.l2(0.001), activation=tf.nn.relu), ]), hidden_net_q_z_x=Sequential([ K.layers.Dense(100, kernel_regularizer=K.regularizers.l2(0.001), activation=tf.nn.relu), K.layers.Dense(100, kernel_regularizer=K.regularizers.l2(0.001), activation=tf.nn.relu), ]), # x维的数量 x_dims=120, # z维的数量 z_dims=5, ) tc.end() model_time = tc.get_s() + "秒" print_info(use_plt, "5.构建Donut模型【共用时{}】".format(model_time)) # 2.构造训练器 tc.start() trainer = DonutTrainer(model=model, model_vs=model_vs) tc.end() trainer_time = tc.get_s() + "秒" print_info(use_plt, "6.构造训练器【共用时{}】".format(trainer_time)) # 3.构造预测器 tc.start() predictor = DonutPredictor(model) tc.end() predictor_time = tc.get_s() + "秒" print_info(use_plt, "7.构造预测器【共用时{}】".format(predictor_time)) with tf.Session().as_default(): # 4.训练器训练模型 tc.start() epoch_list, lr_list, epoch_time, train_message = \ trainer.fit(use_plt, train_values, train_labels, train_missing, test_values, test_labels, test_missing, train_mean, train_std, valid_num) tc.end() fit_time = tc.get_s() + "秒" print_info(use_plt, "8.训练器训练模型【共用时{}】".format(fit_time)) print_text(use_plt, "所有epoch【共用时:{}】".format(epoch_time)) print_text(use_plt, "退火学习率 学习率随epoch变化") show_line_chart(use_plt, epoch_list, lr_list, 'annealing learning rate') # 5.预测器获取重构概率 tc.start() refactor_probability = predictor.get_refactor_probability(test_values, test_missing) tc.end() probability_time = tc.get_s() + "秒" print_info(use_plt, "9.预测器获取重构概率【共用时{}】".format(probability_time)) return refactor_probability, epoch_list, lr_list, epoch_time, model_time, trainer_time, predictor_time, fit_time, probability_time, train_message