if dataset == 'BGL':
data_instances = config.BGL_data
(x_train, y_train), (x_test, y_test), (x_validate, y_validate) = load_BGL(data_instances, 0.35, 0.6)
collector = Collector(result_folder, (1, 1, 1, 1), False, config.BGL_col_header, 100)
if dataset == 'HDFS':
data_instances = config.HDFS_data
(x_train, y_train), (x_test, y_test), (x_validate, y_validate) = dataloader.load_HDFS(data_instances,
train_ratio=0.35,
is_data_instance=True,
test_ratio=0.6)
collector = Collector(result_folder, (1, 1, 1, 1), False, config.HDFS_col_header, 100)
assert FLAGS.h < FLAGS.plb
lstm_preprocessor = preprocessing.LstmPreprocessor(x_train, x_test, x_validate)
sym_count = len(lstm_preprocessor.vectors) - 1
print('Total symbols: %d' % sym_count)
print(lstm_preprocessor.syms)
# pad x_train
x_train = [lstm_preprocessor.pad(t, FLAGS.plb) if len(t) < FLAGS.plb else t for t in x_train]
# throw away anomalies & same event series in x_train
x_train = lstm_preprocessor.process_train_inputs(x_train, y_train, FLAGS.h, True,
FLAGS.no_repeat_series)
x_train = lstm_preprocessor.transform_to_same_length(x_train, FLAGS.h)
model = lstm_attention.LSTMAttention(FLAGS.g, FLAGS.h, FLAGS.L, FLAGS.alpha, FLAGS.batch_size, sym_count).model
# checkpoint = keras.callbacks.ModelCheckpoint(checkpoint_name,
# verbose=1, save_weights_only=True)
result.append(t)
return result
if __name__ == '__main__':
assert FLAGS.h < FLAGS.plb
config.init('testbed')
checkpoint_name = config.path + FLAGS.checkpoint_name
top_counts_file_name = config.path + 'top_counts.pkl'
file = config.testbed_path + 'logstash-2019.07.22_ts-food-service_sorted.csv_structured.csv'
df = pd.read_csv(file)
event_sequence = [list(df['EventId'].values)]
lstm_preprocessor = preprocessing.LstmPreprocessor(event_sequence)
sym_count = len(lstm_preprocessor.vectors) - 1
print('Total symbols: %d' % sym_count)
print(lstm_preprocessor.syms)
model = lstm_attention_count_vector.LSTMAttention(3, FLAGS.h, FLAGS.L, FLAGS.alpha, FLAGS.batch_size,
sym_count).model
checkpoint = keras.callbacks.ModelCheckpoint(checkpoint_name, verbose=1, save_weights_only=True)
if os.path.exists(checkpoint_name):
print('== Reading model parameters from %s ==' % checkpoint_name)
model.load_weights(checkpoint_name)
inputs, labels = lstm_preprocessor.gen_input_and_label(event_sequence)
count_vectors = lstm_preprocessor.gen_count_vectors(event_sequence)