def random_search_unsupervised(data_and_labels: tuple, model: Union[LocalOutlierFactor, IsolationForest], params: Dict) -> Dict: x_train, x_test, _, y_test = data_and_labels scores = [] for experiment in params['experiments']: model.set_params(**experiment['hyperparameters']) print(f'Model current hyperparameters are: {experiment["hyperparameters"]}.') if isinstance(model, LocalOutlierFactor): y_pred = model.fit_predict(x_test) # return labels else: model.fit(x_train) y_pred = model.predict(x_test) # return labels y_pred = convert_predictions(y_pred) metrics_report(y_test, y_pred) model_path = create_model_path(DIR_TO_EXPERIMENTS, str(uuid.uuid4())) torch.save(model, model_path) res = create_experiment_report(get_metrics(y_test, y_pred), experiment['hyperparameters'], file_path=model_path) scores.append(res) create_checkpoint({'experiments': scores}, EXPERIMENT_PATH) return { 'experiments': scores }
def random_search(data_and_labels: tuple, model: TransformerAutoEncoder, params: Dict) -> Dict: x_train, x_test, _, y_test = data_and_labels scores = [] for conf in zip(*params.values()): kwargs = {k: val for k, val in zip(params.keys(), conf)} model.set_params(**kwargs) print(f'Model current hyperparameters are: {kwargs}.') model.fit(x_train) y_pred = model.predict(x_test) # return reconstruction errors theta, f1 = find_optimal_threshold(y_test, y_pred) y_pred = classify(y_pred, theta) metrics_report(y_test, y_pred) scores.append( create_experiment_report(get_metrics(y_test, y_pred), kwargs)) # visualize_distribution_with_labels(y_pred, y_test, to_file=False) from sklearn.metrics import confusion_matrix print(confusion_matrix(y_test, y_pred)) create_checkpoint({'experiments': scores}, EXPERIMENT_PATH) return {'experiments': scores}
def grid_search(data_and_labels: tuple, model: Union[LocalOutlierFactor, IsolationForest], params: Dict) -> Dict: x_train, x_test, _, y_test = data_and_labels scores = [] for conf in itertools.product(*params.values()): kwargs = {k: val for k, val in zip(params.keys(), conf)} model.set_params(**kwargs) print(f'Model (hyper)parameters are: {model.get_params()}.') if isinstance(model, LocalOutlierFactor): y_pred = model.fit_predict(x_test) else: model.fit(x_train) y_pred = model.predict(x_test) y_pred = convert_predictions(y_pred) metrics_report(y_test, y_pred) scores.append( create_experiment_report(get_metrics(y_test, y_pred), model.get_params())) return {'experiments': scores}
def random_search(data_and_labels: tuple, model: Union[AutoEncoder, VanillaTCN, AETCN, AECNN1D, CNN1D, CNN2D, TCNCNN1D, SACNN1D, SACNN2D], params: Dict) -> Dict: x_train, x_test, _, y_test = data_and_labels scores = [] for experiment in params['experiments']: model.set_params(**experiment['hyperparameters']) print(f'Model current hyperparameters are: {experiment["hyperparameters"]}.') model.fit(x_train) y_pred = model.predict(x_test) # return reconstruction errors theta, f1 = find_optimal_threshold(y_test, y_pred) y_pred = classify(y_pred, theta) metrics_report(y_test, y_pred) model_path = create_model_path(DIR_TO_EXPERIMENTS, str(uuid.uuid4())) torch.save(model, model_path) res = create_experiment_report(get_metrics(y_test, y_pred), experiment['hyperparameters'], theta, model_path) scores.append(res) create_checkpoint({'experiments': scores}, EXPERIMENT_PATH) return { 'experiments': scores }
def evaluate_unsupervised(x_test: np.ndarray, y_test: np.array, experiments: Dict) -> Dict: model_config = find_best_model(experiments) model = torch.load(model_config['model_path']) if isinstance(model, LocalOutlierFactor): y_pred = model.fit_predict(x_test) # return labels else: y_pred = model.predict(x_test) # return labels y_pred = convert_predictions(y_pred) auc_score = roc_auc_score(y_test, y_pred) metrics_report(y_test, y_pred) return create_report( model_config, { **get_metrics(y_test, y_pred), 'auc_score': float(auc_score) })
def evaluate(x_test: np.ndarray, y_test: np.array, experiments: Dict) -> Dict: model_config = find_best_model(experiments) model = torch.load(model_config['model_path']) theta = model_config['threshold'] y_pred = model.predict(x_test) # return reconstruction errors np.savez('preds', y_pred=y_pred, y_test=y_test) auc_score = roc_auc_score(y_test, y_pred) y_pred = classify(y_pred, theta) metrics_report(y_test, y_pred) # print('# trainable params:', sum(p.numel() for p in model._model.parameters() if p.requires_grad), ',# params:', sum(p.numel() for p in model._model.parameters())) return create_report( model_config, { **get_metrics(y_test, y_pred), 'auc_score': float(auc_score) })
def get_extracted_features(x_train: List, x_val: List, x_test: List, y_val: np.array): sc = CustomMinMaxScaler() x_train = sc.fit_transform(x_train) x_val = sc.transform(x_val) x_test = sc.transform(x_test) model = torch.load( '../../models/aetcn/5d9ad591-6d3c-428f-894f-02af96ca1930.pt') y_pred = model.predict(x_val) # return reconstruction errors train_features = model.extract_features(x_train).astype(dtype=np.float32) val_features = model.extract_features(x_val).astype(dtype=np.float32) test_features = model.extract_features(x_test).astype(dtype=np.float32) theta, f1 = find_optimal_threshold(y_val, y_pred) y_pred = classify(y_pred, theta) metrics_report(y_val, y_pred) return train_features, val_features, test_features
def random_search(data_and_labels: tuple, model: Union[AutoEncoder, IsolationForest], params: Dict) -> Dict: x_train, x_test, _, y_test = data_and_labels scores = [] for conf in zip(*params.values()): kwargs = {k: val for k, val in zip(params.keys(), conf)} model.set_params(**kwargs) print(f'Model current hyperparameters are: {kwargs}.') model.fit(x_train) y_pred = model.predict(x_test) # return reconstruction errors theta, f1 = find_optimal_threshold(y_test, y_pred) y_pred = classify(y_pred, theta) metrics_report(y_test, y_pred) scores.append( create_experiment_report(get_metrics(y_test, y_pred), kwargs)) create_checkpoint({'experiments': scores}, EXPERIMENT_PATH) return {'experiments': scores}
def train_window(x_train: List, x_test: List, y_train: np.array, y_test: np.array) -> Dict: sc = CustomMinMaxScaler() x_train = sc.fit_transform(x_train) x_test = sc.transform(x_test) scores = [] for w in range(1, 50, 2): print('Window:', w) model = VanillaTCN(epochs=1, window=w) model.fit(x_train[y_train == 0]) y_pred = model.predict(x_test) # return reconstruction errors theta, f1 = find_optimal_threshold(y_test, y_pred) y_pred = classify(y_pred, theta) metrics_report(y_test, y_pred) scores.append( create_experiment_report(get_metrics(y_test, y_pred), {'window': w})) create_checkpoint( {'experiments': scores}, '../../models/TCN-cropped-window-embeddings-HDFS1.json') return {'experiments': scores}
"model_path": "../../models/aetcn/4f5f4682-1ca5-400a-a340-6243716690c0.pt", "threshold": 0.00331703620031476 } X = load_pickle_file( '../../data/processed/HDFS1/X-val-HDFS1-cv1-1-block.npy')[:1000] y = np.load('../../data/processed/HDFS1/y-val-HDFS1-cv1-1-block.npy')[:1000] # list of matrices, a matrix == blk_id -> NumPy [[005515, ...], ...] (n_logs x 100) # F1 = (2 * r * p) / (r + p) n_examples = 700 sc = CustomMinMaxScaler() # range 0 -- 1 x_train = sc.fit_transform(X[:n_examples]) y_train = y[:n_examples] x_test = sc.transform(X[n_examples:]) y_test = y[n_examples:] model = AETCN() model.set_params(**config['hyperparameters']) model.fit(x_train[y_train == 0]) # 0 -> normal, 1 -> anomaly y_pred = model.predict(x_test) # return reconstruction errors theta, f1 = find_optimal_threshold(y_test, y_pred) y_pred = classify(y_pred, theta) metrics_report(y_test, y_pred) confusion_matrix(y_test, y_pred)