Пример #1
0
def luminol_artificial(other=False):
    train = "../data/artificial/artificial_train.csv"
    test = "../data/artificial/artificial_test1.csv"
    if other:
        test = "../data/artificial/artificial_test2.csv"
    train, _ = loader_generic(train)
    test, labels = loader_generic(test)
    lag_size = 1500
    train = Dataset(train, 1, 0, lag_size, 1, 45).data
    test = Dataset(test, 1, 0, lag_size, 1, 45).data

    lumi_params = dict()
    lumi_params["precision"] = 8
    lumi_params["lag_window_size"] = lag_size
    lumi_params["future_window_size"] = 1500
    lumi_params["chunk_size"] = 7

    # put data as a dict, required by luminol
    processed_data = np.concatenate([train, test])
    ts = dict()
    for i, d in enumerate(processed_data):
        ts[i] = d
    detector = luminol.anomaly_detector.AnomalyDetector(
        ts, algorithm_params=lumi_params)

    # get scores
    score = detector.get_all_scores()
    scores = []
    for (timestamp, value) in score.iteritems():
        scores.append(value)
    # keep scores only related to test set
    scores = scores[len(train):]

    # normalize
    scores = np.array(scores)
    if scores.max() != 0:
        scores /= scores.max()

    beta = 1
    thres, f1, fpr, rfpr, p, r, tot_pred, tot_labels, tot_cor, rpc, roc = get_thres(
        labels[:len(scores)], scores, beta)
    print("beta 1")
    print(
        "thres %s |f1 %s |fpr %s |rfpr %s |p %s |r %s |tot_pred %s |tot_labels %s |tot_cor %s |rpc %s |roc %s"
        %
        (thres, f1, fpr, rfpr, p, r, tot_pred, tot_labels, tot_cor, rpc, roc))
    beta = 0.1
    thres, f1, fpr, rfpr, p, r, tot_pred, tot_labels, tot_cor, rpc, roc = get_thres(
        labels[:len(scores)], scores, beta)
    print("beta 0.1")
    print(
        "thres %s |f1 %s |fpr %s |rfpr %s |p %s |r %s |tot_pred %s |tot_labels %s |tot_cor %s |rpc %s |roc %s"
        %
        (thres, f1, fpr, rfpr, p, r, tot_pred, tot_labels, tot_cor, rpc, roc))
Пример #2
0
def som_taxi():
    train = "../data/taxi/nyc_taxi_train.csv"
    test = "../data/taxi/nyc_taxi_test.csv"
    train, _ = loader_generic(train)
    test, labels = loader_generic(test)
    wsize = 175
    train = Dataset(train, 1, 0, wsize, 1, 5)
    test = Dataset(test, 1, 0, wsize, 1, 5)

    detector = SomAnomalyDetector(8,
                                  8,
                                  wsize,
                                  5,
                                  0.001,
                                  1400,
                                  decay_factor=0.5)

    # use train set just to get statistics in the model
    for i in range(len(train)):
        detector.add_data_point(train[i])

    # pass on test set
    scores = np.zeros(len(test.data))
    for i in range(len(test)):
        window_score = detector.add_data_point(test[i])
        # update score of elements in window
        for u in range(i, i + wsize):
            scores[u] = max(scores[u], window_score)

    beta = 1
    thres, f1, fpr, rfpr, p, r, tot_pred, tot_labels, tot_cor, rpc, roc = get_thres(
        labels[:len(scores)], scores, beta)
    print("beta 1")
    print(
        "thres %s |f1 %s |fpr %s |rfpr %s |p %s |r %s |tot_pred %s |tot_labels %s |tot_cor %s |rpc %s |roc %s"
        %
        (thres, f1, fpr, rfpr, p, r, tot_pred, tot_labels, tot_cor, rpc, roc))
    beta = 0.1
    thres, f1, fpr, rfpr, p, r, tot_pred, tot_labels, tot_cor, rpc, roc = get_thres(
        labels[:len(scores)], scores, beta)
    print("beta 0.1")
    print(
        "thres %s |f1 %s |fpr %s |rfpr %s |p %s |r %s |tot_pred %s |tot_labels %s |tot_cor %s |rpc %s |roc %s"
        %
        (thres, f1, fpr, rfpr, p, r, tot_pred, tot_labels, tot_cor, rpc, roc))
Пример #3
0
def loda_artificial(other=False):
    train = "../data/artificial/artificial_train.csv"
    test = "../data/artificial/artificial_test1.csv"
    if other:
        test = "../data/artificial/artificial_test2.csv"
    train, _ = loader_generic(train)
    test, labels = loader_generic(test)
    wsize = 225
    train = Dataset(train, 0, 0, wsize, 1, 1)
    test = Dataset(test, 0, 0, wsize, 1, 1)

    detector = Loda(wsize, 1400)

    # use train set just to get statistics in the model
    for i in range(len(train)):
        detector.add_data_point(train[i])

    # pass on test set
    scores = np.zeros(len(test.data))
    for i in range(len(test)):
        window_score = detector.add_data_point(test[i])
        # update score of elements in window
        for u in range(i, i + wsize):
            scores[u] = max(scores[u], window_score)

    beta = 1
    thres, f1, fpr, rfpr, p, r, tot_pred, tot_labels, tot_cor, rpc, roc = get_thres(
        labels[:len(scores)], scores, beta)
    print("beta 1")
    print(
        "thres %s |f1 %s |fpr %s |rfpr %s |p %s |r %s |tot_pred %s |tot_labels %s |tot_cor %s |rpc %s |roc %s"
        %
        (thres, f1, fpr, rfpr, p, r, tot_pred, tot_labels, tot_cor, rpc, roc))
    beta = 0.1
    thres, f1, fpr, rfpr, p, r, tot_pred, tot_labels, tot_cor, rpc, roc = get_thres(
        labels[:len(scores)], scores, beta)
    print("beta 0.1")
    print(
        "thres %s |f1 %s |fpr %s |rfpr %s |p %s |r %s |tot_pred %s |tot_labels %s |tot_cor %s |rpc %s |roc %s"
        %
        (thres, f1, fpr, rfpr, p, r, tot_pred, tot_labels, tot_cor, rpc, roc))