Пример #1
0
def _evaluate_on_signal(pipeline, signal, metrics):
    data = load_signal(signal)
    anomalies = analyze(pipeline, data)

    truth = load_anomalies(signal)

    return {
        name: scorer(truth, anomalies, data)
        for name, scorer in metrics.items()
    }
Пример #2
0
def _evaluate_signal(pipeline,
                     name,
                     dataset,
                     signal,
                     hyperparameter,
                     metrics,
                     test_split=False,
                     detrend=False):

    train, test = _load_signal(signal, test_split)
    truth = load_anomalies(signal)

    if detrend:
        train = _detrend_signal(train, 'value')
        test = _detrend_signal(test, 'value')

    try:
        LOGGER.info("Scoring pipeline %s on signal %s (test split: %s)", name,
                    signal, test_split)

        start = datetime.utcnow()
        pipeline = _load_pipeline(pipeline, hyperparameter)
        anomalies = analyze(pipeline, train, test)
        elapsed = datetime.utcnow() - start

        scores = {
            name: scorer(truth, anomalies, test)
            for name, scorer in metrics.items()
        }
        scores['status'] = 'OK'

    except Exception as ex:
        LOGGER.exception(
            "Exception scoring pipeline %s on signal %s (test split: %s), error %s.",
            name, signal, test_split, ex)

        elapsed = datetime.utcnow() - start
        scores = {name: 0 for name in metrics.keys()}

        metric_ = 'confusion_matrix'
        if metric_ in metrics.keys():
            fn = len(truth)
            scores[metric_] = (None, 0, fn, 0)  # (tn, fp, fn, tp)

        scores['status'] = 'ERROR'

    scores['elapsed'] = elapsed.total_seconds()
    scores['pipeline'] = name
    scores['split'] = test_split
    scores['dataset'] = dataset
    scores['signal'] = signal

    return scores
Пример #3
0
def _evaluate_signal(pipeline,
                     signal,
                     hyperparameter,
                     metrics,
                     test_split=False,
                     detrend=False,
                     pipeline_path=None):

    train, test = _load_signal(signal, test_split)
    truth = load_anomalies(signal)

    if detrend:
        train = _detrend_signal(train, 'value')
        test = _detrend_signal(test, 'value')

    try:
        LOGGER.info("Scoring pipeline %s on signal %s (test split: %s)",
                    pipeline, signal, test_split)

        start = datetime.utcnow()
        pipeline = _load_pipeline(pipeline, hyperparameter)
        anomalies = analyze(pipeline, train, test)
        elapsed = datetime.utcnow() - start

        scores = {
            name: scorer(truth, anomalies, test)
            for name, scorer in metrics.items()
        }

        status = 'OK'

    except Exception as ex:
        LOGGER.exception(
            "Exception scoring pipeline %s on signal %s (test split: %s), error %s.",
            pipeline, signal, test_split, ex)

        elapsed = datetime.utcnow() - start
        scores = {name: 0 for name in metrics.keys()}

        status = 'ERROR'

    if 'confusion_matrix' in metrics.keys():
        _parse_confusion_matrix(scores, truth)

    scores['status'] = status
    scores['elapsed'] = elapsed.total_seconds()
    scores['split'] = test_split

    if pipeline_path:
        with open(pipeline_path, 'wb') as f:
            pickle.dump(pipeline, f)

    return scores
Пример #4
0
def _evaluate_on_signal(pipeline, signal, metrics, holdout=True):
    if holdout:
        train = load_signal(signal + '-train')
    else:
        train = load_signal(signal)

    test = load_signal(signal + '-test')
    start = datetime.utcnow()
    anomalies = analyze(pipeline, train, test)
    elapsed = datetime.utcnow() - start

    truth = load_anomalies(signal)

    scores = {
        name: scorer(truth, anomalies, test)
        for name, scorer in metrics.items()
    }
    scores['elapsed'] = elapsed.total_seconds()

    return scores