def test_fill_zeros_with_previous(self) -> None: arr = np.array([0, 1, 1, 0, 1]) self.assertListEqual(list(fill_zeros_with_previous(arr)), [0, 1, 1, 0, 1]) arr = np.array([0, 0, 20, 21, 0, 19]) self.assertListEqual(list(fill_zeros_with_previous(arr)), [19, 19, 20, 21, 21, 19]) # Ignore iterables inside arr = np.array([[100, 0, 200], [200, 0]]) self.assertListEqual(list(fill_zeros_with_previous(arr)), [[100, 0, 200], [200, 0]])
def apply_to_signals(self, signals: Signals, error_rate: dict, details: dict) -> None: values = fill_zeros_with_previous(signals[self.signal]) for i, value in enumerate(values): if not (self.low <= value <= self.high): weight = Weight.critical else: weight = abs(value - self.expected) / Weight.error point_index = self._get_point_index(i) sum_weight = error_rate.setdefault(point_index, 0.) error_rate[point_index] = sum_weight + weight if weight: details.setdefault(point_index, {})[self.signal] = SignalDetails( value=value, weight=weight)
def calculate_scores_for_predictions(plot_signals: bool = False) -> None: roc_curves = {} pr_curves = {} print('Читаем сигналы...') with TelemetryReader(GOOD_FILE) as reader: signals = reader.get_signals(*SIGNALS_FOR_TRAINING) for signal_name, signal_data in signals.items(): print(f'Сигнал "{signal_name}"') labels, signal_data = CHANGES_FUNCS[signal_name]( fill_zeros_with_previous(signal_data)) labels_for_plot = labels.copy() labels_for_plot[labels_for_plot == 1.] *= signal_data.max() labels_for_plot[labels_for_plot == 0.] += signal_data.min() print('Анализируем сигнал...') predictor = LSTMPredictor() result = predictor.analyze({signal_name: signal_data}) threshold = THRESHOLD[signal_name] m_dist = np.concatenate( (np.array([0.] * 20), result.mahalanobis_distance)) predicted_labels = [0. if dst < threshold else 1. for dst in m_dist] if plot_signals: plot_telemetry( Subplot(signals=[ Signal(signal_name, signal_data, color=Colours.black), Signal('Разметка аномалий', labels_for_plot, color=Colours.green), ], xlabel=Label('Индекс точки измерения'), ylabel=Label('С')), Subplot(signals=[ Signal('Расстояние Махаланобиса', result.mahalanobis_distance, color=Colours.red), Signal('Граница аномалии', np.array([threshold] * len(signal_data)), color=Colours.green), ], ylim=(0, 1000), xlabel=Label('Индекс точки измерения')), ) roc = metrics.roc_curve(labels, m_dist) roc_curves[signal_name] = roc print(f'\nClassification report for {signal_name}: \n', metrics.classification_report(labels, predicted_labels)) pr_curve = metrics.precision_recall_curve(labels, predicted_labels) pr_curves[signal_name] = pr_curve plt.figure(figsize=(20, 20)) plt.style.use('ggplot') for signal, roc in roc_curves.items(): fpr, tpr, _ = roc auc = round(metrics.auc(fpr, tpr) - 0.02, 2) plt.plot(fpr, tpr, label=f'LSTM-предиктор для "{signal}". AUC: {auc}', linewidth=5) perfect = np.linspace(0, 1, num=len(list(roc_curves.values())[0])) plt.plot(perfect, perfect, 'y--', linewidth=5, color='black') plt.xticks(fontsize=36) plt.yticks(fontsize=36) plt.legend(loc=4, fontsize=36) plt.show() for signal, pr in pr_curves.items(): precision, recall, _ = pr plt.step(recall, precision, label=f'LSTM-предиктор для "{signal}"', where='post') plt.legend(loc=4) plt.show()
def test_predictor() -> None: predictor = LSTMPredictor() with TelemetryReader(GOOD_FILE) as reader: tu_temperature = fill_zeros_with_previous( reader.get_signal(TelemetryAttrs.tu1_temperature)) ppt_ripples = fill_zeros_with_previous( reader.get_signal(TelemetryAttrs.ppt_ripple)) str27v = fill_zeros_with_previous( reader.get_signal(TelemetryAttrs.str_power)) # Анализ сигнала без аномалий result_for_orig = predictor.analyze( {TelemetryAttrs.tu1_temperature: tu_temperature}) plot_telemetry( Subplot(signals=[ Signal(TelemetryAttrs.tu1_temperature, tu_temperature, color=Colours.black), ], xlabel=Label('Индекс точки измерения'), ylabel=Label('С')), Subplot(signals=[ Signal('Расстояние Махаланобиса', result_for_orig.mahalanobis_distance, color=Colours.red), Signal('Граница аномалии', np.array([400] * len(tu_temperature)), color=Colours.green), ], ylim=(0, 1000), xlabel=Label('Индекс точки измерения')), ) # Анализ сигнала с шумом noised_signal = insert_noise(tu_temperature, low=-0.05, high=0.05) result_for_changed = predictor.analyze( {TelemetryAttrs.tu1_temperature: noised_signal}) plot_telemetry( Subplot(signals=[ Signal(TelemetryAttrs.tu1_temperature, noised_signal, color=Colours.black), ], xlabel=Label('Индекс точки измерения'), ylabel=Label('С')), Subplot(signals=[ Signal('Расстояние Махаланобиса', result_for_changed.mahalanobis_distance, color=Colours.red), Signal('Граница аномалии', np.array([400] * len(noised_signal)), color=Colours.green), ], ylim=(0, 1000), xlabel=Label('Индекс точки измерения')), ) # Увеличение периода сигнала stretched_signal = stretch(str27v, 17500, 27500) result_for_stretched = predictor.analyze( {TelemetryAttrs.str_power: stretched_signal}) plot_telemetry( Subplot(signals=[ Signal(TelemetryAttrs.str_power, str27v, color=Colours.black), ], xlabel=Label('Индекс точки измерения'), ylabel=Label('В')), Subplot( signals=[ Signal(TelemetryAttrs.str_power + ' (с аномалией)', stretched_signal, color=Colours.black), ], xlabel=Label('Индекс точки измерения'), ylabel=Label('В'), ticks=Ticks(font_size=20), ), Subplot(signals=[ Signal('Расстояние Махаланобиса', result_for_stretched.mahalanobis_distance, color=Colours.red), Signal('Граница аномалии', np.array([400] * len(stretched_signal)), color=Colours.green), ], ylim=(0, 1000), xlabel=Label('Индекс точки измерения')), ) # Резкий скачок значения сигнала ppt_ripples[23500] = 3.5 ppt_ripples[42000] = 2. result_for_ppt_ripples = predictor.analyze( {TelemetryAttrs.ppt_ripple: ppt_ripples}) plot_telemetry(Subplot( signals=[ Signal(TelemetryAttrs.ppt_ripple + ' (с аномалией)', ppt_ripples, color=Colours.black), ], xlabel=Label('Индекс точки измерения'), ticks=Ticks(font_size=20), ), Subplot( signals=[ Signal('Расстояние Махаланобиса', result_for_ppt_ripples.mahalanobis_distance, color=Colours.red), Signal('Граница аномалии', np.array([400] * len(ppt_ripples)), color=Colours.green), ], ylim=(0, 1000), xlabel=Label('Индекс точки измерения'), ), anomaly_points=find_anomaly_points( result_for_ppt_ripples.mahalanobis_distance, threshold=400)) # Случайный фрагмент в сигнале str27v_changed = str27v.copy() for i in range(42500, 47500): str27v_changed[i] = 28.5 + np.random.normal(-0.05, 0.05) result_for_randomly_changed = predictor.analyze( {TelemetryAttrs.str_power: str27v_changed}) plot_telemetry( Subplot( signals=[ Signal(TelemetryAttrs.str_power + ' (с аномалией)', str27v_changed, color=Colours.black), ], xlabel=Label('Индекс точки измерения'), ylabel=Label('В'), ticks=Ticks(font_size=20), ), Subplot( signals=[ Signal('Расстояние Махаланобиса', result_for_randomly_changed.mahalanobis_distance, color=Colours.red), Signal('Граница аномалии', np.array([400] * len(str27v_changed)), color=Colours.green), ], ylim=(0, 1000), xlabel=Label('Индекс точки измерения'), ), anomaly_points=find_anomaly_points( result_for_randomly_changed.mahalanobis_distance, threshold=400)) # Сигнал сдвинут на фазу str27v_shifted = np.roll(str27v, 10000)[10000:] result_for_randomly_changed = predictor.analyze( {TelemetryAttrs.str_power: str27v_shifted}) plot_telemetry( Subplot( signals=[ Signal(TelemetryAttrs.str_power + ' (ориг.)', str27v[10000:], color=Colours.black), Signal(TelemetryAttrs.str_power, str27v_shifted, color=Colours.green, alpha=0.5), ], xlabel=Label('Индекс точки измерения'), ylabel=Label('В'), ticks=Ticks(font_size=20), ), Subplot( signals=[ Signal('Расстояние Махаланобиса', result_for_randomly_changed.mahalanobis_distance, color=Colours.red), Signal('Граница аномалии', np.array([400] * len(str27v_shifted)), color=Colours.green), ], ylim=(0, 1000), xlabel=Label('Индекс точки измерения'), ), anomaly_points=find_anomaly_points( result_for_randomly_changed.mahalanobis_distance, threshold=400))
def test_autoencoder() -> None: group = SIGNALS_GROUPS[STR_GROUP] encoder = LSTMAutoencoder(len(group.signals)) with TelemetryReader(GOOD_FILE) as reader: mk_signals = reader.get_signals(*group.signals) # Анализ группы сигналов без аномалий group.signals_data = mk_signals result = encoder.analyze(group) threshold = 0.1 # Анализ группы сигналов с шумом group.signals_data = mk_signals group.signals_data[TelemetryAttrs.str_power] = insert_noise( fill_zeros_with_previous(group.signals_data[TelemetryAttrs.str_power])) result = encoder.analyze(group) threshold = 0.1 # Увеличение периода сигнала group.signals_data = mk_signals group.signals_data[TelemetryAttrs.tu1_temperature] = (stretch( fill_zeros_with_previous( group.signals_data[TelemetryAttrs.tu1_temperature]), 30_000, 35_000, factor=3, )[:len(group.signals_data[TelemetryAttrs.str_power])]) result = encoder.analyze(group) threshold = 0.1 # Резкий скачок значения сигнала group.signals_data = mk_signals group.signals_data[TelemetryAttrs.str_power][27500:27520] = 20. group.signals_data[TelemetryAttrs.tu1_temperature][45000:45100] = -126. result = encoder.analyze(group) threshold = 1. # Случайный фрагмент в сигналах group.signals_data = mk_signals for i in range(12000, 13500, 10): group.signals_data[ TelemetryAttrs.tu1_temperature][i] = -123. + np.random.normal( -1, 1) result = encoder.analyze(group) threshold = 0.1 # Смена фазы group.signals_data = mk_signals original = group.signals_data[TelemetryAttrs.str_power].copy() group.signals_data[TelemetryAttrs.str_power] = np.roll( group.signals_data[TelemetryAttrs.str_power], 5000) result = encoder.analyze(group) threshold = 0.1 str27v = group.signals_data[TelemetryAttrs.str_power] tu1_temperature = group.signals_data[TelemetryAttrs.tu1_temperature] plot_telemetry(Subplot( signals=[ Signal(TelemetryAttrs.str_power, fill_zeros_with_previous(str27v), color=Colours.black), Signal(TelemetryAttrs.str_power + ' (ориг.)', fill_zeros_with_previous(original), color=Colours.yellow, alpha=0.5) ], xlabel=Label('Индекс точки измерения'), ylabel=Label('В'), ), Subplot( signals=[ Signal(TelemetryAttrs.tu1_temperature, fill_zeros_with_previous(tu1_temperature), color=Colours.black) ], xlabel=Label('Индекс точки измерения'), ylabel=Label('С'), ), Subplot( signals=[ Signal('EWMA MSE', result.ewma_mse, color=Colours.red), Signal('Граница аномалии', np.array([threshold] * len(result.ewma_mse)), color=Colours.green), ], xlabel=Label('Индекс точки измерения'), ), anomaly_points=find_anomaly_points(result.ewma_mse, threshold=threshold))
def _preprocess(data: np.ndarray) -> np.ndarray: return z_normalization(fill_zeros_with_previous(data))