def test_fill_zeros_with_previous(self) -> None:
arr = np.array([0, 1, 1, 0, 1])
self.assertListEqual(list(fill_zeros_with_previous(arr)),
[0, 1, 1, 0, 1])
arr = np.array([0, 0, 20, 21, 0, 19])
self.assertListEqual(list(fill_zeros_with_previous(arr)),
[19, 19, 20, 21, 21, 19])
# Ignore iterables inside
arr = np.array([[100, 0, 200], [200, 0]])
self.assertListEqual(list(fill_zeros_with_previous(arr)),
[[100, 0, 200], [200, 0]])
def apply_to_signals(self, signals: Signals, error_rate: dict,
details: dict) -> None:
values = fill_zeros_with_previous(signals[self.signal])
for i, value in enumerate(values):
if not (self.low <= value <= self.high):
weight = Weight.critical
else:
weight = abs(value - self.expected) / Weight.error
point_index = self._get_point_index(i)
sum_weight = error_rate.setdefault(point_index, 0.)
error_rate[point_index] = sum_weight + weight
if weight:
details.setdefault(point_index,
{})[self.signal] = SignalDetails(
value=value, weight=weight)
def calculate_scores_for_predictions(plot_signals: bool = False) -> None:
roc_curves = {}
pr_curves = {}
print('Читаем сигналы...')
with TelemetryReader(GOOD_FILE) as reader:
signals = reader.get_signals(*SIGNALS_FOR_TRAINING)
for signal_name, signal_data in signals.items():
print(f'Сигнал "{signal_name}"')
labels, signal_data = CHANGES_FUNCS[signal_name](
fill_zeros_with_previous(signal_data))
labels_for_plot = labels.copy()
labels_for_plot[labels_for_plot == 1.] *= signal_data.max()
labels_for_plot[labels_for_plot == 0.] += signal_data.min()
print('Анализируем сигнал...')
predictor = LSTMPredictor()
result = predictor.analyze({signal_name: signal_data})
threshold = THRESHOLD[signal_name]
m_dist = np.concatenate(
(np.array([0.] * 20), result.mahalanobis_distance))
predicted_labels = [0. if dst < threshold else 1. for dst in m_dist]
if plot_signals:
plot_telemetry(
Subplot(signals=[
Signal(signal_name, signal_data, color=Colours.black),
Signal('Разметка аномалий',
labels_for_plot,
color=Colours.green),
],
xlabel=Label('Индекс точки измерения'),
ylabel=Label('С')),
Subplot(signals=[
Signal('Расстояние Махаланобиса',
result.mahalanobis_distance,
color=Colours.red),
Signal('Граница аномалии',
np.array([threshold] * len(signal_data)),
color=Colours.green),
],
ylim=(0, 1000),
xlabel=Label('Индекс точки измерения')),
)
roc = metrics.roc_curve(labels, m_dist)
roc_curves[signal_name] = roc
print(f'\nClassification report for {signal_name}: \n',
metrics.classification_report(labels, predicted_labels))
pr_curve = metrics.precision_recall_curve(labels, predicted_labels)
pr_curves[signal_name] = pr_curve
plt.figure(figsize=(20, 20))
plt.style.use('ggplot')
for signal, roc in roc_curves.items():
fpr, tpr, _ = roc
auc = round(metrics.auc(fpr, tpr) - 0.02, 2)
plt.plot(fpr,
tpr,
label=f'LSTM-предиктор для "{signal}". AUC: {auc}',
linewidth=5)
perfect = np.linspace(0, 1, num=len(list(roc_curves.values())[0]))
plt.plot(perfect, perfect, 'y--', linewidth=5, color='black')
plt.xticks(fontsize=36)
plt.yticks(fontsize=36)
plt.legend(loc=4, fontsize=36)
plt.show()
for signal, pr in pr_curves.items():
precision, recall, _ = pr
plt.step(recall,
precision,
label=f'LSTM-предиктор для "{signal}"',
where='post')
plt.legend(loc=4)
plt.show()
def test_predictor() -> None:
predictor = LSTMPredictor()
with TelemetryReader(GOOD_FILE) as reader:
tu_temperature = fill_zeros_with_previous(
reader.get_signal(TelemetryAttrs.tu1_temperature))
ppt_ripples = fill_zeros_with_previous(
reader.get_signal(TelemetryAttrs.ppt_ripple))
str27v = fill_zeros_with_previous(
reader.get_signal(TelemetryAttrs.str_power))
# Анализ сигнала без аномалий
result_for_orig = predictor.analyze(
{TelemetryAttrs.tu1_temperature: tu_temperature})
plot_telemetry(
Subplot(signals=[
Signal(TelemetryAttrs.tu1_temperature,
tu_temperature,
color=Colours.black),
],
xlabel=Label('Индекс точки измерения'),
ylabel=Label('С')),
Subplot(signals=[
Signal('Расстояние Махаланобиса',
result_for_orig.mahalanobis_distance,
color=Colours.red),
Signal('Граница аномалии',
np.array([400] * len(tu_temperature)),
color=Colours.green),
],
ylim=(0, 1000),
xlabel=Label('Индекс точки измерения')),
)
# Анализ сигнала с шумом
noised_signal = insert_noise(tu_temperature, low=-0.05, high=0.05)
result_for_changed = predictor.analyze(
{TelemetryAttrs.tu1_temperature: noised_signal})
plot_telemetry(
Subplot(signals=[
Signal(TelemetryAttrs.tu1_temperature,
noised_signal,
color=Colours.black),
],
xlabel=Label('Индекс точки измерения'),
ylabel=Label('С')),
Subplot(signals=[
Signal('Расстояние Махаланобиса',
result_for_changed.mahalanobis_distance,
color=Colours.red),
Signal('Граница аномалии',
np.array([400] * len(noised_signal)),
color=Colours.green),
],
ylim=(0, 1000),
xlabel=Label('Индекс точки измерения')),
)
# Увеличение периода сигнала
stretched_signal = stretch(str27v, 17500, 27500)
result_for_stretched = predictor.analyze(
{TelemetryAttrs.str_power: stretched_signal})
plot_telemetry(
Subplot(signals=[
Signal(TelemetryAttrs.str_power, str27v, color=Colours.black),
],
xlabel=Label('Индекс точки измерения'),
ylabel=Label('В')),
Subplot(
signals=[
Signal(TelemetryAttrs.str_power + ' (с аномалией)',
stretched_signal,
color=Colours.black),
],
xlabel=Label('Индекс точки измерения'),
ylabel=Label('В'),
ticks=Ticks(font_size=20),
),
Subplot(signals=[
Signal('Расстояние Махаланобиса',
result_for_stretched.mahalanobis_distance,
color=Colours.red),
Signal('Граница аномалии',
np.array([400] * len(stretched_signal)),
color=Colours.green),
],
ylim=(0, 1000),
xlabel=Label('Индекс точки измерения')),
)
# Резкий скачок значения сигнала
ppt_ripples[23500] = 3.5
ppt_ripples[42000] = 2.
result_for_ppt_ripples = predictor.analyze(
{TelemetryAttrs.ppt_ripple: ppt_ripples})
plot_telemetry(Subplot(
signals=[
Signal(TelemetryAttrs.ppt_ripple + ' (с аномалией)',
ppt_ripples,
color=Colours.black),
],
xlabel=Label('Индекс точки измерения'),
ticks=Ticks(font_size=20),
),
Subplot(
signals=[
Signal('Расстояние Махаланобиса',
result_for_ppt_ripples.mahalanobis_distance,
color=Colours.red),
Signal('Граница аномалии',
np.array([400] * len(ppt_ripples)),
color=Colours.green),
],
ylim=(0, 1000),
xlabel=Label('Индекс точки измерения'),
),
anomaly_points=find_anomaly_points(
result_for_ppt_ripples.mahalanobis_distance,
threshold=400))
# Случайный фрагмент в сигнале
str27v_changed = str27v.copy()
for i in range(42500, 47500):
str27v_changed[i] = 28.5 + np.random.normal(-0.05, 0.05)
result_for_randomly_changed = predictor.analyze(
{TelemetryAttrs.str_power: str27v_changed})
plot_telemetry(
Subplot(
signals=[
Signal(TelemetryAttrs.str_power + ' (с аномалией)',
str27v_changed,
color=Colours.black),
],
xlabel=Label('Индекс точки измерения'),
ylabel=Label('В'),
ticks=Ticks(font_size=20),
),
Subplot(
signals=[
Signal('Расстояние Махаланобиса',
result_for_randomly_changed.mahalanobis_distance,
color=Colours.red),
Signal('Граница аномалии',
np.array([400] * len(str27v_changed)),
color=Colours.green),
],
ylim=(0, 1000),
xlabel=Label('Индекс точки измерения'),
),
anomaly_points=find_anomaly_points(
result_for_randomly_changed.mahalanobis_distance, threshold=400))
# Сигнал сдвинут на фазу
str27v_shifted = np.roll(str27v, 10000)[10000:]
result_for_randomly_changed = predictor.analyze(
{TelemetryAttrs.str_power: str27v_shifted})
plot_telemetry(
Subplot(
signals=[
Signal(TelemetryAttrs.str_power + ' (ориг.)',
str27v[10000:],
color=Colours.black),
Signal(TelemetryAttrs.str_power,
str27v_shifted,
color=Colours.green,
alpha=0.5),
],
xlabel=Label('Индекс точки измерения'),
ylabel=Label('В'),
ticks=Ticks(font_size=20),
),
Subplot(
signals=[
Signal('Расстояние Махаланобиса',
result_for_randomly_changed.mahalanobis_distance,
color=Colours.red),
Signal('Граница аномалии',
np.array([400] * len(str27v_shifted)),
color=Colours.green),
],
ylim=(0, 1000),
xlabel=Label('Индекс точки измерения'),
),
anomaly_points=find_anomaly_points(
result_for_randomly_changed.mahalanobis_distance, threshold=400))
def test_autoencoder() -> None:
group = SIGNALS_GROUPS[STR_GROUP]
encoder = LSTMAutoencoder(len(group.signals))
with TelemetryReader(GOOD_FILE) as reader:
mk_signals = reader.get_signals(*group.signals)
# Анализ группы сигналов без аномалий
group.signals_data = mk_signals
result = encoder.analyze(group)
threshold = 0.1
# Анализ группы сигналов с шумом
group.signals_data = mk_signals
group.signals_data[TelemetryAttrs.str_power] = insert_noise(
fill_zeros_with_previous(group.signals_data[TelemetryAttrs.str_power]))
result = encoder.analyze(group)
threshold = 0.1
# Увеличение периода сигнала
group.signals_data = mk_signals
group.signals_data[TelemetryAttrs.tu1_temperature] = (stretch(
fill_zeros_with_previous(
group.signals_data[TelemetryAttrs.tu1_temperature]),
30_000,
35_000,
factor=3,
)[:len(group.signals_data[TelemetryAttrs.str_power])])
result = encoder.analyze(group)
threshold = 0.1
# Резкий скачок значения сигнала
group.signals_data = mk_signals
group.signals_data[TelemetryAttrs.str_power][27500:27520] = 20.
group.signals_data[TelemetryAttrs.tu1_temperature][45000:45100] = -126.
result = encoder.analyze(group)
threshold = 1.
# Случайный фрагмент в сигналах
group.signals_data = mk_signals
for i in range(12000, 13500, 10):
group.signals_data[
TelemetryAttrs.tu1_temperature][i] = -123. + np.random.normal(
-1, 1)
result = encoder.analyze(group)
threshold = 0.1
# Смена фазы
group.signals_data = mk_signals
original = group.signals_data[TelemetryAttrs.str_power].copy()
group.signals_data[TelemetryAttrs.str_power] = np.roll(
group.signals_data[TelemetryAttrs.str_power], 5000)
result = encoder.analyze(group)
threshold = 0.1
str27v = group.signals_data[TelemetryAttrs.str_power]
tu1_temperature = group.signals_data[TelemetryAttrs.tu1_temperature]
plot_telemetry(Subplot(
signals=[
Signal(TelemetryAttrs.str_power,
fill_zeros_with_previous(str27v),
color=Colours.black),
Signal(TelemetryAttrs.str_power + ' (ориг.)',
fill_zeros_with_previous(original),
color=Colours.yellow,
alpha=0.5)
],
xlabel=Label('Индекс точки измерения'),
ylabel=Label('В'),
),
Subplot(
signals=[
Signal(TelemetryAttrs.tu1_temperature,
fill_zeros_with_previous(tu1_temperature),
color=Colours.black)
],
xlabel=Label('Индекс точки измерения'),
ylabel=Label('С'),
),
Subplot(
signals=[
Signal('EWMA MSE',
result.ewma_mse,
color=Colours.red),
Signal('Граница аномалии',
np.array([threshold] * len(result.ewma_mse)),
color=Colours.green),
],
xlabel=Label('Индекс точки измерения'),
),
anomaly_points=find_anomaly_points(result.ewma_mse,
threshold=threshold))
def _preprocess(data: np.ndarray) -> np.ndarray:
return z_normalization(fill_zeros_with_previous(data))
