def test_batch_engine_interactor(self):
stream_builder = DummyStreamBuilder()
interactor = StreamEngineInteractor(stream_builder,
DummyEngineBuilder(),
DummyMessageHandler())
interactor.run().to_blocking()
for i, item in enumerate(stream_builder.pushed):
output = json.loads(item)
self.assertEqual(output["agg_value"], i)
self.assertEqual(
AnomalyResult(**output["anomaly_results"]),
AnomalyResult(-10 * i, 10 * i, math.pow(i, 0.5), bool(i % 2)))
class SQLiteObservableRepository(unittest.TestCase, LoggingMixin):
DB_FILE = "/tmp/test_sqlite_observable.sqlite"
ANOM = AnomalyResult(-1, 1, 0.5, False)
@classmethod
def setUpClass(cls):
super().setUpClass()
cls.repo = SQLiteRepository(cls.DB_FILE)
cls.repo.initialize()
cls.repo.insert(
OutputMessage("app",
cls.ANOM,
1,
AggregationFunction.NONE,
1,
ts=datetime.now()))
cls.repo.insert(
OutputMessage("app",
cls.ANOM,
1,
AggregationFunction.NONE,
4,
ts=datetime.now()))
cls.repo.insert(
OutputMessage("app",
cls.ANOM,
1,
AggregationFunction.NONE,
3,
ts=datetime.now()))
cls.repo.insert(
OutputMessage("app",
cls.ANOM,
1,
AggregationFunction.NONE,
2,
ts=datetime.now()))
@classmethod
def tearDownClass(cls):
super().tearDownClass()
os.remove(cls.DB_FILE)
def __init__(self, methodName='runTest'):
super().__init__(methodName)
def test_observable_sqlite(self):
obs_rep = ObservableRepository(self.repo, application="app")
obs_rep.get_observable() \
.subscribe(lambda x: self.logger.debug(str(x)))
def test_get_min(self):
obs_rep = ObservableRepository(self.repo, application="app")
self.assertEqual(1, obs_rep.get_min())
def test_get_max(self):
obs_rep = ObservableRepository(self.repo, application="app")
self.assertEqual(4, obs_rep.get_max())
def predict(self, value: float, **kwargs) -> AnomalyResult:
anomaly_score = self.get_anomaly_score(
{"timestamp": kwargs["ts"], "value": value})
return AnomalyResult(0.0,
0.0,
anomaly_score,
(anomaly_score >= self.threshold))
def test_batch_engine_interactor(self):
interactor = BatchEngineInteractor(DummyBatch(), DummyEngineBuilder(),
DummyMessageHandler())
for i, item in enumerate(interactor.process()):
self.assertEqual(item.agg_value, i)
self.assertEqual(
item.anomaly_results,
AnomalyResult(-10 * i, 10 * i, math.pow(i, 0.5), bool(i % 2)))
def map(self, item: Row) -> OutputMessage:
anom_results = AnomalyResult(value_lower_limit=item[7],
value_upper_limit=item[5],
anomaly_probability=item[6],
is_anomaly=item[8])
fmt = "%Y-%m-%d %H:%M:%S"
return OutputMessage(item[0],
anom_results,
agg_window_millis=item[4],
agg_function=AggregationFunction(item[2]),
agg_value=item[3],
ts=datetime.datetime.strptime(item[1][0:19], fmt))
def test_insert_fetch_map(self):
# Insert
anom_results = AnomalyResult(-10, 10, 0.5, False)
output_message = OutputMessage(self.APP, anom_results, 0,
AggregationFunction.NONE, 0,
datetime(2018, 1, 1, 2, 0))
self.repository.insert(output_message)
# Fetch
values = self.repository.fetch(self.APP, datetime(2018, 1, 1),
datetime(2018, 1, 2))
# Map
output_messages = list(map(self.repository.map, values))
self.assertDictEqual(output_message.to_dict(),
output_messages[0].to_dict())
def setUpClass(cls):
super().setUpClass()
cls.config = Config(
"test", open("{}/anomdec_home_test/anomdec.yml".format(TEST_PATH)))
conf = cls.config.get_as_dict()
repository = conf["test"][3][0].repository
repository.initialize()
anom = AnomalyResult(-1, 1, 0.5, False)
ini_ts = datetime.now() - timedelta(days=1)
for i in range(200):
repository.insert(
OutputMessage("app",
anom,
1,
AggregationFunction.NONE,
1,
ts=ini_ts + timedelta(minutes=1)))
def predict(self, value: float, **kwargs) -> AnomalyResult:
item = [value] + self._extractor(**kwargs)
self._update_buffer(item)
result = {}
if np.isnan(self._data).any():
result['anomaly_probability'] = -1
result['is_anomaly'] = -1
result['value_upper_limit'] = -1
result['value_lower_limit'] = -1
else:
prediction = self._ema(self._data, self.window)[0]
l_b = prediction - self._std * self.threshold
u_b = prediction + self._std * self.threshold
result['anomaly_probability'] = 0 if (u_b > value > l_b) else 1
result['is_anomaly'] = False if (u_b > value > l_b) else True
result['value_upper_limit'] = u_b
result['value_lower_limit'] = l_b
self._update(value=value)
return AnomalyResult(**result.copy())
def test_predict(self):
self.engine = RobustDetector(window=self.WINDOW, threshold=0.999)
test_values = [1, 2, 1, 2, 5, 5, 5, 2]
predictions = [
AnomalyResult(-1, -1, -1, -1),
AnomalyResult(-1, -1, -1, -1),
AnomalyResult(-1, -1, -1, -1),
AnomalyResult(-1, -1, -1, -1),
AnomalyResult(-0.7907926364110414, 3.7907926364110414,
0.9999988290287686, 1),
AnomalyResult(-0.29079263641104136, 4.290792636411041,
0.999974054066204, 1),
AnomalyResult(-3.3723779092331236, 10.372377909233123, 0.75, 0),
AnomalyResult(5.0, 5.0, 1.0, 1),
]
for i, val in enumerate(test_values):
actual = self.engine.predict(val).to_dict()
expected = predictions[i].to_dict()
self.assertDictEqual(actual, expected)
def predict(self, value: float, **kwargs) -> AnomalyResult:
results = {}
if np.isnan(self._data).any():
results['anomaly_probability'] = -1
results['is_anomaly'] = -1
results['value_upper_limit'] = -1
results['value_lower_limit'] = -1
else:
if self._mad != 0:
z_score = np.abs(value - self._median) / self._mad
else:
z_score = np.inf
results['anomaly_probability'] = 1 - st.norm.sf(z_score)
results['is_anomaly'] = int(
results['anomaly_probability'] >= self.threshold)
results['value_upper_limit'] = \
(self._median + self._mad*st.norm.ppf(self.threshold))
results['value_lower_limit'] = \
(self._median - self._mad*st.norm.ppf(self.threshold))
self._update(value=value)
return AnomalyResult(**results)
def predict(self, value: float, **kwargs) -> AnomalyResult:
return AnomalyResult(-1, 1, 0, False)
def predict(self, value: float, **kwargs) -> AnomalyResult:
return AnomalyResult(-10 * value, 10 * value, math.pow(value, 0.5),
bool(value % 2))