def setUp(self):
self.db = "test-{}".format(int(datetime.datetime.now().timestamp()))
self.source = InfluxDataSource({
'name': 'test',
'addr': ADDR,
'database': self.db,
})
self.source.drop()
self.source.init()
self.storage = TempStorage()
generator = SinEventGenerator(base=3, sigma=0.05)
self.to_date = datetime.datetime.now().timestamp()
self.from_date = self.to_date - 3600 * 24 * 7
for ts in generator.generate_ts(
self.from_date,
self.to_date,
step_ms=60000,
):
self.source.insert_times_data(
measurement='measure1',
ts=ts,
data={'foo': random.lognormvariate(10, 1)},
)
self.source.commit()
def test_validation(self):
with self.assertRaises(errors.Invalid):
InfluxDataSource({
'addr': 'localhost',
})
with self.assertRaises(errors.Invalid):
InfluxDataSource({
'database': 'foo',
})
def setUp(self):
self.app = app.test_client()
self.app.testing = True
self.bucket_interval = 20 * 60
t0 = int(datetime.datetime.now().timestamp())
t0 -= t0 % self.bucket_interval
self.t0 = t0
self.db = 'test-{}'.format(t0)
logging.info("creating database %s", self.db)
self.source = InfluxDataSource({
'name': 'nose',
'addr': ADDR,
'database': self.db,
})
self.source.drop()
self.source.init()
self.generator = SinEventGenerator(base=3, amplitude=3, sigma=0.01)
to_date = datetime.datetime.now().timestamp()
# Be sure that date range is aligned
self.to_date = math.floor(
to_date / self.bucket_interval) * self.bucket_interval # noqa E501
self.from_date = self.to_date - 3600 * 24 * 7 * 3
for ts in self.generator.generate_ts(self.from_date,
self.to_date,
step_ms=600000): # noqa E501
self.source.insert_times_data(measurement='bar',
ts=ts,
data={
'foo':
random.normalvariate(10, 1),
})
self.source.commit()
self.dirpath = tempfile.mkdtemp()
configyml = os.path.join(self.dirpath, 'config.yml')
cfg = open(configyml, 'w')
cfg.write(CONFIG.format(ADDR, self.db, self.dirpath))
cfg.close()
g_app_init(configyml)
def test_build_times_queries(self):
where = "time >= 1515404366123400000 and time < 1515423565456000000"
queries = list(
self.source._build_times_queries(
self.model,
from_date=1515404366.1234,
to_date="2018-01-08T14:59:25.456Z",
))
self.assertEqual(
queries,
[
"select MEAN(\"foo\") as \"avg_foo\" from \"measure1\" "
"where {} group by time(3000ms);".format(where),
"select COUNT(\"bar\") as \"count_bar\" from \"measure2\" "
"where {} group by time(3000ms);".format(where),
"select MEAN(\"baz\") as \"avg_baz\" from \"measure1\" "
"where {} and \"mytag\"='myvalue' group by time(3000ms);".
format(where),
],
)
source = InfluxDataSource({
'name': 'test',
'addr': ADDR,
'database': self.db,
'retention_policy': 'custom',
})
queries = list(
source._build_times_queries(
self.model,
from_date=1515404366.1234,
to_date="2018-01-08T14:59:25.456Z",
))
from_prefix = '"{}"."custom".'.format(self.db)
self.assertEqual(
queries,
[
"select MEAN(\"foo\") as \"avg_foo\" from {}\"measure1\" "
"where {} group by time(3000ms);".format(from_prefix, where),
"select COUNT(\"bar\") as \"count_bar\" from {}\"measure2\" "
"where {} group by time(3000ms);".format(from_prefix, where),
"select MEAN(\"baz\") as \"avg_baz\" from {}\"measure1\" "
"where {} and \"mytag\"='myvalue' group by time(3000ms);".
format(from_prefix, where),
],
)
def setUp(self):
if 'LOUDML_ADDR' in os.environ:
self.loudml_addr = os.environ['LOUDML_ADDR']
else:
self.loudml_addr = 'localhost:8077'
self.bucket_interval = 20 * 60
t0 = int(datetime.datetime.now().timestamp())
t0 -= t0 % self.bucket_interval
self.t0 = t0
self.db = 'test-{}'.format(t0)
logging.info("creating database %s", self.db)
if 'INFLUXDB_ADDR' in os.environ:
addr = os.environ['INFLUXDB_ADDR']
else:
addr = 'localhost'
self.source = InfluxDataSource({
'name': 'nosetests',
'addr': addr,
'database': self.db,
})
self.source.drop()
self.source.init()
self.generator = SinEventGenerator(base=3, amplitude=3, sigma=0.01)
to_date = datetime.datetime.now().timestamp()
# Be sure that date range is aligned
self.to_date = math.floor(to_date / self.bucket_interval) * self.bucket_interval # noqa E501
self.from_date = self.to_date - 3600 * 24 * 7 * 3
for ts in self.generator.generate_ts(self.from_date, self.to_date, step_ms=600000): # noqa E501
self.source.insert_times_data(
measurement='bar',
ts=ts,
data={
'foo': random.normalvariate(10, 1),
}
)
self.source.commit()
class TestInfluxLong(unittest.TestCase):
def setUp(self):
self.db = "test-{}".format(int(datetime.datetime.now().timestamp()))
self.source = InfluxDataSource({
'name': 'test',
'addr': ADDR,
'database': self.db,
})
self.source.drop()
self.source.init()
self.storage = TempStorage()
generator = SinEventGenerator(base=3, sigma=0.05)
self.to_date = datetime.datetime.now().timestamp()
self.from_date = self.to_date - 3600 * 24 * 7
for ts in generator.generate_ts(
self.from_date,
self.to_date,
step_ms=60000,
):
self.source.insert_times_data(
measurement='measure1',
ts=ts,
data={'foo': random.lognormvariate(10, 1)},
)
self.source.commit()
def test_train(self):
model = DonutModel(
dict(
name='test',
offset=30,
span=5,
bucket_interval=20 * 60,
interval=60,
features=FEATURES[0:1],
max_evals=1,
))
# Train
model.train(self.source,
from_date=self.from_date,
to_date=self.to_date)
# Check
self.assertTrue(model.is_trained)
def setUp(self):
this_day = int(
datetime.datetime.now(tz=datetime.timezone.utc).replace(
hour=0,
minute=0,
second=0,
microsecond=0,
).timestamp())
self.database = 'test-times-%d' % this_day
logging.info("creating database %s", self.database)
self.source = InfluxDataSource({
'name': 'test',
'type': 'influx',
'addr': ADDR,
'database': self.database,
})
self.source.drop()
self.source.init()
# Sin wave. 600s period.
generator = SinEventGenerator(base=50,
amplitude=50,
period=600,
sigma=0.01)
# Normal data in range 06-12
dt = datetime.datetime(2018, 8, 1, 6, 0)
from_date = dt.replace(tzinfo=timezone.utc).timestamp()
dt = datetime.datetime(2018, 8, 1, 12, 0)
to_date = dt.replace(tzinfo=timezone.utc).timestamp()
for ts, data in self.generate_data(generator,
from_date,
to_date,
step_ms=1000,
errors=0):
self.source.insert_times_data(
measurement='test_auto',
ts=ts,
data=data,
)
# Random 20s drops range 12-13
dt = datetime.datetime(2018, 8, 1, 12, 0)
from_date = dt.replace(tzinfo=timezone.utc).timestamp()
dt = datetime.datetime(2018, 8, 1, 13, 0)
to_date = dt.replace(tzinfo=timezone.utc).timestamp()
for ts, data in self.generate_data(generator,
from_date,
to_date,
step_ms=1000,
errors=0.0001,
burst_ms=20000):
self.source.insert_times_data(
measurement='test_auto',
ts=ts,
data=data,
)
# Again normal data in range 13-14
dt = datetime.datetime(2018, 8, 1, 13, 0)
from_date = dt.replace(tzinfo=timezone.utc).timestamp()
dt = datetime.datetime(2018, 8, 1, 14, 0)
to_date = dt.replace(tzinfo=timezone.utc).timestamp()
for ts, data in self.generate_data(generator,
from_date,
to_date,
step_ms=1000,
errors=0):
self.source.insert_times_data(
measurement='test_auto',
ts=ts,
data=data,
)
# Duplicate normal data in range 06-14 and measurement=normal
dt = datetime.datetime(2018, 8, 1, 6, 0)
from_date = dt.replace(tzinfo=timezone.utc).timestamp()
dt = datetime.datetime(2018, 8, 1, 14, 0)
to_date = dt.replace(tzinfo=timezone.utc).timestamp()
for ts, data in self.generate_data(generator,
from_date,
to_date,
step_ms=1000,
errors=0):
self.source.insert_times_data(
measurement='normal',
ts=ts,
data=data,
)
self.source.commit()
class TestInfluxTimes(unittest.TestCase):
def setUp(self):
this_day = int(
datetime.datetime.now(tz=datetime.timezone.utc).replace(
hour=0,
minute=0,
second=0,
microsecond=0,
).timestamp())
self.database = 'test-times-%d' % this_day
logging.info("creating database %s", self.database)
self.source = InfluxDataSource({
'name': 'test',
'type': 'influx',
'addr': ADDR,
'database': self.database,
})
self.source.drop()
self.source.init()
# Sin wave. 600s period.
generator = SinEventGenerator(base=50,
amplitude=50,
period=600,
sigma=0.01)
# Normal data in range 06-12
dt = datetime.datetime(2018, 8, 1, 6, 0)
from_date = dt.replace(tzinfo=timezone.utc).timestamp()
dt = datetime.datetime(2018, 8, 1, 12, 0)
to_date = dt.replace(tzinfo=timezone.utc).timestamp()
for ts, data in self.generate_data(generator,
from_date,
to_date,
step_ms=1000,
errors=0):
self.source.insert_times_data(
measurement='test_auto',
ts=ts,
data=data,
)
# Random 20s drops range 12-13
dt = datetime.datetime(2018, 8, 1, 12, 0)
from_date = dt.replace(tzinfo=timezone.utc).timestamp()
dt = datetime.datetime(2018, 8, 1, 13, 0)
to_date = dt.replace(tzinfo=timezone.utc).timestamp()
for ts, data in self.generate_data(generator,
from_date,
to_date,
step_ms=1000,
errors=0.0001,
burst_ms=20000):
self.source.insert_times_data(
measurement='test_auto',
ts=ts,
data=data,
)
# Again normal data in range 13-14
dt = datetime.datetime(2018, 8, 1, 13, 0)
from_date = dt.replace(tzinfo=timezone.utc).timestamp()
dt = datetime.datetime(2018, 8, 1, 14, 0)
to_date = dt.replace(tzinfo=timezone.utc).timestamp()
for ts, data in self.generate_data(generator,
from_date,
to_date,
step_ms=1000,
errors=0):
self.source.insert_times_data(
measurement='test_auto',
ts=ts,
data=data,
)
# Duplicate normal data in range 06-14 and measurement=normal
dt = datetime.datetime(2018, 8, 1, 6, 0)
from_date = dt.replace(tzinfo=timezone.utc).timestamp()
dt = datetime.datetime(2018, 8, 1, 14, 0)
to_date = dt.replace(tzinfo=timezone.utc).timestamp()
for ts, data in self.generate_data(generator,
from_date,
to_date,
step_ms=1000,
errors=0):
self.source.insert_times_data(
measurement='normal',
ts=ts,
data=data,
)
self.source.commit()
# queries to plot the data in Chronograf
# SELECT count("foo") AS "count_foo" FROM "test-times-*"."autogen"."test_auto" \
# WHERE time > :dashboardTime: GROUP BY time(10s) FILL(null)
# SELECT last("count_foo") AS "predicted" FROM "test-times-*"."autogen"."prediction_test" \
# WHERE time > :dashboardTime: GROUP BY time(10s) FILL(null)
# def tearDown(self):
# self.source.drop()
def generate_data(self,
generator,
from_date,
to_date,
step_ms=1000,
errors=0,
burst_ms=0):
ano = False
previous_ts = None
for ts in generator.generate_ts(from_date, to_date, step_ms=step_ms):
if ano == False and errors > 0:
val = random.random()
if val < errors:
ano = True
total_burst_ms = 0
previous_ts = ts
if ano == True and total_burst_ms < burst_ms:
total_burst_ms += (ts - previous_ts) * 1000.0
previous_ts = ts
else:
ano = False
yield ts, {
'foo': random.lognormvariate(10, 1),
}
def test_loudmld(self):
model = DonutModel(
dict(
name='test',
offset=30,
span=200,
bucket_interval=10,
interval=10,
features=[{
'measurement': 'test_auto',
'name': 'count_foo',
'metric': 'count',
'field': 'foo',
}],
max_evals=21,
))
model2 = DonutModel(
dict(
name='normal',
offset=30,
span=200,
bucket_interval=10,
interval=10,
features=[{
'measurement': 'normal',
'name': 'count_foo',
'metric': 'count',
'field': 'foo',
}],
max_evals=1,
))
# Normal data in range 06-12
dt = datetime.datetime(2018, 8, 1, 6, 0)
from_date = dt.replace(tzinfo=timezone.utc).timestamp()
dt = datetime.datetime(2018, 8, 1, 12, 0)
to_date = dt.replace(tzinfo=timezone.utc).timestamp()
print("training model")
model.train(self.source, from_date, to_date)
print("training done")
# simulate loudmld loop in range 11h00 - 13h30
dt = datetime.datetime(2018, 8, 1, 11, 00)
from_date = dt.replace(tzinfo=timezone.utc).timestamp()
dt = datetime.datetime(2018, 8, 1, 13, 30)
to_date = dt.replace(tzinfo=timezone.utc).timestamp()
normal = []
data = self.source.get_times_data(
model2,
from_date=from_date,
to_date=to_date,
)
for line in data:
normal.append(line[1])
normal = np.array(normal)
prediction = model.predict2(self.source,
from_date,
to_date,
mse_rtol=4)
model.detect_anomalies(prediction)
self.source.save_timeseries_prediction(prediction, model)
self.source.commit()
self.assertEqual(normal.T[0].shape, prediction.predicted.shape)
for j, _ in enumerate(normal):
np.testing.assert_allclose(
prediction.predicted[j],
normal[j],
rtol=0.10,
atol=20,
)
def setUp(self):
bucket_interval = 3
t0 = int(datetime.datetime.now().timestamp())
# XXX Bucket returned by InfluxDB are aligne on modulo(bucket_interval), that's why
# timestamp must be aligned for unit tests.
t0 -= t0 % bucket_interval
self.t0 = t0
self.db = 'test-{}'.format(t0)
logging.info("creating database %s", self.db)
self.source = InfluxDataSource({
'name': 'test',
'addr': ADDR,
'database': self.db,
})
self.source.drop()
self.source.init()
self.model = Model(
dict(
name="test-model",
offset=30,
span=300,
bucket_interval=3,
interval=60,
features=FEATURES,
))
data = [
# (foo, bar, timestamp)
(1, 33, t0 - 1), # excluded
(2, 120, t0),
(3, 312, t0 + 1),
# empty
(4, 18, t0 + 7),
(5, 78, t0 + 9), # excluded
]
for foo, bar, ts in data:
self.source.insert_times_data(measurement='measure1',
ts=ts,
data={
'foo': foo,
})
self.source.insert_times_data(measurement='measure2',
ts=ts,
data={
'bar': bar,
})
self.source.insert_times_data(measurement='measure3',
ts=ts,
tags={
'tag_kw': 'tag1',
'tag_int': 9,
'tag_bool': False,
},
data={
'baz': bar,
})
self.source.insert_times_data(measurement='measure3',
ts=ts,
tags={
'tag_kw': 'tag2',
'tag_int': 7,
'tag_bool': True,
},
data={
'baz': -bar,
})
self.source.commit()
class TestInfluxQuick(unittest.TestCase):
def setUp(self):
bucket_interval = 3
t0 = int(datetime.datetime.now().timestamp())
# XXX Bucket returned by InfluxDB are aligne on modulo(bucket_interval), that's why
# timestamp must be aligned for unit tests.
t0 -= t0 % bucket_interval
self.t0 = t0
self.db = 'test-{}'.format(t0)
logging.info("creating database %s", self.db)
self.source = InfluxDataSource({
'name': 'test',
'addr': ADDR,
'database': self.db,
})
self.source.drop()
self.source.init()
self.model = Model(
dict(
name="test-model",
offset=30,
span=300,
bucket_interval=3,
interval=60,
features=FEATURES,
))
data = [
# (foo, bar, timestamp)
(1, 33, t0 - 1), # excluded
(2, 120, t0),
(3, 312, t0 + 1),
# empty
(4, 18, t0 + 7),
(5, 78, t0 + 9), # excluded
]
for foo, bar, ts in data:
self.source.insert_times_data(measurement='measure1',
ts=ts,
data={
'foo': foo,
})
self.source.insert_times_data(measurement='measure2',
ts=ts,
data={
'bar': bar,
})
self.source.insert_times_data(measurement='measure3',
ts=ts,
tags={
'tag_kw': 'tag1',
'tag_int': 9,
'tag_bool': False,
},
data={
'baz': bar,
})
self.source.insert_times_data(measurement='measure3',
ts=ts,
tags={
'tag_kw': 'tag2',
'tag_int': 7,
'tag_bool': True,
},
data={
'baz': -bar,
})
self.source.commit()
def tearDown(self):
self.source.drop()
def test_validation(self):
with self.assertRaises(errors.Invalid):
InfluxDataSource({
'addr': 'localhost',
})
with self.assertRaises(errors.Invalid):
InfluxDataSource({
'database': 'foo',
})
def test_build_time_predicates(self):
self.assertEqual(
_build_time_predicates(),
[],
)
self.assertEqual(
_build_time_predicates(
from_date=1515404366.1234,
to_date="2018-01-08T14:59:25.456Z",
),
[
"time >= 1515404366123400000",
"time < 1515423565456000000",
],
)
def test_build_tags_predicates(self):
self.assertEqual(
_build_tags_predicates(),
[],
)
self.assertEqual(
_build_tags_predicates([
{
'tag': 'foo',
'value': 'bar'
},
{
'tag': 'a "',
'value': 'b \''
},
{
'tag': 'int',
'value': 42
},
{
'tag': 'bool',
'value': True
},
]), [
"\"foo\"='bar'",
"\"a \\\"\"='b \\''",
"(\"int\"='42' OR \"int\"=42)",
"(\"bool\"='True' OR \"bool\"=True)",
])
def test_build_times_queries(self):
where = "time >= 1515404366123400000 and time < 1515423565456000000"
queries = list(
self.source._build_times_queries(
self.model,
from_date=1515404366.1234,
to_date="2018-01-08T14:59:25.456Z",
))
self.assertEqual(
queries,
[
"select MEAN(\"foo\") as \"avg_foo\" from \"measure1\" "\
"where {} group by time(3000ms);".format(where),
"select COUNT(\"bar\") as \"count_bar\" from \"measure2\" "\
"where {} group by time(3000ms);".format(where),
"select MEAN(\"baz\") as \"avg_baz\" from \"measure1\" "\
"where {} and \"mytag\"='myvalue' group by time(3000ms);".format(where),
],
)
source = InfluxDataSource({
'name': 'test',
'addr': ADDR,
'database': self.db,
'retention_policy': 'custom',
})
queries = list(
source._build_times_queries(
self.model,
from_date=1515404366.1234,
to_date="2018-01-08T14:59:25.456Z",
))
from_prefix = '"{}"."custom".'.format(self.db)
self.assertEqual(
queries,
[
"select MEAN(\"foo\") as \"avg_foo\" from {}\"measure1\" "\
"where {} group by time(3000ms);".format(from_prefix, where),
"select COUNT(\"bar\") as \"count_bar\" from {}\"measure2\" "\
"where {} group by time(3000ms);".format(from_prefix, where),
"select MEAN(\"baz\") as \"avg_baz\" from {}\"measure1\" "\
"where {} and \"mytag\"='myvalue' group by time(3000ms);".format(from_prefix, where),
],
)
def test_get_times_data(self):
logging.info("[%d %d]", self.t0, self.t0)
res = self.source.get_times_data(
self.model,
from_date=self.t0,
to_date=self.t0 + 8,
)
foo_avg = []
bar_count = []
for line in res:
foo_avg.append(nan_to_none(line[1][0]))
bar_count.append(nan_to_none(line[1][1]))
self.assertEqual(foo_avg, [2.5, None, 4.0])
self.assertEqual(bar_count, [2.0, 0, 1.0])
def test_get_times_data2(self):
res = self.source.get_times_data(
self.model,
from_date=self.t0,
to_date=self.t0 + 8,
)
# _source to write aggregate data to RAM
_source = MemDataSource()
_features = copy.deepcopy(self.model.features)
for _, feature in enumerate(self.model.features):
feature.metric = 'avg'
i = None
for i, (_, val, timeval) in enumerate(res):
bucket = {
feature.field: val[i]
for i, feature in enumerate(self.model.features)
}
bucket.update({'timestamp': make_ts(timeval)})
_source.insert_times_data(bucket)
res2 = _source.get_times_data(
self.model,
from_date=self.t0,
to_date=self.t0 + 8,
)
self.model.features = _features
for i, (_, val2, timeval2) in enumerate(res2):
(_, val, timeval) = res[i]
np.testing.assert_allclose(val, val2)
def test_match_all(self):
model = Model(
dict(
name="test-model",
offset=30,
span=300,
bucket_interval=3,
interval=60,
features=FEATURES_MATCH_ALL_TAG1,
))
res = self.source.get_times_data(
model,
from_date=self.t0,
to_date=self.t0 + 8,
)
baz_avg = []
for line in res:
baz_avg.append(line[1][0])
np.testing.assert_allclose(
np.array(baz_avg),
np.array([216.0, np.nan, 18.0]),
rtol=0,
atol=0,
)
model = Model(
dict(
name="test-model",
offset=30,
span=300,
bucket_interval=3,
interval=60,
features=FEATURES_MATCH_ALL_TAG2,
))
res = self.source.get_times_data(
model,
from_date=self.t0,
to_date=self.t0 + 8,
)
baz_avg = []
for line in res:
baz_avg.append(line[1][0])
np.testing.assert_allclose(
np.array(baz_avg),
np.array([-216.0, np.nan, -18.0]),
rtol=0,
atol=0,
)
class AppTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
pass
@classmethod
def tearDownClass(cls):
pass
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def setUp(self):
self.app = app.test_client()
self.app.testing = True
self.bucket_interval = 20 * 60
t0 = int(datetime.datetime.now().timestamp())
t0 -= t0 % self.bucket_interval
self.t0 = t0
self.db = 'test-{}'.format(t0)
logging.info("creating database %s", self.db)
self.source = InfluxDataSource({
'name': 'nose',
'addr': ADDR,
'database': self.db,
})
self.source.drop()
self.source.init()
self.generator = SinEventGenerator(base=3, amplitude=3, sigma=0.01)
to_date = datetime.datetime.now().timestamp()
# Be sure that date range is aligned
self.to_date = math.floor(
to_date / self.bucket_interval) * self.bucket_interval # noqa E501
self.from_date = self.to_date - 3600 * 24 * 7 * 3
for ts in self.generator.generate_ts(self.from_date,
self.to_date,
step_ms=600000): # noqa E501
self.source.insert_times_data(measurement='bar',
ts=ts,
data={
'foo':
random.normalvariate(10, 1),
})
self.source.commit()
self.dirpath = tempfile.mkdtemp()
configyml = os.path.join(self.dirpath, 'config.yml')
cfg = open(configyml, 'w')
cfg.write(CONFIG.format(ADDR, self.db, self.dirpath))
cfg.close()
g_app_init(configyml)
def tearDown(self):
g_app_stop()
self.source.drop()
shutil.rmtree(self.dirpath)
def _wait_job(self, job_id):
state = None
while not state_is_done(state):
time.sleep(5)
result = self.app.get('/jobs/{}'.format(job_id))
res = json.loads(result.data.decode('utf-8'))
print(res)
state = res['state']
return state
def _get_models(self):
result = self.app.get('/models', )
self.assertEqual(result.status_code, 200)
d = json.loads(result.data.decode('utf-8'))
return d
def _require_model(self):
d = self._get_models()
if len(d) > 0 and d[0]['name'] == 'test-model':
return
model = dict(
name='test-model',
default_datasource='nose',
offset=30,
span=24 * 3,
bucket_interval=self.bucket_interval,
interval=60,
features=FEATURES,
grace_period="140m", # = 7 points
max_threshold=99.7,
min_threshold=68,
max_evals=1,
)
model['type'] = 'donut'
result = self.app.put(
'/models',
follow_redirects=True,
content_type='application/json',
data=json.dumps(model),
)
self.assertEqual(result.status_code, 201)
d = self._get_models()
self.assertEqual(len(d), 1)
self.assertEqual(d[0]['settings']['name'], 'test-model')
return d[0]
def _require_training(self):
model = self._require_model()
if model['state']['trained']:
return
result = self.app.post(
'/models/{}/_train?from={}&to={}'.format(
'test-model',
str(self.from_date),
str(self.to_date),
), )
job_id = read_job_id(result)
status = self._wait_job(job_id)
self.assertEqual(status, 'done')
def test_train(self):
self._require_training()
def test_home(self):
result = self.app.get('/')
self.assertEqual(result.status_code, 200)
d = json.loads(result.data.decode('utf-8'))
self.assertIsNotNone(d.get('host_id'))
self.assertIsNotNone(d.get('version'))
class AppTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
pass
@classmethod
def tearDownClass(cls):
pass
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._jwt = None
def setUp(self):
if 'LOUDML_ADDR' in os.environ:
self.loudml_addr = os.environ['LOUDML_ADDR']
else:
self.loudml_addr = 'localhost:8077'
self.bucket_interval = 20 * 60
t0 = int(datetime.datetime.now().timestamp())
t0 -= t0 % self.bucket_interval
self.t0 = t0
self.db = 'test-{}'.format(t0)
logging.info("creating database %s", self.db)
if 'INFLUXDB_ADDR' in os.environ:
addr = os.environ['INFLUXDB_ADDR']
else:
addr = 'localhost'
self.source = InfluxDataSource({
'name': 'nosetests',
'addr': addr,
'database': self.db,
})
self.source.drop()
self.source.init()
self.generator = SinEventGenerator(base=3, amplitude=3, sigma=0.01)
to_date = datetime.datetime.now().timestamp()
# Be sure that date range is aligned
self.to_date = math.floor(to_date / self.bucket_interval) * self.bucket_interval # noqa E501
self.from_date = self.to_date - 3600 * 24 * 7 * 3
for ts in self.generator.generate_ts(self.from_date, self.to_date, step_ms=600000): # noqa E501
self.source.insert_times_data(
measurement='bar',
ts=ts,
data={
'foo': random.normalvariate(10, 1),
}
)
self.source.commit()
def tearDown(self):
self.source.drop()
def get(self, url, data=None, **kwargs):
headers = kwargs.pop('headers', {})
if self._jwt:
headers['Authorization'] = 'Bearer {}'.format(self._jwt)
return requests.get(
self.get_url(url),
data=data,
headers=headers,
**kwargs
)
def post(self, url, data=None, **kwargs):
headers = kwargs.pop('headers', {})
if self._jwt:
headers['Authorization'] = 'Bearer {}'.format(self._jwt)
return requests.post(
self.get_url(url),
data=data,
headers=headers,
**kwargs
)
def patch(self, url, data=None, **kwargs):
headers = kwargs.pop('headers', {})
if self._jwt:
headers['Authorization'] = 'Bearer {}'.format(self._jwt)
return requests.patch(
self.get_url(url),
data=data,
headers=headers,
**kwargs
)
def delete(self, url, data=None, **kwargs):
headers = kwargs.pop('headers', {})
if self._jwt:
headers['Authorization'] = 'Bearer {}'.format(self._jwt)
return requests.delete(
self.get_url(url),
data=data,
headers=headers,
**kwargs
)
def put(self, url, data=None, content_type=None, **kwargs):
headers = kwargs.pop('headers', {})
if self._jwt:
headers['Authorization'] = 'Bearer {}'.format(self._jwt)
if content_type:
headers['Content-Type'] = content_type
return requests.put(
self.get_url(url),
data=data,
headers=headers,
**kwargs
)
def get_url(self, url):
if 'USE_SSL' in os.environ:
scheme = 'https://'
else:
scheme = 'http://'
return scheme + self.loudml_addr + url
def _wait_job(self, job_id):
state = None
while not state_is_done(state):
time.sleep(5)
response = self.get(
'/jobs/{}'.format(job_id)
)
res = response.json()
# print(res)
state = res['state']
return state
def _get_models(self):
response = self.get(
'/models',
)
self.assertEqual(response.status_code, 200)
return response.json()
def _get_sources(self):
response = self.get(
'/datasources',
)
self.assertEqual(response.status_code, 200)
return response.json()
def _require_source(self):
sources = {
source['name']: source
for source in self._get_sources()
}
if self.db in sources:
return
if 'INFLUXDB_ADDR' in os.environ:
addr = os.environ['INFLUXDB_ADDR']
else:
addr = 'localhost:8086'
source = {
'name': self.db,
'type': 'influxdb',
'addr': addr,
'database': self.db,
'create_database': 'true',
'retention_policy': 'autogen',
'max_series_per_request': 2000,
}
response = self.put(
'/datasources',
content_type='application/json',
data=json.dumps(source),
)
self.assertEqual(response.status_code, 201)
sources = {
source['name']: source
for source in self._get_sources()
}
self.assertTrue(self.db in sources)
def _del_model(self, model_name):
response = self.delete(
'/models/{}'.format(model_name),
)
self.assertTrue(response.status_code in [200, 404])
def _require_model(self):
self._require_source()
models = {
model['settings']['name']: model
for model in self._get_models()
}
if 'test-model' in models:
return models['test-model']
model = dict(
name='test-model',
default_datasource=self.db,
offset=30,
span=24 * 3,
bucket_interval=self.bucket_interval,
interval=60,
features=FEATURES,
grace_period="140m", # = 7 points
max_threshold=99.7,
min_threshold=68,
max_evals=1,
)
model['type'] = 'donut'
response = self.put(
'/models',
content_type='application/json',
data=json.dumps(model),
)
self.assertEqual(response.status_code, 201)
models = {
model['settings']['name']: model
for model in self._get_models()
}
self.assertTrue('test-model' in models)
return models['test-model']
def _require_training(self):
model = self._require_model()
if model['state']['trained']:
return
response = self.post(
'/models/{}/_train?from={}&to={}'.format(
'test-model',
str(self.from_date),
str(self.to_date),
),
)
job_id = read_job_id(response.text)
status = self._wait_job(job_id)
self.assertEqual(status, 'done')
def test_training(self):
self._del_model('test-model')
self._require_training()
def test_home(self):
response = self.get('/')
self.assertEqual(response.status_code, 200)
home = response.json()
self.assertIsNotNone(home.get('host_id'))
self.assertIsNotNone(home.get('version'))