class TestInfluxLong(unittest.TestCase): def setUp(self): self.db = "test-{}".format(int(datetime.datetime.now().timestamp())) self.source = InfluxBucket({ 'name': 'test', 'addr': ADDR, 'database': self.db, 'measurement': 'nosetests', }) self.source.drop() self.source.init() self.storage = TempStorage() generator = SinEventGenerator(base=3, sigma=0.05) self.to_date = datetime.datetime.now().timestamp() self.from_date = self.to_date - 3600 * 24 * 7 for ts in generator.generate_ts( self.from_date, self.to_date, step_ms=60000, ): self.source.insert_times_data( measurement='measure1', ts=ts, data={'foo': random.lognormvariate(10, 1)}, ) self.source.commit() def test_train(self): model = DonutModel( dict( name='test', offset=30, span=5, bucket_interval=20 * 60, interval=60, features=FEATURES[0:1], max_evals=1, )) # Train model.train(self.source, from_date=self.from_date, to_date=self.to_date) # Check self.assertTrue(model.is_trained)
class TestInfluxQuick(unittest.TestCase): def setUp(self): bucket_interval = 3 t0 = int(datetime.datetime.now().timestamp()) # XXX Bucket returned by InfluxDB are aligned on # modulo(bucket_interval), that's why # timestamp must be aligned for unit tests. t0 -= t0 % bucket_interval self.t0 = t0 self.db = 'test-{}'.format(t0) logging.info("creating database %s", self.db) self.source = InfluxBucket({ 'name': 'test', 'addr': ADDR, 'database': self.db, 'measurement': 'nosetests', }) self.source.drop() self.source.init() self.model = Model( dict( name="test-model", offset=30, span=300, bucket_interval=3, interval=60, features=FEATURES, )) data = [ # (foo, bar, timestamp) (1, 33, t0 - 1), # excluded (2, 120, t0), (3, 312, t0 + 1), # empty (4, 18, t0 + 7), (5, 78, t0 + 9), # excluded ] for foo, bar, ts in data: self.source.insert_times_data(measurement='measure1', ts=ts, data={ 'foo': foo, }) self.source.insert_times_data(measurement='measure2', ts=ts, data={ 'bar': bar, }) self.source.insert_times_data(measurement='measure3', ts=ts, tags={ 'tag_kw': 'tag1', 'tag_int': 9, 'tag_bool': False, }, data={ 'baz': bar, }) self.source.insert_times_data(measurement='measure3', ts=ts, tags={ 'tag_kw': 'tag2', 'tag_int': 7, 'tag_bool': True, }, data={ 'baz': -bar, }) self.source.commit() def tearDown(self): self.source.drop() def test_validation(self): with self.assertRaises(errors.Invalid): InfluxBucket({ 'addr': 'localhost', }) with self.assertRaises(errors.Invalid): InfluxBucket({ 'database': 'foo', }) def test_build_time_predicates(self): self.assertEqual( _build_time_predicates(), [], ) self.assertEqual( _build_time_predicates( from_date=1515404366.1234, to_date="2018-01-08T14:59:25.456Z", ), [ "time >= 1515404366123400000", "time < 1515423565456000000", ], ) def test_build_tags_predicates(self): self.assertEqual( _build_tags_predicates(), [], ) self.assertEqual( _build_tags_predicates([ { 'tag': 'foo', 'value': 'bar' }, { 'tag': 'a "', 'value': 'b \'' }, { 'tag': 'int', 'value': 42 }, { 'tag': 'bool', 'value': True }, ]), [ "\"foo\"='bar'", "\"a \\\"\"='b \\''", "(\"int\"='42' OR \"int\"=42)", "(\"bool\"='True' OR \"bool\"=True)", ]) def test_build_times_queries(self): where = "time >= 1515404366123400000 and time < 1515423565456000000" queries = list( self.source._build_times_queries( bucket_interval=self.model.bucket_interval, features=self.model.features, from_date=1515404366.1234, to_date="2018-01-08T14:59:25.456Z", )) self.assertEqual( queries, [ "select MEAN(\"foo\") as \"avg_foo\" from \"measure1\" " "where {} group by time(3000ms);".format(where), "select COUNT(\"bar\") as \"count_bar\" from \"measure2\" " "where {} group by time(3000ms);".format(where), "select MEAN(\"baz\") as \"avg_baz\" from \"measure1\" " "where {} and \"mytag\"='myvalue' group by time(3000ms);". format(where), ], ) source = InfluxBucket({ 'name': 'test', 'addr': ADDR, 'database': self.db, 'retention_policy': 'custom', 'measurement': 'nosetests', }) queries = list( source._build_times_queries( bucket_interval=self.model.bucket_interval, features=self.model.features, from_date=1515404366.1234, to_date="2018-01-08T14:59:25.456Z", )) from_prefix = '"{}"."custom".'.format(self.db) self.assertEqual( queries, [ "select MEAN(\"foo\") as \"avg_foo\" from {}\"measure1\" " "where {} group by time(3000ms);".format(from_prefix, where), "select COUNT(\"bar\") as \"count_bar\" from {}\"measure2\" " "where {} group by time(3000ms);".format(from_prefix, where), "select MEAN(\"baz\") as \"avg_baz\" from {}\"measure1\" " "where {} and \"mytag\"='myvalue' group by time(3000ms);". format(from_prefix, where), ], ) def test_get_times_data(self): logging.info("[%d %d]", self.t0, self.t0) res = self.source.get_times_data( bucket_interval=self.model.bucket_interval, features=self.model.features, from_date=self.t0, to_date=self.t0 + 8, ) foo_avg = [] bar_count = [] for line in res: foo_avg.append(nan_to_none(line[1][0])) bar_count.append(nan_to_none(line[1][1])) self.assertEqual(foo_avg, [2.5, None, 4.0]) self.assertEqual(bar_count, [2.0, 0, 1.0]) def test_get_times_data2(self): res = self.source.get_times_data( bucket_interval=self.model.bucket_interval, features=self.model.features, from_date=self.t0, to_date=self.t0 + 8, ) # _source to write aggregate data to RAM _source = MemBucket() _features = copy.deepcopy(self.model.features) for _, feature in enumerate(self.model.features): feature.metric = 'avg' i = None for i, (_, val, timeval) in enumerate(res): bucket = { feature.field: val[i] for i, feature in enumerate(self.model.features) } bucket.update({'timestamp': make_ts(timeval)}) _source.insert_times_data(bucket) res2 = _source.get_times_data( bucket_interval=self.model.bucket_interval, features=self.model.features, from_date=self.t0, to_date=self.t0 + 8, ) self.model.features = _features for i, (_, val2, timeval2) in enumerate(res2): (_, val, timeval) = res[i] np.testing.assert_allclose(val, val2) def test_match_all(self): model = Model( dict( name="test-model", offset=30, span=300, bucket_interval=3, interval=60, features=FEATURES_MATCH_ALL_TAG1, )) res = self.source.get_times_data( bucket_interval=model.bucket_interval, features=model.features, from_date=self.t0, to_date=self.t0 + 8, ) baz_avg = [] for line in res: baz_avg.append(line[1][0]) np.testing.assert_allclose( np.array(baz_avg), np.array([216.0, np.nan, 18.0]), rtol=0, atol=0, ) model = Model( dict( name="test-model", offset=30, span=300, bucket_interval=3, interval=60, features=FEATURES_MATCH_ALL_TAG2, )) res = self.source.get_times_data( bucket_interval=model.bucket_interval, features=model.features, from_date=self.t0, to_date=self.t0 + 8, ) baz_avg = [] for line in res: baz_avg.append(line[1][0]) np.testing.assert_allclose( np.array(baz_avg), np.array([-216.0, np.nan, -18.0]), rtol=0, atol=0, )
class AppTests(unittest.TestCase): @classmethod def setUpClass(cls): pass @classmethod def tearDownClass(cls): pass def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._jwt = None def setUp(self): if 'LOUDML_ADDR' in os.environ: self.loudml_addr = os.environ['LOUDML_ADDR'] else: self.loudml_addr = 'localhost:8077' self.bucket_interval = 20 * 60 t0 = int(datetime.datetime.now().timestamp()) t0 -= t0 % self.bucket_interval self.t0 = t0 self.db = 'test-{}'.format(t0) logging.info("creating database %s", self.db) if 'INFLUXDB_ADDR' in os.environ: addr = os.environ['INFLUXDB_ADDR'] else: addr = 'localhost' self.source = InfluxBucket({ 'name': 'nosetests', 'addr': addr, 'database': self.db, }) self.source.drop() self.source.init() self.generator = SinEventGenerator(base=3, amplitude=3, sigma=0.01) to_date = datetime.datetime.now().timestamp() # Be sure that date range is aligned self.to_date = math.floor( to_date / self.bucket_interval) * self.bucket_interval # noqa E501 self.from_date = self.to_date - 3600 * 24 * 7 * 3 for ts in self.generator.generate_ts(self.from_date, self.to_date, step_ms=600000): # noqa E501 self.source.insert_times_data(measurement='bar', ts=ts, data={ 'foo': random.normalvariate(10, 1), }) self.source.commit() def tearDown(self): self.source.drop() def get(self, url, data=None, **kwargs): headers = kwargs.pop('headers', {}) if self._jwt: headers['Authorization'] = 'Bearer {}'.format(self._jwt) return requests.get(self.get_url(url), data=data, headers=headers, **kwargs) def post(self, url, data=None, content_type=None, **kwargs): headers = kwargs.pop('headers', {}) if self._jwt: headers['Authorization'] = 'Bearer {}'.format(self._jwt) if content_type: headers['Content-Type'] = content_type return requests.post(self.get_url(url), data=data, headers=headers, **kwargs) def patch(self, url, data=None, **kwargs): headers = kwargs.pop('headers', {}) if self._jwt: headers['Authorization'] = 'Bearer {}'.format(self._jwt) return requests.patch(self.get_url(url), data=data, headers=headers, **kwargs) def delete(self, url, data=None, **kwargs): headers = kwargs.pop('headers', {}) if self._jwt: headers['Authorization'] = 'Bearer {}'.format(self._jwt) return requests.delete(self.get_url(url), data=data, headers=headers, **kwargs) def put(self, url, data=None, content_type=None, **kwargs): headers = kwargs.pop('headers', {}) if self._jwt: headers['Authorization'] = 'Bearer {}'.format(self._jwt) if content_type: headers['Content-Type'] = content_type return requests.put(self.get_url(url), data=data, headers=headers, **kwargs) def get_url(self, url): if 'USE_SSL' in os.environ: scheme = 'https://' else: scheme = 'http://' return scheme + self.loudml_addr + url def _wait_job(self, job_id): state = None while not state_is_done(state): time.sleep(5) response = self.get('/jobs/{}'.format(job_id)) res = response.json() # print(res) state = res['state'] return state def _get_models(self): response = self.get('/models', ) self.assertEqual(response.status_code, 200) return response.json() def _get_sources(self): response = self.get('/buckets', ) self.assertEqual(response.status_code, 200) return response.json() def _require_source(self): sources = {source['name']: source for source in self._get_sources()} if self.db in sources: return if 'INFLUXDB_ADDR' in os.environ: addr = os.environ['INFLUXDB_ADDR'] else: addr = 'localhost:8086' source = { 'name': self.db, 'type': 'influxdb', 'addr': addr, 'database': self.db, 'create_database': 'true', 'retention_policy': 'autogen', 'max_series_per_request': 2000, } response = self.post( '/buckets', content_type='application/json', data=json.dumps(source), ) self.assertEqual(response.status_code, 201) sources = {source['name']: source for source in self._get_sources()} self.assertTrue(self.db in sources) def _del_model(self, model_name): response = self.delete('/models/{}'.format(model_name), ) self.assertTrue(response.status_code in [200, 404]) def _require_model(self): self._require_source() models = { model['settings']['name']: model for model in self._get_models() } if 'test-model' in models: return models['test-model'] model = dict( name='test-model', default_bucket=self.db, offset=30, span=24 * 3, bucket_interval=self.bucket_interval, interval=60, features=FEATURES, grace_period="140m", # = 7 points max_threshold=99.7, min_threshold=68, max_evals=1, ) model['type'] = 'donut' response = self.post( '/models', content_type='application/json', data=json.dumps(model), ) self.assertEqual(response.status_code, 201) models = { model['settings']['name']: model for model in self._get_models() } self.assertTrue('test-model' in models) return models['test-model'] def _require_training(self): model = self._require_model() if model['state']['trained']: return response = self.post( '/models/{}/_train?from={}&to={}'.format( 'test-model', str(self.from_date), str(self.to_date), ), ) job_id = read_job_id(response.text) status = self._wait_job(job_id) self.assertEqual(status, 'done') def test_training(self): self._del_model('test-model') self._require_training() def test_home(self): response = self.get('/') self.assertEqual(response.status_code, 200) home = response.json() self.assertIsNotNone(home.get('host_id')) self.assertIsNotNone(home.get('version'))