def test_weekly(self): d = TSDB(BUCKET_TYPE="weekly") for i in range(0, 20): d._insert("weekly", [(i * 24 * 60 * 60, 1.1)]) buckets = d.storage.query("weekly", 0, 20 * 24 * 60 * 60) self.assertEqual(len(buckets), 4) i = buckets[0] self.assertEqual(len(i), 4) self.assertEqual(i[0][0], 0) self.assertEqual(i[3][0], 3 * 24 * 60 * 60) i2 = buckets[1] self.assertEqual(len(i2), 7) self.assertEqual(i2[0][0], 4 * 24 * 60 * 60) self.assertEqual(i2[6][0], 10 * 24 * 60 * 60)
def test_monthly(self): d = TSDB(BUCKET_TYPE="monthly") for i in range(0, 40): d._insert("monthly", [(i * 24 * 60 * 60, 1.1)]) buckets = d.storage.query("monthly", 0, 40 * 24 * 60 * 60) self.assertEqual(len(buckets), 2) i = buckets[0] self.assertEqual(len(i), 31) self.assertEqual(i[0][0], 0) self.assertEqual(i[30][0], 30 * 24 * 60 * 60) i2 = buckets[1] self.assertEqual(len(i2), 9) self.assertEqual(i2[0][0], 31 * 24 * 60 * 60) self.assertEqual(i2[8][0], 39 * 24 * 60 * 60)
def test_hourly(self): d = TSDB(BUCKET_TYPE="hourly") for i in range(0, 70): d._insert("his", [(i * 60, 1.1)]) buckets = d.storage.query("his", 0, 70*60) self.assertEqual(len(buckets), 2) i = buckets[0] self.assertEqual(len(i), 60) self.assertEqual(i[0][0], 0) self.assertEqual(i[59][0], 59*60) i2 = buckets[1] self.assertEqual(len(i2), 10) self.assertEqual(i2[0][0], 60*60) self.assertEqual(i2[9][0], 69*60)
def test_daily(self): d = TSDB(BUCKET_TYPE="daily") for i in range(0, 50): d._insert("daily", [(i * 60 * 30, 1.1)]) buckets = d.storage.query("daily", 0, 50 * 60 * 30) self.assertEqual(len(buckets), 2) i = buckets[0] self.assertEqual(len(i), 48) self.assertEqual(i[0][0], 0) self.assertEqual(i[47][0], 47*60*30) i2 = buckets[1] self.assertEqual(len(i2), 2) self.assertEqual(i2[0][0], 48 * 30 * 60) self.assertEqual(i2[1][0], 49 * 30 * 60)
def test_merge(self): d = TSDB(BUCKET_TYPE="dynamic", BUCKET_DYNAMIC_TARGET=2, BUCKET_DYNAMIC_MAX=2) d._insert("merge", [(1, 2.0), (2, 3.0), (5, 6.0), (6, 7.0), (9, 10.0), (0, 1.0)]) res = d._query("merge", 0, 10) self.assertEqual(len(res), 6) d._insert("merge", [(3, 4.0), (4, 5.0), (7, 8.0), (8, 9.0)]) buckets = d.storage.query("merge", 0, 10) self.assertEqual(len(buckets), 5) for b in buckets: self.assertEqual(len(b), 2) res = d._query("merge", 0, 10) self.assertEqual(len(res), 10) for ts, v in res.all(): self.assertAlmostEqual(float(ts + 1.0), v)
def test_dynamic(self): d = TSDB(BUCKET_TYPE="dynamic", BUCKET_DYNAMIC_TARGET=3, BUCKET_DYNAMIC_MAX=3) d._insert("hi", [(1, 1.1), (2, 2.2)]) d._insert("hi", [(4, 4.4)]) i = d.storage.last("hi") self.assertEqual(len(i), 3) self.assertEqual(i[0][0], 1) self.assertEqual(i[1][0], 2) self.assertEqual(i[2][0], 4) d._insert("hi", [(3, 3.3)]) buckets = d.storage.query("hi", 0, 10) self.assertEqual(len(buckets), 2) i = buckets[0] self.assertEqual(len(i), 3) self.assertEqual(i[0][0], 1) self.assertEqual(i[1][0], 2) self.assertEqual(i[2][0], 3) i2 = buckets[1] self.assertEqual(len(i2), 1) self.assertEqual(i2[0][0], 4)
def test_largedataset(self): # Generate d = [] for i in range(50000): d.append((i, i * 2.5)) s = [] while len(d) > 0: count = random.randint(3, 30) s.append([]) for _ in range(count): if len(d) < 1: break el = d.pop(0) s[-1].append(el) # Make some holes s.insert(200, s.pop(100)) s.insert(200, s.pop(100)) s.insert(200, s.pop(100)) s.insert(1000, s.pop(1100)) s.insert(1200, s.pop(1300)) s.insert(1400, s.pop(1400)) # Strange Future Hole s.insert(2000, s.pop(1800)) # Insert d = TSDB(BUCKET_TYPE="dynamic", BUCKET_DYNAMIC_TARGET=100) for p in s: d._insert("large", p) buckets = d.storage.query("large", 0, 50000) self.assertGreater(len(buckets), 450) self.assertLess(len(buckets), 550) res = d._query("large", 1, 50000) self.assertEqual(len(res), 49999) res = d._query("large", 0, 49999) self.assertEqual(len(res), 50000)
#!/usr/bin/python # coding: utf8 import time import random import logging from pytsdb import TSDB PATCH_SIZES = [3, 10, 30, 50] DATA_POINTS = 1000 TIME_OFFSET = int(time.time() - 365*24*60*60) TIME_OFFSET = TIME_OFFSET - (TIME_OFFSET % (24 * 60 * 60)) db = TSDB(STORAGE="cassandra", BUCKET_TYPE="daily", ENABLE_CACHING=True) def insertTest(points, patch, sensor="sensor0"): i = 0 while i < points: d = [(TIME_OFFSET+(i+x)*600, float(random.randint(20,25))) for x in range(patch)] key = "{}.phex.{}".format(sensor, patch) db._insert(key, d) i += patch return i if __name__ == '__main__': db.storage._createTable() logging.basicConfig(level=logging.WARNING)
def test_cassandra_rewrite(self): cassandra_host = os.getenv('CASSANDRA_HOST', 'localhost') cassandra_port = os.getenv('CASSANDRA_PORT', 9042) db = TSDB(STORAGE="cassandra", BUCKET_TYPE="daily", CASSANDRA_HOST=cassandra_host, CASSANDRA_PORT=cassandra_port) db.storage._dropTable() db.storage._createTable() for i in range(0, 300): db._insert("cass", [(i * 10 * 60, 1.1)]) res = db.storage.query("cass", 0, 4 * 24 * 60 * 60) self.assertEqual(len(res), 3) self.assertEqual(len(res[0]), 144) self.assertEqual(len(res[1]), 144) self.assertEqual(len(res[2]), 12) res = db._query("cass", 0, 4 * 24 * 60 * 60) self.assertEqual(len(res), 300) self.assertEqual(res[144][0], 24 * 60 * 60) self.assertAlmostEqual(res[144][1], 1.1, 4) for i in range(0, 300): db._insert("cass", [(i * 10 * 60, 2.2)]) res = db.storage.query("cass", 0, 4 * 24 * 60 * 60) self.assertEqual(len(res), 3) self.assertEqual(len(res[0]), 144) self.assertEqual(len(res[1]), 144) self.assertEqual(len(res[2]), 12) res = db._query("cass", 0, 4 * 24 * 60 * 60) self.assertEqual(len(res), 300) self.assertEqual(res[144][0], 24 * 60 * 60) self.assertAlmostEqual(res[144][1], 1.1, 4) for i in range(0, 300): db._insert("cass", [(i * 10 * 60 + 1, 2.2)]) res = db.storage.query("cass", 0, 4 * 24 * 60 * 60 + 1) self.assertEqual(len(res), 3) self.assertEqual(len(res[0]), 288) self.assertEqual(len(res[1]), 288) self.assertEqual(len(res[2]), 24) res = db._query("cass", 0, 4 * 24 * 60 * 60 + 1) self.assertEqual(len(res), 600) self.assertEqual(res[288][0], 24 * 60 * 60) self.assertAlmostEqual(res[288][1], 1.1, 4) self.assertEqual(res[289][0], 24 * 60 * 60 + 1) self.assertAlmostEqual(res[289][1], 2.2, 4)
def test_simple(self): cassandra_host = os.getenv('CASSANDRA_HOST', 'localhost') cassandra_port = os.getenv('CASSANDRA_PORT', 9042) db = TSDB(STORAGE="cassandra", CASSANDRA_HOST=cassandra_host, CASSANDRA_PORT=cassandra_port) db.storage._dropTable() db.storage._createTable() d1 = [(i * 600, 6.5) for i in range(500)] d2 = [(i * 600, 25.5) for i in range(500)] d3 = [(i * 600, 10.5) for i in range(500)] data = [{"key": "sensor1.ph", "data": d1}, {"key": "sensor1.temp", "data": d2}] db.insert_bulk(data) db.insert("sensor1.act", d3) s = db.stats("sensor1.act") self.assertEqual(s["ts_min"], 0) self.assertEqual(s["ts_max"], 499 * 600) self.assertEqual(s["count"], 500) self.assertEqual(s["key"], "sensor1.act") s = db.stats_bulk(["sensor1.ph", "sensor1.temp"]) self.assertEqual(len(s), 2) self.assertEqual(s[0]["ts_min"], 0) self.assertEqual(s[0]["ts_max"], 499 * 600) self.assertEqual(s[0]["count"], 500) self.assertEqual(s[0]["key"], "sensor1.ph") self.assertEqual(s[1]["ts_min"], 0) self.assertEqual(s[1]["ts_max"], 499 * 600) self.assertEqual(s[1]["count"], 500) self.assertEqual(s[1]["key"], "sensor1.temp") r = db.query("sensor1.act", 0, 500*600) a = list(r.all()) d = list(r.aggregation("daily", "mean")) self.assertEqual(len(a), 500) self.assertEqual(len(d), 4) for ts, v in d: self.assertAlmostEqual(v, 10.5, 4) r = db.query("sensor1.ph", 0, 500*600) a = list(r.all()) d = list(r.aggregation("daily", "mean")) self.assertEqual(len(a), 500) self.assertEqual(len(d), 4) for ts, v in d: self.assertAlmostEqual(v, 6.5, 4) r = db.query("sensor1.temp", 0, 500*600) a = list(r.all()) d = list(r.aggregation("daily", "mean")) self.assertEqual(len(a), 500) self.assertEqual(len(d), 4) for ts, v in d: self.assertAlmostEqual(v, 25.5, 4)
def test_invalidmetricname(self): with self.assertRaises(ValueError): d = TSDB() d._insert("hüü", [(1, 1.1)])
#!/usr/bin/python # coding: utf8 import time import random from pytsdb import TSDB db = TSDB(STORAGE="cassandra") def onData(key, event): print("New: {}".format(event)) data = db._query(event.key, event.ts_min, event.ts_max) print(data.pretty_print()) print("---") db._register_data_listener("temp", onData) try: while True: time.sleep(0.1) except KeyboardInterrupt: db._close() print 'bye bye'
#!/usr/bin/python # coding: utf8 import time import random from pytsdb import TSDB db = TSDB(STORAGE="cassandra") try: while True: t = time.time() v = float(random.randint(20,25)) db._insert("temp", [(time.time(), v)]) print("{} temp: {}".format(t, v)) time.sleep(0.7) except KeyboardInterrupt: print 'bye bye'