def test_tsdb_persistentdb():
# synthetic data
t = np.array([1, 1.5, 2, 2.5, 10, 11, 12])
v1 = np.array([10, 12, -11, 1.5, 10, 13, 17])
v2 = np.array([8, 12, -11, 1.5, 10, 13, 17])
a1 = TimeSeries(t, v1)
a2 = TimeSeries(t, v2)
identity = lambda x: x
# index: 1 is binary tree index, 2 is bitmap index
schema = {
"pk": {"type": "str", "convert": identity, "index": None, "values": None},
"ts": {"type": "int", "convert": identity, "index": None, "values": None},
"order": {"type": "int", "convert": int, "index": 1, "values": None},
"blarg": {"type": "int", "convert": int, "index": 1, "values": None},
"useless": {"type": "int", "convert": identity, "index": 1, "values": None},
"mean": {"type": "float", "convert": float, "index": 1, "values": None},
"std": {"type": "float", "convert": float, "index": 1, "values": None},
"vp": {"type": "bool", "convert": bool, "index": 2, "values": [True, False]},
"deleted": {"type": "bool", "convert": bool, "index": 2, "values": [True, False]},
}
data_dir = "db_files/default/"
# set up directory for db data
if not os.path.exists(data_dir):
os.makedirs(data_dir)
# Delete any default db present (otherwise the db creation will load
# the previous one...)
filelist = [data_dir + f for f in os.listdir(data_dir)]
for f in filelist:
os.remove(f)
# create persistent db
ddb = PersistentDB(schema, "pk", len(t))
# CHECK INSERTION/UPSERTION/DELETION -->
# insert two new time series and metadata
ddb.insert_ts("pk1", a1)
ddb.upsert_meta("pk1", {"order": 1, "blarg": 2})
ddb.insert_ts("pk2", a2)
ddb.upsert_meta("pk2", {"order": 2, "blarg": 2})
# try to insert a duplicate primary key
with pytest.raises(ValueError):
ddb.insert_ts("pk2", a2)
# delete a valid time series
ddb.delete_ts("pk1")
# Check the deletion in the primary index
assert "pk1" not in ddb.pks.keys()
# Check the deletion in other index
for field, index in ddb.indexes.items():
if field == "deleted":
continue
for value in index.values():
assert "pk1" not in value
# # check that it isn't present any more
pk, selected = ddb.select({"pk": "pk1"}, [], None)
assert pk == []
assert len(selected) == 0
pk, selected = ddb.select({"pk": "pk2"}, [], None)
assert pk == ["pk2"]
assert len(selected) == 1
# add the time series back in
ddb.insert_ts("pk1", a1)
# Test consecutives meta upsert
ddb.upsert_meta("pk1", {"order": 2, "blarg": 3})
for k, v in ddb.indexes["order"].items():
if k == 2:
assert "pk1" in v
else:
assert "pk1" not in v
ddb.upsert_meta("pk1", {"order": 1, "blarg": 2})
for k, v in ddb.indexes["blarg"].items():
if k == 2:
assert "pk1" in v
else:
assert "pk1" not in v
# check that it's present now
pk, selected = ddb.select({"pk": "pk1"}, [], None)
assert pk == ["pk1"]
assert len(selected) == 1
# delete an invalid time series
with pytest.raises(ValueError):
ddb.delete_ts("not_here")
# try to insert metadata for a time series that isn't present
with pytest.raises(ValueError):
ddb.upsert_meta("pk3", {"order": 2, "blarg": 2})
# extract database entries for testing
db_rows = {pk: ddb._get_meta(pk) for pk in ddb.pks.keys()}
idx = sorted(db_rows.keys()) # sorted primary keys
# check primary keys
assert idx == ["pk1", "pk2"]
# check metadata
assert db_rows["pk1"]["order"] == 1
assert db_rows["pk2"]["order"] == 2
assert db_rows["pk1"]["blarg"] == 2
assert db_rows["pk2"]["blarg"] == 2
# CHECK SELECT OPERATIONS -->
pk, selected = ddb.select({}, None, None)
assert sorted(pk) == ["pk1", "pk2"]
assert selected == [{}, {}]
pk, selected = ddb.select({}, None, {"sort_by": "-order", "limit": 5})
assert sorted(pk) == ["pk1", "pk2"]
assert selected == [{}, {}]
pk, selected = ddb.select({}, None, {"sort_by": "+pk"})
assert pk == ["pk1", "pk2"]
assert selected == [{}, {}]
pk, selected = ddb.select({"order": 1, "blarg": 2}, [], None)
assert pk == ["pk1"]
assert len(selected) == 1
assert selected[0]["pk"] == "pk1"
assert selected[0]["order"] == 1
assert selected[0]["blarg"] == 2
pk, selected = ddb.select({"order": [1, 2], "blarg": 2}, [], None)
assert sorted(pk) == ["pk1", "pk2"]
assert len(selected) == 2
idx = pk.index("pk1")
assert selected[idx]["pk"] == "pk1"
assert selected[idx]["order"] == 1
assert selected[idx]["blarg"] == 2
pk, selected = ddb.select({"order": {">=": 4}}, ["order"], None)
assert len(pk) == 0
assert len(selected) == 0
# field not in schema
with pytest.raises(ValueError):
ddb.select({}, None, {"sort_by": "-unknown", "limit": 5})
# bulk update of indices
ddb.index_bulk()
check = ["blarg", "deleted", "mean", "order", "std", "useless", "vp"]
assert sorted(ddb.indexes.keys()) == check
# check time series select
pk, selected = ddb.select(meta={"pk": "pk1"}, fields=["ts"], additional=None)
assert len(pk) == 1
assert selected[0]["ts"] == a1
# CHECK BITMAP INDICES -->
# insert more data
ddb.insert_ts("pk3", a1)
ddb.insert_ts("pk4", a1)
ddb.insert_ts("pk5", a1)
ddb.insert_ts("pk6", a1)
# boolean fields should be initialized as negative
assert len(ddb.indexes["deleted"][True]) == 0
assert len(ddb.indexes["vp"][True]) == 0
# swap one
ddb.upsert_meta("pk4", {"vp": True})
assert len(ddb.indexes["vp"][True]) == 1
assert len(ddb.indexes["vp"][False]) == 5
assert list(ddb.indexes["vp"][True])[0] == "pk4"
# swap back
ddb.upsert_meta("pk4", {"vp": False})
assert len(ddb.indexes["vp"][True]) == 0
assert len(ddb.indexes["vp"][False]) == 6
# delete entry
ddb.delete_ts("pk6")
for (k, v) in ddb.indexes["vp"].items():
assert "pk6" not in v
# try to add a time series of the wrong length
with pytest.raises(ValueError):
ddb.insert_ts("wrong_length", TimeSeries([1, 2], [3, 4]))
class PersistentDBTests(unittest.TestCase):
def setUp(self):
self.dirPath = "files/testing"
if not os.path.isdir(self.dirPath):
os.makedirs(self.dirPath)
self._createdDirs = True
else:
self._createdDirs = False
schema = {
'pk': {'type': 'string', 'index': None},
'ts': {'type': None, 'index': None},
'order': {'type': 'int', 'index': 2, 'values': [-5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5]},
'blarg': {'type': 'int', 'index': 2, 'values': [1, 2]},
'mean': {'type': 'float', 'index': 1},
'std': {'type': 'float', 'index': 1},
'vp': {'type': 'bool', 'index': 2, 'values': [0,1]},
'd-vp1': {'type': 'float', 'index': 1}
}
self.schema = schema
self.tsLength = 1024
self.db = PersistentDB(schema, pk_field='pk', db_name='testing', ts_length=self.tsLength, testing=True)
for i in range(100):
pk = 'ts-'+str(i)
values = np.array(range(self.tsLength)) + i
series = ts.TimeSeries(values, values)
meta = {}
n_order = len(schema['order']['values'])# 11
meta['order'] = schema['order']['values'][i % n_order]
n_blarg = 2
meta['blarg'] = schema['blarg']['values'][i % n_blarg]
meta['mean'] = float(series.mean())# make sure they are python floats, not numpy floats
meta['std'] = float(series.std())
meta['vp'] = False
self.db.insert_ts(pk, series)
self.db.upsert_meta(pk, meta)
def tearDown(self):
self.db.delete_database()
def test_bad_testing_dbname(self):
with self.assertRaises(ValueError):
otherdb = PersistentDB(self.schema, pk_field='pk', db_name='testing', ts_length=self.tsLength)
def test_meta_save_ts(self):
self.db.close()
self.db = PersistentDB(pk_field='pk', db_name='testing', ts_length=self.tsLength, testing=True)
self.assertEqual(len(self.db),100)
def test_schema_change_good(self):
self.db.close()
self.db = PersistentDB(self.schema, pk_field='pk', db_name='testing', ts_length=self.tsLength, testing=True)
def test_schema_change_bad(self):
badschema = dict(self.schema)
badschema['blarg'] = {'type': 'int', 'index': 2, 'values': [1, 2, 3]}
self.db.close()
with self.assertRaises(ValueError):
self.db = PersistentDB(badschema, pk_field='pk', db_name='testing', ts_length=self.tsLength, testing=True)
def test_bad_insert(self):
pk = 'bad'
existing = 'ts-0'
with self.assertRaises(ValueError):
bad_series = np.array(range(self.tsLength+3))
self.db.insert_ts(pk, bad_series)
with self.assertRaises(ValueError):
values = np.array(range(self.tsLength+5))
bad_series = ts.TimeSeries(values, values)
self.db.insert_ts(pk, bad_series)
with self.assertRaises(ValueError):
values = np.array(range(self.tsLength))
series = ts.TimeSeries(values,values)
self.db.insert_ts('ts-0', series)
def test_read_meta(self):
for i in range(100):
pk = 'ts-'+str(i)
values = np.array(range(self.tsLength)) + i
series = ts.TimeSeries(values, values)
r_meta = self.db._get_meta_list(pk)
n_order = len(self.schema['order']['values'])# 11
self.assertEqual(r_meta[self.db.metaheap.fields.index('order')],self.schema['order']['values'][i % n_order])
n_blarg = 2
self.assertEqual(r_meta[self.db.metaheap.fields.index('blarg')],self.schema['blarg']['values'][i % n_blarg])
self.assertEqual(r_meta[self.db.metaheap.fields.index('mean')],series.mean())
self.assertEqual(r_meta[self.db.metaheap.fields.index('std')],series.std())
def test_read_ts(self):
for i in range(100):
pk = 'ts-'+str(i)
values = np.array(range(self.tsLength)) + i
series = ts.TimeSeries(values, values)
r_ts = self.db._return_ts(pk)
self.assertEqual(series,r_ts)
def test_select(self):
self.db.select({'pk':'ts-0'})
self.db.select({'pk':'ts-35'})
self.db.select({'order':2})
self.db.select({'order':{'>=':2}},['blarg'],{'sort_by':'+blarg'})
self.db.select({'order':{'>=':2}},['blarg'],{'sort_by':'-order'})
val = self.db.select({'order':{'>=':2}},['pk','blarg'],{'sort_by':'-order', 'limit':10})
self.assertTrue(len(val[0]) <= 10)
with self.assertRaises(ValueError):
self.db.select({'order':{'>=':2}},['blarg'],{'sort_by':'=blarg'})
with self.assertRaises(ValueError):
self.db.select({'order':{'>=':2}},['blarg'],{'sort_by':'-blargh'})
self.db.select({'order':{'>=':2}},None,{'sort_by':'-blarg'})
with self.assertRaises(TypeError):
self.db.select({'order':{'>=':2}},('pk','blarg'),{'sort_by':'-order', 'limit':10})
def test_indices(self):
"test indices via the select function, which calls on them"
n_test = 10
for i in range(n_test):
pk = 'ts-'+str(i)
tsmeta = self.db._get_meta_dict(pk)
tsinstance = tsmeta['ts']
# assert values are in indices
for field, value in tsmeta.items():
if field in self.schema.keys() and self.schema[field]['index'] is not None:
self.assertTrue(pk in self.db.select({field:value})[0])
def test_index_bulk(self):
self.db.index_bulk()
def test_upsert_meta(self):
n_test = 10
for i in range(n_test):
pk = 'ts-'+str(i)
tsmeta = self.db._get_meta_dict(pk)
tsinstance = tsmeta['ts']
oldval = tsmeta['order']
orderval = oldval + 1 if oldval != 5 else -5
tsmeta['order'] = orderval
# change value of order and check that indices change
self.db.upsert_meta(pk, tsmeta)
newmeta = self.db._get_meta_dict(pk)
self.assertEqual(newmeta['order'],orderval)
self.assertTrue(pk not in self.db.select({'order':oldval})[0])
self.assertTrue(pk in self.db.select({'order':orderval})[0])
# replace the values
replaceval = orderval - 1 if orderval != -5 else 5
tsmeta['order'] = orderval
self.db.upsert_meta(pk, tsmeta)
with self.assertRaises(TypeError):
tsmeta['order'] = "wrong type"
self.db.upsert_meta(pk, tsmeta)
def test_delete(self):
n_delete = 10
# delete and check to make sure they're gone
for i in range(n_delete):
pk = 'ts-'+str(i)
tsmeta = self.db._get_meta_dict(pk)
tsinstance = tsmeta['ts']
self.db.delete_ts(pk) # delete the timeseries
with self.assertRaises(KeyError):
self.db[pk] # check to make sure it's gone
self.assertEqual(self.db.select({'pk':pk}), ([],[]))
for field, value in tsmeta.items(): # make sure it's gone from indexes
if field in self.schema.keys() and self.schema[field]['index'] is not None:
self.assertTrue(pk not in self.db.select({field:value})[0])
# reinsert
for i in range(n_delete):
pk = 'ts-'+str(i)
values = np.array(range(self.tsLength)) + i
series = ts.TimeSeries(values, values)
meta = {}
n_order = len(self.schema['order']['values'])# 11
meta['order'] = self.schema['order']['values'][i % n_order]
n_blarg = 2
meta['blarg'] = self.schema['blarg']['values'][i % n_blarg]
meta['mean'] = float(series.mean())
meta['std'] = float(series.std())
meta['vp'] = False
self.db.insert_ts(pk, series)
self.db.upsert_meta(pk, meta)
# check to make sure everything is working as before
for i in range(n_delete):
pk = 'ts-'+str(i)
values = np.array(range(self.tsLength)) + i
series = ts.TimeSeries(values, values)
r_meta = self.db._get_meta_list(pk)
n_order = len(self.schema['order']['values'])# 11
self.assertTrue(r_meta[self.db.metaheap.fields.index('order')] == self.schema['order']['values'][i % n_order])
n_blarg = 2
self.assertTrue(r_meta[self.db.metaheap.fields.index('blarg')] == self.schema['blarg']['values'][i % n_blarg])
self.assertTrue(r_meta[self.db.metaheap.fields.index('mean')] == series.mean())
self.assertTrue(r_meta[self.db.metaheap.fields.index('std')] == series.std())
def test_vps(self):
self.db.add_vp('vp-0', 'ts-88')
self.db.add_vp('vp-1', 'ts-87')
with self.assertRaises(IndexError):
self.db.add_vp('vp-2', 'ts-1000')# doesn't exist in db
self.assertTrue('vp-0' in self.db.vps)
self.db.delete_vp('vp-0')
self.assertTrue('vp-0' not in self.db.vps)
with self.assertRaises(IndexError):
self.db.delete_vp('vp-3')# doesn't exist in db
self.db.close()
self.db = PersistentDB(pk_field='pk', db_name='testing', ts_length=self.tsLength, testing=True)
self.assertTrue('vp-1' in self.db.vps)
class PersistentDBTests(unittest.TestCase):
def setUp(self):
self.dirPath = "files/testing"
if not os.path.isdir(self.dirPath):
os.makedirs(self.dirPath)
self._createdDirs = True
else:
self._createdDirs = False
schema = {
'pk': {
'type': 'string',
'index': None
},
'ts': {
'type': None,
'index': None
},
'order': {
'type': 'int',
'index': 2,
'values': [-5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5]
},
'blarg': {
'type': 'int',
'index': 2,
'values': [1, 2]
},
'mean': {
'type': 'float',
'index': 1
},
'std': {
'type': 'float',
'index': 1
},
'vp': {
'type': 'bool',
'index': 2,
'values': [0, 1]
},
'd-vp1': {
'type': 'float',
'index': 1
}
}
self.schema = schema
self.tsLength = 1024
self.db = PersistentDB(schema,
pk_field='pk',
db_name='testing',
ts_length=self.tsLength,
testing=True)
for i in range(100):
pk = 'ts-' + str(i)
values = np.array(range(self.tsLength)) + i
series = ts.TimeSeries(values, values)
meta = {}
n_order = len(schema['order']['values']) # 11
meta['order'] = schema['order']['values'][i % n_order]
n_blarg = 2
meta['blarg'] = schema['blarg']['values'][i % n_blarg]
meta['mean'] = float(series.mean(
)) # make sure they are python floats, not numpy floats
meta['std'] = float(series.std())
meta['vp'] = False
self.db.insert_ts(pk, series)
self.db.upsert_meta(pk, meta)
def tearDown(self):
self.db.delete_database()
def test_bad_testing_dbname(self):
with self.assertRaises(ValueError):
otherdb = PersistentDB(self.schema,
pk_field='pk',
db_name='testing',
ts_length=self.tsLength)
def test_meta_save_ts(self):
self.db.close()
self.db = PersistentDB(pk_field='pk',
db_name='testing',
ts_length=self.tsLength,
testing=True)
self.assertEqual(len(self.db), 100)
def test_schema_change_good(self):
self.db.close()
self.db = PersistentDB(self.schema,
pk_field='pk',
db_name='testing',
ts_length=self.tsLength,
testing=True)
def test_schema_change_bad(self):
badschema = dict(self.schema)
badschema['blarg'] = {'type': 'int', 'index': 2, 'values': [1, 2, 3]}
self.db.close()
with self.assertRaises(ValueError):
self.db = PersistentDB(badschema,
pk_field='pk',
db_name='testing',
ts_length=self.tsLength,
testing=True)
def test_bad_insert(self):
pk = 'bad'
existing = 'ts-0'
with self.assertRaises(ValueError):
bad_series = np.array(range(self.tsLength + 3))
self.db.insert_ts(pk, bad_series)
with self.assertRaises(ValueError):
values = np.array(range(self.tsLength + 5))
bad_series = ts.TimeSeries(values, values)
self.db.insert_ts(pk, bad_series)
with self.assertRaises(ValueError):
values = np.array(range(self.tsLength))
series = ts.TimeSeries(values, values)
self.db.insert_ts('ts-0', series)
def test_read_meta(self):
for i in range(100):
pk = 'ts-' + str(i)
values = np.array(range(self.tsLength)) + i
series = ts.TimeSeries(values, values)
r_meta = self.db._get_meta_list(pk)
n_order = len(self.schema['order']['values']) # 11
self.assertEqual(r_meta[self.db.metaheap.fields.index('order')],
self.schema['order']['values'][i % n_order])
n_blarg = 2
self.assertEqual(r_meta[self.db.metaheap.fields.index('blarg')],
self.schema['blarg']['values'][i % n_blarg])
self.assertEqual(r_meta[self.db.metaheap.fields.index('mean')],
series.mean())
self.assertEqual(r_meta[self.db.metaheap.fields.index('std')],
series.std())
def test_read_ts(self):
for i in range(100):
pk = 'ts-' + str(i)
values = np.array(range(self.tsLength)) + i
series = ts.TimeSeries(values, values)
r_ts = self.db._return_ts(pk)
self.assertEqual(series, r_ts)
def test_select(self):
self.db.select({'pk': 'ts-0'})
self.db.select({'pk': 'ts-35'})
self.db.select({'order': 2})
self.db.select({'order': {'>=': 2}}, ['blarg'], {'sort_by': '+blarg'})
self.db.select({'order': {'>=': 2}}, ['blarg'], {'sort_by': '-order'})
val = self.db.select({'order': {
'>=': 2
}}, ['pk', 'blarg'], {
'sort_by': '-order',
'limit': 10
})
self.assertTrue(len(val[0]) <= 10)
with self.assertRaises(ValueError):
self.db.select({'order': {
'>=': 2
}}, ['blarg'], {'sort_by': '=blarg'})
with self.assertRaises(ValueError):
self.db.select({'order': {
'>=': 2
}}, ['blarg'], {'sort_by': '-blargh'})
self.db.select({'order': {'>=': 2}}, None, {'sort_by': '-blarg'})
with self.assertRaises(TypeError):
self.db.select({'order': {
'>=': 2
}}, ('pk', 'blarg'), {
'sort_by': '-order',
'limit': 10
})
def test_indices(self):
"test indices via the select function, which calls on them"
n_test = 10
for i in range(n_test):
pk = 'ts-' + str(i)
tsmeta = self.db._get_meta_dict(pk)
tsinstance = tsmeta['ts']
# assert values are in indices
for field, value in tsmeta.items():
if field in self.schema.keys(
) and self.schema[field]['index'] is not None:
self.assertTrue(pk in self.db.select({field: value})[0])
def test_index_bulk(self):
self.db.index_bulk()
def test_upsert_meta(self):
n_test = 10
for i in range(n_test):
pk = 'ts-' + str(i)
tsmeta = self.db._get_meta_dict(pk)
tsinstance = tsmeta['ts']
oldval = tsmeta['order']
orderval = oldval + 1 if oldval != 5 else -5
tsmeta['order'] = orderval
# change value of order and check that indices change
self.db.upsert_meta(pk, tsmeta)
newmeta = self.db._get_meta_dict(pk)
self.assertEqual(newmeta['order'], orderval)
self.assertTrue(pk not in self.db.select({'order': oldval})[0])
self.assertTrue(pk in self.db.select({'order': orderval})[0])
# replace the values
replaceval = orderval - 1 if orderval != -5 else 5
tsmeta['order'] = orderval
self.db.upsert_meta(pk, tsmeta)
with self.assertRaises(TypeError):
tsmeta['order'] = "wrong type"
self.db.upsert_meta(pk, tsmeta)
def test_delete(self):
n_delete = 10
# delete and check to make sure they're gone
for i in range(n_delete):
pk = 'ts-' + str(i)
tsmeta = self.db._get_meta_dict(pk)
tsinstance = tsmeta['ts']
self.db.delete_ts(pk) # delete the timeseries
with self.assertRaises(KeyError):
self.db[pk] # check to make sure it's gone
self.assertEqual(self.db.select({'pk': pk}), ([], []))
for field, value in tsmeta.items(
): # make sure it's gone from indexes
if field in self.schema.keys(
) and self.schema[field]['index'] is not None:
self.assertTrue(
pk not in self.db.select({field: value})[0])
# reinsert
for i in range(n_delete):
pk = 'ts-' + str(i)
values = np.array(range(self.tsLength)) + i
series = ts.TimeSeries(values, values)
meta = {}
n_order = len(self.schema['order']['values']) # 11
meta['order'] = self.schema['order']['values'][i % n_order]
n_blarg = 2
meta['blarg'] = self.schema['blarg']['values'][i % n_blarg]
meta['mean'] = float(series.mean())
meta['std'] = float(series.std())
meta['vp'] = False
self.db.insert_ts(pk, series)
self.db.upsert_meta(pk, meta)
# check to make sure everything is working as before
for i in range(n_delete):
pk = 'ts-' + str(i)
values = np.array(range(self.tsLength)) + i
series = ts.TimeSeries(values, values)
r_meta = self.db._get_meta_list(pk)
n_order = len(self.schema['order']['values']) # 11
self.assertTrue(r_meta[self.db.metaheap.fields.index('order')] ==
self.schema['order']['values'][i % n_order])
n_blarg = 2
self.assertTrue(r_meta[self.db.metaheap.fields.index('blarg')] ==
self.schema['blarg']['values'][i % n_blarg])
self.assertTrue(
r_meta[self.db.metaheap.fields.index('mean')] == series.mean())
self.assertTrue(
r_meta[self.db.metaheap.fields.index('std')] == series.std())
def test_vps(self):
self.db.add_vp('vp-0', 'ts-88')
self.db.add_vp('vp-1', 'ts-87')
with self.assertRaises(IndexError):
self.db.add_vp('vp-2', 'ts-1000') # doesn't exist in db
self.assertTrue('vp-0' in self.db.vps)
self.db.delete_vp('vp-0')
self.assertTrue('vp-0' not in self.db.vps)
with self.assertRaises(IndexError):
self.db.delete_vp('vp-3') # doesn't exist in db
self.db.close()
self.db = PersistentDB(pk_field='pk',
db_name='testing',
ts_length=self.tsLength,
testing=True)
self.assertTrue('vp-1' in self.db.vps)