def test_load_del(self):
db = PersistentDB(schema, 'pk', dbname='testdb', overwrite=True)
n_add = 50
mus = np.random.uniform(low=0.0, high=1.0, size=n_add)
sigs = np.random.uniform(low=0.05, high=0.4, size=n_add)
jits = np.random.uniform(low=0.05, high=0.2, size=n_add)
saveinfo = {}
for i, m, s, j in zip(range(n_add), mus, sigs, jits):
new_ts = tsmaker(m, s, j)
db.insert_ts("ts-{}".format(i), tsmaker(m, s, j))
db.upsert_meta("ts-{}".format(i), {
'mean': new_ts.mean(),
'std': new_ts.std()
})
saveinfo["ts-{}".format(i)] = new_ts.mean()
db.add_vp("ts-4")
db.add_vp()
db.delete_ts("ts-4")
pks, fields = db.select(meta={'vp': True}, fields=None)
self.assertEqual(len(pks), 1)
newdb = PersistentDB(schema, 'pk', dbname='testdb', load=True)
pks, fields = db.select(meta={}, fields=['mean'])
self.assertEqual(len(pks), n_add - 1)
self.assertTrue("ts-4" not in pks)
for i in range(0, n_add - 1):
self.assertEqual(fields[i]['mean'], saveinfo[pks[i]])
def test_load_del(self):
db = PersistentDB(schema, 'pk', dbname='testdb', overwrite=True)
n_add = 50
mus = np.random.uniform(low=0.0, high=1.0, size=n_add)
sigs = np.random.uniform(low=0.05, high=0.4, size=n_add)
jits = np.random.uniform(low=0.05, high=0.2, size=n_add)
saveinfo = {}
for i, m, s, j in zip(range(n_add), mus, sigs, jits):
new_ts = tsmaker(m, s, j)
db.insert_ts("ts-{}".format(i), tsmaker(m, s, j))
db.upsert_meta("ts-{}".format(i), {'mean':new_ts.mean(), 'std':new_ts.std()})
saveinfo["ts-{}".format(i)] = new_ts.mean()
db.add_vp("ts-4")
db.add_vp()
db.delete_ts("ts-4")
pks, fields = db.select(meta={'vp':True}, fields=None)
self.assertEqual(len(pks),1)
newdb = PersistentDB(schema, 'pk', dbname='testdb', load=True)
pks, fields = db.select(meta={}, fields=['mean'])
self.assertEqual(len(pks), n_add-1)
self.assertTrue("ts-4" not in pks)
for i in range(0,n_add-1):
self.assertEqual(fields[i]['mean'], saveinfo[pks[i]])
def test_trees(self):
db = PersistentDB(schema, 'pk', dbname='testdb', overwrite=True)
n_add = 50
mus = np.random.uniform(low=0.0, high=1.0, size=n_add)
sigs = np.random.uniform(low=0.05, high=0.4, size=n_add)
jits = np.random.uniform(low=0.05, high=0.2, size=n_add)
for i, m, s, j in zip(range(n_add), mus, sigs, jits):
new_ts = tsmaker(m, s, j)
db.insert_ts("ts-{}".format(i), tsmaker(m, s, j))
db.upsert_meta("ts-{}".format(i), {
'mean': new_ts.mean(),
'std': new_ts.std()
})
randi = set(np.random.randint(0, n_add, size=5))
for i in randi:
db.delete_ts("ts-{}".format(i))
pks, fields = db.select(meta={
'mean': {
'<=': 0.5
},
'std': {
'>': 2
}
},
fields=['mean', 'std'])
for row in fields:
self.assertLessEqual(row['mean'], 0.5)
self.assertGreater(row['std'], 2)
def test_trees(self):
db = PersistentDB(schema, 'pk', dbname='testdb', overwrite=True)
n_add = 50
mus = np.random.uniform(low=0.0, high=1.0, size=n_add)
sigs = np.random.uniform(low=0.05, high=0.4, size=n_add)
jits = np.random.uniform(low=0.05, high=0.2, size=n_add)
for i, m, s, j in zip(range(n_add), mus, sigs, jits):
new_ts = tsmaker(m, s, j)
db.insert_ts("ts-{}".format(i), tsmaker(m, s, j))
db.upsert_meta("ts-{}".format(i), {'mean':new_ts.mean(), 'std':new_ts.std()})
randi = set(np.random.randint(0,n_add, size=5))
for i in randi:
db.delete_ts("ts-{}".format(i))
pks, fields = db.select(meta={'mean':{'<=':0.5}, 'std':{'>':2}}, fields=['mean', 'std'])
for row in fields:
self.assertLessEqual(row['mean'], 0.5)
self.assertGreater(row['std'], 2)
def test_db_upsertmeta(self):
ts1 = TimeSeries([1, 2, 3], [4, 5, 6])
db = PersistentDB(schema, 'pk', dbname='testdb', overwrite=True)
with self.assertRaises(ValueError):
db.upsert_meta('ts1', {'mean': 5})
db.insert_ts('ts1', ts1)
with self.assertRaises(ValueError):
db.upsert_meta('ts1', 'mean' == 5)
db.upsert_meta('ts1', {'mean': 5})
def test_db_upsertmeta(self):
ts1 = TimeSeries([1,2,3],[4,5,6])
db = PersistentDB(schema, 'pk', dbname='testdb', overwrite=True)
with self.assertRaises(ValueError):
db.upsert_meta('ts1', {'mean':5})
db.insert_ts('ts1', ts1)
with self.assertRaises(ValueError):
db.upsert_meta('ts1', 'mean' == 5)
db.upsert_meta('ts1', {'mean':5})
def test_db_select(self):
db = PersistentDB(schema, 'pk', dbname='testdb', overwrite=True)
db.insert_ts('one', TimeSeries([1, 2, 3], [4, 5, 6]))
db.insert_ts('two', TimeSeries([7, 8, 9], [3, 4, 5]))
db.insert_ts('negone', TimeSeries([1, 2, 3], [-4, -5, -6]))
db.upsert_meta('one', {'order': 3})
db.upsert_meta('one', {'order': 1, 'mean': 5})
db.upsert_meta('two', {'order': 2, 'mean': 4})
db.upsert_meta('negone', {'order': -1, 'mean': -5})
with self.assertRaises(ValueError):
db.select(meta=None, fields=None)
with self.assertRaises(ValueError):
db.select(meta=None, fields='mean')
pks, fields = db.select(meta={}, fields=None)
self.assertEqual(set(pks), set(['one', 'two', 'negone']))
self.assertEqual(len(fields[0]), 0)
pks, fields = db.select(meta={}, fields=[])
self.assertEqual(set(pks), set(['one', 'two', 'negone']))
self.assertEqual(fields[pks.index('one')]['order'], 1)
pks, fields = db.select(meta={'mean': 5}, fields=None)
self.assertEqual(set(pks), set(['one']))
pks, fields = db.select(meta={'mean': {'<=': 4}}, fields=None)
self.assertEqual(set(pks), set(['two', 'negone']))
pks, fields = db.select(meta={'mean': 5}, fields=['order'])
self.assertEqual(fields[0]['order'], 1)
pks, fields = db.select(meta={},
fields=None,
additional={'sort_by': '+order'})
self.assertEqual(pks, ['negone', 'one', 'two'])
pks, fields = db.select(meta={},
fields=None,
additional={'sort_by': '-order'})
self.assertEqual(pks, ['two', 'one', 'negone'])
pks, fields = db.select(meta={},
fields=None,
additional={
'sort_by': '-order',
'limit': 2
})
self.assertEqual(pks, ['two', 'one'])
def test_db_select(self):
db = PersistentDB(schema, 'pk', dbname='testdb', overwrite=True)
db.insert_ts('one', TimeSeries([1,2,3],[4,5,6]))
db.insert_ts('two', TimeSeries([7,8,9],[3,4,5]))
db.insert_ts('negone', TimeSeries([1,2,3],[-4,-5,-6]))
db.upsert_meta('one', {'order':3})
db.upsert_meta('one', {'order':1, 'mean':5})
db.upsert_meta('two', {'order':2, 'mean':4})
db.upsert_meta('negone', {'order':-1, 'mean':-5})
with self.assertRaises(ValueError):
db.select(meta=None, fields=None)
with self.assertRaises(ValueError):
db.select(meta=None, fields='mean')
pks, fields = db.select(meta={}, fields=None)
self.assertEqual(set(pks), set(['one', 'two', 'negone']))
self.assertEqual(len(fields[0]), 0)
pks, fields = db.select(meta={}, fields=[])
self.assertEqual(set(pks), set(['one', 'two', 'negone']))
self.assertEqual(fields[pks.index('one')]['order'], 1)
pks, fields = db.select(meta={'mean':5}, fields=None)
self.assertEqual(set(pks), set(['one']))
pks, fields = db.select(meta={'mean':{'<=':4}}, fields=None)
self.assertEqual(set(pks), set(['two', 'negone']))
pks, fields = db.select(meta={'mean':5}, fields=['order'])
self.assertEqual(fields[0]['order'], 1)
pks, fields = db.select(meta={}, fields=None, additional={'sort_by':'+order'})
self.assertEqual(pks, ['negone', 'one', 'two'])
pks, fields = db.select(meta={}, fields=None, additional={'sort_by':'-order'})
self.assertEqual(pks, ['two', 'one', 'negone'])
pks, fields = db.select(meta={}, fields=None, additional={'sort_by':'-order', 'limit':2})
self.assertEqual(pks, ['two', 'one'])
class PersistentDBTests(unittest.TestCase):
def setUp(self):
self.dirPath = "persistent_files/testing"
if not os.path.isdir(self.dirPath):
os.makedirs(self.dirPath)
self._createdDirs = True
else:
self._createdDirs = False
self.schema = schema_type
self.ts_length = 100
self.db = PersistentDB(schema_type, pk_field='pk', db_name='testing', ts_length=self.ts_length)
for i in range(100):
pk = 'ts-'+str(i)
values = np.array(range(self.ts_length)) + i
series = ts.TimeSeries(values, values)
meta = {}
n_order = len(schema_type['order']['values'])# 11
meta['order'] = schema_type['order']['values'][i % n_order]
n_blarg = 2
meta['blarg'] = schema_type['blarg']['values'][i % n_blarg]
meta['mean'] = float(series.mean())# make sure they are python floats, not numpy floats
meta['std'] = float(series.std())
meta['vp'] = False
self.db.insert_ts(pk, series)
self.db.upsert_meta(pk, meta)
def tearDown(self):
self.db.delete_database()
def test_select1(self):
self.db.select({'pk':'ts-0'})
def test_select2(self):
self.db.select({'pk':'ts-35'})
def test_select3(self):
self.db.select({'order':2})
def test_select4(self):
self.db.select({'order':{'>=':1}},['blarg'],{'sort_by':'-order'})
def test_select5(self):
self.db.select({'order':{'<=':2}},['blarg'],{'sort_by':'+blarg'})
def test_select6(self):
val = self.db.select({'order':{'>=':2}},['pk','blarg'],{'sort_by':'-order', 'limit':10})
self.assertTrue(len(val[0]) <= 10)
def test_select7(self):
with self.assertRaises(ValueError):
self.db.select({'order':{'>=':2}},['blarg'],{'sort_by':'=blarg'})
def test_select8(self):
with self.assertRaises(ValueError):
self.db.select({'order':{'>=':2}},['blarg'],{'sort_by':'-none'})
def test_select9(self):
self.db.select({'order':{'>=':2}},None,{'sort_by':'-blarg'})
def test_select10(self):
with self.assertRaises(TypeError):
self.db.select({'order':{'>=':2}},('pk','blarg'),{'sort_by':'-order', 'limit':10})
def test_meta_save_ts(self):
self.db.close()
self.db = PersistentDB(pk_field='pk', db_name='testing', ts_length=self.ts_length)
self.assertEqual(len(self.db),100)
def test_schema_change(self):
self.db.close()
self.db = PersistentDB(self.schema, pk_field='pk', db_name='testing', ts_length=self.ts_length)
def test_schema_change_exception(self):
badschema = dict(self.schema)
badschema['blarg'] = {'type': 'int', 'index': 2, 'values': [1, 2, 3]}
self.db.close()
with self.assertRaises(ValueError):
self.db = PersistentDB(badschema, pk_field='pk', db_name='testing', ts_length=self.ts_length)
def test_insert_exception(self):
pk = 'bad'
existing = 'ts-0'
with self.assertRaises(ValueError):
bad_series = np.array(range(self.ts_length+3))
self.db.insert_ts(pk, bad_series)
with self.assertRaises(ValueError):
values = np.array(range(self.ts_length+5))
bad_series = ts.TimeSeries(values, values)
self.db.insert_ts(pk, bad_series)
with self.assertRaises(ValueError):
values = np.array(range(self.ts_length))
series = ts.TimeSeries(values,values)
self.db.insert_ts('ts-0', series)
def test_metadata(self):
for i in range(100):
pk = 'ts-'+str(i)
values = np.array(range(self.ts_length)) + i
series = ts.TimeSeries(values, values)
r_meta = self.db._get_meta_list(pk)
n_order = len(self.schema['order']['values'])
self.assertEqual(r_meta[self.db.metaheap.fields.index('order')], self.schema['order']['values'][i % n_order])
n_blarg = 2
self.assertEqual(r_meta[self.db.metaheap.fields.index('blarg')],self.schema['blarg']['values'][i % n_blarg])
self.assertEqual(r_meta[self.db.metaheap.fields.index('mean')],series.mean())
self.assertEqual(r_meta[self.db.metaheap.fields.index('std')],series.std())
def test_read_ts(self):
for i in range(100):
pk = 'ts-'+str(i)
values = np.array(range(self.ts_length)) + i
series = ts.TimeSeries(values, values)
r_ts = self.db._return_ts(pk)
self.assertEqual(series,r_ts)
def test_indices(self):
n_test = 10
for i in range(n_test):
pk = 'ts-'+str(i)
tsmeta = self.db._get_meta_dict(pk)
tsinstance = tsmeta['ts']
# assert values are in indices
for field, value in tsmeta.items():
if field in self.schema.keys() and self.schema[field]['index'] is not None:
self.assertTrue(pk in self.db.select({field:value})[0])
def test_index_bulk(self):
self.db.index_bulk()
def test_delete_ts(self):
n_delete = 10
# delete and check to make sure they're gone
for i in range(n_delete):
pk = 'ts-'+str(i)
tsmeta = self.db._get_meta_dict(pk)
self.db.delete_ts(pk) # delete the timeseries
#Check 1: __get__() get by pk fail
with self.assertRaises(KeyError):
self.db[pk] # check to make sure it's gone
#Check 2: db_select return empty sets
self.assertEqual(self.db.select({'pk':pk}), ([],[]))
#Check 3: does not exist in index
for field, value in tsmeta.items(): # make sure it's gone from indexes
if field in self.schema.keys() and self.schema[field]['index'] is not None:
self.assertTrue(pk not in self.db.select({field:value})[0])
#Check 4: check the db after deletion is clean and can hold the same pk and timeseries again
# insert the deleted ts and check to make sure everything is working as before
for i in range(n_delete):
pk = 'ts-'+str(i)
values = np.array(range(self.ts_length)) + i
series = ts.TimeSeries(values, values)
meta = {}
meta['mean'] = float(series.mean())
meta['std'] = float(series.std())
meta['vp'] = False
meta['blarg'] = self.schema['blarg']['values'][i % 2] #blarg only has two value
n_order = len(self.schema['order']['values'])# 11
meta['order'] = self.schema['order']['values'][i % n_order]
self.db.insert_ts(pk, series)
self.db.upsert_meta(pk, meta)
for i in range(n_delete):
pk = 'ts-'+str(i)
values = np.array(range(self.ts_length)) + i
series = ts.TimeSeries(values, values)
r_meta = self.db._get_meta_list(pk)
n_order = len(self.schema['order']['values'])# 11
self.assertTrue(r_meta[self.db.metaheap.fields.index('order')] == self.schema['order']['values'][i % n_order])
n_blarg = 2
self.assertTrue(r_meta[self.db.metaheap.fields.index('blarg')] == self.schema['blarg']['values'][i % n_blarg])
self.assertTrue(r_meta[self.db.metaheap.fields.index('mean')] == series.mean())
self.assertTrue(r_meta[self.db.metaheap.fields.index('std')] == series.std())
class PersistentDBTests(unittest.TestCase):
def setUp(self):
self.dirPath = "persistent_files/testing"
if not os.path.isdir(self.dirPath):
os.makedirs(self.dirPath)
self._createdDirs = True
else:
self._createdDirs = False
self.schema = schema_type
self.ts_length = 100
self.db = PersistentDB(schema_type,
pk_field='pk',
db_name='testing',
ts_length=self.ts_length)
for i in range(100):
pk = 'ts-' + str(i)
values = np.array(range(self.ts_length)) + i
series = ts.TimeSeries(values, values)
meta = {}
n_order = len(schema_type['order']['values']) # 11
meta['order'] = schema_type['order']['values'][i % n_order]
n_blarg = 2
meta['blarg'] = schema_type['blarg']['values'][i % n_blarg]
meta['mean'] = float(series.mean(
)) # make sure they are python floats, not numpy floats
meta['std'] = float(series.std())
meta['vp'] = False
self.db.insert_ts(pk, series)
self.db.upsert_meta(pk, meta)
def tearDown(self):
self.db.delete_database()
def test_select1(self):
self.db.select({'pk': 'ts-0'})
def test_select2(self):
self.db.select({'pk': 'ts-35'})
def test_select3(self):
self.db.select({'order': 2})
def test_select4(self):
self.db.select({'order': {'>=': 1}}, ['blarg'], {'sort_by': '-order'})
def test_select5(self):
self.db.select({'order': {'<=': 2}}, ['blarg'], {'sort_by': '+blarg'})
def test_select6(self):
val = self.db.select({'order': {
'>=': 2
}}, ['pk', 'blarg'], {
'sort_by': '-order',
'limit': 10
})
self.assertTrue(len(val[0]) <= 10)
def test_select7(self):
with self.assertRaises(ValueError):
self.db.select({'order': {
'>=': 2
}}, ['blarg'], {'sort_by': '=blarg'})
def test_select8(self):
with self.assertRaises(ValueError):
self.db.select({'order': {
'>=': 2
}}, ['blarg'], {'sort_by': '-none'})
def test_select9(self):
self.db.select({'order': {'>=': 2}}, None, {'sort_by': '-blarg'})
def test_select10(self):
with self.assertRaises(TypeError):
self.db.select({'order': {
'>=': 2
}}, ('pk', 'blarg'), {
'sort_by': '-order',
'limit': 10
})
def test_meta_save_ts(self):
self.db.close()
self.db = PersistentDB(pk_field='pk',
db_name='testing',
ts_length=self.ts_length)
self.assertEqual(len(self.db), 100)
def test_schema_change(self):
self.db.close()
self.db = PersistentDB(self.schema,
pk_field='pk',
db_name='testing',
ts_length=self.ts_length)
def test_schema_change_exception(self):
badschema = dict(self.schema)
badschema['blarg'] = {'type': 'int', 'index': 2, 'values': [1, 2, 3]}
self.db.close()
with self.assertRaises(ValueError):
self.db = PersistentDB(badschema,
pk_field='pk',
db_name='testing',
ts_length=self.ts_length)
def test_insert_exception(self):
pk = 'bad'
existing = 'ts-0'
with self.assertRaises(ValueError):
bad_series = np.array(range(self.ts_length + 3))
self.db.insert_ts(pk, bad_series)
with self.assertRaises(ValueError):
values = np.array(range(self.ts_length + 5))
bad_series = ts.TimeSeries(values, values)
self.db.insert_ts(pk, bad_series)
with self.assertRaises(ValueError):
values = np.array(range(self.ts_length))
series = ts.TimeSeries(values, values)
self.db.insert_ts('ts-0', series)
def test_metadata(self):
for i in range(100):
pk = 'ts-' + str(i)
values = np.array(range(self.ts_length)) + i
series = ts.TimeSeries(values, values)
r_meta = self.db._get_meta_list(pk)
n_order = len(self.schema['order']['values'])
self.assertEqual(r_meta[self.db.metaheap.fields.index('order')],
self.schema['order']['values'][i % n_order])
n_blarg = 2
self.assertEqual(r_meta[self.db.metaheap.fields.index('blarg')],
self.schema['blarg']['values'][i % n_blarg])
self.assertEqual(r_meta[self.db.metaheap.fields.index('mean')],
series.mean())
self.assertEqual(r_meta[self.db.metaheap.fields.index('std')],
series.std())
def test_read_ts(self):
for i in range(100):
pk = 'ts-' + str(i)
values = np.array(range(self.ts_length)) + i
series = ts.TimeSeries(values, values)
r_ts = self.db._return_ts(pk)
self.assertEqual(series, r_ts)
def test_indices(self):
n_test = 10
for i in range(n_test):
pk = 'ts-' + str(i)
tsmeta = self.db._get_meta_dict(pk)
tsinstance = tsmeta['ts']
# assert values are in indices
for field, value in tsmeta.items():
if field in self.schema.keys(
) and self.schema[field]['index'] is not None:
self.assertTrue(pk in self.db.select({field: value})[0])
def test_index_bulk(self):
self.db.index_bulk()
def test_delete_ts(self):
n_delete = 10
# delete and check to make sure they're gone
for i in range(n_delete):
pk = 'ts-' + str(i)
tsmeta = self.db._get_meta_dict(pk)
self.db.delete_ts(pk) # delete the timeseries
#Check 1: __get__() get by pk fail
with self.assertRaises(KeyError):
self.db[pk] # check to make sure it's gone
#Check 2: db_select return empty sets
self.assertEqual(self.db.select({'pk': pk}), ([], []))
#Check 3: does not exist in index
for field, value in tsmeta.items(
): # make sure it's gone from indexes
if field in self.schema.keys(
) and self.schema[field]['index'] is not None:
self.assertTrue(
pk not in self.db.select({field: value})[0])
#Check 4: check the db after deletion is clean and can hold the same pk and timeseries again
# insert the deleted ts and check to make sure everything is working as before
for i in range(n_delete):
pk = 'ts-' + str(i)
values = np.array(range(self.ts_length)) + i
series = ts.TimeSeries(values, values)
meta = {}
meta['mean'] = float(series.mean())
meta['std'] = float(series.std())
meta['vp'] = False
meta['blarg'] = self.schema['blarg']['values'][
i % 2] #blarg only has two value
n_order = len(self.schema['order']['values']) # 11
meta['order'] = self.schema['order']['values'][i % n_order]
self.db.insert_ts(pk, series)
self.db.upsert_meta(pk, meta)
for i in range(n_delete):
pk = 'ts-' + str(i)
values = np.array(range(self.ts_length)) + i
series = ts.TimeSeries(values, values)
r_meta = self.db._get_meta_list(pk)
n_order = len(self.schema['order']['values']) # 11
self.assertTrue(r_meta[self.db.metaheap.fields.index('order')] ==
self.schema['order']['values'][i % n_order])
n_blarg = 2
self.assertTrue(r_meta[self.db.metaheap.fields.index('blarg')] ==
self.schema['blarg']['values'][i % n_blarg])
self.assertTrue(
r_meta[self.db.metaheap.fields.index('mean')] == series.mean())
self.assertTrue(
r_meta[self.db.metaheap.fields.index('std')] == series.std())