def test_load_del(self):
db = PersistentDB(schema, 'pk', dbname='testdb', overwrite=True)
n_add = 50
mus = np.random.uniform(low=0.0, high=1.0, size=n_add)
sigs = np.random.uniform(low=0.05, high=0.4, size=n_add)
jits = np.random.uniform(low=0.05, high=0.2, size=n_add)
saveinfo = {}
for i, m, s, j in zip(range(n_add), mus, sigs, jits):
new_ts = tsmaker(m, s, j)
db.insert_ts("ts-{}".format(i), tsmaker(m, s, j))
db.upsert_meta("ts-{}".format(i), {
'mean': new_ts.mean(),
'std': new_ts.std()
})
saveinfo["ts-{}".format(i)] = new_ts.mean()
db.add_vp("ts-4")
db.add_vp()
db.delete_ts("ts-4")
pks, fields = db.select(meta={'vp': True}, fields=None)
self.assertEqual(len(pks), 1)
newdb = PersistentDB(schema, 'pk', dbname='testdb', load=True)
pks, fields = db.select(meta={}, fields=['mean'])
self.assertEqual(len(pks), n_add - 1)
self.assertTrue("ts-4" not in pks)
for i in range(0, n_add - 1):
self.assertEqual(fields[i]['mean'], saveinfo[pks[i]])
def test_trees(self):
db = PersistentDB(schema, 'pk', dbname='testdb', overwrite=True)
n_add = 50
mus = np.random.uniform(low=0.0, high=1.0, size=n_add)
sigs = np.random.uniform(low=0.05, high=0.4, size=n_add)
jits = np.random.uniform(low=0.05, high=0.2, size=n_add)
for i, m, s, j in zip(range(n_add), mus, sigs, jits):
new_ts = tsmaker(m, s, j)
db.insert_ts("ts-{}".format(i), tsmaker(m, s, j))
db.upsert_meta("ts-{}".format(i), {
'mean': new_ts.mean(),
'std': new_ts.std()
})
randi = set(np.random.randint(0, n_add, size=5))
for i in randi:
db.delete_ts("ts-{}".format(i))
pks, fields = db.select(meta={
'mean': {
'<=': 0.5
},
'std': {
'>': 2
}
},
fields=['mean', 'std'])
for row in fields:
self.assertLessEqual(row['mean'], 0.5)
self.assertGreater(row['std'], 2)
def main(load=False, dbname="db", overwrite=False, threshold = 10, wordlength = 16, tslen = 256, cardinality = 64):
# we augment the schema by adding columns for 5 vantage points
#for i in range(NUMVPS):
# schema["d_vp-{}".format(i)] = {'convert': float, 'index': 1}
db = PersistentDB(schema, 'pk',load=load, dbname=dbname, overwrite=overwrite, threshold = threshold, wordlength = wordlength, tslen = tslen, cardinality = cardinality)
server = TSDBServer(db)
server.run()
def setUp(self):
self.dirPath = "persistent_files/testing"
if not os.path.isdir(self.dirPath):
os.makedirs(self.dirPath)
self._createdDirs = True
else:
self._createdDirs = False
self.schema = schema_type
self.ts_length = 100
self.db = PersistentDB(schema_type,
pk_field='pk',
db_name='testing',
ts_length=self.ts_length)
for i in range(100):
pk = 'ts-' + str(i)
values = np.array(range(self.ts_length)) + i
series = ts.TimeSeries(values, values)
meta = {}
n_order = len(schema_type['order']['values']) # 11
meta['order'] = schema_type['order']['values'][i % n_order]
n_blarg = 2
meta['blarg'] = schema_type['blarg']['values'][i % n_blarg]
meta['mean'] = float(series.mean(
)) # make sure they are python floats, not numpy floats
meta['std'] = float(series.std())
meta['vp'] = False
self.db.insert_ts(pk, series)
self.db.upsert_meta(pk, meta)
def test_db_upsertmeta(self):
ts1 = TimeSeries([1, 2, 3], [4, 5, 6])
db = PersistentDB(schema, 'pk', dbname='testdb', overwrite=True)
with self.assertRaises(ValueError):
db.upsert_meta('ts1', {'mean': 5})
db.insert_ts('ts1', ts1)
with self.assertRaises(ValueError):
db.upsert_meta('ts1', 'mean' == 5)
db.upsert_meta('ts1', {'mean': 5})
def test_schema_change_exception(self):
badschema = dict(self.schema)
badschema['blarg'] = {'type': 'int', 'index': 2, 'values': [1, 2, 3]}
self.db.close()
with self.assertRaises(ValueError):
self.db = PersistentDB(badschema,
pk_field='pk',
db_name='testing',
ts_length=self.ts_length)
def test_db_tsinsert(self):
ts1 = TimeSeries([1, 2, 3], [4, 5, 6])
ts2 = TimeSeries([1, 2, 3], [4, 5, 6])
db = PersistentDB(schema, 'pk', dbname='testdb', overwrite=True)
db.insert_ts('ts1', ts1)
with self.assertRaises(ValueError):
db.insert_ts('ts1', ts2)
with self.assertRaises(ValueError):
db.insert_ts('ts:1', ts2)
with self.assertRaises(ValueError):
db.insert_ts('ts1', [[1, 2, 3], [4, 5, 6]])
db.insert_ts('ts2', ts2)
db.insert_ts('ts3', ts2)
def test_simsearchSAX(self):
db = PersistentDB(schema, 'pk', dbname='testdb', overwrite=True)
n_add = 50
mus = np.random.uniform(low=0.0, high=1.0, size=n_add)
sigs = np.random.uniform(low=0.05, high=0.4, size=n_add)
jits = np.random.uniform(low=0.05, high=0.2, size=n_add)
for i, m, s, j in zip(range(n_add), mus, sigs, jits):
db.insert_ts("ts-{}".format(i), tsmaker(m, s, j))
m = np.random.uniform(low=0.0, high=1.0)
s = np.random.uniform(low=0.05, high=0.4)
j = np.random.uniform(low=0.05, high=0.2)
query = tsmaker(m, s, j)
closest = db.simsearch_SAX(query)
def main():
db_name = 'testing'
dirPath = FILES_DIR + '/' + db_name
if not os.path.isdir(dirPath):
os.makedirs(dirPath)
_createdDirs = True
else:
_createdDirs = False
db = PersistentDB(schema,
pk_field='pk',
db_name=db_name,
ts_length=TS_LENGTH)
server = TSDBServer(db, port=9999)
server.run()
db.delete_database()
def test_db_select(self):
db = PersistentDB(schema, 'pk', dbname='testdb', overwrite=True)
db.insert_ts('one', TimeSeries([1, 2, 3], [4, 5, 6]))
db.insert_ts('two', TimeSeries([7, 8, 9], [3, 4, 5]))
db.insert_ts('negone', TimeSeries([1, 2, 3], [-4, -5, -6]))
db.upsert_meta('one', {'order': 3})
db.upsert_meta('one', {'order': 1, 'mean': 5})
db.upsert_meta('two', {'order': 2, 'mean': 4})
db.upsert_meta('negone', {'order': -1, 'mean': -5})
with self.assertRaises(ValueError):
db.select(meta=None, fields=None)
with self.assertRaises(ValueError):
db.select(meta=None, fields='mean')
pks, fields = db.select(meta={}, fields=None)
self.assertEqual(set(pks), set(['one', 'two', 'negone']))
self.assertEqual(len(fields[0]), 0)
pks, fields = db.select(meta={}, fields=[])
self.assertEqual(set(pks), set(['one', 'two', 'negone']))
self.assertEqual(fields[pks.index('one')]['order'], 1)
pks, fields = db.select(meta={'mean': 5}, fields=None)
self.assertEqual(set(pks), set(['one']))
pks, fields = db.select(meta={'mean': {'<=': 4}}, fields=None)
self.assertEqual(set(pks), set(['two', 'negone']))
pks, fields = db.select(meta={'mean': 5}, fields=['order'])
self.assertEqual(fields[0]['order'], 1)
pks, fields = db.select(meta={},
fields=None,
additional={'sort_by': '+order'})
self.assertEqual(pks, ['negone', 'one', 'two'])
pks, fields = db.select(meta={},
fields=None,
additional={'sort_by': '-order'})
self.assertEqual(pks, ['two', 'one', 'negone'])
pks, fields = db.select(meta={},
fields=None,
additional={
'sort_by': '-order',
'limit': 2
})
self.assertEqual(pks, ['two', 'one'])
def test_simsearch(self):
db = PersistentDB(schema, 'pk', dbname='testdb', overwrite=True)
n_add = 50
mus = np.random.uniform(low=0.0, high=1.0, size=n_add)
sigs = np.random.uniform(low=0.05, high=0.4, size=n_add)
jits = np.random.uniform(low=0.05, high=0.2, size=n_add)
for i, m, s, j in zip(range(n_add), mus, sigs, jits):
db.insert_ts("ts-{}".format(i), tsmaker(m, s, j))
m = np.random.uniform(low=0.0, high=1.0)
s = np.random.uniform(low=0.05, high=0.4)
j = np.random.uniform(low=0.05, high=0.2)
query = tsmaker(m, s, j)
with self.assertRaises(
ValueError): # No similarity search w/o vantage points
closest = db.simsearch(query)
for i in range(5):
db.add_vp()
closest = db.simsearch(query)
def test_badinput(self):
with self.assertRaises(ValueError):
db = PersistentDB({'pk': {
'type': int
}},
'pk',
dbname='testdb',
overwrite=True)
with self.assertRaises(ValueError):
db = PersistentDB(schema,
'pk',
dbname='testdb',
overwrite=True,
threshold='a')
with self.assertRaises(ValueError):
db = PersistentDB(schema,
12,
dbname='testdb',
overwrite=True,
threshold='a')
with self.assertRaises(ValueError):
db = PersistentDB(schema,
'pk',
dbname='testdb',
overwrite=True,
wordlength='a')
with self.assertRaises(ValueError):
db = PersistentDB(schema,
'pk',
dbname='testdb',
overwrite=True,
threshold=-10)
with self.assertRaises(ValueError):
db = PersistentDB(schema,
'pk',
dbname='testdb',
overwrite=True,
wordlength=-10)
with self.assertRaises(ValueError):
db = PersistentDB(schema,
'pk',
dbname='testdb',
overwrite=True,
cardinality=-10)
with self.assertRaises(ValueError):
db = PersistentDB(schema,
'pk',
dbname='testdb',
overwrite=True,
wordlength=10)
with self.assertRaises(ValueError):
db = PersistentDB(schema,
'pk',
dbname='testdb',
overwrite=True,
tslen=300)
with self.assertRaises(ValueError):
db = PersistentDB(schema,
'pk',
dbname='testdb',
overwrite=True,
tslen='256')
with self.assertRaises(ValueError):
db = PersistentDB(schema,
'pk',
dbname='testdb',
overwrite=True,
tslen=8)
with self.assertRaises(ValueError):
db = PersistentDB(schema,
'pk',
dbname='testdb',
overwrite=True,
cardinality=10.5)
with self.assertRaises(ValueError):
db = PersistentDB(schema,
'pk',
dbname='testdb',
overwrite=True,
cardinality=10)
with self.assertRaises(ValueError):
db = PersistentDB(schema,
'pk',
dbname='testdb',
overwrite=True,
cardinality=128)
with self.assertRaises(ValueError):
db = PersistentDB(schema, 'pk', dbname='testdb', load='yes')
with self.assertRaises(ValueError):
db = PersistentDB(schema, 'pk', dbname='testdb', overwrite='yes')
with self.assertRaises(ValueError):
db = PersistentDB(schema, 'pk', dbname=123, overwrite=True)
with self.assertRaises(ValueError):
db = PersistentDB(
{
'pk': {
'type': str,
'index': None
},
'DELETE': {
'type': bool,
'index': 1
}
},
'pk',
dbname='testdb',
overwrite=True)
with self.assertRaises(ValueError):
db = PersistentDB(
{
'pk': {
'type': str,
'index': None
},
'mean:ie': {
'type': float,
'index': 1
}
},
'pk',
dbname='testdb',
overwrite=True)
with self.assertRaises(ValueError):
db = PersistentDB(
{
'pk': {
'type': str,
'index': None
},
'mean': {
'type': dict,
'index': 1
}
},
'pk',
dbname='testdb',
overwrite=True)
with self.assertRaises(ValueError):
db = PersistentDB([{
'type': str,
'index': None
}, {
'type': float,
'index': 1
}],
'pk',
dbname='testdb',
overwrite=True)
with self.assertRaises(ValueError):
db = PersistentDB(
{
'pk': {
'type': int,
'index': None
},
'mean': {
'type': float,
'index': 1
}
},
'pk',
dbname='testdb',
overwrite=True)
with self.assertRaises(ValueError):
db = PersistentDB(
{
'pk': {
'type': str,
'index': None
},
'd_vp-mean': {
'type': float,
'index': 1
}
},
'pk',
dbname='testdb',
overwrite=True)
with self.assertRaises(ValueError):
db = PersistentDB(
{
'pk': {
'type': str,
'index': None
},
'vp': {
'type': float,
'index': 1
}
},
'pk',
dbname='testdb',
overwrite=True)
with self.assertRaises(ValueError):
db = PersistentDB(
{
'pk': {
'type': str,
'index': None
},
'vp': {
'type': bool,
'index': 1
}
},
'mean',
dbname='testdb',
overwrite=True)
def test_schema_change(self):
self.db.close()
self.db = PersistentDB(self.schema,
pk_field='pk',
db_name='testing',
ts_length=self.ts_length)
def test_meta_save_ts(self):
self.db.close()
self.db = PersistentDB(pk_field='pk',
db_name='testing',
ts_length=self.ts_length)
self.assertEqual(len(self.db), 100)
