def proc_main(pk, row, arg):
"""
including pk and arg because of call in tsdb_server.py
"""
damean = ts.TimeSeries(row['ts']['times'], row['ts']['values']).mean()
dastd = ts.TimeSeries(row['ts']['times'], row['ts']['values']).std()
return [damean, dastd]
def hlm_from_psi4lm(psi4lm, w0, cmplx=False, taper=False, cut=False):
h = integrate_FFI(psi4lm, float(w0), order=2, taper=taper, cut=cut)
if cmplx:
return h
#
hplus = timeseries.TimeSeries(h.t, h.y.real)
hcross = timeseries.TimeSeries(h.t, -h.y.imag)
return hplus, hcross
def test_delete_ts(self):
n_delete = 10
# delete and check to make sure they're gone
for i in range(n_delete):
pk = 'ts-' + str(i)
tsmeta = self.db._get_meta_dict(pk)
self.db.delete_ts(pk) # delete the timeseries
#Check 1: __get__() get by pk fail
with self.assertRaises(KeyError):
self.db[pk] # check to make sure it's gone
#Check 2: db_select return empty sets
self.assertEqual(self.db.select({'pk': pk}), ([], []))
#Check 3: does not exist in index
for field, value in tsmeta.items(
): # make sure it's gone from indexes
if field in self.schema.keys(
) and self.schema[field]['index'] is not None:
self.assertTrue(
pk not in self.db.select({field: value})[0])
#Check 4: check the db after deletion is clean and can hold the same pk and timeseries again
# insert the deleted ts and check to make sure everything is working as before
for i in range(n_delete):
pk = 'ts-' + str(i)
values = np.array(range(self.ts_length)) + i
series = ts.TimeSeries(values, values)
meta = {}
meta['mean'] = float(series.mean())
meta['std'] = float(series.std())
meta['vp'] = False
meta['blarg'] = self.schema['blarg']['values'][
i % 2] #blarg only has two value
n_order = len(self.schema['order']['values']) # 11
meta['order'] = self.schema['order']['values'][i % n_order]
self.db.insert_ts(pk, series)
self.db.upsert_meta(pk, meta)
for i in range(n_delete):
pk = 'ts-' + str(i)
values = np.array(range(self.ts_length)) + i
series = ts.TimeSeries(values, values)
r_meta = self.db._get_meta_list(pk)
n_order = len(self.schema['order']['values']) # 11
self.assertTrue(r_meta[self.db.metaheap.fields.index('order')] ==
self.schema['order']['values'][i % n_order])
n_blarg = 2
self.assertTrue(r_meta[self.db.metaheap.fields.index('blarg')] ==
self.schema['blarg']['values'][i % n_blarg])
self.assertTrue(
r_meta[self.db.metaheap.fields.index('mean')] == series.mean())
self.assertTrue(
r_meta[self.db.metaheap.fields.index('std')] == series.std())
def setUp(self):
identity = lambda x: x
to_int = lambda x: int(x)
to_float = lambda x: float(x)
to_bool = lambda x: bool(x)
schema = {
'pk': {
'convert': identity,
'index': None
}, #will be indexed anyways
'ts': {
'convert': identity,
'index': None
},
'order': {
'convert': to_int,
'index': 1
},
'blarg': {
'convert': to_int,
'index': 1
},
'useless': {
'convert': identity,
'index': None
},
'mean': {
'convert': to_float,
'index': 1
},
'std': {
'convert': to_float,
'index': 1
},
'vp': {
'convert': to_bool,
'index': 1
}
}
self.db = DictDB(schema, 'pk', 3)
self.server = TSDBServer(self.db)
self.prot = TSDBProtocol(self.server)
self.des = Deserializer()
msg = TSDBOp_InsertTS('one', ts.TimeSeries([1, 2, 3], [1, 4, 9]))
status, payload = self._mockSendingMessage(msg)
msg = TSDBOp_InsertTS('two', ts.TimeSeries([2, 3, 4], [4, 9, 16]))
status, payload = self._mockSendingMessage(msg)
msg = TSDBOp_InsertTS('three', ts.TimeSeries([9, 3, 4], [4, 0, 16]))
status, payload = self._mockSendingMessage(msg)
msg = TSDBOp_InsertTS('four', ts.TimeSeries([0, 0, 4], [1, 0, 4]))
status, payload = self._mockSendingMessage(msg)
async def test_REST_simple_run(self):
# Data
t = [0, 1, 2, 3, 4]
v = [1.0, 2.0, 3.0, 2.0, 1.0]
ats = ts.TimeSeries(t, v)
# Setup Client
client = TSDB_REST_Client()
# Add Trigger
await client.add_trigger('stats', 'insert_ts', ['mean', 'std'], None)
# Insert
await client.insert_ts(1, ats)
# Select
status, payload = await client.select({'pk': {
'==': 1
}}, ['ts', 'mean', 'std'], None)
assert (status == 0)
assert (ts.TimeSeries(payload['1']['ts'][0],
payload['1']['ts'][1]) == ats)
assert (payload['1']['std'] == 1.4142135623730951)
assert (payload['1']['mean'] == 2.0)
# FINALLY WORKING!!! YAY!!!
# Upsert
await client.upsert_meta(1, {'order': 1})
status, payload = await client.select({'order': {
'==': 1
}}, ['pk', 'order'], None)
assert (status == 0)
assert (payload['1']['order'] == 1)
# Remove Trigger
await client.remove_trigger('stats', 'insert_ts')
# Insert (No Trigger)
await client.insert_ts(2, ats)
status, payload = await client.select({'pk': {
'==': 2
}}, ['ts', 'mean', 'std'], None)
assert (ts.TimeSeries(payload['2']['ts'][0],
payload['2']['ts'][1]) == ats)
assert ('std' not in payload['2'])
assert ('mean' not in payload['2'])
# Delete
await client.delete_ts(1)
status, payload = await client.select({'pk': {
'==': 1
}}, ['ts', 'mean', 'std'], None)
assert (status == 0)
assert (payload == {})
def test_delete(self):
n_delete = 10
# delete and check to make sure they're gone
for i in range(n_delete):
pk = 'ts-' + str(i)
tsmeta = self.db._get_meta_dict(pk)
tsinstance = tsmeta['ts']
self.db.delete_ts(pk) # delete the timeseries
with self.assertRaises(KeyError):
self.db[pk] # check to make sure it's gone
self.assertEqual(self.db.select({'pk': pk}), ([], []))
for field, value in tsmeta.items(
): # make sure it's gone from indexes
if field in self.schema.keys(
) and self.schema[field]['index'] is not None:
self.assertTrue(
pk not in self.db.select({field: value})[0])
# reinsert
for i in range(n_delete):
pk = 'ts-' + str(i)
values = np.array(range(self.tsLength)) + i
series = ts.TimeSeries(values, values)
meta = {}
n_order = len(self.schema['order']['values']) # 11
meta['order'] = self.schema['order']['values'][i % n_order]
n_blarg = 2
meta['blarg'] = self.schema['blarg']['values'][i % n_blarg]
meta['mean'] = float(series.mean())
meta['std'] = float(series.std())
meta['vp'] = False
self.db.insert_ts(pk, series)
self.db.upsert_meta(pk, meta)
# check to make sure everything is working as before
for i in range(n_delete):
pk = 'ts-' + str(i)
values = np.array(range(self.tsLength)) + i
series = ts.TimeSeries(values, values)
r_meta = self.db._get_meta_list(pk)
n_order = len(self.schema['order']['values']) # 11
self.assertTrue(r_meta[self.db.metaheap.fields.index('order')] ==
self.schema['order']['values'][i % n_order])
n_blarg = 2
self.assertTrue(r_meta[self.db.metaheap.fields.index('blarg')] ==
self.schema['blarg']['values'][i % n_blarg])
self.assertTrue(
r_meta[self.db.metaheap.fields.index('mean')] == series.mean())
self.assertTrue(
r_meta[self.db.metaheap.fields.index('std')] == series.std())
def test_insert_exception(self):
pk = 'bad'
existing = 'ts-0'
with self.assertRaises(ValueError):
bad_series = np.array(range(self.ts_length + 3))
self.db.insert_ts(pk, bad_series)
with self.assertRaises(ValueError):
values = np.array(range(self.ts_length + 5))
bad_series = ts.TimeSeries(values, values)
self.db.insert_ts(pk, bad_series)
with self.assertRaises(ValueError):
values = np.array(range(self.ts_length))
series = ts.TimeSeries(values, values)
self.db.insert_ts('ts-0', series)
def setUp(self):
identity = lambda x: x
to_int = lambda x: int(x)
to_float = lambda x: float(x)
to_bool = lambda x: bool(x)
schema = {
'pk': {
'convert': identity,
'index': None
}, #will be indexed anyways
'ts': {
'convert': identity,
'index': None
},
'order': {
'convert': to_int,
'index': 1
},
'blarg': {
'convert': to_int,
'index': 1
},
'useless': {
'convert': identity,
'index': None
},
'mean': {
'convert': to_float,
'index': 1
},
'std': {
'convert': to_float,
'index': 1
},
'vp': {
'convert': to_bool,
'index': 1
}
}
self.db = DictDB(schema, 'pk', 3)
self.db.insert_ts('one', ts.TimeSeries([1, 2, 3], [1, 4, 9]))
self.db.insert_ts('two', ts.TimeSeries([2, 3, 4], [4, 9, 16]))
self.db.insert_ts('three', ts.TimeSeries([9, 3, 4], [4, 0, 16]))
self.db.insert_ts('four', ts.TimeSeries([0, 0, 4], [1, 0, 4]))
self.db.upsert_meta('one', {'order': 1, 'blarg': 1})
self.db.upsert_meta('two', {'order': 2})
self.db.upsert_meta('three', {'order': 1, 'blarg': 2})
self.db.upsert_meta('four', {'order': 2, 'blarg': 2})
def __getitem__(self, key):
if key in self._src_inf:
return self._src_inf[key]
tsmin = self._src_min[key]
tsmax = self._src_max[key]
infn = numpy.maximum(abs(tsmax.y), abs(tsmin.y))
return timeseries.TimeSeries(tsmin.t, infn)
def test_serialize():
t = [0, 1, 2, 3, 4]
v = [1.0, 2.0, 3.0, 2.0, 1.0]
ats = ts.TimeSeries(t, v)
data_dict = {}
data_dict['pk'] = 1
data_dict['ts'] = ats
obj = TSDBOp.to_json(data_dict)
# Serialize
s = serialize(obj)
s_half = len(s) // 2
first_half = s[:s_half]
second_half = s[s_half:]
# Deserialize
# Break with only first half
d = Deserializer()
d.append(first_half)
first_ready = d.ready()
assert (first_ready == False)
r = d.deserialize()
assert (r == None)
# Now re-try with both halves
d.append(first_half)
d.append(second_half)
second_ready = d.ready()
assert (second_ready == True)
r = d.deserialize()
assert (obj == r)
def proc_main(pk, row, arg):
#your code here
argts = ts.TimeSeries(*arg)
series = stand(argts,argts.mean(),argts.std())
a = 4
w = 32
symbols = ['{0:b}'.format(i).zfill(int(np.log(a-1)/np.log(2))+1) for i in range(a)]
if a in Breakpoints:
breakpoints = Breakpoints[a]#norm.ppf(np.array([i/a for i in range(1,a)]))
else:
raise ValueError('Breakpoints do not exist for cardinality {}'.format(a))
breakpoints = np.array([*breakpoints,np.inf])
T = np.zeros((w))
n = len(series)
SAX = []
for i in range(w):
T[i] = w/n*sum([series.data[j] for j in range(int(n/w*i),int(n/w*(i+1)))])
for j in range(a):
if j == a-1:
SAX.append(symbols[j])
break
if T[i]<breakpoints[0]:
SAX.append(symbols[0])
break
if T[i]>breakpoints[j] and T[i]<breakpoints[j+1]:
SAX.append(symbols[j+1])
break
return SAX
async def tsdb_augmented_select(self, request):
if 'query' not in request.GET:
view = AUGMENTED_SELECT_VIEW
return web.Response(body=view.encode('utf-8'))
else: # if there is query parameter in URL
try:
query = json.loads(request.GET['query'])
proc = query['proc']
target = query['target']
if 'where' in query:
metadata_dict = query['where']
else:
metadata_dict = {}
if 'additional' in query:
additional = query['additional']
else:
additional = None
if 'arg' in query: # arg should be put as TimeSeries object form. Last time we only put json form of it and that aroused errors.
arg = ts.TimeSeries(*query['arg'])
else:
arg = None
status, result = await self.client.augmented_select(
proc, target, arg, metadata_dict, additional)
if status != TSDBStatus.OK:
result = "Augmented Selection failed"
except Exception as error:
result = {"msg": "Cannot parse the Request"}
result["type"] = str(type(error))
result["args"] = str(error.args)
finally:
return web.Response(body=json.dumps(result).encode('utf-8'))
def setUp(self):
self.dirPath = "persistent_files/testing"
if not os.path.isdir(self.dirPath):
os.makedirs(self.dirPath)
self._createdDirs = True
else:
self._createdDirs = False
self.schema = schema_type
self.ts_length = 100
self.db = PersistentDB(schema_type,
pk_field='pk',
db_name='testing',
ts_length=self.ts_length)
for i in range(100):
pk = 'ts-' + str(i)
values = np.array(range(self.ts_length)) + i
series = ts.TimeSeries(values, values)
meta = {}
n_order = len(schema_type['order']['values']) # 11
meta['order'] = schema_type['order']['values'][i % n_order]
n_blarg = 2
meta['blarg'] = schema_type['blarg']['values'][i % n_blarg]
meta['mean'] = float(series.mean(
)) # make sure they are python floats, not numpy floats
meta['std'] = float(series.std())
meta['vp'] = False
self.db.insert_ts(pk, series)
self.db.upsert_meta(pk, meta)
def integrate_FFI(ts, w0, order=1, taper=False, cut=False):
regts = ts.regular_sample()
t,z = regts.t, regts.y
p = 2*math.pi/w0
eps = p / (t[-1]-t[0])
if (eps>0.3):
raise RuntimeError("FFI: waveform too short")
#
if taper:
pw = planck_window(eps)
z *= pw(len(z))
#
dt = t[1]-t[0]
zt = np.fft.fft(z)
w = np.fft.fftfreq(len(t), d=dt) * (2*math.pi)
wa = np.abs(w)
fac1 = -1j * np.sign(w) / np.where(wa>w0, wa, w0)
faco = fac1**int(order)
ztf = zt * faco
zf = np.fft.ifft(ztf)
g = timeseries.TimeSeries(t, zf)
if cut:
g.clip(tmin=g.tmin()+p, tmax=g.tmax()-p)
#
return g
def test_find_similar(self):
query = ts.TimeSeries([1, 2, 3], [4, 0, 3])
msg = TSDBOp_FindSimilar(query, self.vpkeys)
status, payload = self._send(msg)
self.assertEqual(payload, None)
def test_read_ts(self):
for i in range(100):
pk = 'ts-' + str(i)
values = np.array(range(self.ts_length)) + i
series = ts.TimeSeries(values, values)
r_ts = self.db._return_ts(pk)
self.assertEqual(series, r_ts)
def test_del2(self):
self.db.delete_ts('one')
self.db.delete_ts('two')
self.db.delete_ts('three')
self.db.delete_ts('four')
pks, payload = self.db.select({}, None, None)
self.assertEqual(set(pks), set([]))
self.db.insert_ts('one', ts.TimeSeries([1, 2, 3], [1, 4, 9]))
self.db.insert_ts('two', ts.TimeSeries([2, 3, 4], [4, 9, 16]))
self.db.insert_ts('three', ts.TimeSeries([9, 3, 4], [4, 0, 16]))
self.db.insert_ts('four', ts.TimeSeries([0, 0, 4], [1, 0, 4]))
self.db.upsert_meta('one', {'order': 1, 'blarg': 1})
self.db.upsert_meta('two', {'order': 2})
self.db.upsert_meta('three', {'order': 1, 'blarg': 2})
self.db.upsert_meta('four', {'order': 2, 'blarg': 2})
def test_find_similar2(self):
query = ts.TimeSeries([0, 5, 10],[15, 25, 50])
msg = TSDBOp_FindSimilar(query)
status, payload = self._mockSendingMessage(msg)
near = list(payload.keys())[0]
self.assertTrue(near in ['one','two','three','four'])
def tsmaker(m, s, j):
"returns metadata and a time series in the shape of a jittered normal"
meta={}
meta['order'] = int(np.random.choice([-5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5]))
meta['blarg'] = int(np.random.choice([1, 2]))
t = np.arange(0.0, 1.0, 0.01)
v = norm.pdf(t, m, s) + j*np.random.randn(100)
return meta, ts.TimeSeries(t, v)
def _populate(self):
msg = TSDBOp_AddTrigger('corr', 'insert_ts', ['d_vp-1'],
ts.TimeSeries([1, 2, 3], [1, 4, 9]))
status, payload = self._send(msg)
msg = TSDBOp_InsertTS('one', ts.TimeSeries([1, 2, 3], [1, 4, 9]))
status, payload = self._send(msg)
msg = TSDBOp_InsertTS('two', ts.TimeSeries([1, 2, 3], [4, 9, 16]))
status, payload = self._send(msg)
msg = TSDBOp_InsertTS('three', ts.TimeSeries([1, 2, 3], [4, 0, 16]))
status, payload = self._send(msg)
msg = TSDBOp_InsertTS('four', ts.TimeSeries([1, 2, 3], [1, 0, 4]))
status, payload = self._send(msg)
def read_and_decode_ts(self, offset):
self.fd.seek(offset)
# ts_length = int.from_bytes(self.fd.read(TS_FIELD_LENGTH), byteorder='little')
# self.fd.seek(offset + TS_FIELD_LENGTH)
buff = self.fd.read(self.byteArrayLength)
items = struct.unpack('%sd' % (2 * self.ts_length), buff)
return timeseries.TimeSeries(items[:self.ts_length],
items[self.ts_length:])
async def get(self):
client = rest.TSDBrestapi()
timeseries = ts.TimeSeries(*ast.literal_eval(self.get_argument('ts')))
r = client.simsearch(timeseries)
status, payload = await tcp_echo_client(r,self.loop)
status = TSDBStatus(status)
print(status)
self.write({'status':str(status),'payload':payload})
def tsmaker(m, s, j):
meta = {}
meta['order'] = int(
np.random.choice([-5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5]))
meta['blarg'] = int(np.random.choice([1, 2]))
t = np.arange(0.0, 1.0, 0.01)
v = norm.pdf(t, m, s) + j * np.random.randn(100)
return meta, ts.TimeSeries(t, v)
def tsmaker(m, s, j):
"generates a new TimeSeries using a normal distribution with a given mean, sd and jitter"
meta = {}
meta['order'] = int(
np.random.choice([-5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5]))
meta['blarg'] = int(np.random.choice([1, 2]))
t = np.arange(0.0, 1.0, 0.01)
v = norm.pdf(t, m, s) + j * np.random.randn(100)
return meta, ts.TimeSeries(t, v)
def test_corr2():
t = [1, 2, 3, 4]
v = [40, 50, 60, 70]
t2 = [1, 2, 3, 4]
v2 = [40, 50, 60, 70]
row = {}
row['ts'] = ts.TimeSeries(t, v)
#Since its the same time series, the correlation here should be zero
assert (corr.proc_main(1, row, (t2, v2))[0] == 0)
def proc_main(pk, row, arg):
# The argument is a time series. But due to serialization it does
# not come out as the "instance", and must be cast
argts = ts.TimeSeries(**arg)
# compute a standardized time series
stand_argts = stand(argts, argts.mean(), argts.std())
# for each row in our select/etc, standardize the time series
mean = ts.TimeSeries(row['ts']['times'], row['ts']['values']).mean()
std = ts.TimeSeries(row['ts']['times'], row['ts']['values']).std()
stand_rowts = stand(ts.TimeSeries(row['ts']['times'], row['ts']['values']),
mean, std)
# compute the normalozed kernelized cross-correlation
kerncorr = kernel_corr(stand_rowts, stand_argts, 1)
# compute a distance from it.
# The distance is given by np.sqrt(K(x,x) + K(y,y) - 2*K(x,y))
# since we are normalized the autocorrs are 1
kerndist = np.sqrt(2 * (1 - kerncorr))
return [kerndist]
def test_protocol_delete(self):
db = DictDB(schema, 'pk')
server = TSDBServer(db)
prot = TSDBProtocol(server)
t1 = [0,1,2,3,4]
v1 = [1.0,2.0,3.0,2.0,1.0]
ats1 = ts.TimeSeries(t1, v1)
t2 = [10,11,12,13,14]
v2 = [-1.0,-2.0,-3.0,-2.0,-1.0]
ats2 = ts.TimeSeries(t2, v2)
insert_op = {}
insert_op['pk'] = 1
insert_op['ts'] = ats1
insert_op['op'] = 'insert_ts'
# Test Protocol Insert
insert_return = prot._insert_ts(insert_op)
assert(insert_return['op'] == 'insert_ts')
assert(insert_return['status'] == TSDBStatus.OK)
assert(insert_return['payload'] == None)
inserted_row = server.db.rows[1]
assert(inserted_row['pk'] == 1)
assert(inserted_row['ts'] == ats1)
insert_return2 = prot._insert_ts(insert_op)
assert(insert_return2['op'] == 'insert_ts')
assert(insert_return2['status'] == TSDBStatus.INVALID_KEY)
delete_op = {}
delete_op['pk'] = 1
delete_op['op'] = 'delete_ts'
delete_return = prot._delete_ts(delete_op)
assert(delete_return['op'] == 'delete_ts')
assert(delete_return['status'] == TSDBStatus.OK)
assert(delete_return['payload'] == None)
assert (len(server.db.rows) == 0)
delete_return2 = prot._delete_ts(delete_op)
assert(delete_return2['op'] == 'delete_ts')
assert(delete_return2['status'] == TSDBStatus.INVALID_KEY)
def test_corr_small():
from procs.corr import proc_main
ts_l = [10, 22, 26, 4, 18]
ts_l_x = range(len(ts_l))
ts_ts = ts.TimeSeries(ts_l_x, ts_l).to_json()
# print (ts_ts)
res = proc_main(None, {'ts': ts_ts}, {'times': ts_l_x, 'values': ts_l})
# print (res)
assert np.abs(res) < EPS
def test_p():
stopTime = 20
numPoints = 200
numSelPoints = 60
t2 = [1, 2, 3, 4]
v2 = [40, 50, 60, 70]
row = {}
data_irr, time_irr = basic_irregular(stopTime, numPoints, numSelPoints)
row['ts'] = ts.TimeSeries(time_irr, data_irr)
assert (np.abs(period.proc_main(1, row, (t2, v2))[0] - 6.28) < 0.5)
def test_client():
client = test_db_init()
# test adding triggers
assert client.add_trigger('junk', 'insert_ts', None, 23) is None
# test insert
assert client.insert_ts('one', ts.TimeSeries([1, 2, 3], [1, 4, 9])) is None
# test removing triggers
assert client.remove_trigger('junk', 'insert_ts') is None
# test upsert
assert client.upsert_meta('one', {'order': 1, 'blarg': 1}) is None
# test select
assert client.select({'order': 1, 'blarg': 1}) == (TSDBStatus(0), {})
