def test_decode_data_from_get(self):
colnames = ["varchar", "sint64", "double", "timestamp",
"boolean", "varchar", "varchar"]
coltypes = [varchar_a, sint64_a, double_a, timestamp_a,
boolean_a, varchar_a, varchar_a]
r0 = (bd0, 0, 1.2, unix_time_millis(ts0), True,
[], str1, None)
r1 = (bd1, 3, 4.5, unix_time_millis(ts1), False,
[], str1, None)
rows = [r0, r1]
# { tsgetresp, { [colnames], [coltypes], [rows] } }
data_t = colnames, coltypes, rows
rsp_data = tsgetresp_a, data_t
rsp_ttb = encode(rsp_data)
tsobj = TsObject(None, self.table)
c = TtbCodec()
c.decode_timeseries(decode(rsp_ttb), tsobj)
for i in range(0, 1):
dr = rows[i]
r = tsobj.rows[i] # encoded
self.assertEqual(r[0], dr[0].encode('utf-8'))
self.assertEqual(r[1], dr[1])
self.assertEqual(r[2], dr[2])
# NB *not* decoding timestamps
# dt = datetime_from_unix_time_millis(dr[3])
self.assertEqual(r[3], dr[3])
if i == 0:
self.assertEqual(r[4], True)
else:
self.assertEqual(r[4], False)
self.assertEqual(r[5], None)
self.assertEqual(r[6], dr[6].encode('ascii'))
self.assertEqual(r[7], None)
def setUpClass(cls):
super(TimeseriesPbufTests, cls).setUpClass()
cls.now = datetime.datetime.utcfromtimestamp(144379690.987000)
fiveMinsAgo = cls.now - fiveMins
tenMinsAgo = fiveMinsAgo - fiveMins
fifteenMinsAgo = tenMinsAgo - fiveMins
twentyMinsAgo = fifteenMinsAgo - fiveMins
twentyFiveMinsAgo = twentyMinsAgo - fiveMins
client = cls.create_client()
table = client.table(table_name)
rows = [['hash1', 'user2', twentyFiveMinsAgo, 'typhoon', 90.3],
['hash1', 'user2', twentyMinsAgo, 'hurricane', 82.3],
['hash1', 'user2', fifteenMinsAgo, 'rain', 79.0],
['hash1', 'user2', fiveMinsAgo, 'wind', None],
['hash1', 'user2', cls.now, 'snow', 20.1]]
try:
ts_obj = table.new(rows)
result = ts_obj.store()
except (RiakError, NotImplementedError) as e:
raise unittest.SkipTest(e)
finally:
client.close()
if result is not True:
raise AssertionError("expected success")
cls.nowMsec = unix_time_millis(cls.now)
cls.fiveMinsAgo = fiveMinsAgo
cls.twentyMinsAgo = twentyMinsAgo
cls.twentyFiveMinsAgo = twentyFiveMinsAgo
cls.tenMinsAgoMsec = unix_time_millis(tenMinsAgo)
cls.twentyMinsAgoMsec = unix_time_millis(twentyMinsAgo)
cls.numCols = len(rows[0])
cls.rows = rows
encoded_rows = [[
str_to_bytes('hash1'),
str_to_bytes('user2'), twentyFiveMinsAgo,
str_to_bytes('typhoon'), 90.3
],
[
str_to_bytes('hash1'),
str_to_bytes('user2'), twentyMinsAgo,
str_to_bytes('hurricane'), 82.3
],
[
str_to_bytes('hash1'),
str_to_bytes('user2'), fifteenMinsAgo,
str_to_bytes('rain'), 79.0
],
[
str_to_bytes('hash1'),
str_to_bytes('user2'), fiveMinsAgo,
str_to_bytes('wind'), None
],
[
str_to_bytes('hash1'),
str_to_bytes('user2'), cls.now,
str_to_bytes('snow'), 20.1
]]
cls.encoded_rows = encoded_rows
def test_encode_data_for_put(self):
r0 = (bd0, 0, 1.2, unix_time_millis(ts0), True, [])
r1 = (bd1, 3, 4.5, unix_time_millis(ts1), False, [])
rows = [r0, r1]
req = tsputreq_a, str_to_bytes(table_name), [], rows
req_test = encode(req)
rows_to_encode = [[bd0, 0, 1.2, ts0, True, None],
[bd1, 3, 4.5, ts1, False, None]]
tsobj = TsObject(None, self.table, rows_to_encode, None)
c = TtbCodec()
msg = c.encode_timeseries_put(tsobj)
self.assertEqual(req_test, msg.data)
def setUpClass(cls):
super(TimeseriesPbufTests, cls).setUpClass()
cls.now = datetime.datetime.utcfromtimestamp(144379690.987000)
fiveMinsAgo = cls.now - fiveMins
tenMinsAgo = fiveMinsAgo - fiveMins
fifteenMinsAgo = tenMinsAgo - fiveMins
twentyMinsAgo = fifteenMinsAgo - fiveMins
twentyFiveMinsAgo = twentyMinsAgo - fiveMins
client = cls.create_client()
table = client.table(table_name)
rows = [
['hash1', 'user2', twentyFiveMinsAgo, 'typhoon', 90.3],
['hash1', 'user2', twentyMinsAgo, 'hurricane', 82.3],
['hash1', 'user2', fifteenMinsAgo, 'rain', 79.0],
['hash1', 'user2', fiveMinsAgo, 'wind', None],
['hash1', 'user2', cls.now, 'snow', 20.1]
]
try:
ts_obj = table.new(rows)
result = ts_obj.store()
except (RiakError, NotImplementedError) as e:
raise unittest.SkipTest(e)
finally:
client.close()
if result is not True:
raise AssertionError("expected success")
cls.nowMsec = unix_time_millis(cls.now)
cls.fiveMinsAgo = fiveMinsAgo
cls.twentyMinsAgo = twentyMinsAgo
cls.twentyFiveMinsAgo = twentyFiveMinsAgo
cls.tenMinsAgoMsec = unix_time_millis(tenMinsAgo)
cls.twentyMinsAgoMsec = unix_time_millis(twentyMinsAgo)
cls.numCols = len(rows[0])
cls.rows = rows
encoded_rows = [
[str_to_bytes('hash1'), str_to_bytes('user2'),
twentyFiveMinsAgo, str_to_bytes('typhoon'), 90.3],
[str_to_bytes('hash1'), str_to_bytes('user2'),
twentyMinsAgo, str_to_bytes('hurricane'), 82.3],
[str_to_bytes('hash1'), str_to_bytes('user2'),
fifteenMinsAgo, str_to_bytes('rain'), 79.0],
[str_to_bytes('hash1'), str_to_bytes('user2'),
fiveMinsAgo, str_to_bytes('wind'), None],
[str_to_bytes('hash1'), str_to_bytes('user2'),
cls.now, str_to_bytes('snow'), 20.1]
]
cls.encoded_rows = encoded_rows
def setUpClass(cls):
cls.ts0ms = unix_time_millis(ts0)
if cls.ts0ms != ex0ms:
raise AssertionError('expected {:d} to equal {:d}'.format(
cls.ts0ms, ex0ms))
cls.ts1ms = unix_time_millis(ts1)
if cls.ts1ms != ex1ms:
raise AssertionError('expected {:d} to equal {:d}'.format(
cls.ts1ms, ex1ms))
cls.rows = [[bd0, 0, 1.2, ts0, True, None],
[bd1, 3, 4.5, ts1, False, blob0]]
cls.test_key = ['hash1', 'user2', ts0]
cls.table = Table(None, table_name)
def test_encode_data_for_put(self):
r0 = (bd0, 0, 1.2, unix_time_millis(ts0), True, [])
r1 = (bd1, 3, 4.5, unix_time_millis(ts1), False, [])
rows = [r0, r1]
req = tsputreq_a, str_to_bytes(table_name), [], rows
req_test = encode(req)
rows_to_encode = [
[bd0, 0, 1.2, ts0, True, None],
[bd1, 3, 4.5, ts1, False, None]
]
tsobj = TsObject(None, self.table, rows_to_encode, None)
c = TtbCodec()
msg = c.encode_timeseries_put(tsobj)
self.assertEqual(req_test, msg.data)
def setUpClass(cls):
cls.ts0ms = unix_time_millis(ts0)
if cls.ts0ms != ex0ms:
raise AssertionError(
'expected {:d} to equal {:d}'.format(cls.ts0ms, ex0ms))
cls.ts1ms = unix_time_millis(ts1)
if cls.ts1ms != ex1ms:
raise AssertionError(
'expected {:d} to equal {:d}'.format(cls.ts1ms, ex1ms))
cls.rows = [
[bd0, 0, 1.2, ts0, True, None],
[bd1, 3, 4.5, ts1, False, blob0]
]
cls.test_key = ['hash1', 'user2', ts0]
cls.table = Table(None, table_name)
def test_get_unix_time_with_tzinfo(self):
try:
import pytz
tz = pytz.timezone('America/Los_Angeles')
ts0_pst = tz.localize(ts0)
utm = unix_time_millis(ts0_pst)
self.assertEqual(utm, ts0_ts_pst)
except ImportError:
pass
def test_encode_data_for_get(self):
keylist = [
str_to_bytes('hash1'), str_to_bytes('user2'), unix_time_millis(ts0)
]
req = tsgetreq_a, str_to_bytes(table_name), keylist, udef_a
req_test = encode(req)
test_key = ['hash1', 'user2', ts0]
c = TtbCodec()
msg = c.encode_timeseries_keyreq(self.table, test_key)
self.assertEqual(req_test, msg.data)
def test_encode_data_for_get(self):
keylist = [
str_to_bytes('hash1'), str_to_bytes('user2'), unix_time_millis(ts0)
]
req = tsgetreq_a, str_to_bytes(table_name), keylist, udef_a
req_test = encode(req)
test_key = ['hash1', 'user2', ts0]
c = TtbCodec()
msg = c.encode_timeseries_keyreq(self.table, test_key)
self.assertEqual(req_test, msg.data)
def test_conv_datetime_to_unix_millis(self):
# This is the "native" Python unix timestamp including
# microseconds, as float. timedelta "total_seconds()"
# returns a value like this
if is_timeseries_supported():
v = 144379690.987000
d = datetime.datetime.utcfromtimestamp(v)
utm = unix_time_millis(d)
self.assertEqual(utm, 144379690987)
else:
pass
def test_decode_data_from_get(self):
colnames = [
"varchar", "sint64", "double", "timestamp", "boolean", "varchar",
"varchar", "blob"
]
coltypes = [
varchar_a, sint64_a, double_a, timestamp_a, boolean_a, varchar_a,
varchar_a
]
r0 = (bd0, 0, 1.2, unix_time_millis(ts0), True, [], str1, None, None)
r1 = (bd1, 3, 4.5, unix_time_millis(ts1), False, [], str1, None, blob0)
rows = [r0, r1]
# { tsgetresp, { [colnames], [coltypes], [rows] } }
data_t = colnames, coltypes, rows
rsp_data = tsgetresp_a, data_t
rsp_ttb = encode(rsp_data)
tsobj = TsObject(None, self.table)
c = TtbCodec()
c.decode_timeseries(decode(rsp_ttb), tsobj)
for i in range(0, 1):
dr = rows[i]
r = tsobj.rows[i] # encoded
self.assertEqual(r[0], dr[0].encode('utf-8'))
self.assertEqual(r[1], dr[1])
self.assertEqual(r[2], dr[2])
# NB *not* decoding timestamps
# dt = datetime_from_unix_time_millis(dr[3])
self.assertEqual(r[3], dr[3])
if i == 0:
self.assertEqual(r[4], True)
else:
self.assertEqual(r[4], False)
self.assertEqual(r[5], None)
self.assertEqual(r[6], dr[6].encode('ascii'))
self.assertEqual(r[7], None)
self.assertEqual(r[8], dr[8])
def test_unix_millis_small_value(self):
if is_timeseries_supported():
# this is what would be stored in Riak TS
v = 1001
dt = datetime_from_unix_time_millis(v)
# This is how Python represents the above
utp = 1.001
dtp = datetime.datetime.utcfromtimestamp(utp)
self.assertEqual(dt, dtp)
utm = unix_time_millis(dt)
self.assertEqual(v, utm)
else:
pass
def test_conv_ms_timestamp_to_datetime_and_back(self):
if is_timeseries_supported():
# this is what would be stored in Riak TS
v = 144379690987
dt = datetime_from_unix_time_millis(v)
# This is how Python represents the above
utp = 144379690.987000
dtp = datetime.datetime.utcfromtimestamp(utp)
self.assertEqual(dt, dtp)
utm = unix_time_millis(dt)
self.assertEqual(v, utm)
else:
pass
def encode_to_ts_cell(self, cell, ts_cell):
if cell is not None:
if isinstance(cell, datetime.datetime):
ts_cell.timestamp_value = unix_time_millis(cell)
elif isinstance(cell, bool):
ts_cell.boolean_value = cell
elif isinstance(cell, six.binary_type):
ts_cell.varchar_value = cell
elif isinstance(cell, six.text_type):
ts_cell.varchar_value = str_to_bytes(cell)
elif isinstance(cell, six.string_types):
ts_cell.varchar_value = str_to_bytes(cell)
elif (isinstance(cell, six.integer_types)):
ts_cell.sint64_value = cell
elif isinstance(cell, float):
ts_cell.double_value = cell
else:
t = type(cell)
raise RiakError("can't serialize type '{}', value '{}'"
.format(t, cell))
def encode_to_ts_cell(self, cell, ts_cell):
if cell is not None:
if isinstance(cell, datetime.datetime):
ts_cell.timestamp_value = unix_time_millis(cell)
elif isinstance(cell, bool):
ts_cell.boolean_value = cell
elif isinstance(cell, six.binary_type):
ts_cell.varchar_value = cell
elif isinstance(cell, six.text_type):
ts_cell.varchar_value = str_to_bytes(cell)
elif isinstance(cell, six.string_types):
ts_cell.varchar_value = str_to_bytes(cell)
elif (isinstance(cell, six.integer_types)):
ts_cell.sint64_value = cell
elif isinstance(cell, float):
ts_cell.double_value = cell
else:
t = type(cell)
raise RiakError("can't serialize type '{}', value '{}'"
.format(t, cell))
def encode_to_ts_cell(self, cell):
if cell is None:
return []
else:
if isinstance(cell, datetime.datetime):
ts = unix_time_millis(cell)
# logging.debug('encoded datetime %s as %s', cell, ts)
return ts
elif isinstance(cell, bool):
return cell
elif isinstance(cell, six.text_type) or \
isinstance(cell, six.binary_type) or \
isinstance(cell, six.string_types):
return cell
elif (isinstance(cell, six.integer_types)):
return cell
elif isinstance(cell, float):
return cell
else:
t = type(cell)
raise RiakError("can't serialize type '{}', value '{}'"
.format(t, cell))
def test_store_and_fetch_and_query(self):
now = datetime.datetime.utcfromtimestamp(144379690.987000)
fiveMinsAgo = now - fiveMins
tenMinsAgo = fiveMinsAgo - fiveMins
fifteenMinsAgo = tenMinsAgo - fiveMins
twentyMinsAgo = fifteenMinsAgo - fiveMins
twentyFiveMinsAgo = twentyMinsAgo - fiveMins
table = self.client.table(table_name)
rows = [['hash1', 'user2', twentyFiveMinsAgo, 'typhoon', 90.3],
['hash1', 'user2', twentyMinsAgo, 'hurricane', 82.3],
['hash1', 'user2', fifteenMinsAgo, 'rain', 79.0],
['hash1', 'user2', fiveMinsAgo, 'wind', None],
['hash1', 'user2', now, 'snow', 20.1]]
# NB: response data is binary
exp_rows = [[
six.b('hash1'),
six.b('user2'), twentyFiveMinsAgo,
six.b('typhoon'), 90.3
],
[
six.b('hash1'),
six.b('user2'), twentyMinsAgo,
six.b('hurricane'), 82.3
],
[
six.b('hash1'),
six.b('user2'), fifteenMinsAgo,
six.b('rain'), 79.0
],
[
six.b('hash1'),
six.b('user2'), fiveMinsAgo,
six.b('wind'), None
],
[six.b('hash1'),
six.b('user2'), now,
six.b('snow'), 20.1]]
ts_obj = table.new(rows)
result = ts_obj.store()
self.assertTrue(result)
for i, r in enumerate(rows):
k = r[0:3]
ts_obj = self.client.ts_get(table_name, k)
self.assertIsNotNone(ts_obj)
ts_cols = ts_obj.columns
self.assertEqual(len(ts_cols.names), 5)
self.assertEqual(len(ts_cols.types), 5)
self.assertEqual(len(ts_obj.rows), 1)
row = ts_obj.rows[0]
exp = exp_rows[i]
self.assertEqual(len(row), 5)
self.assertEqual(row, exp)
fmt = """
select * from {table} where
time > {t1} and time < {t2} and
geohash = 'hash1' and
user = '******'
"""
query = fmt.format(table=table_name,
t1=unix_time_millis(tenMinsAgo),
t2=unix_time_millis(now))
ts_obj = self.client.ts_query(table_name, query)
if ts_obj.columns is not None:
self.assertEqual(len(ts_obj.columns.names), 5)
self.assertEqual(len(ts_obj.columns.types), 5)
self.assertEqual(len(ts_obj.rows), 1)
row = ts_obj.rows[0]
self.assertEqual(bytes_to_str(row[0]), 'hash1')
self.assertEqual(bytes_to_str(row[1]), 'user2')
self.assertEqual(row[2], fiveMinsAgo)
self.assertEqual(row[2].microsecond, 987000)
self.assertEqual(bytes_to_str(row[3]), 'wind')
self.assertIsNone(row[4])
def test_encode_decode_timestamp(self):
ts0ms = unix_time_millis(ts0)
self.assertEqual(ts0ms, ex0ms)
ts0_d = datetime_from_unix_time_millis(ts0ms)
self.assertEqual(ts0, ts0_d)
def test_get_unix_time_without_tzinfo(self):
self.assertIsNone(epoch.tzinfo)
self.assertIsNotNone(epoch_tz.tzinfo)
self.assertIsNone(ts0.tzinfo)
utm = unix_time_millis(ts0)
self.assertEqual(utm, ts0_ts)
def test_encode_decode_timestamp(self):
ts0ms = unix_time_millis(ts0)
self.assertEqual(ts0ms, ex0ms)
ts0_d = datetime_from_unix_time_millis(ts0ms)
self.assertEqual(ts0, ts0_d)
def test_store_and_fetch_and_query(self):
now = datetime.datetime.utcfromtimestamp(144379690.987000)
fiveMinsAgo = now - fiveMins
tenMinsAgo = fiveMinsAgo - fiveMins
fifteenMinsAgo = tenMinsAgo - fiveMins
twentyMinsAgo = fifteenMinsAgo - fiveMins
twentyFiveMinsAgo = twentyMinsAgo - fiveMins
table = self.client.table(table_name)
rows = [
['hash1', 'user2', twentyFiveMinsAgo, 'typhoon', 90.3],
['hash1', 'user2', twentyMinsAgo, 'hurricane', 82.3],
['hash1', 'user2', fifteenMinsAgo, 'rain', 79.0],
['hash1', 'user2', fiveMinsAgo, 'wind', None],
['hash1', 'user2', now, 'snow', 20.1]
]
# NB: response data is binary
exp_rows = [
[six.b('hash1'), six.b('user2'), twentyFiveMinsAgo,
six.b('typhoon'), 90.3],
[six.b('hash1'), six.b('user2'), twentyMinsAgo,
six.b('hurricane'), 82.3],
[six.b('hash1'), six.b('user2'), fifteenMinsAgo,
six.b('rain'), 79.0],
[six.b('hash1'), six.b('user2'), fiveMinsAgo,
six.b('wind'), None],
[six.b('hash1'), six.b('user2'), now,
six.b('snow'), 20.1]
]
ts_obj = table.new(rows)
result = ts_obj.store()
self.assertTrue(result)
for i, r in enumerate(rows):
k = r[0:3]
ts_obj = self.client.ts_get(table_name, k)
self.assertIsNotNone(ts_obj)
ts_cols = ts_obj.columns
self.assertEqual(len(ts_cols.names), 5)
self.assertEqual(len(ts_cols.types), 5)
self.assertEqual(len(ts_obj.rows), 1)
row = ts_obj.rows[0]
exp = exp_rows[i]
self.assertEqual(len(row), 5)
self.assertEqual(row, exp)
fmt = """
select * from {table} where
time > {t1} and time < {t2} and
geohash = 'hash1' and
user = '******'
"""
query = fmt.format(
table=table_name,
t1=unix_time_millis(tenMinsAgo),
t2=unix_time_millis(now))
ts_obj = self.client.ts_query(table_name, query)
if ts_obj.columns is not None:
self.assertEqual(len(ts_obj.columns.names), 5)
self.assertEqual(len(ts_obj.columns.types), 5)
self.assertEqual(len(ts_obj.rows), 1)
row = ts_obj.rows[0]
self.assertEqual(bytes_to_str(row[0]), 'hash1')
self.assertEqual(bytes_to_str(row[1]), 'user2')
self.assertEqual(row[2], fiveMinsAgo)
self.assertEqual(row[2].microsecond, 987000)
self.assertEqual(bytes_to_str(row[3]), 'wind')
self.assertIsNone(row[4])
