def test_decode_data_from_get(self): colnames = ["varchar", "sint64", "double", "timestamp", "boolean", "varchar", "varchar"] coltypes = [varchar_a, sint64_a, double_a, timestamp_a, boolean_a, varchar_a, varchar_a] r0 = (bd0, 0, 1.2, unix_time_millis(ts0), True, [], str1, None) r1 = (bd1, 3, 4.5, unix_time_millis(ts1), False, [], str1, None) rows = [r0, r1] # { tsgetresp, { [colnames], [coltypes], [rows] } } data_t = colnames, coltypes, rows rsp_data = tsgetresp_a, data_t rsp_ttb = encode(rsp_data) tsobj = TsObject(None, self.table) c = TtbCodec() c.decode_timeseries(decode(rsp_ttb), tsobj) for i in range(0, 1): dr = rows[i] r = tsobj.rows[i] # encoded self.assertEqual(r[0], dr[0].encode('utf-8')) self.assertEqual(r[1], dr[1]) self.assertEqual(r[2], dr[2]) # NB *not* decoding timestamps # dt = datetime_from_unix_time_millis(dr[3]) self.assertEqual(r[3], dr[3]) if i == 0: self.assertEqual(r[4], True) else: self.assertEqual(r[4], False) self.assertEqual(r[5], None) self.assertEqual(r[6], dr[6].encode('ascii')) self.assertEqual(r[7], None)
def setUpClass(cls): super(TimeseriesPbufTests, cls).setUpClass() cls.now = datetime.datetime.utcfromtimestamp(144379690.987000) fiveMinsAgo = cls.now - fiveMins tenMinsAgo = fiveMinsAgo - fiveMins fifteenMinsAgo = tenMinsAgo - fiveMins twentyMinsAgo = fifteenMinsAgo - fiveMins twentyFiveMinsAgo = twentyMinsAgo - fiveMins client = cls.create_client() table = client.table(table_name) rows = [['hash1', 'user2', twentyFiveMinsAgo, 'typhoon', 90.3], ['hash1', 'user2', twentyMinsAgo, 'hurricane', 82.3], ['hash1', 'user2', fifteenMinsAgo, 'rain', 79.0], ['hash1', 'user2', fiveMinsAgo, 'wind', None], ['hash1', 'user2', cls.now, 'snow', 20.1]] try: ts_obj = table.new(rows) result = ts_obj.store() except (RiakError, NotImplementedError) as e: raise unittest.SkipTest(e) finally: client.close() if result is not True: raise AssertionError("expected success") cls.nowMsec = unix_time_millis(cls.now) cls.fiveMinsAgo = fiveMinsAgo cls.twentyMinsAgo = twentyMinsAgo cls.twentyFiveMinsAgo = twentyFiveMinsAgo cls.tenMinsAgoMsec = unix_time_millis(tenMinsAgo) cls.twentyMinsAgoMsec = unix_time_millis(twentyMinsAgo) cls.numCols = len(rows[0]) cls.rows = rows encoded_rows = [[ str_to_bytes('hash1'), str_to_bytes('user2'), twentyFiveMinsAgo, str_to_bytes('typhoon'), 90.3 ], [ str_to_bytes('hash1'), str_to_bytes('user2'), twentyMinsAgo, str_to_bytes('hurricane'), 82.3 ], [ str_to_bytes('hash1'), str_to_bytes('user2'), fifteenMinsAgo, str_to_bytes('rain'), 79.0 ], [ str_to_bytes('hash1'), str_to_bytes('user2'), fiveMinsAgo, str_to_bytes('wind'), None ], [ str_to_bytes('hash1'), str_to_bytes('user2'), cls.now, str_to_bytes('snow'), 20.1 ]] cls.encoded_rows = encoded_rows
def test_encode_data_for_put(self): r0 = (bd0, 0, 1.2, unix_time_millis(ts0), True, []) r1 = (bd1, 3, 4.5, unix_time_millis(ts1), False, []) rows = [r0, r1] req = tsputreq_a, str_to_bytes(table_name), [], rows req_test = encode(req) rows_to_encode = [[bd0, 0, 1.2, ts0, True, None], [bd1, 3, 4.5, ts1, False, None]] tsobj = TsObject(None, self.table, rows_to_encode, None) c = TtbCodec() msg = c.encode_timeseries_put(tsobj) self.assertEqual(req_test, msg.data)
def setUpClass(cls): super(TimeseriesPbufTests, cls).setUpClass() cls.now = datetime.datetime.utcfromtimestamp(144379690.987000) fiveMinsAgo = cls.now - fiveMins tenMinsAgo = fiveMinsAgo - fiveMins fifteenMinsAgo = tenMinsAgo - fiveMins twentyMinsAgo = fifteenMinsAgo - fiveMins twentyFiveMinsAgo = twentyMinsAgo - fiveMins client = cls.create_client() table = client.table(table_name) rows = [ ['hash1', 'user2', twentyFiveMinsAgo, 'typhoon', 90.3], ['hash1', 'user2', twentyMinsAgo, 'hurricane', 82.3], ['hash1', 'user2', fifteenMinsAgo, 'rain', 79.0], ['hash1', 'user2', fiveMinsAgo, 'wind', None], ['hash1', 'user2', cls.now, 'snow', 20.1] ] try: ts_obj = table.new(rows) result = ts_obj.store() except (RiakError, NotImplementedError) as e: raise unittest.SkipTest(e) finally: client.close() if result is not True: raise AssertionError("expected success") cls.nowMsec = unix_time_millis(cls.now) cls.fiveMinsAgo = fiveMinsAgo cls.twentyMinsAgo = twentyMinsAgo cls.twentyFiveMinsAgo = twentyFiveMinsAgo cls.tenMinsAgoMsec = unix_time_millis(tenMinsAgo) cls.twentyMinsAgoMsec = unix_time_millis(twentyMinsAgo) cls.numCols = len(rows[0]) cls.rows = rows encoded_rows = [ [str_to_bytes('hash1'), str_to_bytes('user2'), twentyFiveMinsAgo, str_to_bytes('typhoon'), 90.3], [str_to_bytes('hash1'), str_to_bytes('user2'), twentyMinsAgo, str_to_bytes('hurricane'), 82.3], [str_to_bytes('hash1'), str_to_bytes('user2'), fifteenMinsAgo, str_to_bytes('rain'), 79.0], [str_to_bytes('hash1'), str_to_bytes('user2'), fiveMinsAgo, str_to_bytes('wind'), None], [str_to_bytes('hash1'), str_to_bytes('user2'), cls.now, str_to_bytes('snow'), 20.1] ] cls.encoded_rows = encoded_rows
def setUpClass(cls): cls.ts0ms = unix_time_millis(ts0) if cls.ts0ms != ex0ms: raise AssertionError('expected {:d} to equal {:d}'.format( cls.ts0ms, ex0ms)) cls.ts1ms = unix_time_millis(ts1) if cls.ts1ms != ex1ms: raise AssertionError('expected {:d} to equal {:d}'.format( cls.ts1ms, ex1ms)) cls.rows = [[bd0, 0, 1.2, ts0, True, None], [bd1, 3, 4.5, ts1, False, blob0]] cls.test_key = ['hash1', 'user2', ts0] cls.table = Table(None, table_name)
def test_encode_data_for_put(self): r0 = (bd0, 0, 1.2, unix_time_millis(ts0), True, []) r1 = (bd1, 3, 4.5, unix_time_millis(ts1), False, []) rows = [r0, r1] req = tsputreq_a, str_to_bytes(table_name), [], rows req_test = encode(req) rows_to_encode = [ [bd0, 0, 1.2, ts0, True, None], [bd1, 3, 4.5, ts1, False, None] ] tsobj = TsObject(None, self.table, rows_to_encode, None) c = TtbCodec() msg = c.encode_timeseries_put(tsobj) self.assertEqual(req_test, msg.data)
def setUpClass(cls): cls.ts0ms = unix_time_millis(ts0) if cls.ts0ms != ex0ms: raise AssertionError( 'expected {:d} to equal {:d}'.format(cls.ts0ms, ex0ms)) cls.ts1ms = unix_time_millis(ts1) if cls.ts1ms != ex1ms: raise AssertionError( 'expected {:d} to equal {:d}'.format(cls.ts1ms, ex1ms)) cls.rows = [ [bd0, 0, 1.2, ts0, True, None], [bd1, 3, 4.5, ts1, False, blob0] ] cls.test_key = ['hash1', 'user2', ts0] cls.table = Table(None, table_name)
def test_get_unix_time_with_tzinfo(self): try: import pytz tz = pytz.timezone('America/Los_Angeles') ts0_pst = tz.localize(ts0) utm = unix_time_millis(ts0_pst) self.assertEqual(utm, ts0_ts_pst) except ImportError: pass
def test_encode_data_for_get(self): keylist = [ str_to_bytes('hash1'), str_to_bytes('user2'), unix_time_millis(ts0) ] req = tsgetreq_a, str_to_bytes(table_name), keylist, udef_a req_test = encode(req) test_key = ['hash1', 'user2', ts0] c = TtbCodec() msg = c.encode_timeseries_keyreq(self.table, test_key) self.assertEqual(req_test, msg.data)
def test_conv_datetime_to_unix_millis(self): # This is the "native" Python unix timestamp including # microseconds, as float. timedelta "total_seconds()" # returns a value like this if is_timeseries_supported(): v = 144379690.987000 d = datetime.datetime.utcfromtimestamp(v) utm = unix_time_millis(d) self.assertEqual(utm, 144379690987) else: pass
def test_decode_data_from_get(self): colnames = [ "varchar", "sint64", "double", "timestamp", "boolean", "varchar", "varchar", "blob" ] coltypes = [ varchar_a, sint64_a, double_a, timestamp_a, boolean_a, varchar_a, varchar_a ] r0 = (bd0, 0, 1.2, unix_time_millis(ts0), True, [], str1, None, None) r1 = (bd1, 3, 4.5, unix_time_millis(ts1), False, [], str1, None, blob0) rows = [r0, r1] # { tsgetresp, { [colnames], [coltypes], [rows] } } data_t = colnames, coltypes, rows rsp_data = tsgetresp_a, data_t rsp_ttb = encode(rsp_data) tsobj = TsObject(None, self.table) c = TtbCodec() c.decode_timeseries(decode(rsp_ttb), tsobj) for i in range(0, 1): dr = rows[i] r = tsobj.rows[i] # encoded self.assertEqual(r[0], dr[0].encode('utf-8')) self.assertEqual(r[1], dr[1]) self.assertEqual(r[2], dr[2]) # NB *not* decoding timestamps # dt = datetime_from_unix_time_millis(dr[3]) self.assertEqual(r[3], dr[3]) if i == 0: self.assertEqual(r[4], True) else: self.assertEqual(r[4], False) self.assertEqual(r[5], None) self.assertEqual(r[6], dr[6].encode('ascii')) self.assertEqual(r[7], None) self.assertEqual(r[8], dr[8])
def test_unix_millis_small_value(self): if is_timeseries_supported(): # this is what would be stored in Riak TS v = 1001 dt = datetime_from_unix_time_millis(v) # This is how Python represents the above utp = 1.001 dtp = datetime.datetime.utcfromtimestamp(utp) self.assertEqual(dt, dtp) utm = unix_time_millis(dt) self.assertEqual(v, utm) else: pass
def test_conv_ms_timestamp_to_datetime_and_back(self): if is_timeseries_supported(): # this is what would be stored in Riak TS v = 144379690987 dt = datetime_from_unix_time_millis(v) # This is how Python represents the above utp = 144379690.987000 dtp = datetime.datetime.utcfromtimestamp(utp) self.assertEqual(dt, dtp) utm = unix_time_millis(dt) self.assertEqual(v, utm) else: pass
def encode_to_ts_cell(self, cell, ts_cell): if cell is not None: if isinstance(cell, datetime.datetime): ts_cell.timestamp_value = unix_time_millis(cell) elif isinstance(cell, bool): ts_cell.boolean_value = cell elif isinstance(cell, six.binary_type): ts_cell.varchar_value = cell elif isinstance(cell, six.text_type): ts_cell.varchar_value = str_to_bytes(cell) elif isinstance(cell, six.string_types): ts_cell.varchar_value = str_to_bytes(cell) elif (isinstance(cell, six.integer_types)): ts_cell.sint64_value = cell elif isinstance(cell, float): ts_cell.double_value = cell else: t = type(cell) raise RiakError("can't serialize type '{}', value '{}'" .format(t, cell))
def encode_to_ts_cell(self, cell): if cell is None: return [] else: if isinstance(cell, datetime.datetime): ts = unix_time_millis(cell) # logging.debug('encoded datetime %s as %s', cell, ts) return ts elif isinstance(cell, bool): return cell elif isinstance(cell, six.text_type) or \ isinstance(cell, six.binary_type) or \ isinstance(cell, six.string_types): return cell elif (isinstance(cell, six.integer_types)): return cell elif isinstance(cell, float): return cell else: t = type(cell) raise RiakError("can't serialize type '{}', value '{}'" .format(t, cell))
def test_store_and_fetch_and_query(self): now = datetime.datetime.utcfromtimestamp(144379690.987000) fiveMinsAgo = now - fiveMins tenMinsAgo = fiveMinsAgo - fiveMins fifteenMinsAgo = tenMinsAgo - fiveMins twentyMinsAgo = fifteenMinsAgo - fiveMins twentyFiveMinsAgo = twentyMinsAgo - fiveMins table = self.client.table(table_name) rows = [['hash1', 'user2', twentyFiveMinsAgo, 'typhoon', 90.3], ['hash1', 'user2', twentyMinsAgo, 'hurricane', 82.3], ['hash1', 'user2', fifteenMinsAgo, 'rain', 79.0], ['hash1', 'user2', fiveMinsAgo, 'wind', None], ['hash1', 'user2', now, 'snow', 20.1]] # NB: response data is binary exp_rows = [[ six.b('hash1'), six.b('user2'), twentyFiveMinsAgo, six.b('typhoon'), 90.3 ], [ six.b('hash1'), six.b('user2'), twentyMinsAgo, six.b('hurricane'), 82.3 ], [ six.b('hash1'), six.b('user2'), fifteenMinsAgo, six.b('rain'), 79.0 ], [ six.b('hash1'), six.b('user2'), fiveMinsAgo, six.b('wind'), None ], [six.b('hash1'), six.b('user2'), now, six.b('snow'), 20.1]] ts_obj = table.new(rows) result = ts_obj.store() self.assertTrue(result) for i, r in enumerate(rows): k = r[0:3] ts_obj = self.client.ts_get(table_name, k) self.assertIsNotNone(ts_obj) ts_cols = ts_obj.columns self.assertEqual(len(ts_cols.names), 5) self.assertEqual(len(ts_cols.types), 5) self.assertEqual(len(ts_obj.rows), 1) row = ts_obj.rows[0] exp = exp_rows[i] self.assertEqual(len(row), 5) self.assertEqual(row, exp) fmt = """ select * from {table} where time > {t1} and time < {t2} and geohash = 'hash1' and user = '******' """ query = fmt.format(table=table_name, t1=unix_time_millis(tenMinsAgo), t2=unix_time_millis(now)) ts_obj = self.client.ts_query(table_name, query) if ts_obj.columns is not None: self.assertEqual(len(ts_obj.columns.names), 5) self.assertEqual(len(ts_obj.columns.types), 5) self.assertEqual(len(ts_obj.rows), 1) row = ts_obj.rows[0] self.assertEqual(bytes_to_str(row[0]), 'hash1') self.assertEqual(bytes_to_str(row[1]), 'user2') self.assertEqual(row[2], fiveMinsAgo) self.assertEqual(row[2].microsecond, 987000) self.assertEqual(bytes_to_str(row[3]), 'wind') self.assertIsNone(row[4])
def test_encode_decode_timestamp(self): ts0ms = unix_time_millis(ts0) self.assertEqual(ts0ms, ex0ms) ts0_d = datetime_from_unix_time_millis(ts0ms) self.assertEqual(ts0, ts0_d)
def test_get_unix_time_without_tzinfo(self): self.assertIsNone(epoch.tzinfo) self.assertIsNotNone(epoch_tz.tzinfo) self.assertIsNone(ts0.tzinfo) utm = unix_time_millis(ts0) self.assertEqual(utm, ts0_ts)
def test_store_and_fetch_and_query(self): now = datetime.datetime.utcfromtimestamp(144379690.987000) fiveMinsAgo = now - fiveMins tenMinsAgo = fiveMinsAgo - fiveMins fifteenMinsAgo = tenMinsAgo - fiveMins twentyMinsAgo = fifteenMinsAgo - fiveMins twentyFiveMinsAgo = twentyMinsAgo - fiveMins table = self.client.table(table_name) rows = [ ['hash1', 'user2', twentyFiveMinsAgo, 'typhoon', 90.3], ['hash1', 'user2', twentyMinsAgo, 'hurricane', 82.3], ['hash1', 'user2', fifteenMinsAgo, 'rain', 79.0], ['hash1', 'user2', fiveMinsAgo, 'wind', None], ['hash1', 'user2', now, 'snow', 20.1] ] # NB: response data is binary exp_rows = [ [six.b('hash1'), six.b('user2'), twentyFiveMinsAgo, six.b('typhoon'), 90.3], [six.b('hash1'), six.b('user2'), twentyMinsAgo, six.b('hurricane'), 82.3], [six.b('hash1'), six.b('user2'), fifteenMinsAgo, six.b('rain'), 79.0], [six.b('hash1'), six.b('user2'), fiveMinsAgo, six.b('wind'), None], [six.b('hash1'), six.b('user2'), now, six.b('snow'), 20.1] ] ts_obj = table.new(rows) result = ts_obj.store() self.assertTrue(result) for i, r in enumerate(rows): k = r[0:3] ts_obj = self.client.ts_get(table_name, k) self.assertIsNotNone(ts_obj) ts_cols = ts_obj.columns self.assertEqual(len(ts_cols.names), 5) self.assertEqual(len(ts_cols.types), 5) self.assertEqual(len(ts_obj.rows), 1) row = ts_obj.rows[0] exp = exp_rows[i] self.assertEqual(len(row), 5) self.assertEqual(row, exp) fmt = """ select * from {table} where time > {t1} and time < {t2} and geohash = 'hash1' and user = '******' """ query = fmt.format( table=table_name, t1=unix_time_millis(tenMinsAgo), t2=unix_time_millis(now)) ts_obj = self.client.ts_query(table_name, query) if ts_obj.columns is not None: self.assertEqual(len(ts_obj.columns.names), 5) self.assertEqual(len(ts_obj.columns.types), 5) self.assertEqual(len(ts_obj.rows), 1) row = ts_obj.rows[0] self.assertEqual(bytes_to_str(row[0]), 'hash1') self.assertEqual(bytes_to_str(row[1]), 'user2') self.assertEqual(row[2], fiveMinsAgo) self.assertEqual(row[2].microsecond, 987000) self.assertEqual(bytes_to_str(row[3]), 'wind') self.assertIsNone(row[4])