def setUpClass(cls):
cls.handle = None
cls.set_up_class()
create_statement = ('CREATE TABLE ' + table_name +
'(fld_id INTEGER, fld_long LONG, \
fld_float FLOAT, fld_double DOUBLE, fld_bool BOOLEAN, fld_str STRING, \
fld_bin BINARY, fld_time TIMESTAMP(7), fld_num NUMBER, fld_json JSON, \
fld_arr ARRAY(STRING), fld_map MAP(STRING), \
fld_rec RECORD(fld_id LONG, fld_bool BOOLEAN, fld_str STRING), \
PRIMARY KEY(fld_id)) USING TTL 1 HOURS')
limits = TableLimits(5000, 5000, 50)
create_request = TableRequest().set_statement(
create_statement).set_table_limits(limits)
cls.table_request(create_request)
# put and get some data, read_units = 100, write_units = 199
row = get_row()
key = {'fld_id': 1}
put_request = PutRequest().set_value(row).set_table_name(table_name)
get_request = GetRequest().set_key(key).set_table_name(table_name)
count = 0
while count < 100:
cls.handle.put(put_request)
cls.handle.get(get_request)
count += 1
# sleep to allow records to accumulate over time, but not if
# using Cloudsim.
if not_cloudsim() and not is_onprem():
sleep(2)
# need to sleep to allow usage records to accumulate but not if
# using CloudSim, which doesn't generate usage records.
if not_cloudsim() and not is_onprem():
sleep(40)
def setUp(self):
TestBase.set_up(self)
self.handle_config = get_handle_config(tenant_id)
shardkeys = [0, 1]
ids = [0, 1, 2, 3, 4, 5]
write_multiple_request = WriteMultipleRequest()
for sk in shardkeys:
for i in ids:
row = {
'fld_sid': sk,
'fld_id': i,
'fld_long': 2147483648,
'fld_float': 3.1414999961853027,
'fld_double': 3.1415,
'fld_bool': True,
'fld_str': '{"name": u1, "phone": null}',
'fld_bin': bytearray(pack('>i', 4)),
'fld_time': datetime.now(),
'fld_num': Decimal(5),
'fld_json': {
'a': '1',
'b': None,
'c': '3'
},
'fld_arr': ['a', 'b', 'c'],
'fld_map': {
'a': '1',
'b': '2',
'c': '3'
},
'fld_rec': {
'fld_id': 1,
'fld_bool': False,
'fld_str': None
}
}
write_multiple_request.add(
PutRequest().set_value(row).set_table_name(table_name),
True)
self.handle.write_multiple(write_multiple_request)
write_multiple_request.clear()
prepare_statement_update = (
'DECLARE $fld_sid INTEGER; $fld_id INTEGER; UPDATE ' + table_name +
' u SET u.fld_long = u.fld_long + 1 WHERE fld_sid = $fld_sid ' +
'AND fld_id = $fld_id')
prepare_request_update = PrepareRequest().set_statement(
prepare_statement_update)
self.prepare_result_update = self.handle.prepare(
prepare_request_update)
prepare_statement_select = (
'DECLARE $fld_long LONG; SELECT fld_sid, fld_id, fld_long FROM ' +
table_name + ' WHERE fld_long = $fld_long')
prepare_request_select = PrepareRequest().set_statement(
prepare_statement_select)
self.prepare_result_select = self.handle.prepare(
prepare_request_select)
self.query_request = QueryRequest().set_timeout(timeout)
self.get_request = GetRequest().set_table_name(table_name)
def setUpClass(cls):
TestBase.set_up_class()
create_statement = ('CREATE TABLE ' + table_name +
'(fld_id INTEGER, fld_long LONG, \
fld_float FLOAT, fld_double DOUBLE, fld_bool BOOLEAN, fld_str STRING, \
fld_bin BINARY, fld_time TIMESTAMP(7), fld_num NUMBER, fld_json JSON, \
fld_arr ARRAY(STRING), fld_map MAP(STRING), \
fld_rec RECORD(fld_id LONG, fld_bool BOOLEAN, fld_str STRING), \
PRIMARY KEY(fld_id)) USING TTL 1 HOURS')
limits = TableLimits(5000, 5000, 50)
create_request = TableRequest().set_statement(
create_statement).set_table_limits(limits)
cls._result = TestBase.table_request(create_request, State.ACTIVE)
# put and get some data, read_units = 100, write_units = 199
row = {
'fld_id': 1,
'fld_long': 2147483648,
'fld_float': 3.1414999961853027,
'fld_double': 3.1415,
'fld_bool': True,
'fld_str': '{"name": u1, "phone": null}',
'fld_bin': bytearray(pack('>i', 4)),
'fld_time': datetime.now(),
'fld_num': Decimal(5),
'fld_json': {
'a': '1',
'b': None,
'c': '3'
},
'fld_arr': ['a', 'b', 'c'],
'fld_map': {
'a': '1',
'b': '2',
'c': '3'
},
'fld_rec': {
'fld_id': 1,
'fld_bool': False,
'fld_str': None
}
}
key = {'fld_id': 1}
put_request = PutRequest().set_value(row).set_table_name(table_name)
get_request = GetRequest().set_key(key).set_table_name(table_name)
count = 0
while count < 100:
cls._handle.put(put_request)
cls._handle.get(get_request)
count += 1
# sleep to allow records to accumulate over time, but not if
# using Cloudsim.
if not_cloudsim():
sleep(2)
# need to sleep to allow usage records to accumulate but not if
# using CloudSim, which doesn't generate usage records.
if not_cloudsim():
sleep(40)
def setUp(self):
self.set_up()
self.row = get_row(with_sid=False)
self.key = {'fld_id': 1}
self.put_request = PutRequest().set_value(
self.row).set_table_name(table_name)
self.version = self.handle.put(self.put_request).get_version()
self.get_request = GetRequest().set_key(
self.key).set_table_name(table_name)
self.delete_request = DeleteRequest().set_key(
self.key).set_table_name(table_name).set_timeout(timeout)
def setUp(self):
self.set_up()
self.row = get_row(with_sid=False)
self.key = {'fld_id': 1}
self.put_request = PutRequest().set_value(
self.row).set_table_name(table_name).set_timeout(timeout)
self.get_request = GetRequest().set_key(
self.key).set_table_name(table_name)
self.ttl = TimeToLive.of_hours(24)
self.hour_in_milliseconds = 60 * 60 * 1000
self.day_in_milliseconds = 24 * 60 * 60 * 1000
def setUp(self):
self.set_up()
self.handle_config = get_handle_config(tenant_id)
self.min_time = list()
self.max_time = list()
shardkeys = 2
ids = 6
write_multiple_request = WriteMultipleRequest()
for sk in range(shardkeys):
for i in range(ids):
row = get_row()
if i == 0:
self.min_time.append(row['fld_time'])
elif i == ids - 1:
self.max_time.append(row['fld_time'])
row['fld_sid'] = sk
row['fld_id'] = i
row['fld_bool'] = False if sk == 0 else True
row['fld_str'] = (
'{"name": u' +
str(shardkeys * ids - sk * ids - i - 1).zfill(2) + '}')
row['fld_json']['location']['coordinates'] = ([
23.549 - sk * 0.5 - i, 35.2908 + sk * 0.5 + i
])
write_multiple_request.add(
PutRequest().set_value(row).set_table_name(table_name),
True)
self.handle.write_multiple(write_multiple_request)
write_multiple_request.clear()
prepare_statement_update = (
'DECLARE $fld_sid INTEGER; $fld_id INTEGER; UPDATE ' + table_name +
' u SET u.fld_long = u.fld_long + 1 WHERE fld_sid = $fld_sid ' +
'AND fld_id = $fld_id')
prepare_request_update = PrepareRequest().set_statement(
prepare_statement_update)
self.prepare_result_update = self.handle.prepare(
prepare_request_update)
prepare_statement_select = (
'DECLARE $fld_long LONG; SELECT fld_sid, fld_id, fld_long FROM ' +
table_name + ' WHERE fld_long = $fld_long')
prepare_request_select = PrepareRequest().set_statement(
prepare_statement_select)
self.prepare_result_select = self.handle.prepare(
prepare_request_select)
self.query_request = QueryRequest().set_timeout(timeout)
self.get_request = GetRequest().set_table_name(table_name)
def setUp(self):
TestBase.set_up(self)
self.row = {'fld_id': 1, 'fld_long': 2147483648,
'fld_float': 3.1414999961853027, 'fld_double': 3.1415,
'fld_bool': True, 'fld_str': '{"name": u1, "phone": null}',
'fld_bin': bytearray(pack('>i', 4)),
'fld_time': datetime.now(), 'fld_num': Decimal(5),
'fld_json': {'a': '1', 'b': None, 'c': '3'},
'fld_arr': ['a', 'b', 'c'],
'fld_map': {'a': '1', 'b': '2', 'c': '3'},
'fld_rec': {'fld_id': 1, 'fld_bool': False,
'fld_str': None}}
self.key = {'fld_id': 1}
self.put_request = PutRequest().set_value(self.row).set_table_name(
table_name)
self.version = self.handle.put(self.put_request).get_version()
self.get_request = GetRequest().set_key(self.key).set_table_name(
table_name)
self.delete_request = DeleteRequest().set_key(self.key).set_table_name(
table_name).set_timeout(timeout)
def setUp(self):
TestBase.set_up(self)
self.row = {
'fld_id': 1,
'fld_long': 2147483648,
'fld_float': 3.1414999961853027,
'fld_double': 3.1415,
'fld_bool': True,
'fld_str': '{"name": u1, "phone": null}',
'fld_bin': bytearray(pack('>i', 4)),
'fld_time': datetime.now(),
'fld_num': Decimal(5),
'fld_json': {
'a': '1',
'b': None,
'c': '3'
},
'fld_arr': ['a', 'b', 'c'],
'fld_map': {
'a': '1',
'b': '2',
'c': '3'
},
'fld_rec': {
'fld_id': 1,
'fld_bool': False,
'fld_str': None
}
}
self.key = {'fld_id': 1}
self.put_request = PutRequest().set_value(
self.row).set_table_name(table_name).set_timeout(timeout)
self.get_request = GetRequest().set_key(
self.key).set_table_name(table_name)
self.ttl = TimeToLive.of_hours(24)
self.hour_in_milliseconds = 60 * 60 * 1000
self.day_in_milliseconds = 24 * 60 * 60 * 1000
def main():
handle = None
try:
#
# Create a handle
#
handle = get_handle(tenant_id)
#
# Create a table
#
statement = 'Create table if not exists ' + table_name + '(id integer, \
sid integer, name string, primary key(shard(sid), id))'
print('Creating table: ' + statement)
request = TableRequest().set_statement(statement).set_table_limits(
TableLimits(30, 10, 1))
result = handle.table_request(request)
#
# Table creation can take time, depending on the state of the system.
# If if fails after 40s, re-run the program
#
result.wait_for_state(handle, table_name, State.ACTIVE, 50000, 3000)
print('After create table')
#
# Put a few rows
#
request = PutRequest().set_table_name(table_name)
for i in range(10):
value = {'id': i, 'sid': 0, 'name': 'myname' + str(i)}
request.set_value(value)
handle.put(request)
print('After put of 10 rows')
#
# Get the row
#
request = GetRequest().set_key({
'id': 1,
'sid': 0
}).set_table_name(table_name)
result = handle.get(request)
print('After get: ' + str(result))
#
# Query, using a range
#
statement = 'select * from ' + table_name + ' where id > 2 and id < 8'
request = QueryRequest().set_statement(statement)
result = handle.query(request)
print('Query results for: ' + statement)
for r in result.get_results():
print('\t' + str(r))
#
# Delete the row
#
request = DeleteRequest().set_key({
'id': 1,
'sid': 0
}).set_table_name(table_name)
result = handle.delete(request)
print('After delete: ' + str(result))
#
# Get again to show deletion
#
request = GetRequest().set_key({
'id': 1,
'sid': 0
}).set_table_name(table_name)
result = handle.get(request)
print('After get (should be None): ' + str(result))
#
# Drop the table
#
if drop_table:
request = TableRequest().set_statement('drop table if exists ' +
table_name)
result = handle.table_request(request)
#
# Table drop can take time, depending on the state of the system.
# If this wait fails the table will still probably been dropped
#
result.wait_for_state(handle, table_name, State.DROPPED, 40000,
2000)
print('After drop table')
else:
print('Not dropping table')
print('Example is complete')
except Exception as e:
print(e)
traceback.print_exc()
finally:
# If the handle isn't closed Python will not exit properly
if handle is not None:
handle.close()
def setUp(self):
self.config = get_simple_handle_config(tenant_id)
self.table_request = TableRequest().set_statement(
'DROP TABLE IF EXISTS ' + table_name)
self.get_request = GetRequest()
def setUp(self):
self.set_up()
self.key = {'fld_sid': 1, 'fld_id': 1}
self.get_request = GetRequest().set_key(
self.key).set_table_name(table_name).set_timeout(timeout)
def setUp(self):
self.set_up()
self.shardkeys = [0, 1]
self.ids = [0, 1, 2, 3, 4, 5]
self.rows = list()
self.new_rows = list()
self.versions = list()
self.requests = list()
self.illegal_requests = list()
ttl = TimeToLive.of_days(16)
for sk in self.shardkeys:
self.rows.append(list())
self.new_rows.append(list())
self.versions.append(list())
for i in self.ids:
row = get_row()
row['fld_sid'] = sk
row['fld_id'] = i
new_row = deepcopy(row)
new_row['fld_long'] = 2147483649
self.rows[sk].append(row)
self.new_rows[sk].append(new_row)
put_request = PutRequest().set_value(row).set_table_name(
table_name).set_ttl(ttl)
self.versions[sk].append(
self.handle.put(put_request).get_version())
self.old_expect_expiration = ttl.to_expiration_time(
int(round(time() * 1000)))
self.ttl = TimeToLive.of_hours(1)
self.ops_sk = 0
illegal_sk = 1
self.requests.append(PutRequest().set_value(
self.new_rows[self.ops_sk][0]).set_table_name(table_name).set_ttl(
self.ttl).set_return_row(True))
self.requests.append(PutRequest().set_value(self.new_rows[
self.ops_sk][1]).set_table_name(table_name).set_option(
PutOption.IF_ABSENT).set_ttl(self.ttl).set_return_row(True))
self.requests.append(PutRequest().set_value(
self.new_rows[self.ops_sk][2]).set_use_table_default_ttl(
True).set_table_name(table_name).set_option(
PutOption.IF_PRESENT).set_return_row(True))
self.requests.append(PutRequest().set_value(
self.new_rows[self.ops_sk][3]).set_table_name(table_name).set_ttl(
self.ttl).set_option(PutOption.IF_VERSION).set_match_version(
self.versions[self.ops_sk][3]).set_return_row(True))
self.requests.append(DeleteRequest().set_key({
'fld_sid': self.ops_sk,
'fld_id': 4
}).set_table_name(table_name).set_return_row(True))
self.requests.append(DeleteRequest().set_key({
'fld_sid': self.ops_sk,
'fld_id': 5
}).set_table_name(table_name).set_return_row(True).set_match_version(
self.versions[self.ops_sk][0]))
self.illegal_requests.append(DeleteRequest().set_key({
'fld_sid': self.ops_sk,
'fld_id': 0
}).set_table_name('IllegalUsers'))
self.illegal_requests.append(DeleteRequest().set_key({
'fld_sid': illegal_sk,
'fld_id': 0
}).set_table_name(table_name))
self.write_multiple_request = WriteMultipleRequest().set_timeout(
timeout)
self.get_request = GetRequest().set_table_name(table_name)
self.hour_in_milliseconds = 60 * 60 * 1000
self.day_in_milliseconds = 24 * 60 * 60 * 1000
def main():
handle = None
try:
#
# Create a handle
#
handle = get_handle(tenant_id)
#
# Create a table
#
statement = 'Create table if not exists ' + table_name + '(id integer, \
sid integer, name string, primary key(shard(sid), id))'
print('Creating table: ' + statement)
request = TableRequest().set_statement(statement).set_table_limits(
TableLimits(30, 10, 1))
handle.do_table_request(request, 50000, 3000)
print('After create table')
#
# Put a few rows
#
request = PutRequest().set_table_name(table_name)
for i in range(10):
value = {'id': i, 'sid': 0, 'name': 'myname' + str(i)}
request.set_value(value)
handle.put(request)
print('After put of 10 rows')
#
# Get the row
#
request = GetRequest().set_key({
'id': 1,
'sid': 0
}).set_table_name(table_name)
result = handle.get(request)
print('After get: ' + str(result))
#
# Query, using a range
#
statement = 'select * from ' + table_name + ' where id > 2 and id < 8'
request = QueryRequest().set_statement(statement)
print('Query results for: ' + statement)
while True:
result = handle.query(request)
for r in result.get_results():
print('\t' + str(r))
if request.is_done():
break
#
# Delete the row
#
request = DeleteRequest().set_key({
'id': 1,
'sid': 0
}).set_table_name(table_name)
result = handle.delete(request)
print('After delete: ' + str(result))
#
# Get again to show deletion
#
request = GetRequest().set_key({
'id': 1,
'sid': 0
}).set_table_name(table_name)
result = handle.get(request)
print('After get (should be None): ' + str(result))
#
# Drop the table
#
if drop_table:
request = TableRequest().set_statement('drop table if exists ' +
table_name)
handle.do_table_request(request, 40000, 2000)
print('After drop table')
else:
print('Not dropping table')
print('Example is complete')
except Exception as e:
print(e)
traceback.print_exc()
finally:
# If the handle isn't closed Python will not exit properly
if handle is not None:
handle.close()
def _do_rate_limited_ops(self, num_seconds, read_limit, write_limit,
max_rows, check_units, use_percent,
use_external_limiters):
"""
Runs puts and gets continuously for N seconds.
Verify that the resultant RUs/WUs used match the given rate limits.
"""
if read_limit == 0 and write_limit == 0:
return
put_request = PutRequest().set_table_name(table_name)
get_request = GetRequest().set_table_name(table_name)
key = dict()
# TODO: random sizes 0-nKB.
value = dict()
value['name'] = 'jane'
start_time = int(round(time() * 1000))
end_time = start_time + num_seconds * 1000
read_units_used = 0
write_units_used = 0
total_delayed_ms = 0
throttle_exceptions = 0
rlim = None
wlim = None
max_val = float(read_limit + write_limit)
if not use_external_limiters:
# Reset internal limiters so they don't have unused units.
self.handle.get_client().reset_rate_limiters(table_name)
else:
rlim = SimpleRateLimiter(read_limit * use_percent / 100.0, 1)
wlim = SimpleRateLimiter(write_limit * use_percent / 100.0, 1)
while True:
fld_id = int(random() * max_rows)
if read_limit == 0:
do_put = True
elif write_limit == 0:
do_put = False
else:
v = int(random() * max_val)
do_put = v >= read_limit
try:
if do_put:
value['id'] = fld_id
put_request.set_value(value).set_read_rate_limiter(
None).set_write_rate_limiter(wlim)
pres = self.handle.put(put_request)
write_units_used += pres.get_write_units()
total_delayed_ms += pres.get_rate_limit_delayed_ms()
rs = pres.get_retry_stats()
if rs is not None:
throttle_exceptions += rs.get_num_exceptions(
WriteThrottlingException.__class__.__name__)
else:
key['id'] = fld_id
get_request.set_key(key).set_read_rate_limiter(
rlim).set_write_rate_limiter(None)
gres = self.handle.get(get_request)
read_units_used += gres.get_read_units()
total_delayed_ms += gres.get_rate_limit_delayed_ms()
rs = gres.get_retry_stats()
if rs is not None:
throttle_exceptions += rs.get_num_exceptions(
ReadThrottlingException.__class__.__name__)
except ReadThrottlingException:
self.fail(
'Expected no read throttling exceptions, got one.')
except WriteThrottlingException:
self.fail(
'Expected no write throttling exceptions, got one.')
if int(round(time() * 1000)) >= end_time:
break
num_seconds = (int(round(time() * 1000)) - start_time) / 1000
rus = read_units_used / num_seconds
wus = write_units_used / num_seconds
if not check_units:
return
use_percent /= 100.0
if (rus < read_limit * use_percent * 0.8
or rus > read_limit * use_percent * 1.2):
self.fail('Gets: Expected around ' +
str(read_limit * use_percent) + ' RUs, got ' +
str(rus))
if (wus < write_limit * use_percent * 0.8
or wus > write_limit * use_percent * 1.2):
self.fail('Puts: Expected around ' +
str(write_limit * use_percent) + ' WUs, got ' +
str(wus))
def setUp(self):
TestBase.set_up(self)
self.shardkeys = [0, 1]
self.ids = [0, 1, 2, 3, 4, 5]
self.rows = list()
self.new_rows = list()
self.versions = list()
self.requests = list()
self.illegal_requests = list()
ttl = TimeToLive.of_days(16)
for sk in self.shardkeys:
self.rows.append(list())
self.new_rows.append(list())
self.versions.append(list())
for i in self.ids:
row = {'fld_sid': sk, 'fld_id': i, 'fld_long': 2147483648,
'fld_float': 3.1414999961853027, 'fld_double': 3.1415,
'fld_bool': True,
'fld_str': '{"name": u1, "phone": null}',
'fld_bin': bytearray(pack('>i', 4)),
'fld_time': datetime.now(), 'fld_num': Decimal(5),
'fld_json': {'a': '1', 'b': None, 'c': '3'},
'fld_arr': ['a', 'b', 'c'],
'fld_map': {'a': '1', 'b': '2', 'c': '3'},
'fld_rec': {'fld_id': 1, 'fld_bool': False,
'fld_str': None}}
new_row = deepcopy(row)
new_row.update({'fld_long': 2147483649})
self.rows[sk].append(row)
self.new_rows[sk].append(new_row)
put_request = PutRequest().set_value(row).set_table_name(
table_name).set_ttl(ttl)
self.versions[sk].append(
self.handle.put(put_request).get_version())
self.old_expect_expiration = ttl.to_expiration_time(
int(round(time() * 1000)))
self.ttl = TimeToLive.of_hours(1)
self.ops_sk = 0
illegal_sk = 1
self.requests.append(PutRequest().set_value(
self.new_rows[self.ops_sk][0]).set_table_name(table_name).set_ttl(
self.ttl).set_return_row(True))
self.requests.append(PutRequest().set_value(
self.new_rows[self.ops_sk][1]).set_table_name(
table_name).set_option(PutOption.IF_ABSENT).set_ttl(
self.ttl).set_return_row(True))
self.requests.append(PutRequest().set_value(
self.new_rows[self.ops_sk][2]).set_use_table_default_ttl(
True).set_table_name(table_name).set_option(
PutOption.IF_PRESENT).set_return_row(True))
self.requests.append(PutRequest().set_value(
self.new_rows[self.ops_sk][3]).set_table_name(
table_name).set_option(PutOption.IF_VERSION).set_ttl(
self.ttl).set_match_version(
self.versions[self.ops_sk][3]).set_return_row(True))
self.requests.append(DeleteRequest().set_key(
{'fld_sid': self.ops_sk, 'fld_id': 4}).set_table_name(
table_name).set_return_row(True))
self.requests.append(DeleteRequest().set_key(
{'fld_sid': self.ops_sk, 'fld_id': 5}).set_table_name(
table_name).set_match_version(
self.versions[self.ops_sk][0]).set_return_row(True))
self.illegal_requests.append(DeleteRequest().set_key(
{'fld_sid': self.ops_sk, 'fld_id': 0}).set_table_name(
'IllegalUsers'))
self.illegal_requests.append(DeleteRequest().set_key(
{'fld_sid': illegal_sk, 'fld_id': 0}).set_table_name(table_name))
self.write_multiple_request = WriteMultipleRequest().set_timeout(
timeout)
self.get_request = GetRequest().set_table_name(table_name)
self.hour_in_milliseconds = 60 * 60 * 1000
self.day_in_milliseconds = 24 * 60 * 60 * 1000