def testWriteMultipleAddIllegalRequestAndAbortIfUnsuccessful(self): self.assertRaises(IllegalArgumentException, self.write_multiple_request.add, 'IllegalRequest', True) self.assertRaises(IllegalArgumentException, self.write_multiple_request.add, PutRequest(), 'IllegalAbortIfUnsuccessful') # add two operations with different table name self.write_multiple_request.add(self.requests[0], True) self.assertRaises(IllegalArgumentException, self.write_multiple_request.add, self.illegal_requests[0], False) self.write_multiple_request.clear() # add two operations with different major paths self.write_multiple_request.add(self.requests[0], True).add(self.illegal_requests[1], False) self.assertRaises(IllegalArgumentException, self.handle.write_multiple, self.write_multiple_request) self.write_multiple_request.clear() # add operations when sub requests reached the max number if not is_onprem(): count = 0 while count <= 50: row = get_row() row['fld_id'] = count self.write_multiple_request.add( PutRequest().set_value(row).set_table_name(table_name), True) count += 1 self.assertRaises(BatchOperationNumberLimitException, self.handle.write_multiple, self.write_multiple_request)
def handler(ctx, data: io.BytesIO=None): logger = logging.getLogger() logger.setLevel(logging.INFO) try: store_handle = get_handle() logs = json.loads(data.getvalue()) logger.info('Received {} entries.'.format(len(logs))) for item in logs: if 'value' in item: item['value'] = base64_decode(item['value']) if 'key' in item: item['key'] = base64_decode(item['key']) # # For demo purpose, we are inserting the data as received, no processing # Put rows in NoSQL # request = PutRequest().set_table_name('demoKeyVal') for item in logs: if 'value' in item: value = { 'value': json.loads(item['value'])} request.set_value(value) store_handle.put(request) request = PutRequest().set_table_name('demo') for item in logs: if 'value' in item: request.set_value_from_json(item['value']) store_handle.put(request) # # return data in CSV mode # df = pd.json_normalize(logs) csv_result = df.to_csv(index=False) return response.Response(ctx, status_code=200, response_data=csv_result, headers={"Content-Type": "text/csv"}) except (Exception, ValueError) as e: # For demo purpose, I am ignoring all errors # During tests, I sent some no JSON messages to my Stream that were broken all # If there is an error, in the next iteration I will receive all pending message from the Stream # so continue to have the same error logger.info('Logging and ignore the error') logger.error(str(e))
def setUpClass(cls): cls.handle = None cls.set_up_class() create_statement = ('CREATE TABLE ' + table_name + '(fld_id INTEGER, fld_long LONG, \ fld_float FLOAT, fld_double DOUBLE, fld_bool BOOLEAN, fld_str STRING, \ fld_bin BINARY, fld_time TIMESTAMP(7), fld_num NUMBER, fld_json JSON, \ fld_arr ARRAY(STRING), fld_map MAP(STRING), \ fld_rec RECORD(fld_id LONG, fld_bool BOOLEAN, fld_str STRING), \ PRIMARY KEY(fld_id)) USING TTL 1 HOURS') limits = TableLimits(5000, 5000, 50) create_request = TableRequest().set_statement( create_statement).set_table_limits(limits) cls.table_request(create_request) # put and get some data, read_units = 100, write_units = 199 row = get_row() key = {'fld_id': 1} put_request = PutRequest().set_value(row).set_table_name(table_name) get_request = GetRequest().set_key(key).set_table_name(table_name) count = 0 while count < 100: cls.handle.put(put_request) cls.handle.get(get_request) count += 1 # sleep to allow records to accumulate over time, but not if # using Cloudsim. if not_cloudsim() and not is_onprem(): sleep(2) # need to sleep to allow usage records to accumulate but not if # using CloudSim, which doesn't generate usage records. if not_cloudsim() and not is_onprem(): sleep(40)
def testWriteMultipleAddIllegalRequestAndAbortIfUnsuccessful(self): self.assertRaises(IllegalArgumentException, self.write_multiple_request.add, 'IllegalRequest', True) self.assertRaises(IllegalArgumentException, self.write_multiple_request.add, PutRequest(), 'IllegalAbortIfUnsuccessful') # add two operations with different table name self.write_multiple_request.add(self.requests[0], True) self.assertRaises(IllegalArgumentException, self.write_multiple_request.add, self.illegal_requests[0], False) self.write_multiple_request.clear() # add two operations with different major paths self.write_multiple_request.add( self.requests[0], True).add(self.illegal_requests[1], False) self.assertRaises(IllegalArgumentException, self.handle.write_multiple, self.write_multiple_request) self.write_multiple_request.clear() # add operations when sub requests reached the max number count = 0 while count < 50: self.write_multiple_request.add(self.requests[0], True) count += 1 self.assertRaises(BatchOperationNumberLimitException, self.write_multiple_request.add, self.requests[0], True)
def testWriteMultipleWithIdentityColumn(self): num_operations = 10 id_table = table_prefix + 'Identity' create_request = TableRequest().set_statement( 'CREATE TABLE ' + id_table + '(sid INTEGER, id LONG GENERATED \ ALWAYS AS IDENTITY, name STRING, PRIMARY KEY(SHARD(sid), id))') create_request.set_table_limits(TableLimits(5000, 5000, 50)) self.table_request(create_request) # add ten operations row = {'name': 'myname', 'sid': 1} for idx in range(num_operations): put_request = PutRequest().set_table_name(id_table).set_value(row) put_request.set_identity_cache_size(idx) self.write_multiple_request.add(put_request, False) # execute the write multiple request versions = list() result = self.handle.write_multiple(self.write_multiple_request) op_results = self._check_write_multiple_result(result, num_operations) for idx in range(result.size()): versions.append( self._check_operation_result(op_results[idx], True, True, idx + 1)) self.check_cost(result, 0, 0, num_operations, num_operations) # check the records after write_multiple request succeed self.get_request.set_table_name(id_table) for idx in range(num_operations): self.get_request.set_key({'sid': 1, 'id': idx + 1}) result = self.handle.get(self.get_request) expected = OrderedDict() expected['sid'] = 1 expected['id'] = idx + 1 expected['name'] = 'myname' self.check_get_result(result, expected, versions[idx]) self.check_cost(result, 1, 2, 0, 0)
def setUp(self): TestBase.set_up(self) self.shardkeys = [0, 1] ids = [0, 1, 2, 3, 4, 5] write_multiple_request = WriteMultipleRequest() for sk in self.shardkeys: for i in ids: row = {'fld_sid': sk, 'fld_id': i, 'fld_long': 2147483648, 'fld_float': 3.1414999961853027, 'fld_double': 3.1415, 'fld_bool': True, 'fld_str': '{"name": u1, "phone": null}', 'fld_bin': bytearray(pack('>i', 4)), 'fld_time': datetime.now(), 'fld_num': Decimal(5), 'fld_json': {'a': '1', 'b': None, 'c': '3'}, 'fld_arr': ['a', 'b', 'c'], 'fld_map': {'a': '1', 'b': '2', 'c': '3'}, 'fld_rec': {'fld_id': 1, 'fld_bool': False, 'fld_str': None}} write_multiple_request.add( PutRequest().set_value(row).set_table_name(table_name), True) self.handle.write_multiple(write_multiple_request) write_multiple_request.clear() self.key = {'fld_sid': 1} self.multi_delete_request = MultiDeleteRequest().set_timeout(timeout) prep_request = PrepareRequest().set_statement( 'SELECT fld_sid, fld_id FROM ' + table_name) prep_result = self.handle.prepare(prep_request) self.query_request = QueryRequest().set_prepared_statement(prep_result)
def add_new_user(user_data_dict): nosql_request = PutRequest().set_table_name('fotogal_ntable_users') nosql_request.set_value(user_data_dict) nosql_put_result = nosql_handler.put(nosql_request) user_saved_id = nosql_put_result.get_generated_value() return user_saved_id
def setUp(self): TestBase.set_up(self) self.handle_config = get_handle_config(tenant_id) shardkeys = [0, 1] ids = [0, 1, 2, 3, 4, 5] write_multiple_request = WriteMultipleRequest() for sk in shardkeys: for i in ids: row = { 'fld_sid': sk, 'fld_id': i, 'fld_long': 2147483648, 'fld_float': 3.1414999961853027, 'fld_double': 3.1415, 'fld_bool': True, 'fld_str': '{"name": u1, "phone": null}', 'fld_bin': bytearray(pack('>i', 4)), 'fld_time': datetime.now(), 'fld_num': Decimal(5), 'fld_json': { 'a': '1', 'b': None, 'c': '3' }, 'fld_arr': ['a', 'b', 'c'], 'fld_map': { 'a': '1', 'b': '2', 'c': '3' }, 'fld_rec': { 'fld_id': 1, 'fld_bool': False, 'fld_str': None } } write_multiple_request.add( PutRequest().set_value(row).set_table_name(table_name), True) self.handle.write_multiple(write_multiple_request) write_multiple_request.clear() prepare_statement_update = ( 'DECLARE $fld_sid INTEGER; $fld_id INTEGER; UPDATE ' + table_name + ' u SET u.fld_long = u.fld_long + 1 WHERE fld_sid = $fld_sid ' + 'AND fld_id = $fld_id') prepare_request_update = PrepareRequest().set_statement( prepare_statement_update) self.prepare_result_update = self.handle.prepare( prepare_request_update) prepare_statement_select = ( 'DECLARE $fld_long LONG; SELECT fld_sid, fld_id, fld_long FROM ' + table_name + ' WHERE fld_long = $fld_long') prepare_request_select = PrepareRequest().set_statement( prepare_statement_select) self.prepare_result_select = self.handle.prepare( prepare_request_select) self.query_request = QueryRequest().set_timeout(timeout) self.get_request = GetRequest().set_table_name(table_name)
def setUpClass(cls): TestBase.set_up_class() create_statement = ('CREATE TABLE ' + table_name + '(fld_id INTEGER, fld_long LONG, \ fld_float FLOAT, fld_double DOUBLE, fld_bool BOOLEAN, fld_str STRING, \ fld_bin BINARY, fld_time TIMESTAMP(7), fld_num NUMBER, fld_json JSON, \ fld_arr ARRAY(STRING), fld_map MAP(STRING), \ fld_rec RECORD(fld_id LONG, fld_bool BOOLEAN, fld_str STRING), \ PRIMARY KEY(fld_id)) USING TTL 1 HOURS') limits = TableLimits(5000, 5000, 50) create_request = TableRequest().set_statement( create_statement).set_table_limits(limits) cls._result = TestBase.table_request(create_request, State.ACTIVE) # put and get some data, read_units = 100, write_units = 199 row = { 'fld_id': 1, 'fld_long': 2147483648, 'fld_float': 3.1414999961853027, 'fld_double': 3.1415, 'fld_bool': True, 'fld_str': '{"name": u1, "phone": null}', 'fld_bin': bytearray(pack('>i', 4)), 'fld_time': datetime.now(), 'fld_num': Decimal(5), 'fld_json': { 'a': '1', 'b': None, 'c': '3' }, 'fld_arr': ['a', 'b', 'c'], 'fld_map': { 'a': '1', 'b': '2', 'c': '3' }, 'fld_rec': { 'fld_id': 1, 'fld_bool': False, 'fld_str': None } } key = {'fld_id': 1} put_request = PutRequest().set_value(row).set_table_name(table_name) get_request = GetRequest().set_key(key).set_table_name(table_name) count = 0 while count < 100: cls._handle.put(put_request) cls._handle.get(get_request) count += 1 # sleep to allow records to accumulate over time, but not if # using Cloudsim. if not_cloudsim(): sleep(2) # need to sleep to allow usage records to accumulate but not if # using CloudSim, which doesn't generate usage records. if not_cloudsim(): sleep(40)
def load_img_objectstorage(img_filepath, user_id, username, is_profile=False, comment=''): """ Load an image to OCI Object Storage """ img_content = open(img_filepath, 'rb').read() original_filename = img_filepath[img_filepath.index('/') + 1:] new_filename = return_new_filename(original_filename) oci_namespace = object_storage.get_namespace().data objs_response = object_storage.put_object(oci_namespace, 'fotogal_bucket_images', new_filename, img_content) # Calcula uma data de criacao. datetime_now_obj = datetime.datetime.now() timedelta_obj = datetime.timedelta(hours=(random.randint(1, 6))) datetime_calc = datetime_now_obj - timedelta_obj img_created_ts = datetime_calc.strftime('%s') nosql_img_data = { 'image_url': '', 'image_filename': new_filename, 'image_original_filename': original_filename, 'image_host_fqdn': '', 'image_uri': '', 'image_type': 'jpeg', 'created_ts': img_created_ts, 'user_id': user_id, 'liked_list': [], 'disliked_list': [], 'main_comment': comment, 'is_profile': is_profile } if objs_response.status == 200: nosql_img_data['image_url'] = objs_response.request.url nosql_img_data[ 'image_host_fqdn'] = return_objectstorage_image_hostname( nosql_img_data['image_url']) nosql_img_data['image_uri'] = return_objectstorage_image_uri( nosql_img_data['image_url']) nosql_request = PutRequest().set_table_name('fotogal_ntable_images') nosql_request.set_value(nosql_img_data) nosql_put_result = nosql_handler.put(nosql_request) return new_filename else: print('Error! Cannot save the image into Object Storage!') sys.exit(1)
def testWriteMultipleAddIllegalRequestAndAbortIfUnsuccessful(self): self.assertRaises(IllegalArgumentException, self.write_multiple_request.add, 'IllegalRequest', True) self.assertRaises(IllegalArgumentException, self.write_multiple_request.add, PutRequest(), 'IllegalAbortIfUnsuccessful') # add two operations with different table name self.write_multiple_request.add(self.requests[0], True) self.assertRaises(IllegalArgumentException, self.write_multiple_request.add, self.illegal_requests[0], False) self.write_multiple_request.clear() # add two operations with different major paths self.write_multiple_request.add(self.requests[0], True).add(self.illegal_requests[1], False) self.assertRaises(IllegalArgumentException, self.handle.write_multiple, self.write_multiple_request) if not is_onprem(): # add operations when the request size exceeded the limit self.write_multiple_request.clear() for op in range(64): row = get_row() row['fld_str'] = self.get_random_str(0.4) self.write_multiple_request.add( PutRequest().set_value(row).set_table_name(table_name), True) self.assertRaises(RequestSizeLimitException, self.handle.write_multiple, self.write_multiple_request) # add operations when sub requests reached the max number self.write_multiple_request.clear() for op in range(51): row = get_row() row['fld_id'] = op self.write_multiple_request.add( PutRequest().set_value(row).set_table_name(table_name), True) self.assertRaises(BatchOperationNumberLimitException, self.handle.write_multiple, self.write_multiple_request)
def setUp(self): self.set_up() self.row = get_row(with_sid=False) self.key = {'fld_id': 1} self.put_request = PutRequest().set_value( self.row).set_table_name(table_name).set_timeout(timeout) self.get_request = GetRequest().set_key( self.key).set_table_name(table_name) self.ttl = TimeToLive.of_hours(24) self.hour_in_milliseconds = 60 * 60 * 1000 self.day_in_milliseconds = 24 * 60 * 60 * 1000
def setUp(self): self.set_up() self.row = get_row(with_sid=False) self.key = {'fld_id': 1} self.put_request = PutRequest().set_value( self.row).set_table_name(table_name) self.version = self.handle.put(self.put_request).get_version() self.get_request = GetRequest().set_key( self.key).set_table_name(table_name) self.delete_request = DeleteRequest().set_key( self.key).set_table_name(table_name).set_timeout(timeout)
def setUpClass(cls): TestBase.set_up_class() table_ttl = TimeToLive.of_hours(16) create_statement = ('CREATE TABLE ' + table_name + '(fld_sid INTEGER, fld_id INTEGER, \ fld_long LONG, fld_float FLOAT, fld_double DOUBLE, fld_bool BOOLEAN, \ fld_str STRING, fld_bin BINARY, fld_time TIMESTAMP(7), fld_num NUMBER, \ fld_json JSON, fld_arr ARRAY(STRING), fld_map MAP(STRING), \ fld_rec RECORD(fld_id LONG, fld_bool BOOLEAN, fld_str STRING), \ PRIMARY KEY(SHARD(fld_sid), fld_id)) USING TTL ' + str(table_ttl)) create_request = TableRequest().set_statement( create_statement).set_table_limits(TableLimits(5000, 5000, 50)) cls._result = TestBase.table_request(create_request, State.ACTIVE) global row, tb_expect_expiration, hour_in_milliseconds row = { 'fld_sid': 1, 'fld_id': 1, 'fld_long': 2147483648, 'fld_float': 3.1414999961853027, 'fld_double': 3.1415, 'fld_bool': True, 'fld_str': '{"name": u1, "phone": null}', 'fld_bin': bytearray(pack('>i', 4)), 'fld_time': datetime.now(), 'fld_num': Decimal(5), 'fld_json': { 'a': '1', 'b': None, 'c': '3' }, 'fld_arr': ['a', 'b', 'c'], 'fld_map': { 'a': '1', 'b': '2', 'c': '3' }, 'fld_rec': { 'fld_id': 1, 'fld_bool': False, 'fld_str': None } } put_request = PutRequest().set_value(row).set_table_name(table_name) cls._handle.put(put_request) tb_expect_expiration = table_ttl.to_expiration_time( int(round(time() * 1000))) hour_in_milliseconds = 60 * 60 * 1000
def setUp(self): self.set_up() self.handle_config = get_handle_config(tenant_id) self.min_time = list() self.max_time = list() shardkeys = 2 ids = 6 write_multiple_request = WriteMultipleRequest() for sk in range(shardkeys): for i in range(ids): row = get_row() if i == 0: self.min_time.append(row['fld_time']) elif i == ids - 1: self.max_time.append(row['fld_time']) row['fld_sid'] = sk row['fld_id'] = i row['fld_bool'] = False if sk == 0 else True row['fld_str'] = ( '{"name": u' + str(shardkeys * ids - sk * ids - i - 1).zfill(2) + '}') row['fld_json']['location']['coordinates'] = ([ 23.549 - sk * 0.5 - i, 35.2908 + sk * 0.5 + i ]) write_multiple_request.add( PutRequest().set_value(row).set_table_name(table_name), True) self.handle.write_multiple(write_multiple_request) write_multiple_request.clear() prepare_statement_update = ( 'DECLARE $fld_sid INTEGER; $fld_id INTEGER; UPDATE ' + table_name + ' u SET u.fld_long = u.fld_long + 1 WHERE fld_sid = $fld_sid ' + 'AND fld_id = $fld_id') prepare_request_update = PrepareRequest().set_statement( prepare_statement_update) self.prepare_result_update = self.handle.prepare( prepare_request_update) prepare_statement_select = ( 'DECLARE $fld_long LONG; SELECT fld_sid, fld_id, fld_long FROM ' + table_name + ' WHERE fld_long = $fld_long') prepare_request_select = PrepareRequest().set_statement( prepare_statement_select) self.prepare_result_select = self.handle.prepare( prepare_request_select) self.query_request = QueryRequest().set_timeout(timeout) self.get_request = GetRequest().set_table_name(table_name)
def setUp(self): TestBase.set_up(self) self.row = {'fld_id': 1, 'fld_long': 2147483648, 'fld_float': 3.1414999961853027, 'fld_double': 3.1415, 'fld_bool': True, 'fld_str': '{"name": u1, "phone": null}', 'fld_bin': bytearray(pack('>i', 4)), 'fld_time': datetime.now(), 'fld_num': Decimal(5), 'fld_json': {'a': '1', 'b': None, 'c': '3'}, 'fld_arr': ['a', 'b', 'c'], 'fld_map': {'a': '1', 'b': '2', 'c': '3'}, 'fld_rec': {'fld_id': 1, 'fld_bool': False, 'fld_str': None}} self.key = {'fld_id': 1} self.put_request = PutRequest().set_value(self.row).set_table_name( table_name) self.version = self.handle.put(self.put_request).get_version() self.get_request = GetRequest().set_key(self.key).set_table_name( table_name) self.delete_request = DeleteRequest().set_key(self.key).set_table_name( table_name).set_timeout(timeout)
def setUpClass(cls): cls.handle = None cls.set_up_class() table_ttl = TimeToLive.of_hours(16) create_statement = ('CREATE TABLE ' + table_name + '(fld_sid INTEGER, fld_id INTEGER, \ fld_long LONG, fld_float FLOAT, fld_double DOUBLE, fld_bool BOOLEAN, \ fld_str STRING, fld_bin BINARY, fld_time TIMESTAMP(7), fld_num NUMBER, \ fld_json JSON, fld_arr ARRAY(STRING), fld_map MAP(STRING), \ fld_rec RECORD(fld_id LONG, fld_bool BOOLEAN, fld_str STRING), \ PRIMARY KEY(SHARD(fld_sid), fld_id)) USING TTL ' + str(table_ttl)) create_request = TableRequest().set_statement( create_statement).set_table_limits(TableLimits(100, 100, 1)) cls.table_request(create_request) global row, tb_expect_expiration, version row = get_row() put_request = PutRequest().set_value(row).set_table_name(table_name) version = cls.handle.put(put_request).get_version() tb_expect_expiration = table_ttl.to_expiration_time( int(round(time() * 1000)))
def add(self, table=None, data_dict=None): """Insert new single row. """ put_request = PutRequest() put_request.set_table_name(table) put_request.set_option(PutOption.IF_ABSENT) put_request.set_value(data_dict) nosql_result = self._nosql_handle.put(put_request) # a successful put returns a non-empty version if nosql_result.get_version() is not None: generated_value = nosql_result.get_generated_value() if generated_value: return generated_value else: return True else: return False
def setUp(self): self.set_up() self.shardkeys = [0, 1] ids = [0, 1, 2, 3, 4, 5] write_multiple_request = WriteMultipleRequest() for sk in self.shardkeys: for i in ids: row = get_row() row['fld_sid'] = sk row['fld_id'] = i write_multiple_request.add( PutRequest().set_value(row).set_table_name(table_name), True) self.handle.write_multiple(write_multiple_request) write_multiple_request.clear() self.key = {'fld_sid': 1} self.multi_delete_request = MultiDeleteRequest().set_timeout(timeout) prep_request = PrepareRequest().set_statement( 'SELECT fld_sid, fld_id FROM ' + table_name) prep_result = self.handle.prepare(prep_request) self.query_request = QueryRequest().set_prepared_statement(prep_result)
def update(self, table=None, data_dict=None): """Update existing single row. """ put_request = PutRequest() put_request.set_table_name(table) put_request.set_option(PutOption.IF_PRESENT) put_request.set_value(data_dict) nosql_result = self._nosql_handle.put(put_request) # a successful put returns a non-empty version if nosql_result.get_version() is not None: existing_value = nosql_result.get_existing_value() if existing_value: return existing_value else: return True else: return False
def setUp(self): TestBase.set_up(self) self.row = { 'fld_id': 1, 'fld_long': 2147483648, 'fld_float': 3.1414999961853027, 'fld_double': 3.1415, 'fld_bool': True, 'fld_str': '{"name": u1, "phone": null}', 'fld_bin': bytearray(pack('>i', 4)), 'fld_time': datetime.now(), 'fld_num': Decimal(5), 'fld_json': { 'a': '1', 'b': None, 'c': '3' }, 'fld_arr': ['a', 'b', 'c'], 'fld_map': { 'a': '1', 'b': '2', 'c': '3' }, 'fld_rec': { 'fld_id': 1, 'fld_bool': False, 'fld_str': None } } self.key = {'fld_id': 1} self.put_request = PutRequest().set_value( self.row).set_table_name(table_name).set_timeout(timeout) self.get_request = GetRequest().set_key( self.key).set_table_name(table_name) self.ttl = TimeToLive.of_hours(24) self.hour_in_milliseconds = 60 * 60 * 1000 self.day_in_milliseconds = 24 * 60 * 60 * 1000
def main(): handle = None try: # # Create a handle # handle = get_handle(tenant_id) # # Create a table # statement = 'Create table if not exists ' + table_name + '(id integer, \ sid integer, name string, primary key(shard(sid), id))' print('Creating table: ' + statement) request = TableRequest().set_statement(statement).set_table_limits( TableLimits(30, 10, 1)) handle.do_table_request(request, 50000, 3000) print('After create table') # # Put a few rows # request = PutRequest().set_table_name(table_name) for i in range(10): value = {'id': i, 'sid': 0, 'name': 'myname' + str(i)} request.set_value(value) handle.put(request) print('After put of 10 rows') # # Multiple write a few rows # request = WriteMultipleRequest() for i in range(10): value = {'id': i, 'sid': 0, 'name': 'newname' + str(i)} request.add( PutRequest().set_value(value).set_table_name(table_name), True) result = handle.write_multiple(request) print('After multiple write: ' + str(result)) # # Prepare a statement # statement = 'select * from ' + table_name + ' where id > 2 and id < 8' request = PrepareRequest().set_statement(statement) prepared_result = handle.prepare(request) print('After prepare the statement: ' + statement) # # Query, using the prepared statement # request = QueryRequest().set_prepared_statement(prepared_result) print('Query results for the prepared statement: ') while True: result = handle.query(request) for r in result.get_results(): print('\t' + str(r)) if request.is_done(): break # # Multiple delete the rows # request = MultiDeleteRequest().set_table_name(table_name).set_key( {'sid': 0}) result = handle.multi_delete(request) print('After multiple delete: ' + str(result)) # # Query again to show deletions, using the prepared statement # request = QueryRequest().set_prepared_statement(prepared_result) print('Query results for the prepared statement (should be no rows): ') while True: result = handle.query(request) for r in result.get_results(): print('\t' + str(r)) if request.is_done(): break # # Drop the table # if drop_table: request = TableRequest().set_statement('drop table if exists ' + table_name) handle.do_table_request(request, 40000, 2000) print('After drop table') else: print('Not dropping table') print('Example is complete') except Exception as e: print(e) traceback.print_exc() finally: # If the handle isn't closed Python will not exit properly if handle is not None: handle.close()
def setUp(self): TestBase.set_up(self) self.shardkeys = [0, 1] self.ids = [0, 1, 2, 3, 4, 5] self.rows = list() self.new_rows = list() self.versions = list() self.requests = list() self.illegal_requests = list() ttl = TimeToLive.of_days(16) for sk in self.shardkeys: self.rows.append(list()) self.new_rows.append(list()) self.versions.append(list()) for i in self.ids: row = {'fld_sid': sk, 'fld_id': i, 'fld_long': 2147483648, 'fld_float': 3.1414999961853027, 'fld_double': 3.1415, 'fld_bool': True, 'fld_str': '{"name": u1, "phone": null}', 'fld_bin': bytearray(pack('>i', 4)), 'fld_time': datetime.now(), 'fld_num': Decimal(5), 'fld_json': {'a': '1', 'b': None, 'c': '3'}, 'fld_arr': ['a', 'b', 'c'], 'fld_map': {'a': '1', 'b': '2', 'c': '3'}, 'fld_rec': {'fld_id': 1, 'fld_bool': False, 'fld_str': None}} new_row = deepcopy(row) new_row.update({'fld_long': 2147483649}) self.rows[sk].append(row) self.new_rows[sk].append(new_row) put_request = PutRequest().set_value(row).set_table_name( table_name).set_ttl(ttl) self.versions[sk].append( self.handle.put(put_request).get_version()) self.old_expect_expiration = ttl.to_expiration_time( int(round(time() * 1000))) self.ttl = TimeToLive.of_hours(1) self.ops_sk = 0 illegal_sk = 1 self.requests.append(PutRequest().set_value( self.new_rows[self.ops_sk][0]).set_table_name(table_name).set_ttl( self.ttl).set_return_row(True)) self.requests.append(PutRequest().set_value( self.new_rows[self.ops_sk][1]).set_table_name( table_name).set_option(PutOption.IF_ABSENT).set_ttl( self.ttl).set_return_row(True)) self.requests.append(PutRequest().set_value( self.new_rows[self.ops_sk][2]).set_use_table_default_ttl( True).set_table_name(table_name).set_option( PutOption.IF_PRESENT).set_return_row(True)) self.requests.append(PutRequest().set_value( self.new_rows[self.ops_sk][3]).set_table_name( table_name).set_option(PutOption.IF_VERSION).set_ttl( self.ttl).set_match_version( self.versions[self.ops_sk][3]).set_return_row(True)) self.requests.append(DeleteRequest().set_key( {'fld_sid': self.ops_sk, 'fld_id': 4}).set_table_name( table_name).set_return_row(True)) self.requests.append(DeleteRequest().set_key( {'fld_sid': self.ops_sk, 'fld_id': 5}).set_table_name( table_name).set_match_version( self.versions[self.ops_sk][0]).set_return_row(True)) self.illegal_requests.append(DeleteRequest().set_key( {'fld_sid': self.ops_sk, 'fld_id': 0}).set_table_name( 'IllegalUsers')) self.illegal_requests.append(DeleteRequest().set_key( {'fld_sid': illegal_sk, 'fld_id': 0}).set_table_name(table_name)) self.write_multiple_request = WriteMultipleRequest().set_timeout( timeout) self.get_request = GetRequest().set_table_name(table_name) self.hour_in_milliseconds = 60 * 60 * 1000 self.day_in_milliseconds = 24 * 60 * 60 * 1000
def main(): handle = None try: # # Create a handle # handle = get_handle(tenant_id) # # Create a table # statement = 'Create table if not exists ' + table_name + '(id integer, \ sid integer, name string, primary key(shard(sid), id))' print('Creating table: ' + statement) request = TableRequest().set_statement(statement).set_table_limits( TableLimits(30, 10, 1)) result = handle.table_request(request) # # Table creation can take time, depending on the state of the system. # If if fails after 40s, re-run the program # result.wait_for_state(handle, table_name, State.ACTIVE, 50000, 3000) print('After create table') # # Put a few rows # request = PutRequest().set_table_name(table_name) for i in range(10): value = {'id': i, 'sid': 0, 'name': 'myname' + str(i)} request.set_value(value) handle.put(request) print('After put of 10 rows') # # Get the row # request = GetRequest().set_key({ 'id': 1, 'sid': 0 }).set_table_name(table_name) result = handle.get(request) print('After get: ' + str(result)) # # Query, using a range # statement = 'select * from ' + table_name + ' where id > 2 and id < 8' request = QueryRequest().set_statement(statement) result = handle.query(request) print('Query results for: ' + statement) for r in result.get_results(): print('\t' + str(r)) # # Delete the row # request = DeleteRequest().set_key({ 'id': 1, 'sid': 0 }).set_table_name(table_name) result = handle.delete(request) print('After delete: ' + str(result)) # # Get again to show deletion # request = GetRequest().set_key({ 'id': 1, 'sid': 0 }).set_table_name(table_name) result = handle.get(request) print('After get (should be None): ' + str(result)) # # Drop the table # if drop_table: request = TableRequest().set_statement('drop table if exists ' + table_name) result = handle.table_request(request) # # Table drop can take time, depending on the state of the system. # If this wait fails the table will still probably been dropped # result.wait_for_state(handle, table_name, State.DROPPED, 40000, 2000) print('After drop table') else: print('Not dropping table') print('Example is complete') except Exception as e: print(e) traceback.print_exc() finally: # If the handle isn't closed Python will not exit properly if handle is not None: handle.close()
def _do_rate_limited_ops(self, num_seconds, read_limit, write_limit, max_rows, check_units, use_percent, use_external_limiters): """ Runs puts and gets continuously for N seconds. Verify that the resultant RUs/WUs used match the given rate limits. """ if read_limit == 0 and write_limit == 0: return put_request = PutRequest().set_table_name(table_name) get_request = GetRequest().set_table_name(table_name) key = dict() # TODO: random sizes 0-nKB. value = dict() value['name'] = 'jane' start_time = int(round(time() * 1000)) end_time = start_time + num_seconds * 1000 read_units_used = 0 write_units_used = 0 total_delayed_ms = 0 throttle_exceptions = 0 rlim = None wlim = None max_val = float(read_limit + write_limit) if not use_external_limiters: # Reset internal limiters so they don't have unused units. self.handle.get_client().reset_rate_limiters(table_name) else: rlim = SimpleRateLimiter(read_limit * use_percent / 100.0, 1) wlim = SimpleRateLimiter(write_limit * use_percent / 100.0, 1) while True: fld_id = int(random() * max_rows) if read_limit == 0: do_put = True elif write_limit == 0: do_put = False else: v = int(random() * max_val) do_put = v >= read_limit try: if do_put: value['id'] = fld_id put_request.set_value(value).set_read_rate_limiter( None).set_write_rate_limiter(wlim) pres = self.handle.put(put_request) write_units_used += pres.get_write_units() total_delayed_ms += pres.get_rate_limit_delayed_ms() rs = pres.get_retry_stats() if rs is not None: throttle_exceptions += rs.get_num_exceptions( WriteThrottlingException.__class__.__name__) else: key['id'] = fld_id get_request.set_key(key).set_read_rate_limiter( rlim).set_write_rate_limiter(None) gres = self.handle.get(get_request) read_units_used += gres.get_read_units() total_delayed_ms += gres.get_rate_limit_delayed_ms() rs = gres.get_retry_stats() if rs is not None: throttle_exceptions += rs.get_num_exceptions( ReadThrottlingException.__class__.__name__) except ReadThrottlingException: self.fail( 'Expected no read throttling exceptions, got one.') except WriteThrottlingException: self.fail( 'Expected no write throttling exceptions, got one.') if int(round(time() * 1000)) >= end_time: break num_seconds = (int(round(time() * 1000)) - start_time) / 1000 rus = read_units_used / num_seconds wus = write_units_used / num_seconds if not check_units: return use_percent /= 100.0 if (rus < read_limit * use_percent * 0.8 or rus > read_limit * use_percent * 1.2): self.fail('Gets: Expected around ' + str(read_limit * use_percent) + ' RUs, got ' + str(rus)) if (wus < write_limit * use_percent * 0.8 or wus > write_limit * use_percent * 1.2): self.fail('Puts: Expected around ' + str(write_limit * use_percent) + ' WUs, got ' + str(wus))
from borneo import PutRequest, PutOption # create and close AuthorizationProvider at_provider = SignatureProvider(config_file='~/.oci/config') at_provider.close() # create handle config using a desired region as endpoint and set a # default compartment. handle_config = NoSQLHandleConfig(Regions.SA_SAOPAULO_1) handle_config.set_authorization_provider(at_provider) handle_config.set_default_compartment('<your-compartment-id>') # create the handle. nosql_handle = NoSQLHandle(handle_config) put_request = PutRequest() put_request.set_table_name('produtos') put_request.set_option(PutOption.IF_ABSENT) put_request.set_value({ "id": 1, "propriedades": { "nome": "Óleo Motul 5100 4T (Semi-sintético)", "Tipo": "15W50" }, "valor": 67.90, "frete_gratis": False, "imagens": ["Motul-1.jpg"] }) # write data result = nosql_handle.put(put_request)
def setUp(self): self.set_up() self.shardkeys = [0, 1] self.ids = [0, 1, 2, 3, 4, 5] self.rows = list() self.new_rows = list() self.versions = list() self.requests = list() self.illegal_requests = list() ttl = TimeToLive.of_days(16) for sk in self.shardkeys: self.rows.append(list()) self.new_rows.append(list()) self.versions.append(list()) for i in self.ids: row = get_row() row['fld_sid'] = sk row['fld_id'] = i new_row = deepcopy(row) new_row['fld_long'] = 2147483649 self.rows[sk].append(row) self.new_rows[sk].append(new_row) put_request = PutRequest().set_value(row).set_table_name( table_name).set_ttl(ttl) self.versions[sk].append( self.handle.put(put_request).get_version()) self.old_expect_expiration = ttl.to_expiration_time( int(round(time() * 1000))) self.ttl = TimeToLive.of_hours(1) self.ops_sk = 0 illegal_sk = 1 self.requests.append(PutRequest().set_value( self.new_rows[self.ops_sk][0]).set_table_name(table_name).set_ttl( self.ttl).set_return_row(True)) self.requests.append(PutRequest().set_value(self.new_rows[ self.ops_sk][1]).set_table_name(table_name).set_option( PutOption.IF_ABSENT).set_ttl(self.ttl).set_return_row(True)) self.requests.append(PutRequest().set_value( self.new_rows[self.ops_sk][2]).set_use_table_default_ttl( True).set_table_name(table_name).set_option( PutOption.IF_PRESENT).set_return_row(True)) self.requests.append(PutRequest().set_value( self.new_rows[self.ops_sk][3]).set_table_name(table_name).set_ttl( self.ttl).set_option(PutOption.IF_VERSION).set_match_version( self.versions[self.ops_sk][3]).set_return_row(True)) self.requests.append(DeleteRequest().set_key({ 'fld_sid': self.ops_sk, 'fld_id': 4 }).set_table_name(table_name).set_return_row(True)) self.requests.append(DeleteRequest().set_key({ 'fld_sid': self.ops_sk, 'fld_id': 5 }).set_table_name(table_name).set_return_row(True).set_match_version( self.versions[self.ops_sk][0])) self.illegal_requests.append(DeleteRequest().set_key({ 'fld_sid': self.ops_sk, 'fld_id': 0 }).set_table_name('IllegalUsers')) self.illegal_requests.append(DeleteRequest().set_key({ 'fld_sid': illegal_sk, 'fld_id': 0 }).set_table_name(table_name)) self.write_multiple_request = WriteMultipleRequest().set_timeout( timeout) self.get_request = GetRequest().set_table_name(table_name) self.hour_in_milliseconds = 60 * 60 * 1000 self.day_in_milliseconds = 24 * 60 * 60 * 1000