def test_background_execute_predexp_and_predicate(self): """ Ensure that Scan.execute_background() gets applied to records that match the predicate NOTE: the predicate overrides the predexp """ test_bin = 'Stpredold' keys = [(TEST_NS, TEST_SET, i) for i in range(50)] predexp = [ as_predexp.integer_bin('number'), as_predexp.integer_value(2), as_predexp.integer_equal(), as_predexp.integer_bin('number'), as_predexp.integer_value(3), as_predexp.integer_equal(), as_predexp.predexp_or(2) ] number_predicate = predicates.equals('number', 4) policy = {'predexp': predexp} scan = self.as_connection.scan(TEST_NS, TEST_SET) scan.where(number_predicate) scan.apply(TEST_UDF_MODULE, TEST_UDF_FUNCTION, [test_bin]) job_id = scan.execute_background(policy) # Give time for the scan to finish wait_for_job_completion(self.as_connection, job_id) for key in keys: _, _, bins = self.as_connection.get(key) if bins['number'] == 4: assert (bins[test_bin] == 'aerospike') else: assert (bins.get(test_bin) is None)
def _commit_release(self, txn_id, new_head_offset=None, new_tail_offset=None): """ If the lock is still held by this requester (txn-id), update the new positions of head/tail and r elease the lock. Otherwise abort the request as timed out. :param txn_id: lock owner id, must be unique among concurrent requests :param new_head_offset: new head offset to be updated :param new_tail_offset: new tail offset to be updated :return: throws ASAborted('Timed out') """ metadata_key = (self.namespace, self.name, LargeQueue.META_REC_KEY) predexps = [ predexp.integer_bin("locked"), predexp.integer_value(1), predexp.integer_equal(), predexp.integer_bin("lock-owner"), predexp.integer_value(txn_id), predexp.integer_equal(), predexp.predexp_and(2)] ops = [ op_helpers.write('locked', 0), op_helpers.write('lock-owner', None), op_helpers.write('lock-time-ms', None) ] if new_head_offset is not None: ops.append(op_helpers.write('head-offset', new_head_offset)) if new_tail_offset is not None: ops.append(op_helpers.write('tail-offset', new_tail_offset)) try: _ = self.client.operate(metadata_key, ops, policy={'predexp': predexps}) except exception.FilteredOut as ex: # predexp failed raise ASAborted('Timed out') return
def test_background_execute_predexp_everywhere(self): """ Ensure that Scan.execute_background() gets applied to records that match the predexp """ test_bin = 'number' keys = [(TEST_NS, TEST_SET, i) for i in range(50)] predexp = [ as_predexp.integer_bin('number'), as_predexp.integer_value(2), as_predexp.integer_equal(), as_predexp.integer_bin('number'), as_predexp.integer_value(3), as_predexp.integer_equal(), as_predexp.predexp_or(2) ] #number_predicate = predicates.equals('number', 3) policy = {'predexp': predexp} scan = self.as_connection.scan(TEST_NS, TEST_SET) #scan.where(number_predicate) scan.apply(TEST_UDF_MODULE, TEST_UDF_FUNCTION, [test_bin, 1]) job_id = scan.execute_background(policy) # Give time for the scan to finish wait_for_job_completion(self.as_connection, job_id) for i, key in enumerate(keys): _, _, bins = self.as_connection.get(key) if i == 2 or i == 3: assert (bins[test_bin] == i + 1) else: assert (bins.get(test_bin) == i)
def test_background_execute_with_ops_and_predexp(self): """ Ensure that Scan.execute_background() applies ops to records that match the predexp """ test_bin = 'Stops_preds' keys = [(TEST_NS, TEST_SET, i) for i in range(50)] scan = self.as_connection.scan(TEST_NS, TEST_SET) # scan.apply(TEST_UDF_MODULE, TEST_UDF_FUNCTION, [test_bin]) ops = [operations.write(test_bin, 'new_val')] predexp = [ as_predexp.integer_bin('number'), as_predexp.integer_value(2), as_predexp.integer_equal(), as_predexp.integer_bin('number'), as_predexp.integer_value(3), as_predexp.integer_equal(), as_predexp.predexp_or(2) ] policy = {'predexp': predexp} scan.add_ops(ops) job_id = scan.execute_background(policy) # Give time for the scan to finish wait_for_job_completion(self.as_connection, job_id) for key in keys: _, _, bins = self.as_connection.get(key) if bins['number'] == 2 or bins['number'] == 3: assert (bins[test_bin] == 'new_val') else: assert (bins.get(test_bin) is None)
def test_rec_void_time(self): ''' This could fail due to clock skew ''' for i in range(7): key = 'test', 'ttl', i self.as_connection.put(key, {'time': 'earlier'}, meta={'ttl': 100}) # 150 second range for record TTLs should be enough, we are storing with # Current time + 100s and current time +5000s, so only one of the group should be found void_time_range_start = seconds_to_nanos(int(time.time() + 50)) void_time_range_end = seconds_to_nanos(int(time.time() + 150)) # Store 5 records after the cutoff for i in range(7, 12): key = 'test', 'ttl', i self.as_connection.put(key, {'time': 'later'}, meta={'ttl': 1000}) query = self.as_connection.query('test', 'ttl') predexps = [ predexp.rec_void_time(), predexp.integer_value(void_time_range_start), predexp.integer_greater(), predexp.rec_void_time(), predexp.integer_value(void_time_range_end), predexp.integer_less(), predexp.predexp_and(2) ] query.predexp(predexps) results = query.results() assert len(results) == 7 assert_each_record_bins(results, lambda b: b['time'] == 'earlier')
def test_exists_many_with_large_predexp(self): ''' Proper call to exists_many with predexp in policy. ''' predexp = [ as_predexp.integer_bin('account_id'), as_predexp.integer_value(4), as_predexp.integer_equal(), as_predexp.string_bin('user_name'), as_predexp.string_value('user3'), as_predexp.string_equal(), as_predexp.integer_var('list_val'), as_predexp.integer_value(12), as_predexp.integer_less(), as_predexp.list_bin('charges'), as_predexp.list_iterate_and('list_val'), as_predexp.predexp_or(3) ] matched_recs = [] records = self.as_connection.exists_many(self.keys, {'predexp': predexp}) for rec in records: if rec[1] is not None: matched_recs.append(rec[1]) assert len(matched_recs) == 3
def test_query_apply_with_new_predexp(self): """ Invoke query_apply() with correct policy and predexp """ predexp = [ as_predexp.integer_bin('age'), as_predexp.integer_value(2), as_predexp.integer_equal(), as_predexp.integer_bin('val'), as_predexp.integer_value(3), as_predexp.integer_equal(), as_predexp.predexp_or(2) ] policy = {'total_timeout': 0, 'predexp': predexp} query_id = self.as_connection.query_apply( "test", "demo", self.age_range_pred, "query_apply", "mark_as_applied", ['name', 2], policy) self._wait_for_query_complete(query_id) recs = [] for i in range(1, 10): key = ('test', 'demo', i) _, _, bins = self.as_connection.get(key) if bins['name'] == 'aerospike': recs.append(bins) assert len(recs) == 2 for rec in recs: assert rec['age'] == 2 or rec['val'] == 3
def test_or(self): predexps = [ predexp.integer_bin('positive_i'), predexp.integer_value(5), predexp.integer_equal(), predexp.integer_bin('positive_i'), predexp.integer_value(10), predexp.integer_equal(), predexp.predexp_or(2) ] self.query.predexp(predexps) results = self.query.results() assert len(results) == 2 assert_each_record_bins(results, lambda b: b['positive_i'] in (5, 10))
def test_and(self): predexps = [ predexp.integer_bin('positive_i'), predexp.integer_value(10), predexp.integer_greater(), predexp.integer_bin('positive_i'), predexp.integer_value(20), predexp.integer_less(), predexp.predexp_and(2) ] self.query.predexp(predexps) results = self.query.results() assert len(results) == 9 assert_each_record_bins( results, lambda b: b['positive_i'] > 10 and b['positive_i'] < 20)
def test_background_execute_sindex_predexp(self, clean_test_background): """ Ensure that Query.execute_background() only applies to records matched by the specified predicate """ test_bin = 't4' keys = [(TEST_NS, TEST_SET, i) for i in range(500)] # rec['number'] < 10 predexps = [ predexp.integer_bin('number'), predexp.integer_value(10), predexp.integer_less() ] query = self.as_connection.query(TEST_NS, TEST_SET) query.predexp(predexps) query.apply(TEST_UDF_MODULE, TEST_UDF_FUNCTION, [test_bin]) query.execute_background() # Give time for the query to finish time.sleep(5) # Records with number > 10 should not have had the UDF applied validate_records(self.as_connection, keys[10:], lambda rec: test_bin not in rec) # Records with number < 10 should have had the udf applied validate_records(self.as_connection, keys[:10], lambda rec: rec[test_bin] == 'aerospike')
def test_scan_apply_with_none_set_and_predexp(self): """ Invoke scan_apply() with set argument as None It should invoke the function on all records in NS that match the predexp """ predexp = [ as_predexp.string_bin('name'), as_predexp.string_value('name2'), as_predexp.string_equal(), as_predexp.integer_bin('age'), as_predexp.integer_value(3), as_predexp.integer_unequal(), as_predexp.predexp_and(2) ] policy = {'timeout': 1000, 'predexp': predexp} scan_id = self.as_connection.scan_apply("test", None, "bin_lua", "mytransform", ['age', 2], policy) wait_for_job_completion(self.as_connection, scan_id) for i in range(5): key = ('test', 'demo', i) _, _, bins = self.as_connection.get(key) if bins['name'] == 'name2': assert bins['age'] == i + 2 else: assert bins['age'] == i _, _, rec = self.as_connection.get(('test', None, 'no_set')) assert rec['age'] == 10
def test_or(self): predexps = [ predexp.integer_bin('positive_i'), predexp.integer_value(5), predexp.integer_equal(), predexp.integer_bin('positive_i'), predexp.integer_value(10), predexp.integer_equal(), predexp.predexp_or(2) ] self.query.predexp(predexps) results = self.query.results() assert len(results) == 2 assert_each_record_bins( results, lambda b: b['positive_i'] in (5, 10))
def test_rec_last_update(self): ''' This could fail due to clock skew ''' for i in range(7): key = 'test', 'lut', i self.as_connection.put(key, {'time': 'earlier'}) cutoff_nanos = seconds_to_nanos(int(time.time() + 2)) time.sleep(5) # Make sure that we wait long enough # Store 5 records after the cutoff for i in range(7, 12): key = 'test', 'lut', i self.as_connection.put(key, {'time': 'later'}) query = self.as_connection.query('test', 'lut') predexps = [ predexp.rec_last_update(), predexp.integer_value(cutoff_nanos), predexp.integer_less() ] query.predexp(predexps) results = query.results() assert len(results) == 7 assert_each_record_bins( results, lambda b: b['time'] == 'earlier')
def test_predexp_key_operate_record_digest_modulo(self): """ Invoke the C client aerospike_key_operate with a rec_digest_modulo predexp. """ less_than_128 = 0 for i in range(100): key = 'test', 'demo', i if aerospike.calc_digest(*key)[-1] < 128: less_than_128 += 1 self.as_connection.put(key, {'dig_id': i}) results = [] predexp = [ as_predexp.rec_digest_modulo(256), as_predexp.integer_value(128), as_predexp.integer_less() ] ops = [operations.read('dig_id')] for i in range(100): try: key = 'test', 'demo', i _, _, res = self.as_connection.operate( key, ops, policy={'predexp': predexp}) results.append(res) except: pass self.as_connection.remove(key) assert len(results) == less_than_128
def test_rec_device_size(self): long_str_len = 65 * 1024 long_str = long_str_len * 'a' # A 65K string # Store 5 records with a string of 65K for i in range(5): key = 'test', 'dev_size', i self.as_connection.put(key, {'string': long_str}) # Store 3 records with a size much less than 65K for i in range(5, 8): key = 'test', 'dev_size', i self.as_connection.put(key, {'string': "short"}) query = self.as_connection.query('test', 'dev_size') predexps = [ predexp.rec_device_size(), predexp.integer_value(64 * 1024), predexp.integer_greater() ] query.predexp(predexps) results = query.results() assert len(results) == 5 assert_each_record_bins( results, lambda b: len(b['string']) == long_str_len) # This is faster than doing a string compare
def test_predexp_key_operate_record_void_time(self): """ Invoke the C client aerospike_key_operate with a rec_void_time predexp. """ for i in range(5): key = 'test', 'pred_ttl', i self.as_connection.put(key, {'time': 'earlier'}, meta={'ttl': 100}) # 150 second range for record TTLs should be enough, we are storing with # Current time + 100s and current time +5000s, so only one of the group should be found void_time_range_start = (10**9) * int(time.time() + 50) void_time_range_end = (10**9) * int(time.time() + 150) for i in range(5, 10): key = 'test', 'pred_ttl', i self.as_connection.put(key, {'time': 'later'}, meta={'ttl': 1000}) results = [] predexp = [ as_predexp.rec_void_time(), as_predexp.integer_value(void_time_range_start), as_predexp.integer_greater(), as_predexp.rec_void_time(), as_predexp.integer_value(void_time_range_end), as_predexp.integer_less(), as_predexp.predexp_and(2) ] ops = [operations.read('time')] for i in range(10): try: key = 'test', 'pred_ttl', i _, _, res = self.as_connection.operate( key, ops, policy={'predexp': predexp}) results.append(res) except: pass self.as_connection.remove(key) assert len(results) == 5 for res in results: assert res['time'] == 'earlier'
class TestApply(TestBaseClass): def setup_class(cls): # Register setup and teardown functions cls.connection_setup_functions = [add_indexes_and_udfs] cls.connection_teardown_functions = [remove_indexes_and_udfs] @pytest.fixture(autouse=True) def setup(self, request, connection_with_config_funcs): as_connection = connection_with_config_funcs for i in range(5): key = ('test', 'demo', i) rec = { 'name': ['name%s' % (str(i))], 'addr': 'name%s' % (str(i)), 'age': i, 'no': i, 'basic_map': { "k30": 6, "k20": 5, "k10": 1 } } as_connection.put(key, rec) def teardown(): for i in range(5): key = ('test', 'demo', i) as_connection.remove(key) request.addfinalizer(teardown) @pytest.mark.parametrize("func_args, test_bin, predexp, expected", ((['name', 1], 'name', [ as_predexp.integer_bin('age'), as_predexp.integer_value(1), as_predexp.integer_equal() ], ['name1', 1]), ), ids=[ "Integer", ]) def test_apply_causing_list_append_with_correct_params_with_predexp( self, func_args, test_bin, predexp, expected): key = ('test', 'demo', 1) retval = self.as_connection.apply(key, 'sample', 'list_append', func_args, policy={'predexp': predexp}) _, _, bins = self.as_connection.get(key) print(expected) print(bins[test_bin]) assert bins[test_bin] == expected assert retval == 0 # the list_append UDF returns 0
def test_integer_equals(self): predexps = [ predexp.integer_bin('positive_i'), predexp.integer_value(5), predexp.integer_equal() ] self.query.predexp(predexps) results = self.query.results() assert len(results) == 1 assert_each_record_bins(results, lambda b: b['positive_i'] == 5)
def test_integer_greatereq(self): predexps = [ predexp.integer_bin('positive_i'), predexp.integer_value(49), predexp.integer_greatereq() ] self.query.predexp(predexps) results = self.query.results() assert len(results) == 51 assert_each_record_bins(results, lambda b: b['positive_i'] >= 49)
def test_integer_lesseq(self): predexps = [ predexp.integer_bin('positive_i'), predexp.integer_value(10), predexp.integer_lesseq() ] self.query.predexp(predexps) results = self.query.results() assert len(results) == 11 assert_each_record_bins(results, lambda b: b['positive_i'] <= 10)
def test_get_with_predexp_filtered_out(self): ''' Call to get with predexp in policy with expected failures. ''' predexp = [ as_predexp.integer_bin('account_id'), as_predexp.integer_value(3), as_predexp.integer_equal() ] with pytest.raises(e.FilteredOut): self.as_connection.get(self.keys[0], {'predexp': predexp})
def test_select_with_predexp(self): ''' Call to select with predexp in policy. ''' predexp = [ as_predexp.integer_bin('acct_balance'), as_predexp.integer_value(20), as_predexp.integer_equal(), as_predexp.integer_var('charge'), as_predexp.integer_value(20), as_predexp.integer_less(), as_predexp.list_bin('charges'), as_predexp.list_iterate_and('charge'), as_predexp.predexp_and(2) ] result = self.as_connection.select(self.keys[1], ['account_id', 'acct_balance'], {'predexp': predexp}) assert result[2]['account_id'] == 2 and result[2]['acct_balance'] == 20
def test_with_invalid_predicate(self): ''' This passes something which isn't a predicate ''' predexps = [ predexp.integer_var('map_value'), predexp.integer_value(3), predexp.integer_equal(), 5 ] with pytest.raises(e.ParamError): self.query.predexp(predexps)
def main(): # variables to connect to Aerospike host = '127.0.0.1' port = 3000 namespace = 'ns2' set_name = 'example' # connect to Aerospike client = aerospike.client({'hosts': [(host, port)]}).connect() # variables to control which account and locations are corrected account_to_correct = '00007' incorrect_location = 5 correct_location = 2 # only update records that match on 'acct' AND 'loc' predicate_expressions = [ predexp.integer_bin('loc'), predexp.integer_value(incorrect_location), predexp.integer_equal(), predexp.string_bin('acct'), predexp.string_value(account_to_correct), predexp.string_equal(), predexp.predexp_and(2) ] policy = {'predexp': predicate_expressions} # Do a standard scan with the predicate expressions and count the results. scan = client.scan(namespace, set_name) records = scan.results(policy) print("{} records found".format(len(records))) # Do a background scan, which runs server-side, to update the records that # match the predicate expression with the correct value for 'loc'. ops = [operations.write('loc', correct_location)] bgscan = client.scan(namespace, set_name) bgscan.add_ops(ops) scan_id = bgscan.execute_background(policy) print("Running background read/write scan. ID: {}".format(scan_id)) # Wait for the background scan to complete. while True: response = client.job_info(scan_id, aerospike.JOB_SCAN) if response["status"] != aerospike.JOB_STATUS_INPROGRESS: break sleep(0.25) # Do a standard scan with the predicate expressions and count the results. scan = client.scan(namespace, set_name) records = scan.results(policy) print("{} records found".format(len(records)))
def test_list_and_int(self): predexps = [ predexp.integer_var('list_val'), predexp.integer_value(10), predexp.integer_greater(), predexp.list_bin('plus_five_l'), predexp.list_iterate_and('list_val') ] self.query.predexp(predexps) results = self.query.results() assert len(results) == 89 # This isn't true for the first 11 assert_each_record_bins(results, lambda b: b['plus_five_l'][-1] > 10)
def test_list_or_int(self): predexps = [ predexp.integer_var('list_val'), predexp.integer_value(10), predexp.integer_greater(), predexp.list_bin('plus_five_l'), predexp.list_iterate_or('list_val') ] self.query.predexp(predexps) results = self.query.results() assert len(results) == 94 # This isn't true for 0,1,2,3,4,5 so 100 - 6 assert_each_record_bins(results, lambda b: b['plus_five_l'][-1] > 10)
def test_get_with_predexp(self): ''' Call to get with predexp in policy. ''' predexp = [ as_predexp.integer_bin('account_id'), as_predexp.integer_value(1), as_predexp.integer_equal() ] record = self.as_connection.get(self.keys[0], {'predexp': predexp}) assert record[2]['account_id'] == 1
def test_select_with_predexp_filtered_out(self): ''' Call to select with predexp in policy with expected failures. ''' predexp = [ as_predexp.integer_bin('acct_balance'), as_predexp.integer_value(20), as_predexp.integer_equal(), as_predexp.integer_var('charge'), as_predexp.integer_value( 10), # charge should trigger a filtered_out as_predexp.integer_less(), as_predexp.list_bin('charges'), as_predexp.list_iterate_and('charge'), as_predexp.predexp_and(2) ] with pytest.raises(e.FilteredOut): self.as_connection.select(self.keys[1], ['account_id', 'acct_balance'], {'predexp': predexp})
def test_put_with_predexp_filtered_out(self): ''' Call put with predexp in policy with expected failure. ''' predexp = [ as_predexp.integer_bin('account_id'), as_predexp.integer_value(4), as_predexp.integer_equal() ] with pytest.raises(e.FilteredOut): self.as_connection.put(self.keys[0], {'newkey': 'newval'}, policy={'predexp': predexp})
def test_remove_bin_with_predexp_filtered_out(self): ''' Call remove_bin with predexp in policy with expected failure. ''' predexp = [ as_predexp.integer_bin('account_id'), as_predexp.integer_value(4), as_predexp.integer_equal() ] with pytest.raises(e.FilteredOut): self.as_connection.remove_bin(self.keys[0], ['account_id', 'user_name'], policy={'predexp': predexp})
def test_pos_remove_with_predexp(self): ''' Call remove with predexp in policy. ''' predexp = [ as_predexp.integer_bin('account_id'), as_predexp.integer_value(1), as_predexp.integer_equal() ] records = self.as_connection.remove(self.keys[0]) rec = self.as_connection.exists(self.keys[0]) assert rec[1] is None
def test_list_or_int(self): predexps = [ predexp.integer_var('list_val'), predexp.integer_value(10), predexp.integer_greater(), predexp.list_bin('plus_five_l'), predexp.list_iterate_or('list_val') ] self.query.predexp(predexps) results = self.query.results() assert len(results) == 94 # This isn't true for 0,1,2,3,4,5 so 100 - 6 assert_each_record_bins( results, lambda b: b['plus_five_l'][-1] > 10)
def test_list_and_int(self): predexps = [ predexp.integer_var('list_val'), predexp.integer_value(10), predexp.integer_greater(), predexp.list_bin('plus_five_l'), predexp.list_iterate_and('list_val') ] self.query.predexp(predexps) results = self.query.results() assert len(results) == 89 # This isn't true for the first 11 assert_each_record_bins( results, lambda b: b['plus_five_l'][-1] > 10)
def test_mapvalue_iterate_and(self): predexps = [ predexp.integer_var('map_value'), predexp.integer_value(3), predexp.integer_unequal(), predexp.map_bin('map'), predexp.mapval_iterate_and('map_value') ] query = self.as_connection.query('test', 'demo3') query.predexp(predexps) results = query.results() assert len(results) == 1 assert_each_record_bins( results, lambda b: all([b['map'][key] != 3 for key in b['map']]))
def test_rec_void_time(self): ''' This could fail due to clock skew ''' for i in range(7): key = 'test', 'ttl', i self.as_connection.put(key, {'time': 'earlier'}, meta={'ttl': 100}) # 150 second range for record TTLs should be enough, we are storing with # Current time + 100s and current time +5000s, so only one of the group should be found void_time_range_start = seconds_to_nanos(int(time.time() + 50)) void_time_range_end = seconds_to_nanos(int(time.time() + 150)) # Store 5 records after the cutoff for i in range(7, 12): key = 'test', 'ttl', i self.as_connection.put(key, {'time': 'later'}, meta={'ttl': 1000}) query = self.as_connection.query('test', 'ttl') predexps = [ predexp.rec_void_time(), predexp.integer_value(void_time_range_start), predexp.integer_greater(), predexp.rec_void_time(), predexp.integer_value(void_time_range_end), predexp.integer_less(), predexp.predexp_and(2) ] query.predexp(predexps) results = query.results() assert len(results) == 7 assert_each_record_bins( results, lambda b: b['time'] == 'earlier')
def test_digest_modulo(self): # Count of digests whose last byte is < 128 less_than_128 = 0 expected_ids = set([]) for i in range(100): key = 'test', 'demo', i if aerospike.calc_digest(*key)[-1] < 128: expected_ids.add(i) less_than_128 += 1 predexps = [ predexp.rec_digest_modulo(256), predexp.integer_value(128), predexp.integer_less() ] self.query.predexp(predexps) results = self.query.results() assert len(results) == less_than_128