def _commit_release(self, txn_id, new_head_offset=None, new_tail_offset=None):
"""
If the lock is still held by this requester (txn-id), update the new positions of head/tail and r
elease the lock. Otherwise abort the request as timed out.
:param txn_id: lock owner id, must be unique among concurrent requests
:param new_head_offset: new head offset to be updated
:param new_tail_offset: new tail offset to be updated
:return: throws ASAborted('Timed out')
"""
metadata_key = (self.namespace, self.name, LargeQueue.META_REC_KEY)
predexps = [ predexp.integer_bin("locked"),
predexp.integer_value(1),
predexp.integer_equal(),
predexp.integer_bin("lock-owner"),
predexp.integer_value(txn_id),
predexp.integer_equal(),
predexp.predexp_and(2)]
ops = [ op_helpers.write('locked', 0),
op_helpers.write('lock-owner', None),
op_helpers.write('lock-time-ms', None) ]
if new_head_offset is not None:
ops.append(op_helpers.write('head-offset', new_head_offset))
if new_tail_offset is not None:
ops.append(op_helpers.write('tail-offset', new_tail_offset))
try:
_ = self.client.operate(metadata_key, ops, policy={'predexp': predexps})
except exception.FilteredOut as ex: # predexp failed
raise ASAborted('Timed out')
return
def test_background_execute_with_ops_and_predexp(self):
"""
Ensure that Scan.execute_background() applies ops to records that match the predexp
"""
test_bin = 'Stops_preds'
keys = [(TEST_NS, TEST_SET, i) for i in range(50)]
scan = self.as_connection.scan(TEST_NS, TEST_SET)
# scan.apply(TEST_UDF_MODULE, TEST_UDF_FUNCTION, [test_bin])
ops = [operations.write(test_bin, 'new_val')]
predexp = [
as_predexp.integer_bin('number'),
as_predexp.integer_value(2),
as_predexp.integer_equal(),
as_predexp.integer_bin('number'),
as_predexp.integer_value(3),
as_predexp.integer_equal(),
as_predexp.predexp_or(2)
]
policy = {'predexp': predexp}
scan.add_ops(ops)
job_id = scan.execute_background(policy)
# Give time for the scan to finish
wait_for_job_completion(self.as_connection, job_id)
for key in keys:
_, _, bins = self.as_connection.get(key)
if bins['number'] == 2 or bins['number'] == 3:
assert (bins[test_bin] == 'new_val')
else:
assert (bins.get(test_bin) is None)
def test_background_execute_predexp_and_predicate(self):
"""
Ensure that Scan.execute_background() gets applied to records that match the predicate
NOTE: the predicate overrides the predexp
"""
test_bin = 'Stpredold'
keys = [(TEST_NS, TEST_SET, i) for i in range(50)]
predexp = [
as_predexp.integer_bin('number'),
as_predexp.integer_value(2),
as_predexp.integer_equal(),
as_predexp.integer_bin('number'),
as_predexp.integer_value(3),
as_predexp.integer_equal(),
as_predexp.predexp_or(2)
]
number_predicate = predicates.equals('number', 4)
policy = {'predexp': predexp}
scan = self.as_connection.scan(TEST_NS, TEST_SET)
scan.where(number_predicate)
scan.apply(TEST_UDF_MODULE, TEST_UDF_FUNCTION, [test_bin])
job_id = scan.execute_background(policy)
# Give time for the scan to finish
wait_for_job_completion(self.as_connection, job_id)
for key in keys:
_, _, bins = self.as_connection.get(key)
if bins['number'] == 4:
assert (bins[test_bin] == 'aerospike')
else:
assert (bins.get(test_bin) is None)
def test_query_apply_with_new_predexp(self):
"""
Invoke query_apply() with correct policy and predexp
"""
predexp = [
as_predexp.integer_bin('age'),
as_predexp.integer_value(2),
as_predexp.integer_equal(),
as_predexp.integer_bin('val'),
as_predexp.integer_value(3),
as_predexp.integer_equal(),
as_predexp.predexp_or(2)
]
policy = {'total_timeout': 0, 'predexp': predexp}
query_id = self.as_connection.query_apply(
"test", "demo", self.age_range_pred, "query_apply",
"mark_as_applied", ['name', 2], policy)
self._wait_for_query_complete(query_id)
recs = []
for i in range(1, 10):
key = ('test', 'demo', i)
_, _, bins = self.as_connection.get(key)
if bins['name'] == 'aerospike':
recs.append(bins)
assert len(recs) == 2
for rec in recs:
assert rec['age'] == 2 or rec['val'] == 3
def test_background_execute_predexp_everywhere(self):
"""
Ensure that Scan.execute_background() gets applied to records that match the predexp
"""
test_bin = 'number'
keys = [(TEST_NS, TEST_SET, i) for i in range(50)]
predexp = [
as_predexp.integer_bin('number'),
as_predexp.integer_value(2),
as_predexp.integer_equal(),
as_predexp.integer_bin('number'),
as_predexp.integer_value(3),
as_predexp.integer_equal(),
as_predexp.predexp_or(2)
]
#number_predicate = predicates.equals('number', 3)
policy = {'predexp': predexp}
scan = self.as_connection.scan(TEST_NS, TEST_SET)
#scan.where(number_predicate)
scan.apply(TEST_UDF_MODULE, TEST_UDF_FUNCTION, [test_bin, 1])
job_id = scan.execute_background(policy)
# Give time for the scan to finish
wait_for_job_completion(self.as_connection, job_id)
for i, key in enumerate(keys):
_, _, bins = self.as_connection.get(key)
if i == 2 or i == 3:
assert (bins[test_bin] == i + 1)
else:
assert (bins.get(test_bin) == i)
def test_or(self):
predexps = [
predexp.integer_bin('positive_i'),
predexp.integer_value(5),
predexp.integer_equal(),
predexp.integer_bin('positive_i'),
predexp.integer_value(10),
predexp.integer_equal(),
predexp.predexp_or(2)
]
self.query.predexp(predexps)
results = self.query.results()
assert len(results) == 2
assert_each_record_bins(results, lambda b: b['positive_i'] in (5, 10))
def test_and(self):
predexps = [
predexp.integer_bin('positive_i'),
predexp.integer_value(10),
predexp.integer_greater(),
predexp.integer_bin('positive_i'),
predexp.integer_value(20),
predexp.integer_less(),
predexp.predexp_and(2)
]
self.query.predexp(predexps)
results = self.query.results()
assert len(results) == 9
assert_each_record_bins(
results, lambda b: b['positive_i'] > 10 and b['positive_i'] < 20)
def test_or(self):
predexps = [
predexp.integer_bin('positive_i'),
predexp.integer_value(5),
predexp.integer_equal(),
predexp.integer_bin('positive_i'),
predexp.integer_value(10),
predexp.integer_equal(),
predexp.predexp_or(2)
]
self.query.predexp(predexps)
results = self.query.results()
assert len(results) == 2
assert_each_record_bins(
results,
lambda b: b['positive_i'] in (5, 10))
def test_and(self):
predexps = [
predexp.integer_bin('positive_i'),
predexp.integer_value(10),
predexp.integer_greater(),
predexp.integer_bin('positive_i'),
predexp.integer_value(20),
predexp.integer_less(),
predexp.predexp_and(2)
]
self.query.predexp(predexps)
results = self.query.results()
assert len(results) == 9
assert_each_record_bins(
results,
lambda b: b['positive_i'] > 10 and b['positive_i'] < 20)
def test_background_execute_sindex_predexp(self, clean_test_background):
"""
Ensure that Query.execute_background() only applies to records matched by
the specified predicate
"""
test_bin = 't4'
keys = [(TEST_NS, TEST_SET, i) for i in range(500)]
# rec['number'] < 10
predexps = [
predexp.integer_bin('number'),
predexp.integer_value(10),
predexp.integer_less()
]
query = self.as_connection.query(TEST_NS, TEST_SET)
query.predexp(predexps)
query.apply(TEST_UDF_MODULE, TEST_UDF_FUNCTION, [test_bin])
query.execute_background()
# Give time for the query to finish
time.sleep(5)
# Records with number > 10 should not have had the UDF applied
validate_records(self.as_connection, keys[10:],
lambda rec: test_bin not in rec)
# Records with number < 10 should have had the udf applied
validate_records(self.as_connection, keys[:10],
lambda rec: rec[test_bin] == 'aerospike')
def test_scan_apply_with_none_set_and_predexp(self):
"""
Invoke scan_apply() with set argument as None
It should invoke the function on all records in NS that match the predexp
"""
predexp = [
as_predexp.string_bin('name'),
as_predexp.string_value('name2'),
as_predexp.string_equal(),
as_predexp.integer_bin('age'),
as_predexp.integer_value(3),
as_predexp.integer_unequal(),
as_predexp.predexp_and(2)
]
policy = {'timeout': 1000, 'predexp': predexp}
scan_id = self.as_connection.scan_apply("test", None, "bin_lua",
"mytransform", ['age', 2],
policy)
wait_for_job_completion(self.as_connection, scan_id)
for i in range(5):
key = ('test', 'demo', i)
_, _, bins = self.as_connection.get(key)
if bins['name'] == 'name2':
assert bins['age'] == i + 2
else:
assert bins['age'] == i
_, _, rec = self.as_connection.get(('test', None, 'no_set'))
assert rec['age'] == 10
def test_exists_many_with_large_predexp(self):
'''
Proper call to exists_many with predexp in policy.
'''
predexp = [
as_predexp.integer_bin('account_id'),
as_predexp.integer_value(4),
as_predexp.integer_equal(),
as_predexp.string_bin('user_name'),
as_predexp.string_value('user3'),
as_predexp.string_equal(),
as_predexp.integer_var('list_val'),
as_predexp.integer_value(12),
as_predexp.integer_less(),
as_predexp.list_bin('charges'),
as_predexp.list_iterate_and('list_val'),
as_predexp.predexp_or(3)
]
matched_recs = []
records = self.as_connection.exists_many(self.keys,
{'predexp': predexp})
for rec in records:
if rec[1] is not None:
matched_recs.append(rec[1])
assert len(matched_recs) == 3
class TestApply(TestBaseClass):
def setup_class(cls):
# Register setup and teardown functions
cls.connection_setup_functions = [add_indexes_and_udfs]
cls.connection_teardown_functions = [remove_indexes_and_udfs]
@pytest.fixture(autouse=True)
def setup(self, request, connection_with_config_funcs):
as_connection = connection_with_config_funcs
for i in range(5):
key = ('test', 'demo', i)
rec = {
'name': ['name%s' % (str(i))],
'addr': 'name%s' % (str(i)),
'age': i,
'no': i,
'basic_map': {
"k30": 6,
"k20": 5,
"k10": 1
}
}
as_connection.put(key, rec)
def teardown():
for i in range(5):
key = ('test', 'demo', i)
as_connection.remove(key)
request.addfinalizer(teardown)
@pytest.mark.parametrize("func_args, test_bin, predexp, expected",
((['name', 1], 'name', [
as_predexp.integer_bin('age'),
as_predexp.integer_value(1),
as_predexp.integer_equal()
], ['name1', 1]), ),
ids=[
"Integer",
])
def test_apply_causing_list_append_with_correct_params_with_predexp(
self, func_args, test_bin, predexp, expected):
key = ('test', 'demo', 1)
retval = self.as_connection.apply(key,
'sample',
'list_append',
func_args,
policy={'predexp': predexp})
_, _, bins = self.as_connection.get(key)
print(expected)
print(bins[test_bin])
assert bins[test_bin] == expected
assert retval == 0 # the list_append UDF returns 0
def test_integer_equals(self):
predexps = [
predexp.integer_bin('positive_i'),
predexp.integer_value(5),
predexp.integer_equal()
]
self.query.predexp(predexps)
results = self.query.results()
assert len(results) == 1
assert_each_record_bins(results, lambda b: b['positive_i'] == 5)
def test_integer_greatereq(self):
predexps = [
predexp.integer_bin('positive_i'),
predexp.integer_value(49),
predexp.integer_greatereq()
]
self.query.predexp(predexps)
results = self.query.results()
assert len(results) == 51
assert_each_record_bins(results, lambda b: b['positive_i'] >= 49)
def test_integer_lesseq(self):
predexps = [
predexp.integer_bin('positive_i'),
predexp.integer_value(10),
predexp.integer_lesseq()
]
self.query.predexp(predexps)
results = self.query.results()
assert len(results) == 11
assert_each_record_bins(results, lambda b: b['positive_i'] <= 10)
def test_get_with_predexp_filtered_out(self):
'''
Call to get with predexp in policy with expected failures.
'''
predexp = [
as_predexp.integer_bin('account_id'),
as_predexp.integer_value(3),
as_predexp.integer_equal()
]
with pytest.raises(e.FilteredOut):
self.as_connection.get(self.keys[0], {'predexp': predexp})
def test_query_apply_with_bad_new_predexp(self):
"""
Invoke query_apply() with correct policy and predexp
"""
predexp = [
as_predexp.integer_bin('age'),
as_predexp.string_value(2),
as_predexp.integer_equal(),
as_predexp.integer_bin('val'),
as_predexp.integer_value(3),
as_predexp.integer_equal(),
as_predexp.predexp_or(2)
]
policy = {'total_timeout': 0, 'predexp': predexp}
with pytest.raises(e.ParamError):
query_id = self.as_connection.query_apply(
"test", "demo", self.age_range_pred, "query_apply",
"mark_as_applied", ['name', 2], policy)
def main():
# variables to connect to Aerospike
host = '127.0.0.1'
port = 3000
namespace = 'ns2'
set_name = 'example'
# connect to Aerospike
client = aerospike.client({'hosts': [(host, port)]}).connect()
# variables to control which account and locations are corrected
account_to_correct = '00007'
incorrect_location = 5
correct_location = 2
# only update records that match on 'acct' AND 'loc'
predicate_expressions = [
predexp.integer_bin('loc'),
predexp.integer_value(incorrect_location),
predexp.integer_equal(),
predexp.string_bin('acct'),
predexp.string_value(account_to_correct),
predexp.string_equal(),
predexp.predexp_and(2)
]
policy = {'predexp': predicate_expressions}
# Do a standard scan with the predicate expressions and count the results.
scan = client.scan(namespace, set_name)
records = scan.results(policy)
print("{} records found".format(len(records)))
# Do a background scan, which runs server-side, to update the records that
# match the predicate expression with the correct value for 'loc'.
ops = [operations.write('loc', correct_location)]
bgscan = client.scan(namespace, set_name)
bgscan.add_ops(ops)
scan_id = bgscan.execute_background(policy)
print("Running background read/write scan. ID: {}".format(scan_id))
# Wait for the background scan to complete.
while True:
response = client.job_info(scan_id, aerospike.JOB_SCAN)
if response["status"] != aerospike.JOB_STATUS_INPROGRESS:
break
sleep(0.25)
# Do a standard scan with the predicate expressions and count the results.
scan = client.scan(namespace, set_name)
records = scan.results(policy)
print("{} records found".format(len(records)))
def test_get_with_predexp(self):
'''
Call to get with predexp in policy.
'''
predexp = [
as_predexp.integer_bin('account_id'),
as_predexp.integer_value(1),
as_predexp.integer_equal()
]
record = self.as_connection.get(self.keys[0], {'predexp': predexp})
assert record[2]['account_id'] == 1
def test_put_with_predexp_filtered_out(self):
'''
Call put with predexp in policy with expected failure.
'''
predexp = [
as_predexp.integer_bin('account_id'),
as_predexp.integer_value(4),
as_predexp.integer_equal()
]
with pytest.raises(e.FilteredOut):
self.as_connection.put(self.keys[0], {'newkey': 'newval'},
policy={'predexp': predexp})
def test_remove_bin_with_predexp_filtered_out(self):
'''
Call remove_bin with predexp in policy with expected failure.
'''
predexp = [
as_predexp.integer_bin('account_id'),
as_predexp.integer_value(4),
as_predexp.integer_equal()
]
with pytest.raises(e.FilteredOut):
self.as_connection.remove_bin(self.keys[0],
['account_id', 'user_name'],
policy={'predexp': predexp})
def test_pos_remove_with_predexp(self):
'''
Call remove with predexp in policy.
'''
predexp = [
as_predexp.integer_bin('account_id'),
as_predexp.integer_value(1),
as_predexp.integer_equal()
]
records = self.as_connection.remove(self.keys[0])
rec = self.as_connection.exists(self.keys[0])
assert rec[1] is None
def test_put_with_predexp(self):
'''
Call put with predexp in policy.
'''
predexp = [
as_predexp.integer_bin('account_id'),
as_predexp.integer_value(1),
as_predexp.integer_equal()
]
self.as_connection.put(self.keys[0], {'newkey': 'newval'},
policy={'predexp': predexp})
rec = self.as_connection.get(self.keys[0])
assert rec[2]['newkey'] == 'newval'
def _lock(self, txn_id, op):
"""
Atomically check if the queue is locked, break an expired lock, lock the queue and
set the lock-owner and lock-time, and if the operation is enqueue, also increment and
return the fencing counter.
Try multiple times if the lock is not available, and wait before subsequent attempt.
:param txn_id: lock owner id, must be unique among concurrent requests
:param op: enqueue or dequeue
:return: dict with head and tail positions on success
throws ASAborted('Failed to acquire lock') on failure
"""
metadata_key = (self.namespace, self.name, LargeQueue.META_REC_KEY)
for _ in range(LargeQueue.LOCK_MAX_RETRIES):
curr_time_ms = LargeQueue._curr_time_milliseconds()
predexps = [ predexp.integer_bin("locked"),
predexp.integer_value(0),
predexp.integer_equal(),
predexp.integer_bin("lock-time-ms"),
predexp.integer_value(curr_time_ms-LargeQueue.LOCK_EXPIRATION_MS),
predexp.integer_less(),
predexp.predexp_or(2) ]
ops = [ op_helpers.read('head-offset'),
op_helpers.read('tail-offset'),
op_helpers.write('locked', 1),
op_helpers.write('lock-owner', txn_id),
op_helpers.write('lock-time-ms', curr_time_ms) ]
if op == LargeQueue.Ops.Enqueue:
ops.append(op_helpers.increment('fencing-ctr', 1))
ops.append(op_helpers.read('fencing-ctr'))
try:
_, _, record = self.client.operate(metadata_key, ops, policy={'predexp': predexps})
except exception.FilteredOut as ex: # predexp failed
time.sleep(LargeQueue.LOCK_POLL_WAIT_MS/1000.0)
continue
return record
raise ASAborted('Failed to acquire lock')
def test_background_execute_predexp_and_predicate(self,
clean_test_background):
"""
Ensure that Query.execute_background() gets applied to records that match the predicate
NOTE: the predicate overrides the predexp
"""
test_bin = 'tpredold'
keys = [(TEST_NS, TEST_SET, i) for i in range(500)]
predexp = [
as_predexp.integer_bin('number'),
as_predexp.integer_value(2),
as_predexp.integer_equal(),
as_predexp.integer_bin('number'),
as_predexp.integer_value(3),
as_predexp.integer_equal(),
as_predexp.predexp_or(2)
]
number_predicate = predicates.equals('number', 4)
policy = {'predexp': predexp}
query = self.as_connection.query(TEST_NS, TEST_SET)
query.where(number_predicate)
query.apply(TEST_UDF_MODULE, TEST_UDF_FUNCTION, [test_bin])
query.execute_background(policy)
# Give time for the query to finish
time.sleep(5)
for key in keys:
_, _, bins = self.as_connection.get(key)
if bins['number'] == 4:
assert (bins[test_bin] == 'aerospike')
else:
assert (bins.get(test_bin) is None)
def test_background_execute_with_ops_and_predexp(self,
clean_test_background):
"""
Ensure that Query.execute_background() applies ops to records that match the predexp
"""
test_bin = 'tops_preds'
keys = [(TEST_NS, TEST_SET, i) for i in range(500)]
query = self.as_connection.query(TEST_NS, TEST_SET)
# query.apply(TEST_UDF_MODULE, TEST_UDF_FUNCTION, [test_bin])
ops = [operations.write(test_bin, 'new_val')]
predexp = [
as_predexp.integer_bin('number'),
as_predexp.integer_value(2),
as_predexp.integer_equal(),
as_predexp.integer_bin('number'),
as_predexp.integer_value(3),
as_predexp.integer_equal(),
as_predexp.predexp_or(2)
]
policy = {'predexp': predexp}
query.add_ops(ops)
query.execute_background(policy)
# Give time for the query to finish
time.sleep(5)
for key in keys:
_, _, bins = self.as_connection.get(key)
if bins['number'] == 2 or bins['number'] == 3:
assert (bins[test_bin] == 'new_val')
else:
assert (bins.get(test_bin) is None)
def test_remove_bin_with_predexp(self):
'''
Call remove_bin with predexp in policy.
'''
predexp = [
as_predexp.integer_bin('account_id'),
as_predexp.integer_value(1),
as_predexp.integer_equal()
]
self.as_connection.remove_bin(self.keys[0],
['account_id', 'user_name'],
policy={'predexp': predexp})
rec = self.as_connection.get(self.keys[0])
assert rec[2].get('account_id') is None and rec[2].get(
'user_name') is None
def test_put_new_record_with_predexp(self): # should this fail?
'''
Call put a new record with predexp in policy.
'''
predexp = [
as_predexp.integer_bin('account_id'),
as_predexp.integer_value(1),
as_predexp.integer_equal()
]
key = ("test", "demo", 10)
self.as_connection.put(key, {'newkey': 'newval'},
policy={'predexp': predexp})
rec = self.as_connection.get(key)
self.as_connection.remove(key)
assert rec[2]['newkey'] == 'newval'
def test_query_with_results_method_and_invalid_predexp(self):
"""
Invoke query() with correct arguments
"""
predexp = [
as_predexp.integer_bin('test_age'),
as_predexp.integer_value('1'),
as_predexp.integer_equal()
]
policy = {'predexp': predexp}
query = self.as_connection.query('test', 'demo')
query.select('name', 'test_age')
with pytest.raises(e.ParamError):
query.results(policy)
def test_query_with_results_method_and_predexp(self):
"""
Invoke query() with correct arguments
"""
predexp = [
as_predexp.integer_bin('test_age'),
as_predexp.integer_value(1),
as_predexp.integer_equal()
]
policy = {'predexp': predexp}
query = self.as_connection.query('test', 'demo')
query.select('name', 'test_age')
records = query.results(policy)
assert len(records) == 1
def test_select_with_predexp(self):
'''
Call to select with predexp in policy.
'''
predexp = [
as_predexp.integer_bin('acct_balance'),
as_predexp.integer_value(20),
as_predexp.integer_equal(),
as_predexp.integer_var('charge'),
as_predexp.integer_value(20),
as_predexp.integer_less(),
as_predexp.list_bin('charges'),
as_predexp.list_iterate_and('charge'),
as_predexp.predexp_and(2)
]
result = self.as_connection.select(self.keys[1],
['account_id', 'acct_balance'],
{'predexp': predexp})
assert result[2]['account_id'] == 2 and result[2]['acct_balance'] == 20