def test_rec_last_update(self):
'''
This could fail due to clock skew
'''
for i in range(7):
key = 'test', 'lut', i
self.as_connection.put(key, {'time': 'earlier'})
cutoff_nanos = seconds_to_nanos(int(time.time() + 2))
time.sleep(5) # Make sure that we wait long enough
# Store 5 records after the cutoff
for i in range(7, 12):
key = 'test', 'lut', i
self.as_connection.put(key, {'time': 'later'})
query = self.as_connection.query('test', 'lut')
predexps = [
predexp.rec_last_update(),
predexp.integer_value(cutoff_nanos),
predexp.integer_less()
]
query.predexp(predexps)
results = query.results()
assert len(results) == 7
assert_each_record_bins(
results,
lambda b: b['time'] == 'earlier')
def test_predexp_key_operate_record_digest_modulo(self):
"""
Invoke the C client aerospike_key_operate with a rec_digest_modulo predexp.
"""
less_than_128 = 0
for i in range(100):
key = 'test', 'demo', i
if aerospike.calc_digest(*key)[-1] < 128:
less_than_128 += 1
self.as_connection.put(key, {'dig_id': i})
results = []
predexp = [
as_predexp.rec_digest_modulo(256),
as_predexp.integer_value(128),
as_predexp.integer_less()
]
ops = [operations.read('dig_id')]
for i in range(100):
try:
key = 'test', 'demo', i
_, _, res = self.as_connection.operate(
key, ops, policy={'predexp': predexp})
results.append(res)
except:
pass
self.as_connection.remove(key)
assert len(results) == less_than_128
def test_exists_many_with_large_predexp(self):
'''
Proper call to exists_many with predexp in policy.
'''
predexp = [
as_predexp.integer_bin('account_id'),
as_predexp.integer_value(4),
as_predexp.integer_equal(),
as_predexp.string_bin('user_name'),
as_predexp.string_value('user3'),
as_predexp.string_equal(),
as_predexp.integer_var('list_val'),
as_predexp.integer_value(12),
as_predexp.integer_less(),
as_predexp.list_bin('charges'),
as_predexp.list_iterate_and('list_val'),
as_predexp.predexp_or(3)
]
matched_recs = []
records = self.as_connection.exists_many(self.keys,
{'predexp': predexp})
for rec in records:
if rec[1] is not None:
matched_recs.append(rec[1])
assert len(matched_recs) == 3
def test_rec_void_time(self):
'''
This could fail due to clock skew
'''
for i in range(7):
key = 'test', 'ttl', i
self.as_connection.put(key, {'time': 'earlier'}, meta={'ttl': 100})
# 150 second range for record TTLs should be enough, we are storing with
# Current time + 100s and current time +5000s, so only one of the group should be found
void_time_range_start = seconds_to_nanos(int(time.time() + 50))
void_time_range_end = seconds_to_nanos(int(time.time() + 150))
# Store 5 records after the cutoff
for i in range(7, 12):
key = 'test', 'ttl', i
self.as_connection.put(key, {'time': 'later'}, meta={'ttl': 1000})
query = self.as_connection.query('test', 'ttl')
predexps = [
predexp.rec_void_time(),
predexp.integer_value(void_time_range_start),
predexp.integer_greater(),
predexp.rec_void_time(),
predexp.integer_value(void_time_range_end),
predexp.integer_less(),
predexp.predexp_and(2)
]
query.predexp(predexps)
results = query.results()
assert len(results) == 7
assert_each_record_bins(results, lambda b: b['time'] == 'earlier')
def test_background_execute_sindex_predexp(self, clean_test_background):
"""
Ensure that Query.execute_background() only applies to records matched by
the specified predicate
"""
test_bin = 't4'
keys = [(TEST_NS, TEST_SET, i) for i in range(500)]
# rec['number'] < 10
predexps = [
predexp.integer_bin('number'),
predexp.integer_value(10),
predexp.integer_less()
]
query = self.as_connection.query(TEST_NS, TEST_SET)
query.predexp(predexps)
query.apply(TEST_UDF_MODULE, TEST_UDF_FUNCTION, [test_bin])
query.execute_background()
# Give time for the query to finish
time.sleep(5)
# Records with number > 10 should not have had the UDF applied
validate_records(self.as_connection, keys[10:],
lambda rec: test_bin not in rec)
# Records with number < 10 should have had the udf applied
validate_records(self.as_connection, keys[:10],
lambda rec: rec[test_bin] == 'aerospike')
def test_integer_less(self):
predexps = [
predexp.integer_bin('positive_i'),
predexp.integer_value(10),
predexp.integer_less()
]
self.query.predexp(predexps)
results = self.query.results()
assert len(results) == 10
assert_each_record_bins(results, lambda b: b['positive_i'] < 10)
def test_and(self):
predexps = [
predexp.integer_bin('positive_i'),
predexp.integer_value(10),
predexp.integer_greater(),
predexp.integer_bin('positive_i'),
predexp.integer_value(20),
predexp.integer_less(),
predexp.predexp_and(2)
]
self.query.predexp(predexps)
results = self.query.results()
assert len(results) == 9
assert_each_record_bins(
results, lambda b: b['positive_i'] > 10 and b['positive_i'] < 20)
def test_and(self):
predexps = [
predexp.integer_bin('positive_i'),
predexp.integer_value(10),
predexp.integer_greater(),
predexp.integer_bin('positive_i'),
predexp.integer_value(20),
predexp.integer_less(),
predexp.predexp_and(2)
]
self.query.predexp(predexps)
results = self.query.results()
assert len(results) == 9
assert_each_record_bins(
results,
lambda b: b['positive_i'] > 10 and b['positive_i'] < 20)
def test_predexp_key_operate_record_void_time(self):
"""
Invoke the C client aerospike_key_operate with a rec_void_time predexp.
"""
for i in range(5):
key = 'test', 'pred_ttl', i
self.as_connection.put(key, {'time': 'earlier'}, meta={'ttl': 100})
# 150 second range for record TTLs should be enough, we are storing with
# Current time + 100s and current time +5000s, so only one of the group should be found
void_time_range_start = (10**9) * int(time.time() + 50)
void_time_range_end = (10**9) * int(time.time() + 150)
for i in range(5, 10):
key = 'test', 'pred_ttl', i
self.as_connection.put(key, {'time': 'later'}, meta={'ttl': 1000})
results = []
predexp = [
as_predexp.rec_void_time(),
as_predexp.integer_value(void_time_range_start),
as_predexp.integer_greater(),
as_predexp.rec_void_time(),
as_predexp.integer_value(void_time_range_end),
as_predexp.integer_less(),
as_predexp.predexp_and(2)
]
ops = [operations.read('time')]
for i in range(10):
try:
key = 'test', 'pred_ttl', i
_, _, res = self.as_connection.operate(
key, ops, policy={'predexp': predexp})
results.append(res)
except:
pass
self.as_connection.remove(key)
assert len(results) == 5
for res in results:
assert res['time'] == 'earlier'
def test_digest_modulo(self):
# Count of digests whose last byte is < 128
less_than_128 = 0
expected_ids = set([])
for i in range(100):
key = 'test', 'demo', i
if aerospike.calc_digest(*key)[-1] < 128:
expected_ids.add(i)
less_than_128 += 1
predexps = [
predexp.rec_digest_modulo(256),
predexp.integer_value(128),
predexp.integer_less()
]
self.query.predexp(predexps)
results = self.query.results()
assert len(results) == less_than_128
def test_select_with_predexp(self):
'''
Call to select with predexp in policy.
'''
predexp = [
as_predexp.integer_bin('acct_balance'),
as_predexp.integer_value(20),
as_predexp.integer_equal(),
as_predexp.integer_var('charge'),
as_predexp.integer_value(20),
as_predexp.integer_less(),
as_predexp.list_bin('charges'),
as_predexp.list_iterate_and('charge'),
as_predexp.predexp_and(2)
]
result = self.as_connection.select(self.keys[1],
['account_id', 'acct_balance'],
{'predexp': predexp})
assert result[2]['account_id'] == 2 and result[2]['acct_balance'] == 20
def test_select_with_predexp_filtered_out(self):
'''
Call to select with predexp in policy with expected failures.
'''
predexp = [
as_predexp.integer_bin('acct_balance'),
as_predexp.integer_value(20),
as_predexp.integer_equal(),
as_predexp.integer_var('charge'),
as_predexp.integer_value(
10), # charge should trigger a filtered_out
as_predexp.integer_less(),
as_predexp.list_bin('charges'),
as_predexp.list_iterate_and('charge'),
as_predexp.predexp_and(2)
]
with pytest.raises(e.FilteredOut):
self.as_connection.select(self.keys[1],
['account_id', 'acct_balance'],
{'predexp': predexp})
def test_predexp_key_operate_record_last_updated(self):
"""
Invoke the C client aerospike_key_operate with a record_last_updated predexp.
"""
for i in range(5):
key = 'test', 'pred_lut', i
self.as_connection.put(key, {'time': 'earlier'})
cutoff_nanos = (10**9) * int(time.time() + 2)
time.sleep(5)
for i in range(5, 10):
key = 'test', 'pred_lut', i
self.as_connection.put(key, {'time': 'later'})
results = []
predexp = [
as_predexp.rec_last_update(),
as_predexp.integer_value(cutoff_nanos),
as_predexp.integer_less()
]
ops = [operations.read('time')]
for i in range(10):
try:
key = 'test', 'pred_lut', i
_, _, res = self.as_connection.operate(
key, ops, policy={'predexp': predexp})
results.append(res)
except:
pass
self.as_connection.remove(key)
assert len(results) == 5
for res in results:
assert res['time'] == 'earlier'
def enqueue(self, entry, txn_id, overwrite_if_full=False):
"""
Append a new entry to the queue. Fails if the queue lock cannot be acquired. Can fail if the queue is full.
If the fencing counter has wrapped around, reset all fencing values.
:param entry: new entry to be enqueued
:param txn_id: lock owner id, must be unique among concurrent requests
:param overwrite_if_full: flag indicating if the head position should be overwritten if the queue is full
:return: Offset position of the enqueued entry. throws: ASAborted('Queue is full'), ASAborted('Timed out')
"""
q_state = self._lock(txn_id, LargeQueue.Ops.Enqueue)
# compute the record and list indices
head_offset = long(q_state['head-offset'])
tail_offset = long(q_state['tail-offset'])
fencing_ctr = q_state['fencing-ctr']
# if the fencing counter has a non-positive value, it has wrapped around past the max value; reset
if fencing_ctr <= 0:
self._reset_fencing_marks()
buf_rec_index, entry_index = self._get_entry_location(tail_offset) # tail points to where the new entry will go
entry_val = {'offset': tail_offset, 'entry': entry}
queue_is_full = self._queue_is_full(head_offset, tail_offset)
if queue_is_full and not overwrite_if_full:
self._commit_release(txn_id)
raise ASAborted('Queue is full')
predexps = [ predexp.integer_bin("fencing-mark"),
predexp.integer_value(fencing_ctr),
predexp.integer_less() ]
ops = [ op_helpers.write('fencing-mark', fencing_ctr),
list_helpers.list_set('entries', entry_index, entry_val) ]
buf_rec_key = (self.namespace, self.name, LargeQueue._buf_record_key(buf_rec_index))
try:
(_, _, record) = self.client.operate(buf_rec_key, ops, policy={'predexp': predexps})
except exception.FilteredOut as ex:
raise ASAborted('Timed out')
self._commit_release(txn_id, new_head_offset=head_offset + 1 if queue_is_full else None,
new_tail_offset=tail_offset + 1)
return tail_offset
def test_rec_void_time(self):
'''
This could fail due to clock skew
'''
for i in range(7):
key = 'test', 'ttl', i
self.as_connection.put(key, {'time': 'earlier'}, meta={'ttl': 100})
# 150 second range for record TTLs should be enough, we are storing with
# Current time + 100s and current time +5000s, so only one of the group should be found
void_time_range_start = seconds_to_nanos(int(time.time() + 50))
void_time_range_end = seconds_to_nanos(int(time.time() + 150))
# Store 5 records after the cutoff
for i in range(7, 12):
key = 'test', 'ttl', i
self.as_connection.put(key, {'time': 'later'}, meta={'ttl': 1000})
query = self.as_connection.query('test', 'ttl')
predexps = [
predexp.rec_void_time(),
predexp.integer_value(void_time_range_start),
predexp.integer_greater(),
predexp.rec_void_time(),
predexp.integer_value(void_time_range_end),
predexp.integer_less(),
predexp.predexp_and(2)
]
query.predexp(predexps)
results = query.results()
assert len(results) == 7
assert_each_record_bins(
results,
lambda b: b['time'] == 'earlier')
def _lock(self, txn_id, op):
"""
Atomically check if the queue is locked, break an expired lock, lock the queue and
set the lock-owner and lock-time, and if the operation is enqueue, also increment and
return the fencing counter.
Try multiple times if the lock is not available, and wait before subsequent attempt.
:param txn_id: lock owner id, must be unique among concurrent requests
:param op: enqueue or dequeue
:return: dict with head and tail positions on success
throws ASAborted('Failed to acquire lock') on failure
"""
metadata_key = (self.namespace, self.name, LargeQueue.META_REC_KEY)
for _ in range(LargeQueue.LOCK_MAX_RETRIES):
curr_time_ms = LargeQueue._curr_time_milliseconds()
predexps = [ predexp.integer_bin("locked"),
predexp.integer_value(0),
predexp.integer_equal(),
predexp.integer_bin("lock-time-ms"),
predexp.integer_value(curr_time_ms-LargeQueue.LOCK_EXPIRATION_MS),
predexp.integer_less(),
predexp.predexp_or(2) ]
ops = [ op_helpers.read('head-offset'),
op_helpers.read('tail-offset'),
op_helpers.write('locked', 1),
op_helpers.write('lock-owner', txn_id),
op_helpers.write('lock-time-ms', curr_time_ms) ]
if op == LargeQueue.Ops.Enqueue:
ops.append(op_helpers.increment('fencing-ctr', 1))
ops.append(op_helpers.read('fencing-ctr'))
try:
_, _, record = self.client.operate(metadata_key, ops, policy={'predexp': predexps})
except exception.FilteredOut as ex: # predexp failed
time.sleep(LargeQueue.LOCK_POLL_WAIT_MS/1000.0)
continue
return record
raise ASAborted('Failed to acquire lock')
class TestPredEveryWhere(object):
@pytest.fixture(autouse=True)
def setup(self, request, as_connection):
"""
Setup Method
"""
self.keys = []
self.test_data = [{
'account_id': j,
'user_name': 'user' + str(j),
'acct_balance': j * 10,
'charges': [j + 5, j + 10],
'meta': {
'date': '11/4/2019'
}
} for j in range(1, 5)]
self.test_data.append({'string_list': ['s1', 's2', 's3', 's4']})
self.test_data.append(
{'map_bin': {
'k1': 1,
'k2': 2,
'k3': 3,
'k4': 4
}})
georec = {
'id': 1,
'point': geo_point,
'region': geo_circle,
'geolist': [geo_point]
}
self.test_data.append(georec)
self.test_data_bin = 'test_data'
self.keys = [('test', 'pred_evry', i + 1)
for i, _ in enumerate(self.test_data)]
# print('self keys is: ', self.keys)
for key, data in zip(self.keys, self.test_data):
self.as_connection.put(key, data)
# cleanup
yield
for key in self.keys:
try:
self.as_connection.remove(key)
except e.AerospikeError:
pass
@pytest.mark.parametrize(
"ops, predexp, expected_bins, expected_res, key_num",
[
( # test integer equal
[operations.increment("account_id", 1)], [
as_predexp.integer_bin('account_id'),
as_predexp.integer_value(3),
as_predexp.integer_equal()
], {
'account_id': 4,
'user_name': 'user3',
'acct_balance': 30,
'charges': [8, 13],
'meta': {
'date': '11/4/2019'
}
}, {}, 3),
( # test string equal
[operations.increment("account_id", 1)], [
as_predexp.string_bin('user_name'),
as_predexp.string_value('user3'),
as_predexp.string_equal()
], {
'account_id': 4,
'user_name': 'user3',
'acct_balance': 30,
'charges': [8, 13],
'meta': {
'date': '11/4/2019'
}
}, {}, 3),
( # test and
[
list_operations.list_remove_by_index_range(
'charges', 0, 3, aerospike.LIST_RETURN_COUNT),
operations.increment("acct_balance", -23)
], [
as_predexp.integer_bin('acct_balance'),
as_predexp.integer_value(10),
as_predexp.integer_greatereq(),
as_predexp.integer_bin('acct_balance'),
as_predexp.integer_value(50),
as_predexp.integer_lesseq(),
as_predexp.predexp_and(2)
], {
'account_id': 4,
'user_name': 'user4',
'acct_balance': 17,
'charges': [],
'meta': {
'date': '11/4/2019'
}
}, {
'charges': [0, 1]
}, 4),
( # test or
[map_operations.map_put('meta', 'lupdated', 'now')], [
as_predexp.string_bin('user_name'),
as_predexp.string_value('user2'),
as_predexp.string_equal(),
as_predexp.integer_bin('acct_balance'),
as_predexp.integer_value(50),
as_predexp.integer_greatereq(),
as_predexp.predexp_or(2)
], {
'account_id': 2,
'user_name': 'user2',
'acct_balance': 20,
'charges': [7, 12],
'meta': {
'date': '11/4/2019',
'lupdated': 'now'
}
}, {
'meta': 2
}, 2),
( # test integer greater
[map_operations.map_clear('meta')], [
as_predexp.integer_bin('account_id'),
as_predexp.integer_value(2),
as_predexp.integer_greater()
], {
'account_id': 3,
'user_name': 'user3',
'acct_balance': 30,
'charges': [8, 13],
'meta': {}
}, {
'meta': None
}, 3),
( # test integer greatereq
[map_operations.map_clear('meta')], [
as_predexp.integer_bin('account_id'),
as_predexp.integer_value(2),
as_predexp.integer_greatereq()
], {
'account_id': 3,
'user_name': 'user3',
'acct_balance': 30,
'charges': [8, 13],
'meta': {}
}, {
'meta': None
}, 3),
( # test integer less
[list_operations.list_clear('charges')], [
as_predexp.integer_bin('account_id'),
as_predexp.integer_value(5),
as_predexp.integer_less()
], {
'account_id': 4,
'user_name': 'user4',
'acct_balance': 40,
'charges': [],
'meta': {
'date': '11/4/2019'
}
}, {}, 4),
( # test integer lesseq
[list_operations.list_clear('charges')], [
as_predexp.integer_bin('account_id'),
as_predexp.integer_value(4),
as_predexp.integer_lesseq()
], {
'account_id': 4,
'user_name': 'user4',
'acct_balance': 40,
'charges': [],
'meta': {
'date': '11/4/2019'
}
}, {}, 4),
( # test string unequal
[list_operations.list_append('charges', 2)], [
as_predexp.string_bin('user_name'),
as_predexp.string_value('user2'),
as_predexp.string_unequal()
], {
'account_id': 4,
'user_name': 'user4',
'acct_balance': 40,
'charges': [9, 14, 2],
'meta': {
'date': '11/4/2019'
}
}, {
'charges': 3
}, 4),
( # test not
[list_operations.list_append('charges', 2)], [
as_predexp.string_bin('user_name'),
as_predexp.string_value('user4'),
as_predexp.string_unequal(),
as_predexp.predexp_not()
], {
'account_id': 4,
'user_name': 'user4',
'acct_balance': 40,
'charges': [9, 14, 2],
'meta': {
'date': '11/4/2019'
}
}, {
'charges': 3
}, 4),
( # test string regex
[list_operations.list_append('charges', 2)], [
as_predexp.string_bin('user_name'),
as_predexp.string_value('.*4.*'),
as_predexp.string_regex(aerospike.REGEX_ICASE)
], {
'account_id': 4,
'user_name': 'user4',
'acct_balance': 40,
'charges': [9, 14, 2],
'meta': {
'date': '11/4/2019'
}
}, {
'charges': 3
}, 4),
( # test list or int
[list_operations.list_append('charges', 2)], [
as_predexp.integer_var('list_val'),
as_predexp.integer_value(14),
as_predexp.integer_equal(),
as_predexp.list_bin('charges'),
as_predexp.list_iterate_or('list_val')
], {
'account_id': 4,
'user_name': 'user4',
'acct_balance': 40,
'charges': [9, 14, 2],
'meta': {
'date': '11/4/2019'
}
}, {
'charges': 3
}, 4),
( # test list and int
[list_operations.list_append('charges', 2)], [
as_predexp.integer_var('list_val'),
as_predexp.integer_value(120),
as_predexp.integer_less(),
as_predexp.list_bin('charges'),
as_predexp.list_iterate_or('list_val')
], {
'account_id': 4,
'user_name': 'user4',
'acct_balance': 40,
'charges': [9, 14, 2],
'meta': {
'date': '11/4/2019'
}
}, {
'charges': 3
}, 4),
( # test list or str
[list_operations.list_append('string_list', 's5')], [
as_predexp.string_var('list_val'),
as_predexp.string_value('s2'),
as_predexp.string_equal(),
as_predexp.list_bin('string_list'),
as_predexp.list_iterate_or('list_val')
], {
'string_list': ['s1', 's2', 's3', 's4', 's5']
}, {
'string_list': 5
}, 5),
( # test list and str
[
list_operations.list_remove_by_index_range(
'string_list', 0, aerospike.LIST_RETURN_VALUE, 2)
], [
as_predexp.string_var('list_val'),
as_predexp.string_value('.*s.*'),
as_predexp.string_regex(aerospike.REGEX_ICASE),
as_predexp.list_bin('string_list'),
as_predexp.list_iterate_and('list_val')
], {
'string_list': ['s3', 's4']
}, {
'string_list': ['s1', 's2']
}, 5),
( # test map_key_iterate_or
[map_operations.map_put('map_bin', 'k5', 5)], [
as_predexp.string_var('map_key'),
as_predexp.string_value('k3'),
as_predexp.string_equal(),
as_predexp.map_bin('map_bin'),
as_predexp.mapkey_iterate_or('map_key')
], {
'map_bin': {
'k1': 1,
'k2': 2,
'k3': 3,
'k4': 4,
'k5': 5
}
}, {
'map_bin': 5
}, 6),
( # test map_key_iterate_and
[map_operations.map_put('map_bin', 'k5', 5)], [
as_predexp.string_var('map_key'),
as_predexp.string_value('k7'),
as_predexp.string_unequal(),
as_predexp.map_bin('map_bin'),
as_predexp.mapkey_iterate_and('map_key')
], {
'map_bin': {
'k1': 1,
'k2': 2,
'k3': 3,
'k4': 4,
'k5': 5
}
}, {
'map_bin': 5
}, 6),
( # test mapkey_iterate_and
[map_operations.map_put('map_bin', 'k5', 5)], [
as_predexp.string_var('map_key'),
as_predexp.string_value('k7'),
as_predexp.string_unequal(),
as_predexp.map_bin('map_bin'),
as_predexp.mapkey_iterate_and('map_key')
], {
'map_bin': {
'k1': 1,
'k2': 2,
'k3': 3,
'k4': 4,
'k5': 5
}
}, {
'map_bin': 5
}, 6),
( # test mapval_iterate_and
[map_operations.map_put('map_bin', 'k5', 5)], [
as_predexp.integer_var('map_val'),
as_predexp.integer_value(7),
as_predexp.integer_unequal(),
as_predexp.map_bin('map_bin'),
as_predexp.mapval_iterate_and('map_val')
], {
'map_bin': {
'k1': 1,
'k2': 2,
'k3': 3,
'k4': 4,
'k5': 5
}
}, {
'map_bin': 5
}, 6),
( # test mapval_iterate_or
[
map_operations.map_get_by_key('map_bin', 'k1',
aerospike.MAP_RETURN_VALUE)
], [
as_predexp.integer_var('map_val'),
as_predexp.integer_value(3),
as_predexp.integer_less(),
as_predexp.map_bin('map_bin'),
as_predexp.mapval_iterate_or('map_val')
], {
'map_bin': {
'k1': 1,
'k2': 2,
'k3': 3,
'k4': 4
}
}, {
'map_bin': 1
}, 6)
])
def test_predexp_key_operate(self, ops, predexp, expected_bins,
expected_res, key_num):
"""
Invoke the C client aerospike_key_operate with predexp.
"""
key = ('test', 'pred_evry', key_num)
_, _, res = self.as_connection.operate(key,
ops,
policy={'predexp': predexp})
assert res == expected_res
_, _, bins = self.as_connection.get(key)
assert bins == expected_bins
@pytest.mark.parametrize(
"ops, predexp, expected_bins, expected_res, key_num",
[( # test mapval_iterate_or
[
map_operations.map_put_items('map_bin', {
'k5': 5,
'k6': 6
}),
map_operations.map_get_by_key('map_bin', 'k1',
aerospike.MAP_RETURN_VALUE)
], [
as_predexp.integer_var('map_val'),
as_predexp.integer_value(3),
as_predexp.integer_less(),
as_predexp.map_bin('map_bin'),
as_predexp.mapval_iterate_or('map_val')
], {
'map_bin': {
'k1': 1,
'k2': 2,
'k3': 3,
'k4': 4,
'k5': 5,
'k6': 6
}
}, [('map_bin', 6), ('map_bin', 1)], 6)])
def test_predexp_key_operate_ordered(self, ops, predexp, expected_bins,
expected_res, key_num):
"""
Invoke the C client aerospike_key_operate with predexp using operate_ordered.
"""
key = ('test', 'pred_evry', key_num)
_, _, res = self.as_connection.operate_ordered(
key, ops, policy={'predexp': predexp})
assert res == expected_res
_, _, bins = self.as_connection.get(key)
assert bins == expected_bins
@pytest.mark.parametrize(
"ops, predexp, expected, key_num",
[( # test mapval_iterate_or
[
map_operations.map_get_by_key('map_bin', 'k1',
aerospike.MAP_RETURN_VALUE)
], [
as_predexp.integer_var('map_val'),
as_predexp.integer_value(3),
as_predexp.integer_less(),
as_predexp.map_bin('map_bin'),
as_predexp.mapval_iterate_or('map_val'),
as_predexp.predexp_not()
], e.FilteredOut, 6)])
def test_predexp_key_operate_ordered_negative(self, ops, predexp, expected,
key_num):
"""
Invoke the C client aerospike_key_operate with predexp using operate_ordered with expected failures.
"""
key = ('test', 'pred_evry', key_num)
with pytest.raises(expected):
_, _, res = self.as_connection.operate_ordered(
key, ops, policy={'predexp': predexp})
@pytest.mark.parametrize(
"ops, predexp, key_num, bin",
[
( # test geojson_within
[operations.increment('id', 1)], [
as_predexp.geojson_bin('point'),
as_predexp.geojson_value(geo_circle.dumps()),
as_predexp.geojson_within()
], 7, 'point'),
( # test geojson_contains
[operations.increment('id', 1)], [
as_predexp.geojson_bin('region'),
as_predexp.geojson_value(geo_point.dumps()),
as_predexp.geojson_contains()
], 7, 'point'),
])
def test_predexp_key_operate_geojson(self, ops, predexp, key_num, bin):
"""
Invoke the C client aerospike_key_operate with predexp.
"""
key = ('test', 'pred_evry', key_num)
_, _, _ = self.as_connection.operate(key,
ops,
policy={'predexp': predexp})
_, _, bins = self.as_connection.get(key)
assert bins['id'] == 2
# NOTE: may fail due to clock skew
def test_predexp_key_operate_record_last_updated(self):
"""
Invoke the C client aerospike_key_operate with a record_last_updated predexp.
"""
for i in range(5):
key = 'test', 'pred_lut', i
self.as_connection.put(key, {'time': 'earlier'})
cutoff_nanos = (10**9) * int(time.time() + 2)
time.sleep(5)
for i in range(5, 10):
key = 'test', 'pred_lut', i
self.as_connection.put(key, {'time': 'later'})
results = []
predexp = [
as_predexp.rec_last_update(),
as_predexp.integer_value(cutoff_nanos),
as_predexp.integer_less()
]
ops = [operations.read('time')]
for i in range(10):
try:
key = 'test', 'pred_lut', i
_, _, res = self.as_connection.operate(
key, ops, policy={'predexp': predexp})
results.append(res)
except:
pass
self.as_connection.remove(key)
assert len(results) == 5
for res in results:
assert res['time'] == 'earlier'
# NOTE: may fail due to clock skew
def test_predexp_key_operate_record_void_time(self):
"""
Invoke the C client aerospike_key_operate with a rec_void_time predexp.
"""
for i in range(5):
key = 'test', 'pred_ttl', i
self.as_connection.put(key, {'time': 'earlier'}, meta={'ttl': 100})
# 150 second range for record TTLs should be enough, we are storing with
# Current time + 100s and current time +5000s, so only one of the group should be found
void_time_range_start = (10**9) * int(time.time() + 50)
void_time_range_end = (10**9) * int(time.time() + 150)
for i in range(5, 10):
key = 'test', 'pred_ttl', i
self.as_connection.put(key, {'time': 'later'}, meta={'ttl': 1000})
results = []
predexp = [
as_predexp.rec_void_time(),
as_predexp.integer_value(void_time_range_start),
as_predexp.integer_greater(),
as_predexp.rec_void_time(),
as_predexp.integer_value(void_time_range_end),
as_predexp.integer_less(),
as_predexp.predexp_and(2)
]
ops = [operations.read('time')]
for i in range(10):
try:
key = 'test', 'pred_ttl', i
_, _, res = self.as_connection.operate(
key, ops, policy={'predexp': predexp})
results.append(res)
except:
pass
self.as_connection.remove(key)
assert len(results) == 5
for res in results:
assert res['time'] == 'earlier'
def test_predexp_key_operate_record_digest_modulo(self):
"""
Invoke the C client aerospike_key_operate with a rec_digest_modulo predexp.
"""
less_than_128 = 0
for i in range(100):
key = 'test', 'demo', i
if aerospike.calc_digest(*key)[-1] < 128:
less_than_128 += 1
self.as_connection.put(key, {'dig_id': i})
results = []
predexp = [
as_predexp.rec_digest_modulo(256),
as_predexp.integer_value(128),
as_predexp.integer_less()
]
ops = [operations.read('dig_id')]
for i in range(100):
try:
key = 'test', 'demo', i
_, _, res = self.as_connection.operate(
key, ops, policy={'predexp': predexp})
results.append(res)
except:
pass
self.as_connection.remove(key)
assert len(results) == less_than_128
@pytest.mark.parametrize(
"ops, predexp, expected, key_num",
[
( # filtered out
[operations.increment("account_id", 1)], [
as_predexp.integer_bin('account_id'),
as_predexp.integer_value(5),
as_predexp.integer_equal()
], e.FilteredOut, 3),
( # incorrect bin type
[
list_operations.list_remove_by_index_range(
'charges', 0, 3, aerospike.LIST_RETURN_COUNT),
operations.increment("acct_balance", -23)
],
[
as_predexp.integer_bin('acct_balance'),
as_predexp.string_value(10), #incorrect bin type
as_predexp.integer_greatereq(),
as_predexp.integer_bin('acct_balance'),
as_predexp.integer_value(50),
as_predexp.integer_lesseq(),
as_predexp.predexp_and(2)
],
e.ParamError,
4),
( # filtered out
[map_operations.map_put('meta', 'lupdated', 'now')], [
as_predexp.string_bin('user_name'),
as_predexp.string_value('user2'),
as_predexp.string_equal(),
as_predexp.integer_bin('acct_balance'),
as_predexp.integer_value(50),
as_predexp.integer_greatereq(),
as_predexp.predexp_or(2),
as_predexp.predexp_not()
], e.FilteredOut, 2),
( # empty predexp list
[map_operations.map_put('meta', 'lupdated', 'now')
], [], e.InvalidRequest, 2),
( # predexp not in list
[map_operations.map_put('meta', 'lupdated', 'now')
], 'bad predexp', e.ParamError, 2),
])
def test_predexp_key_operate_negative(self, ops, predexp, expected,
key_num):
"""
Invoke the C client aerospike_key_operate with predexp. Expecting failures.
"""
key = ('test', 'pred_evry', key_num)
with pytest.raises(expected):
self.as_connection.operate(key, ops, policy={'predexp': predexp})
@pytest.mark.parametrize("predexp, rec_place, rec_bin, expected",
[([
as_predexp.integer_bin('account_id'),
as_predexp.integer_value(2),
as_predexp.integer_equal()
], 1, 'account_id', 2),
([
as_predexp.string_bin('user_name'),
as_predexp.string_value('user2'),
as_predexp.string_equal(),
], 1, 'account_id', 2),
([
as_predexp.string_bin('user_name'),
as_predexp.string_value('user2'),
as_predexp.string_equal(),
as_predexp.integer_bin('acct_balance'),
as_predexp.integer_value(30),
as_predexp.integer_greatereq(),
as_predexp.predexp_or(2)
], 2, 'account_id', 3)])
def test_pos_get_many_with_predexp(self, predexp, rec_place, rec_bin,
expected):
'''
Proper call to get_many with predexp in policy
'''
records = self.as_connection.get_many(self.keys, {'predexp': predexp})
#assert isinstance(records, list)
# assert records[2][2]['age'] == 2
assert records[rec_place][2][rec_bin] == expected
def test_pos_get_many_with_large_predexp(self):
'''
Proper call to get_many with predexp in policy.
'''
predexp = [
as_predexp.integer_bin('account_id'),
as_predexp.integer_value(4),
as_predexp.integer_equal(),
as_predexp.string_bin('user_name'),
as_predexp.string_value('user3'),
as_predexp.string_equal(),
as_predexp.integer_var('list_val'),
as_predexp.integer_value(12),
as_predexp.integer_less(),
as_predexp.list_bin('charges'),
as_predexp.list_iterate_and('list_val'),
as_predexp.predexp_or(3)
]
matched_recs = []
records = self.as_connection.get_many(self.keys, {'predexp': predexp})
for rec in records:
if rec[2] is not None:
matched_recs.append(rec[2])
assert len(matched_recs) == 3
for rec in matched_recs:
assert rec['account_id'] == 1 or rec['account_id'] == 3 or rec[
'account_id'] == 4
def test_pos_select_many_with_large_predexp(self):
'''
Proper call to select_many with predexp in policy.
'''
predexp = [
as_predexp.integer_bin('account_id'),
as_predexp.integer_value(4),
as_predexp.integer_equal(),
as_predexp.string_bin('user_name'),
as_predexp.string_value('user3'),
as_predexp.string_equal(),
as_predexp.integer_var('list_val'),
as_predexp.integer_value(12),
as_predexp.integer_less(),
as_predexp.list_bin('charges'),
as_predexp.list_iterate_and('list_val'),
as_predexp.predexp_or(3)
]
matched_recs = []
records = self.as_connection.select_many(self.keys, ['account_id'],
{'predexp': predexp})
for rec in records:
if rec[2] is not None:
matched_recs.append(rec[2])
assert len(matched_recs) == 3
for rec in matched_recs:
assert rec['account_id'] == 1 or rec['account_id'] == 3 or rec[
'account_id'] == 4
def test_pos_remove_with_predexp(self):
'''
Call remove with predexp in policy.
'''
predexp = [
as_predexp.integer_bin('account_id'),
as_predexp.integer_value(1),
as_predexp.integer_equal()
]
records = self.as_connection.remove(self.keys[0])
rec = self.as_connection.exists(self.keys[0])
assert rec[1] is None
def test_remove_with_predexp_filtered_out(self):
'''
Call remove with predexp in policy with expected failure.
'''
predexp = [
as_predexp.integer_bin('account_id'),
as_predexp.integer_value(3),
as_predexp.integer_equal()
]
with pytest.raises(e.FilteredOut):
self.as_connection.remove(self.keys[0],
policy={'predexp': predexp})
def test_remove_bin_with_predexp(self):
'''
Call remove_bin with predexp in policy.
'''
predexp = [
as_predexp.integer_bin('account_id'),
as_predexp.integer_value(1),
as_predexp.integer_equal()
]
self.as_connection.remove_bin(self.keys[0],
['account_id', 'user_name'],
policy={'predexp': predexp})
rec = self.as_connection.get(self.keys[0])
assert rec[2].get('account_id') is None and rec[2].get(
'user_name') is None
def test_remove_bin_with_predexp_filtered_out(self):
'''
Call remove_bin with predexp in policy with expected failure.
'''
predexp = [
as_predexp.integer_bin('account_id'),
as_predexp.integer_value(4),
as_predexp.integer_equal()
]
with pytest.raises(e.FilteredOut):
self.as_connection.remove_bin(self.keys[0],
['account_id', 'user_name'],
policy={'predexp': predexp})
def test_put_with_predexp(self):
'''
Call put with predexp in policy.
'''
predexp = [
as_predexp.integer_bin('account_id'),
as_predexp.integer_value(1),
as_predexp.integer_equal()
]
self.as_connection.put(self.keys[0], {'newkey': 'newval'},
policy={'predexp': predexp})
rec = self.as_connection.get(self.keys[0])
assert rec[2]['newkey'] == 'newval'
def test_put_new_record_with_predexp(self): # should this fail?
'''
Call put a new record with predexp in policy.
'''
predexp = [
as_predexp.integer_bin('account_id'),
as_predexp.integer_value(1),
as_predexp.integer_equal()
]
key = ("test", "demo", 10)
self.as_connection.put(key, {'newkey': 'newval'},
policy={'predexp': predexp})
rec = self.as_connection.get(key)
self.as_connection.remove(key)
assert rec[2]['newkey'] == 'newval'
def test_put_with_predexp_filtered_out(self):
'''
Call put with predexp in policy with expected failure.
'''
predexp = [
as_predexp.integer_bin('account_id'),
as_predexp.integer_value(4),
as_predexp.integer_equal()
]
with pytest.raises(e.FilteredOut):
self.as_connection.put(self.keys[0], {'newkey': 'newval'},
policy={'predexp': predexp})
def test_get_with_predexp(self):
'''
Call to get with predexp in policy.
'''
predexp = [
as_predexp.integer_bin('account_id'),
as_predexp.integer_value(1),
as_predexp.integer_equal()
]
record = self.as_connection.get(self.keys[0], {'predexp': predexp})
assert record[2]['account_id'] == 1
def test_get_with_predexp_filtered_out(self):
'''
Call to get with predexp in policy with expected failures.
'''
predexp = [
as_predexp.integer_bin('account_id'),
as_predexp.integer_value(3),
as_predexp.integer_equal()
]
with pytest.raises(e.FilteredOut):
self.as_connection.get(self.keys[0], {'predexp': predexp})
def test_select_with_predexp(self):
'''
Call to select with predexp in policy.
'''
predexp = [
as_predexp.integer_bin('acct_balance'),
as_predexp.integer_value(20),
as_predexp.integer_equal(),
as_predexp.integer_var('charge'),
as_predexp.integer_value(20),
as_predexp.integer_less(),
as_predexp.list_bin('charges'),
as_predexp.list_iterate_and('charge'),
as_predexp.predexp_and(2)
]
result = self.as_connection.select(self.keys[1],
['account_id', 'acct_balance'],
{'predexp': predexp})
assert result[2]['account_id'] == 2 and result[2]['acct_balance'] == 20
def test_select_with_predexp_filtered_out(self):
'''
Call to select with predexp in policy with expected failures.
'''
predexp = [
as_predexp.integer_bin('acct_balance'),
as_predexp.integer_value(20),
as_predexp.integer_equal(),
as_predexp.integer_var('charge'),
as_predexp.integer_value(
10), # charge should trigger a filtered_out
as_predexp.integer_less(),
as_predexp.list_bin('charges'),
as_predexp.list_iterate_and('charge'),
as_predexp.predexp_and(2)
]
with pytest.raises(e.FilteredOut):
self.as_connection.select(self.keys[1],
['account_id', 'acct_balance'],
{'predexp': predexp})
def test_exists_many_with_large_predexp(self):
'''
Proper call to exists_many with predexp in policy.
'''
predexp = [
as_predexp.integer_bin('account_id'),
as_predexp.integer_value(4),
as_predexp.integer_equal(),
as_predexp.string_bin('user_name'),
as_predexp.string_value('user3'),
as_predexp.string_equal(),
as_predexp.integer_var('list_val'),
as_predexp.integer_value(12),
as_predexp.integer_less(),
as_predexp.list_bin('charges'),
as_predexp.list_iterate_and('list_val'),
as_predexp.predexp_or(3)
]
matched_recs = []
records = self.as_connection.exists_many(self.keys,
{'predexp': predexp})
for rec in records:
if rec[1] is not None:
matched_recs.append(rec[1])
assert len(matched_recs) == 3
