def _commit_release(self, txn_id, new_head_offset=None, new_tail_offset=None):
"""
If the lock is still held by this requester (txn-id), update the new positions of head/tail and r
elease the lock. Otherwise abort the request as timed out.
:param txn_id: lock owner id, must be unique among concurrent requests
:param new_head_offset: new head offset to be updated
:param new_tail_offset: new tail offset to be updated
:return: throws ASAborted('Timed out')
"""
metadata_key = (self.namespace, self.name, LargeQueue.META_REC_KEY)
predexps = [ predexp.integer_bin("locked"),
predexp.integer_value(1),
predexp.integer_equal(),
predexp.integer_bin("lock-owner"),
predexp.integer_value(txn_id),
predexp.integer_equal(),
predexp.predexp_and(2)]
ops = [ op_helpers.write('locked', 0),
op_helpers.write('lock-owner', None),
op_helpers.write('lock-time-ms', None) ]
if new_head_offset is not None:
ops.append(op_helpers.write('head-offset', new_head_offset))
if new_tail_offset is not None:
ops.append(op_helpers.write('tail-offset', new_tail_offset))
try:
_ = self.client.operate(metadata_key, ops, policy={'predexp': predexps})
except exception.FilteredOut as ex: # predexp failed
raise ASAborted('Timed out')
return
def test_background_execute_with_ops_and_predexp(self):
"""
Ensure that Scan.execute_background() applies ops to records that match the predexp
"""
test_bin = 'Stops_preds'
keys = [(TEST_NS, TEST_SET, i) for i in range(50)]
scan = self.as_connection.scan(TEST_NS, TEST_SET)
# scan.apply(TEST_UDF_MODULE, TEST_UDF_FUNCTION, [test_bin])
ops = [operations.write(test_bin, 'new_val')]
predexp = [
as_predexp.integer_bin('number'),
as_predexp.integer_value(2),
as_predexp.integer_equal(),
as_predexp.integer_bin('number'),
as_predexp.integer_value(3),
as_predexp.integer_equal(),
as_predexp.predexp_or(2)
]
policy = {'predexp': predexp}
scan.add_ops(ops)
job_id = scan.execute_background(policy)
# Give time for the scan to finish
wait_for_job_completion(self.as_connection, job_id)
for key in keys:
_, _, bins = self.as_connection.get(key)
if bins['number'] == 2 or bins['number'] == 3:
assert (bins[test_bin] == 'new_val')
else:
assert (bins.get(test_bin) is None)
def test_background_execute_with_ops_and_predexp(self, clean_test_background):
"""
Ensure that Query.execute_background() applies ops to records that match the expressions.
"""
test_bin = 'tops_preds'
keys = [(TEST_NS, TEST_SET, i) for i in range(500)]
query = self.as_connection.query(TEST_NS, TEST_SET)
ops = [
operations.write(test_bin, 'new_val')
]
expr = exp.Or(
exp.Eq(exp.IntBin('number'), 2),
exp.Eq(exp.IntBin('number'), 3)
)
policy = {
'expressions': expr.compile()
}
query.add_ops(ops)
query.execute_background(policy=policy)
# Give time for the query to finish
time.sleep(5)
for key in keys:
_, _, bins = self.as_connection.get(key)
if bins['number'] == 2 or bins['number'] == 3:
assert(bins[test_bin] == 'new_val')
else:
assert(bins.get(test_bin) is None)
def test_background_execute_with_ops_and_expressions(self):
"""
Ensure that Scan.execute_background() applies ops to records that match the expressions.
"""
test_bin = 'Stops_preds'
keys = [(TEST_NS, TEST_SET, i) for i in range(50)]
scan = self.as_connection.scan(TEST_NS, TEST_SET)
# scan.apply(TEST_UDF_MODULE, TEST_UDF_FUNCTION, [test_bin])
ops = [
operations.write(test_bin, 'new_val')
]
expr = exp.Or(
exp.Eq(exp.IntBin('number'), 2),
exp.Eq(exp.IntBin('number'), 3)
)
policy = {
'expressions': expr.compile()
}
scan.add_ops(ops)
job_id = scan.execute_background(policy)
wait_for_job_completion(self.as_connection, job_id)
for key in keys:
_, _, bins = self.as_connection.get(key)
if bins['number'] == 2 or bins['number'] == 3:
assert(bins[test_bin] == 'new_val')
else:
assert(bins.get(test_bin) is None)
def test_background_execute_with_ops_and_preds(self):
"""
Ensure that Scan.execute_background() applies ops to records that match the predicate
"""
test_bin = 'St1'
keys = [(TEST_NS, TEST_SET, i) for i in range(50)]
scan = self.as_connection.scan(TEST_NS, TEST_SET)
number_predicate = predicates.equals('number', 3)
# scan.apply(TEST_UDF_MODULE, TEST_UDF_FUNCTION, [test_bin])
ops = [operations.append(test_bin, 'new_val')]
scan.add_ops(ops)
scan.where(number_predicate)
job_id = scan.execute_background()
# Give time for the scan to finish
wait_for_job_completion(self.as_connection, job_id)
_, _, num_5_record = self.as_connection.get((TEST_NS, TEST_SET, 5))
assert num_5_record[test_bin] == 'aerospike'
_, _, num_3_record = self.as_connection.get((TEST_NS, TEST_SET, 3))
assert num_3_record[test_bin] == 'aerospikenew_val'
# cleanup
ops = [operations.write(test_bin, 'aerospike')]
scan.add_ops(ops)
job_id = scan.execute_background()
wait_for_job_completion(self.as_connection, job_id)
validate_records(self.as_connection, keys,
lambda rec: rec[test_bin] == 'aerospike')
def main():
# variables to connect to Aerospike
host = '127.0.0.1'
port = 3000
namespace = 'ns2'
set_name = 'example'
# connect to Aerospike
client = aerospike.client({'hosts': [(host, port)]}).connect()
# variables to control which account and locations are corrected
account_to_correct = '00007'
incorrect_location = 5
correct_location = 2
# only update records that match on 'acct' AND 'loc'
predicate_expressions = [
predexp.integer_bin('loc'),
predexp.integer_value(incorrect_location),
predexp.integer_equal(),
predexp.string_bin('acct'),
predexp.string_value(account_to_correct),
predexp.string_equal(),
predexp.predexp_and(2)
]
policy = {'predexp': predicate_expressions}
# Do a standard scan with the predicate expressions and count the results.
scan = client.scan(namespace, set_name)
records = scan.results(policy)
print("{} records found".format(len(records)))
# Do a background scan, which runs server-side, to update the records that
# match the predicate expression with the correct value for 'loc'.
ops = [operations.write('loc', correct_location)]
bgscan = client.scan(namespace, set_name)
bgscan.add_ops(ops)
scan_id = bgscan.execute_background(policy)
print("Running background read/write scan. ID: {}".format(scan_id))
# Wait for the background scan to complete.
while True:
response = client.job_info(scan_id, aerospike.JOB_SCAN)
if response["status"] != aerospike.JOB_STATUS_INPROGRESS:
break
sleep(0.25)
# Do a standard scan with the predicate expressions and count the results.
scan = client.scan(namespace, set_name)
records = scan.results(policy)
print("{} records found".format(len(records)))
def _lock(self, txn_id, op):
"""
Atomically check if the queue is locked, break an expired lock, lock the queue and
set the lock-owner and lock-time, and if the operation is enqueue, also increment and
return the fencing counter.
Try multiple times if the lock is not available, and wait before subsequent attempt.
:param txn_id: lock owner id, must be unique among concurrent requests
:param op: enqueue or dequeue
:return: dict with head and tail positions on success
throws ASAborted('Failed to acquire lock') on failure
"""
metadata_key = (self.namespace, self.name, LargeQueue.META_REC_KEY)
for _ in range(LargeQueue.LOCK_MAX_RETRIES):
curr_time_ms = LargeQueue._curr_time_milliseconds()
predexps = [ predexp.integer_bin("locked"),
predexp.integer_value(0),
predexp.integer_equal(),
predexp.integer_bin("lock-time-ms"),
predexp.integer_value(curr_time_ms-LargeQueue.LOCK_EXPIRATION_MS),
predexp.integer_less(),
predexp.predexp_or(2) ]
ops = [ op_helpers.read('head-offset'),
op_helpers.read('tail-offset'),
op_helpers.write('locked', 1),
op_helpers.write('lock-owner', txn_id),
op_helpers.write('lock-time-ms', curr_time_ms) ]
if op == LargeQueue.Ops.Enqueue:
ops.append(op_helpers.increment('fencing-ctr', 1))
ops.append(op_helpers.read('fencing-ctr'))
try:
_, _, record = self.client.operate(metadata_key, ops, policy={'predexp': predexps})
except exception.FilteredOut as ex: # predexp failed
time.sleep(LargeQueue.LOCK_POLL_WAIT_MS/1000.0)
continue
return record
raise ASAborted('Failed to acquire lock')
def test_background_execute_with_ops(self):
"""
Ensure that Scan.execute_background() applies ops to all records
"""
test_bin = 'Stops'
keys = [(TEST_NS, TEST_SET, i) for i in range(50)]
scan = self.as_connection.scan(TEST_NS, TEST_SET)
# scan.apply(TEST_UDF_MODULE, TEST_UDF_FUNCTION, [test_bin])
ops = [operations.write(test_bin, 'new_val')]
scan.add_ops(ops)
job_id = scan.execute_background()
# Give time for the scan to finish
wait_for_job_completion(self.as_connection, job_id)
validate_records(self.as_connection, keys,
lambda rec: rec[test_bin] == 'new_val')
def test_pos_operate_write_set_to_aerospike_null(self):
"""
Invoke operate() with write command with bin set to aerospike_null
"""
key = ('test', 'demo', 'null_record')
bins = {"name": "John", "no": 3}
self.as_connection.put(key, bins)
llist = [
operations.write("no", aerospike.null()),
operations.read("no")
]
_, _, bins = self.as_connection.operate(key, llist)
assert {} == bins
self.as_connection.remove(key)
def test_pos_operate_write_set_to_aerospike_null(self):
"""
Invoke operate() with write command with bin set to aerospike_null
"""
key = ('test', 'demo', 'null_record')
bins = {"name": "John", "no": 3}
self.as_connection.put(key, bins)
llist = [
operations.write("no", aerospike.null()),
operations.read("no")
]
_, _, bins = self.as_connection.operate(key, llist)
assert {} == bins
self.as_connection.remove(key)
def test_background_execute_with_ops(self, clean_test_background):
"""
Ensure that Query.execute_background() applies ops to all records
"""
test_bin = 'tops'
keys = [(TEST_NS, TEST_SET, i) for i in range(500)]
query = self.as_connection.query(TEST_NS, TEST_SET)
# query.apply(TEST_UDF_MODULE, TEST_UDF_FUNCTION, [test_bin])
ops = [operations.write(test_bin, 'new_val')]
query.add_ops(ops)
query.execute_background()
# Give time for the query to finish
time.sleep(5)
validate_records(self.as_connection, keys,
lambda rec: rec[test_bin] == 'new_val')
def test_background_execute_with_ops_and_preds(self, clean_test_background):
"""
Ensure that Query.execute_background() applies ops to records that match the predicate
"""
test_bin = 't1'
keys = [(TEST_NS, TEST_SET, i) for i in range(500)]
query = self.as_connection.query(TEST_NS, TEST_SET)
number_predicate = predicates.equals('number', 3)
# query.apply(TEST_UDF_MODULE, TEST_UDF_FUNCTION, [test_bin])
ops = [
operations.append(test_bin, 'new_val')
]
query.add_ops(ops)
query.where(number_predicate)
query.execute_background()
# Give time for the query to finish
time.sleep(5)
_, _, num_5_record = self.as_connection.get((TEST_NS, TEST_SET, 5))
assert num_5_record.get(test_bin) is None
_, _, num_3_record = self.as_connection.get((TEST_NS, TEST_SET, 3))
assert num_3_record[test_bin] == 'new_val'
# cleanup
ops = [
operations.write(test_bin, 'aerospike')
]
query = self.as_connection.query(TEST_NS, TEST_SET)
query.add_ops(ops)
query.execute_background()
time.sleep(3)
validate_records(
self.as_connection, keys,
lambda rec: rec[test_bin] == 'aerospike'
)
def test_background_execute_with_ops_and_predexp(self,
clean_test_background):
"""
Ensure that Query.execute_background() applies ops to records that match the predexp
"""
test_bin = 'tops_preds'
keys = [(TEST_NS, TEST_SET, i) for i in range(500)]
query = self.as_connection.query(TEST_NS, TEST_SET)
# query.apply(TEST_UDF_MODULE, TEST_UDF_FUNCTION, [test_bin])
ops = [operations.write(test_bin, 'new_val')]
predexp = [
as_predexp.integer_bin('number'),
as_predexp.integer_value(2),
as_predexp.integer_equal(),
as_predexp.integer_bin('number'),
as_predexp.integer_value(3),
as_predexp.integer_equal(),
as_predexp.predexp_or(2)
]
policy = {'predexp': predexp}
query.add_ops(ops)
query.execute_background(policy)
# Give time for the query to finish
time.sleep(5)
for key in keys:
_, _, bins = self.as_connection.get(key)
if bins['number'] == 2 or bins['number'] == 3:
assert (bins[test_bin] == 'new_val')
else:
assert (bins.get(test_bin) is None)
def enqueue(self, entry, txn_id, overwrite_if_full=False):
"""
Append a new entry to the queue. Fails if the queue lock cannot be acquired. Can fail if the queue is full.
If the fencing counter has wrapped around, reset all fencing values.
:param entry: new entry to be enqueued
:param txn_id: lock owner id, must be unique among concurrent requests
:param overwrite_if_full: flag indicating if the head position should be overwritten if the queue is full
:return: Offset position of the enqueued entry. throws: ASAborted('Queue is full'), ASAborted('Timed out')
"""
q_state = self._lock(txn_id, LargeQueue.Ops.Enqueue)
# compute the record and list indices
head_offset = long(q_state['head-offset'])
tail_offset = long(q_state['tail-offset'])
fencing_ctr = q_state['fencing-ctr']
# if the fencing counter has a non-positive value, it has wrapped around past the max value; reset
if fencing_ctr <= 0:
self._reset_fencing_marks()
buf_rec_index, entry_index = self._get_entry_location(tail_offset) # tail points to where the new entry will go
entry_val = {'offset': tail_offset, 'entry': entry}
queue_is_full = self._queue_is_full(head_offset, tail_offset)
if queue_is_full and not overwrite_if_full:
self._commit_release(txn_id)
raise ASAborted('Queue is full')
predexps = [ predexp.integer_bin("fencing-mark"),
predexp.integer_value(fencing_ctr),
predexp.integer_less() ]
ops = [ op_helpers.write('fencing-mark', fencing_ctr),
list_helpers.list_set('entries', entry_index, entry_val) ]
buf_rec_key = (self.namespace, self.name, LargeQueue._buf_record_key(buf_rec_index))
try:
(_, _, record) = self.client.operate(buf_rec_key, ops, policy={'predexp': predexps})
except exception.FilteredOut as ex:
raise ASAborted('Timed out')
self._commit_release(txn_id, new_head_offset=head_offset + 1 if queue_is_full else None,
new_tail_offset=tail_offset + 1)
return tail_offset
print("Error: {0} [{1}]".format(e.msg, e.code))
sys.exit(2)
key = (options.namespace, options.set, "list-remove-by-rank")
try:
client.remove(key)
except ex.RecordError as e:
pass
try:
print("\nremove_by_rank(bin, rank[, returnType, context])\n")
sample = [6, 10, 5, 11, 4, 12, 3]
# set the list value, try the VALUE return type
ops = [
operations.write("l", sample),
operations.read("l"),
listops.list_remove_by_rank("l", 0, aerospike.LIST_RETURN_VALUE),
]
key, metadata, bins = client.operate_ordered(key, ops)
print("\n{}\nremove_by_rank('l', 0, VALUE): {}".format(bins[0][1], bins[1][1]))
# [6, 10, 5, 11, 4, 12, 3]
# remove_by_rank('l', 0, VALUE): 3
# reset the list value, try the INDEX return type
ops = [
operations.write("l", sample),
operations.read("l"),
listops.list_remove_by_rank("l", 0, aerospike.LIST_RETURN_INDEX),
]
key, metadata, bins = client.operate_ordered(key, ops)
class TestOperate(object):
def setup_class(cls):
"""
Setup class.
"""
# hostlist, user, password = TestBaseClass.get_hosts()
# config_no_typechecks = {'hosts': hostlist, 'strict_types': False}
# if user is None and password is None:
# TestOperate.client_no_typechecks = aerospike.client(
# config_no_typechecks).connect()
# else:
# TestOperate.client_no_typechecks = aerospike.client(
# config_no_typechecks).connect(user, password)
cls.client_no_typechecks = TestBaseClass.get_new_connection(
{'strict_types': False})
def teardown_class(cls):
TestOperate.client_no_typechecks.close()
@pytest.fixture(autouse=True)
def setup(self, request, as_connection):
"""
Setup Method
"""
keys = []
for i in range(5):
key = ('test', 'demo', i)
rec = {'name': 'name%s' % (str(i)), 'age': i}
as_connection.put(key, rec)
key = ('test', 'demo', 6)
rec = {"age": 6.3}
as_connection.put(key, rec)
keys.append(key)
key = ('test', 'demo', 'bytearray_key')
rec = {"bytearray_bin": bytearray("asd;as[d'as;d", "utf-8")}
as_connection.put(key, rec)
keys.append(key)
key = ('test', 'demo', 'bytes_key')
rec = {"bytes_bin": b''}
as_connection.put(key, rec)
keys.append(key)
key = ('test', 'demo', 'list_key')
rec = {"int_bin": [1, 2, 3, 4], "string_bin": ['a', 'b', 'c', 'd']}
as_connection.put(key, rec)
keys.append(key)
key = ('test', 'demo', 'existing_key')
rec = {
"dict": {
"a": 1
},
"bytearray": bytearray("abc", "utf-8"),
"float": 3.4,
"list": ['a']
}
as_connection.put(key, rec)
keys.append(key)
def teardown():
"""
Teardown Method
"""
for i in range(5):
key = ('test', 'demo', i)
try:
as_connection.remove(key)
except e.RecordNotFound:
pass
for key in keys:
try:
as_connection.remove(key)
except e.RecordNotFound:
pass
request.addfinalizer(teardown)
@pytest.mark.parametrize(
"key, llist, expected",
[
(('test', 'demo', 1), [
operations.prepend("name", u"ram"),
operations.increment("age", 3),
operations.read("name")
], {
'name': 'ramname1'
}),
(
('test', 'demo', 1), # with_write_float_value
[
operations.write("write_bin", {"no": 89.8}),
operations.read("write_bin")
],
{
'write_bin': {
u'no': 89.8
}
}),
(
('test', 'demo', 1), # write positive
[
operations.write("write_bin", {"no": 89}),
operations.read("write_bin")
],
{
'write_bin': {
u'no': 89
}
}),
(
('test', 'demo', 1), # write_tuple_positive
[
operations.write("write_bin", ('a', 'b', 'c')),
operations.read("write_bin")
],
{
'write_bin': ('a', 'b', 'c')
}),
(
('test', 'demo', 1), # with_bin_bytearray
[
operations.prepend("asd[;asjk", "ram"),
operations.read("asd[;asjk")
],
{
'asd[;asjk': 'ram'
}),
(
('test', 'demo',
'bytearray_key'), # with_operator append_val bytearray
[
operations.append("bytearray_bin", bytearray(
"abc", "utf-8")),
operations.read("bytearray_bin")
],
{
'bytearray_bin': bytearray("asd;as[d'as;dabc", "utf-8")
}),
(
('test', 'demo', 'bytearray_new'
), # with_operator append_val bytearray_newrecord
[
operations.append("bytearray_bin",
bytearray("asd;as[d'as;d", "utf-8")),
operations.read("bytearray_bin")
],
{
'bytearray_bin': bytearray("asd;as[d'as;d", "utf-8")
}),
(
('test', 'demo',
'bytes_key'), # with_operator append_val bytes
[
operations.append("bytes_bin", b"abc"),
operations.read("bytes_bin")
],
{
'bytes_bin': b'abc'
}),
(
('test', 'demo',
'bytes_new'), # with_operator append_val bytes_newrecord
[
operations.append("bytes_bin", b"asd;as[d'as;d"),
operations.read("bytes_bin")
],
{
'bytes_bin': b"asd;as[d'as;d"
}),
(
('test', 'demo',
'bytearray_key'), # with_operatorprepend_valbytearray
[
operations.prepend("bytearray_bin",
bytearray("abc", "utf-8")),
operations.read("bytearray_bin")
],
{
'bytearray_bin': bytearray("abcasd;as[d'as;d", "utf-8")
}),
(
('test', 'demo', 'bytearray_new'
), # with_operatorprepend_valbytearray_newrecord
[
operations.prepend("bytearray_bin",
bytearray("asd;as[d'as;d", "utf-8")),
operations.read("bytearray_bin")
],
{
'bytearray_bin': bytearray("asd;as[d'as;d", "utf-8")
}),
(
('test', 'demo',
'bytes_key'), # with_operator prepend_val bytes
[
operations.prepend("bytes_bin", b"abc"),
operations.read("bytes_bin")
],
{
'bytes_bin': b'abc'
}),
(
('test', 'demo',
'bytes_new'), # with_operator prepend_val bytes_newrecord
[
operations.prepend("bytes_bin", b"asd;as[d'as;d"),
operations.read("bytes_bin")
],
{
'bytes_bin': b"asd;as[d'as;d"
})
])
def test_pos_operate_with_correct_paramters(self, key, llist, expected):
"""
Invoke operate() with correct parameters
"""
key, _, bins = self.as_connection.operate(key, llist)
assert bins == expected
self.as_connection.remove(key)
@pytest.mark.parametrize("key, llist, expected", [
(('test', 'demo', 1), [
operations.write("write_bin", {"no": 89.8}),
operations.write("write_bin2", {"no": 100}),
operations.delete(),
], {}),
])
def test_pos_operate_delete_with_correct_paramters(self, key, llist,
expected):
"""
Invoke operate() with correct parameters
"""
key, _, bins = self.as_connection.operate(key, llist)
assert bins == expected
def test_pos_operate_with_increment_positive_float_value(self):
"""
Invoke operate() with correct parameters
"""
if TestOperate.skip_old_server is True:
pytest.skip("Server does not support increment on float type")
key = ('test', 'demo', 6)
llist = [operations.increment("age", 3.5), operations.read("age")]
_, _, bins = self.as_connection.operate(key, llist)
assert bins == {'age': 9.8}
def test_pos_operate_with_correct_policy(self):
"""
Invoke operate() with correct policy
"""
key = ('test', 'demo', 1)
policy = {
'timeout': 1000,
'key': aerospike.POLICY_KEY_SEND,
'commit_level': aerospike.POLICY_COMMIT_LEVEL_MASTER
}
llist = [
operations.append("name", "aa"),
operations.increment("age", 3),
operations.read("name")
]
_, _, bins = self.as_connection.operate(key, llist, {}, policy)
assert bins == {'name': 'name1aa'}
self.as_connection.remove(key)
@pytest.mark.parametrize("key, policy, meta, llist", [
(('test', 'demo', 1), {
'total_timeout': 1000,
'key': aerospike.POLICY_KEY_SEND,
'gen': aerospike.POLICY_GEN_IGNORE,
'commit_level': aerospike.POLICY_COMMIT_LEVEL_ALL
}, {
'gen': 10,
'ttl': 1200
}, [
operations.append("name", "aa"),
operations.increment("age", 3),
operations.read("name")
]),
])
def test_pos_operate_with_policy_gen_ignore(self, key, policy, meta,
llist):
"""
Invoke operate() with gen ignore.
"""
key, meta, bins = self.as_connection.operate(key, llist, meta, policy)
assert bins == {'name': 'name1aa'}
def test_pos_operate_with_policy_gen_EQ(self):
"""
Invoke operate() with gen EQ positive.
"""
key = ('test', 'demo', 1)
policy = {'gen': aerospike.POLICY_GEN_EQ}
(key, meta) = self.as_connection.exists(key)
gen = meta['gen']
meta = {'gen': gen}
llist = [
operations.append("name", "aa"),
operations.increment("age", 3),
operations.read("name")
]
(key, meta, bins) = self.as_connection.operate(key, llist, meta,
policy)
assert bins == {'name': 'name1aa'}
@pytest.mark.parametrize("key, llist",
[(('test', 'demo', 1), [operations.touch(4000)]),
(('test', 'demo', 1), [operations.touch(4000)])])
def test_pos_operate_touch_operation_with_bin_and_value_combination(
self, key, llist):
"""
Invoke operate() with touch value with bin and value combination.
"""
self.as_connection.operate(key, llist)
(key, meta) = self.as_connection.exists(key)
assert meta['ttl'] is not None
def test_pos_operate_with_policy_gen_EQ_not_equal(self):
"""
Invoke operate() with a mismatched generation, and verify
it does not succeed
"""
key = ('test', 'demo', 1)
policy = {'gen': aerospike.POLICY_GEN_EQ}
(_, meta) = self.as_connection.exists(key)
gen = meta['gen']
meta = {
'gen': gen + 5,
}
llist = [
operations.append("name", "aa"),
operations.increment("age", 3),
]
with pytest.raises(e.RecordGenerationError):
self.as_connection.operate(key, llist, meta, policy)
_, _, bins = self.as_connection.get(key)
assert bins == {"age": 1, 'name': 'name1'}
def test_pos_operate_with_policy_gen_GT_mismatch(self):
"""
Invoke operate() with gen GT policy, with amatching generation
then verify that the operation was rejected properly
"""
key = ('test', 'demo', 1)
policy = {'gen': aerospike.POLICY_GEN_GT}
_, meta = self.as_connection.exists(key)
gen = meta['gen']
meta = {'gen': gen}
llist = [
operations.append("name", "aa"),
operations.increment("age", 3)
]
with pytest.raises(e.RecordGenerationError):
self.as_connection.operate(key, llist, meta, policy)
_, _, bins = self.as_connection.get(key)
assert bins == {'age': 1, 'name': 'name1'}
def test_pos_operate_touch_with_meta(self):
"""
Invoke operate() OPERATE_TOUCH using meta to pass in ttl.
"""
key = ('test', 'demo', 1)
meta = {'ttl': 1200}
llist = [operations.touch()]
self.as_connection.operate(key, llist, meta)
_, meta = self.as_connection.exists(key)
assert meta['ttl'] <= 1200 and meta['ttl'] >= 1150
def test_pos_operate_with_policy_gen_GT(self):
"""
Invoke operate() with gen GT positive.
"""
key = ('test', 'demo', 1)
policy = {'gen': aerospike.POLICY_GEN_GT}
_, meta = self.as_connection.exists(key)
gen = meta['gen']
meta = {'gen': gen + 5}
llist = [
operations.append("name", "aa"),
operations.increment("age", 3),
operations.read("name")
]
_, _, bins = self.as_connection.operate(key, llist, meta, policy)
assert bins == {'name': 'name1aa'}
def test_pos_operate_with_nonexistent_key(self):
"""
Invoke operate() with non-existent key
"""
new_key = ('test', 'demo', "key11")
llist = [operations.prepend("loc", "mumbai"), operations.read("loc")]
_, _, bins = self.as_connection.operate(new_key, llist)
assert bins == {'loc': 'mumbai'}
self.as_connection.remove(new_key)
def test_pos_operate_with_nonexistent_bin(self):
"""
Invoke operate() with non-existent bin
"""
key = ('test', 'demo', 1)
llist = [operations.append("addr", "pune"), operations.read("addr")]
_, _, bins = self.as_connection.operate(key, llist)
assert bins == {'addr': 'pune'}
def test_pos_operate_increment_nonexistent_key(self):
"""
Invoke operate() with increment with nonexistent_key
"""
key = ('test', 'demo', "non_existentkey")
llist = [operations.increment("age", 5)]
self.as_connection.operate(key, llist)
_, _, bins = self.as_connection.get(key)
assert bins == {"age": 5}
self.as_connection.remove(key)
def test_pos_operate_with_correct_paramters_without_connection(self):
"""
Invoke operate() with correct parameters without connection
"""
key = ('test', 'demo', 1)
config = {'hosts': [('127.0.0.1', 3000)]}
client1 = aerospike.client(config)
llist = [operations.touch()]
with pytest.raises(e.ClusterError):
client1.operate(key, llist)
def test_pos_operate_write_set_to_aerospike_null(self):
"""
Invoke operate() with write command with bin set to aerospike_null
"""
key = ('test', 'demo', 'null_record')
bins = {"name": "John", "no": 3}
self.as_connection.put(key, bins)
llist = [
operations.write("no", aerospike.null()),
operations.read("no")
]
_, _, bins = self.as_connection.operate(key, llist)
assert {} == bins
self.as_connection.remove(key)
@pytest.mark.parametrize(
"key, llist, expected",
[
(
('test', 'demo', 'prepend_int'), # prepend_with_int
[operations.prepend("age", 4),
operations.read("age")],
{
'age': 4
}),
(
('test', 'demo', 'append_dict'), # append_with_dict
[
operations.append("dict", {
"a": 1,
"b": 2
}),
operations.read("dict")
],
{
'dict': {
"a": 1,
"b": 2
}
}),
(
('test', 'demo', 'incr_string'), # incr_with_string
[
operations.increment("name", "aerospike"),
operations.read("name")
],
{
'name': 'aerospike'
}),
])
def test_pos_operate_new_record(self, key, llist, expected):
"""
Invoke operate() with prepend command on a new record
"""
_, _, bins = TestOperate.client_no_typechecks.operate(key, llist)
assert expected == bins
TestOperate.client_no_typechecks.remove(key)
@pytest.mark.parametrize(
"key, llist",
[
(('test', 'demo', 1),
[operations.prepend("age", 4),
operations.read("age")]),
(
('test', 'demo', 'existing_key'), # Existing list
[operations.prepend("list", ['c']),
operations.read("list")]),
(
('test', 'demo', 'existing_key'), # Existing dict
[operations.append("dict", {"c": 2}),
operations.read("dict")]),
(
('test', 'demo', 'existing_key'), # Exiting float
[operations.append("float", 3.4),
operations.read("float")]),
(
('test', 'demo', 1), # Existing string
[
operations.increment("name", "aerospike"),
operations.read("name")
]),
(
('test', 'demo', 'existing_key'), # Existing Bytearray
[
operations.increment("bytearray", bytearray(
"abc", "utf-8")),
operations.read("bytearray")
]),
])
def test_pos_operate_prepend_with_existing_record(self, key, llist):
"""
Invoke operate() with prepend command on a existing record
"""
with pytest.raises(e.BinIncompatibleType):
TestOperate.client_no_typechecks.operate(key, llist)
TestOperate.client_no_typechecks.remove(key)
def test_pos_operate_incr_with_geospatial_new_record(self):
"""
Invoke operate() with incr command on a new record
"""
key = ('test', 'demo', 'geospatial_key')
llist = [
operations.increment(
"geospatial",
aerospike.GeoJSON({
"type": "Point",
"coordinates": [42.34, 58.62]
})),
operations.read("geospatial")
]
_, _, bins = TestOperate.client_no_typechecks.operate(key, llist)
assert bins['geospatial'].unwrap() == {
'coordinates': [42.34, 58.62],
'type': 'Point'
}
TestOperate.client_no_typechecks.remove(key)
def test_pos_operate_with_bin_length_extra_nostricttypes(self):
"""
Invoke operate() with bin length extra. Strict types disabled
"""
key = ('test', 'demo', 1)
max_length = 'a' * 21
llist = [
operations.prepend("name", "ram"),
operations.increment(max_length, 3)
]
TestOperate.client_no_typechecks.operate(key, llist)
_, _, bins = TestOperate.client_no_typechecks.get(key)
assert bins == {"name": "ramname1", "age": 1}
def test_pos_operate_prepend_set_to_aerospike_null(self):
"""
Invoke operate() with prepend command with bin set to aerospike_null
"""
key = ('test', 'demo', 'null_record')
bins = {"name": "John", "no": 3}
assert 0 == self.as_connection.put(key, bins)
(key, _, bins) = self.as_connection.get(key)
assert {"name": "John", "no": 3} == bins
llist = [
operations.prepend("no", aerospike.null()),
operations.read("no")
]
try:
(key, _, bins) = self.as_connection.operate(key, llist)
except e.InvalidRequest as exception:
assert exception.code == 4
self.as_connection.remove(key)
@pytest.mark.parametrize(
"list, result, bin, expected",
[
([
list_operations.list_append("int_bin", 7),
list_operations.list_get("int_bin", 4)
], {
"int_bin": 7
}, "int_bin", [1, 2, 3, 4, 7]),
([
list_operations.list_append_items("int_bin", [7, 9]),
list_operations.list_get_range("int_bin", 3, 3),
], {
'int_bin': [4, 7, 9]
}, "int_bin", [1, 2, 3, 4, 7, 9]),
([
list_operations.list_insert("int_bin", 2, 7),
list_operations.list_pop("int_bin", 2)
], {
'int_bin': 7
}, "int_bin", [1, 2, 3, 4]),
([
list_operations.list_insert_items("int_bin", 2, [7, 9]),
list_operations.list_pop_range("int_bin", 2, 2)
], {
'int_bin': [7, 9]
}, "int_bin", [1, 2, 3, 4]),
([
list_operations.list_set("int_bin", 2, 18),
list_operations.list_get("int_bin", 2)
], {
'int_bin': 18
}, "int_bin", [1, 2, 18, 4]),
(
[
list_operations.list_set("int_bin", 6, 10),
list_operations.list_get("int_bin", 6)
],
{
'int_bin': 10
},
"int_bin",
[1, 2, 3, 4, None, None, 10
] # Inserting outside of the range adds nils in between
)
])
def test_pos_operate_with_list_addition_operations(self, list, result, bin,
expected):
"""
Invoke operate() with list addition operations
"""
key = ('test', 'demo', 'list_key')
key, _, bins = self.as_connection.operate(key, list)
assert bins == result
key, _, bins = self.as_connection.get(key)
assert bins[bin] == expected
@pytest.mark.parametrize(
"list, bin, expected",
[([list_operations.list_remove("int_bin", 2)], "int_bin", [1, 2, 4]),
([list_operations.list_remove_range("int_bin", 2, 2)
], "int_bin", [1, 2]),
([list_operations.list_trim("int_bin", 2, 2)], "int_bin", [3, 4]),
([list_operations.list_clear("int_bin")], "int_bin", [])])
def test_pos_operate_with_list_remove_operations(self, list, bin,
expected):
"""
Invoke operate() with list remove operations
"""
key = ('test', 'demo', 'list_key')
self.as_connection.operate(key, list)
key, _, bins = self.as_connection.get(key)
assert bins[bin] == expected
def test_pos_operate_with_list_size(self):
"""
Invoke operate() with list_size operation
"""
key = ('test', 'demo', 'list_key')
list = [list_operations.list_size("int_bin")]
key, _, bins = self.as_connection.operate(key, list)
assert bins == {'int_bin': 4}
def test_list_increment_with_valid_value(self):
'''
previous list was [1, 2, 3, 4]
new should be [1, 2, 23, 4]
'''
key = ('test', 'demo', 'list_key')
list = [list_operations.list_increment("int_bin", 2, 20)]
_, _, bins = self.as_connection.operate(key, list)
assert bins == {'int_bin': 23}
_, _, bins = self.as_connection.get(key)
assert bins['int_bin'] == [1, 2, 23, 4]
def test_list_increment_with_incorrect_value_type(self):
'''
previous list was [1, 2, 3, 4]
new should be [1, 2, 23, 4]
'''
key = ('test', 'demo', 'list_key')
list = [list_operations.list_increment("int_bin", 2, "twenty")]
with pytest.raises(e.AerospikeError):
self.as_connection.operate(key, list)
def test_pos_operate_with_list_get_range_val_out_of_bounds(self):
"""
Invoke operate() with list_get_range operation and value out of bounds
"""
key = ('test', 'demo', 'list_key')
list = [list_operations.list_get_range("int_bin", 2, 9)]
_, _, bins = self.as_connection.operate(key, list)
assert bins == {'int_bin': [3, 4]}
def test_pos_operate_with_list_trim_val_with_negative_value(self):
"""
Invoke operate() with list_trimoperation and value is negative
"""
key = ('test', 'demo', 'list_key')
list = [list_operations.list_trim("int_bin", 1, -9)]
self.as_connection.operate(key, list)
_, _, bins = self.as_connection.get(key)
assert bins['int_bin'] == [2, 3, 4]
def test_pos_operate_with_list_insert_index_negative(self):
"""
Invoke operate() with list_insert and item index is a negative value
"""
key = ('test', 'demo', 'list_key')
list = [list_operations.list_insert("int_bin", -2, 9)]
self.as_connection.operate(key, list)
_, _, bins = self.as_connection.get(key)
assert bins['int_bin'] == [1, 2, 9, 3, 4]
@pytest.mark.parametrize("list, result, bin, expected", [
([
list_operations.list_append("string_bin", {"new_val": 1}),
list_operations.list_get("string_bin", 4)
], {
"string_bin": {
"new_val": 1
}
}, "string_bin", ['a', 'b', 'c', 'd', {
'new_val': 1
}]),
([
list_operations.list_append_items("string_bin", [['z', 'x'],
('y', 'w')]),
list_operations.list_get_range("string_bin", 3, 3)
], {
"string_bin": ['d', ['z', 'x'], ('y', 'w')]
}, "string_bin", ['a', 'b', 'c', 'd', ['z', 'x'], ('y', 'w')]),
([
list_operations.list_insert("string_bin", 2, True),
list_operations.list_pop("string_bin", 2)
], {
'string_bin': True
}, "string_bin", ['a', 'b', 'c', 'd']),
([
list_operations.list_insert_items(
"string_bin", 2, [bytearray("abc", "utf-8"), u"xyz"]),
list_operations.list_pop_range("string_bin", 2, 2)
], {
'string_bin': [bytearray(b'abc'), 'xyz']
}, "string_bin", ['a', 'b', 'c', 'd']),
])
def test_pos_operate_with_list_operations_different_datatypes(
self, list, result, bin, expected):
"""
Invoke operate() with list operations using different datatypes
"""
key = ('test', 'demo', 'list_key')
_, _, bins = self.as_connection.operate(key, list)
assert bins == result
_, _, bins = self.as_connection.get(key)
assert bins[bin] == expected
# Negative Tests
def test_neg_operate_with_no_parameters(self):
"""
Invoke operate() without any mandatory parameters.
"""
with pytest.raises(TypeError) as typeError:
self.as_connection.operate()
assert "key" in str(typeError.value)
@pytest.mark.parametrize("key, llist, expected",
[(('test', 'demo', 'bad_key'), [
operations.delete(),
], e.RecordNotFound)])
def test_pos_operate_delete_with_incorrect_paramters(
self, key, llist, expected):
"""
Invoke operate() with correct parameters
"""
with pytest.raises(expected):
self.as_connection.operate(key, llist)
def test_neg_operate_list_operation_bin_notlist(self):
"""
Invoke operate() with a list operation and bin does not contain list
"""
key = ('test', 'demo', 1)
list = [list_operations.list_insert("age", 2, 9)]
with pytest.raises(e.BinIncompatibleType):
self.as_connection.operate(key, list)
def test_neg_operate_append_items_not_a_list(self):
"""
Invoke operate() with list addition operations negative
"""
key = ('test', 'demo', 1)
ops = [list_operations.list_append_items("int_bin", 7)]
with pytest.raises(e.ParamError):
self.as_connection.operate(key, ops)
@pytest.mark.parametrize("list",
[([list_operations.list_get("int_bin", 7)]),
([
list_operations.list_clear("int_bin"),
list_operations.list_pop("int_bin", 2)
]),
([
list_operations.list_clear("int_bin"),
list_operations.list_remove("int_bin", 2)
])])
def test_neg_operate_list_invalid_requests(self, list):
"""
Invoke operate() with list addition operations negative
"""
key = ('test', 'demo', 'list_key')
with pytest.raises(e.OpNotApplicable):
self.as_connection.operate(key, list)
def test_neg_operate_with_bin_length_extra(self):
"""
Invoke operate() with bin length extra. Strict types enabled
"""
key = ('test', 'demo', 1)
max_length = 'a' * 21
llist = [
operations.prepend("name", "ram"),
operations.increment(max_length, 3)
]
with pytest.raises(e.BinNameError):
self.as_connection.operate(key, llist)
def test_neg_operate_empty_string_key(self):
"""
Invoke operate() with empty string key
"""
llist = [operations.prepend("name", "ram")]
with pytest.raises(e.ParamError):
self.as_connection.operate("", llist)
def test_neg_operate_with_extra_parameter(self):
"""
Invoke operate() with extra parameter.
"""
key = ('test', 'demo', 1)
policy = {'timeout': 1000}
llist = [operations.prepend("name", "ram")]
with pytest.raises(TypeError) as typeError:
self.as_connection.operate(key, llist, {}, policy, "")
def test_neg_operate_policy_is_string(self):
"""
Invoke operate() with policy is string
"""
key = ('test', 'demo', 1)
llist = [operations.prepend("name", "ram")]
try:
self.as_connection.operate(key, llist, {}, "")
except e.ParamError as exception:
assert exception.code == -2
def test_neg_operate_key_is_none(self):
"""
Invoke operate() with key is none
"""
llist = [operations.prepend("name", "ram")]
try:
self.as_connection.operate(None, llist)
except e.ParamError as exception:
assert exception.code == -2
def test_neg_operate_append_value_integer(self):
"""
Invoke operate() with append value is of type integer
"""
key = ('test', 'demo', 1)
llist = [operations.append("name", 12)]
try:
self.as_connection.operate(key, llist)
except e.ParamError as exception:
assert exception.code == -2
def test_neg_operate_with_incorrect_polic(self):
"""
Invoke operate() with incorrect policy
"""
key = ('test', 'demo', 1)
policy = {'total_timeout': 0.5}
llist = [operations.prepend("name", "ram")]
with pytest.raises(e.ParamError):
self.as_connection.operate(key, llist, {}, policy)
bins[2][1]))
# get_all_by_value_list('l', [1,2], REVERSE_INDEX): [6, 4, 3, 2, 0]
print("get_all_by_value_list('l', [1,2], RANK): {}".format(bins[3][1]))
# get_all_by_value_list('l', [1,2], RANK): [0, 1, 2, 3, 4]
print("get_all_by_value_list('l', [1,2], REVERSE_RANK): {}".format(
bins[4][1]))
# get_all_by_value_list('l', [1,2], REVERSE_RANK): [6, 5, 4, 3, 2]
print("get_all_by_value_list('l', [1,2], COUNT): {}".format(bins[5][1]))
# get_all_by_value_list('l', [1,2], COUNT): 5
# switch to more complex values, a list of (mostly) ordered pairs
tuples = [["v1", 1], ["v2", 1], ["v1", 2], ["v2", 2], ["v1", 3, "z", 8],
["v1"]]
wildcard = aerospike.CDTWildcard()
ops = [
operations.write("l", tuples),
operations.read("l"),
listops.list_get_by_value_list("l", [["v1"]],
aerospike.LIST_RETURN_INDEX),
listops.list_get_by_value_list("l", [["v1"], ["v1", wildcard]],
aerospike.LIST_RETURN_VALUE),
listops.list_get_by_value_list("l", [["v1"], ["v1", wildcard]],
aerospike.LIST_RETURN_VALUE,
inverted=True),
listops.list_get_by_value_list("l", [["v1", 3, wildcard]],
aerospike.LIST_RETURN_VALUE),
listops.list_get_by_value_list("l", [[wildcard, 2]],
aerospike.LIST_RETURN_VALUE),
listops.list_get_by_value_list("l", [["z"], ["y"]],
aerospike.LIST_RETURN_COUNT),
]
client = aerospike.client(config).connect(options.username, options.password)
except ex.ClientError as e:
print("Error: {0} [{1}]".format(e.msg, e.code))
sys.exit(2)
key = (options.namespace, options.set, "list-remove-by-value-rel-rank-range")
try:
client.remove(key)
except ex.RecordError as e:
pass
try:
print("\nremove_by_value_rel_rank_range(bin, value, rank, returnType[, count])\n")
ops = [
operations.write("l", [2, 4, 7, 3, 8, 9, 26, 11]),
operations.read("l"),
lops.list_remove_by_value_rank_range_relative(
"l", 5, -1, aerospike.LIST_RETURN_VALUE, 2
),
operations.read("l"),
]
key, metadata, bins = client.operate_ordered(key, ops)
print(
"\n{}\nremove_by_value_rel_rank_range('l', 5, -1, VALUE, 2): {}\n{}".format(
bins[0][1], bins[1][1], bins[2][1]
)
)
# [2, 4, 7, 3, 8, 9, 26, 11]
# remove_by_value_rel_rank_range('l', 5, -1, VALUE, 2): [7, 4]
# [2, 3, 8, 9, 26, 11]
print("Error: {0} [{1}]".format(e.msg, e.code))
sys.exit(2)
key = (options.namespace, options.set, "list-remove-by-value-interval")
try:
client.remove(key)
except ex.RecordError as e:
pass
try:
print(
"\nremove_by_value_interval(bin, valueStart, valueStop[, returnType, context])\n"
)
# set the list, remove_by_value_interval('l', 2, VALUE, 4)
ops = [
operations.write("l", [1, 2, 3, 4, 5, 4, 3, 2, 1]),
operations.read("l"),
listops.list_remove_by_value_range("l", aerospike.LIST_RETURN_VALUE, 2,
4),
operations.read("l"),
]
key, metadata, bins = client.operate_ordered(key, ops)
print("\n{}\nremove_by_value_interval('l', 2, VALUE, 4): {}\n{}".format(
bins[0][1], bins[1][1], bins[2][1]))
# [1, 2, 3, 4, 5, 4, 3, 2, 1]
# remove_by_value_interval('l', 2, VALUE, 4): [2, 3, 3, 2]
# [1, 4, 5, 4, 1]
# set the list, remove_by_value_interval('l', 2, INDEX, 4)
ops = [
operations.write("l", [1, 2, 3, 4, 5, 4, 3, 2, 1]),
