def test_xattrs_basic(self):
cb = self.cb
k = self.gen_key('xattrs' + random().__str__())
cb.upsert(k, {})
verbs = ((SD.insert, True), (SD.upsert, True), (SD.replace, False))
for verb in verbs:
kwargs = dict(create_parents=True) if verb[1] else {}
# Operate on a single xattr
rv = cb.mutate_in(
k, [verb[0]('my.attr', 'value', xattr=True, **kwargs)])
self.assertTrue(rv.success)
body = cb.get(k)
self.assertFalse('my' in body.content)
self.assertFalse('my.attr' in body.content)
# Try using lookup_in
rv = cb.lookup_in(k, (SD.get('my.attr'), ))
self.assertRaises(PathNotFoundException, rv.exists, 0)
# Finally, use lookup_in with 'xattrs' attribute enabled
rv = cb.lookup_in(k, (SD.get('my.attr', xattr=True), ))
self.assertTrue(rv.exists(0))
self.assertEqual('value', rv.content_as[str](0))
def test_eventing_does_not_use_xattrs(self):
body = self.create_save_function_body(
self.function_name,
HANDLER_CODE.BUCKET_OPS_WITH_TIMERS,
dcp_stream_boundary="from_now")
# deploy eventing function
self.deploy_function(body)
url = 'couchbase://{ip}/{name}'.format(ip=self.master.ip,
name=self.src_bucket_name)
bucket = Bucket(url, username="******", password="******")
for docid in ['customer123', 'customer1234', 'customer12345']:
bucket.upsert(docid, {'a': 1})
self.verify_eventing_results(self.function_name,
3,
skip_stats_validation=True)
# add new multiple xattrs , delete old xattrs and delete the documents
for docid in ['customer123', 'customer1234', 'customer12345']:
r = bucket.mutate_in(docid, SD.get('eventing', xattr=True))
log.info(r)
if "Could not execute one or more multi lookups or mutations" not in str(
r):
self.fail("eventing is still using xattrs for timers")
r = bucket.mutate_in(docid, SD.get('_eventing', xattr=True))
log.info(r)
if "Could not execute one or more multi lookups or mutations" not in str(
r):
self.fail("eventing is still using xattrs for timers")
self.undeploy_and_delete_function(body)
def test_multi_lookup(self):
cb = self.cb
key = self.gen_key('sdmlookup')
cb.upsert(
key, {
'field1': 'value1',
'field2': 'value2',
'array': [1, 2, 3],
'boolean': False
})
rvs = cb.lookup_in(key,
SD.get('field1'),
SD.exists('field2'),
SD.exists('field3'),
quiet=True)
self.assertFalse(rvs.success)
self.assertEqual(3, rvs.result_count)
self.assertEqual((0, 'value1'), rvs.get(0))
self.assertEqual((0, 'value1'), rvs.get('field1'))
self.assertEqual('value1', rvs[0])
self.assertEqual('value1', rvs['field1'])
self.assertEqual((0, None), rvs.get(1))
self.assertEqual((0, None), rvs.get('field2'))
self.assertEqual(None, rvs[1])
self.assertEqual(None, rvs['field2'])
self.assertTrue(rvs.exists('field2'))
self.assertTrue(rvs.exists(1))
self.assertTrue(1 in rvs)
self.assertTrue('field2' in rvs)
self.assertEqual((E.SubdocPathNotFoundError.CODE, None),
rvs.get('field3'))
self.assertEqual((E.SubdocPathNotFoundError.CODE, None), rvs.get(2))
self.assertFalse(rvs.exists('field3'))
self.assertFalse(rvs.exists(2))
def _getix(rv_, ix):
return rv_[ix]
self.assertRaises(E.SubdocPathNotFoundError, _getix, rvs, 2)
self.assertRaises(E.SubdocPathNotFoundError, _getix, rvs, 'field3')
self.assertFalse(rvs.exists('field3'))
# See what happens when we mix operations
self.assertRaises(E.CouchbaseError, cb.lookup_in, key,
SD.get('field1'), SD.insert('a', 'b'))
# Empty path (invalid)
self.assertRaises(E.CouchbaseError, cb.lookup_in, SD.get(''))
def test_access_ok(self):
cb = self.cb
key = self.gen_key('non-exist')
try:
cb.lookup_in(key, SD.get('pth1'), quiet=True)
except E.NotFoundError as e:
rv = e.all_results[key]
self.assertFalse(rv.access_ok)
cb.upsert(key, {'hello': 'world'})
rv = cb.lookup_in(key, SD.get('nonexist'))
self.assertTrue(rv.access_ok)
def test_multi_lookup(self):
cb = self.cb
key = self.gen_key('sdmlookup')
cb.upsert(key, {
'field1': 'value1',
'field2': 'value2',
'array': [1, 2, 3],
'boolean': False
})
rvs = cb.lookup_in(
key, SD.get('field1'), SD.exists('field2'), SD.exists('field3'),
quiet=True
)
self.assertFalse(rvs.success)
self.assertEqual(3, rvs.result_count)
self.assertEqual((0, 'value1'), rvs.get(0))
self.assertEqual((0, 'value1'), rvs.get('field1'))
self.assertEqual('value1', rvs[0])
self.assertEqual('value1', rvs['field1'])
self.assertEqual((0, None), rvs.get(1))
self.assertEqual((0, None), rvs.get('field2'))
self.assertEqual(None, rvs[1])
self.assertEqual(None, rvs['field2'])
self.assertTrue(rvs.exists('field2'))
self.assertTrue(rvs.exists(1))
self.assertTrue(1 in rvs)
self.assertTrue('field2' in rvs)
self.assertEqual((E.SubdocPathNotFoundError.CODE, None),
rvs.get('field3'))
self.assertEqual((E.SubdocPathNotFoundError.CODE, None),
rvs.get(2))
self.assertFalse(rvs.exists('field3'))
self.assertFalse(rvs.exists(2))
def _getix(rv_, ix):
return rv_[ix]
self.assertRaises(E.SubdocPathNotFoundError, _getix, rvs, 2)
self.assertRaises(E.SubdocPathNotFoundError, _getix, rvs, 'field3')
self.assertFalse(rvs.exists('field3'))
# See what happens when we mix operations
self.assertRaises(E.CouchbaseError, cb.lookup_in, key,
SD.get('field1'), SD.insert('a', 'b'))
# Empty path (invalid)
self.assertRaises(E.CouchbaseError, cb.lookup_in, SD.get(''))
def test_access_ok(self):
cb = self.cb
key = self.gen_key('non-exist')
try:
cb.lookup_in(key, SD.get('pth1'), quiet=True)
except E.NotFoundError as e:
rv = e.all_results[key]
self.assertFalse(rv.access_ok)
cb.upsert(key, {'hello': 'world'})
rv = cb.lookup_in(key, SD.get('nonexist'))
self.assertTrue(rv.access_ok)
def test_lookup_in_timeout(self):
self.coll.upsert("id", {'someArray': ['wibble', 'gronk']})
# wait till it is there
self.try_n_times(10, 1, self.coll.get, "id")
# ok, it is there...
self.coll.get("id", GetOptions(project=["someArray"], timeout=timedelta(seconds=1.0)))
self.assertRaisesRegex(InvalidArgumentException, "Expected timedelta", self.coll.get, "id",
GetOptions(project=["someArray"], timeout=456))
sdresult_2 = self.coll.lookup_in("id", (SD.get("someArray"),), LookupInOptions(timeout=timedelta(microseconds=1)))
self.assertEqual(['wibble', 'gronk'],sdresult_2.content_as[list](0))
sdresult_2 = self.coll.lookup_in("id", (SD.get("someArray"),), LookupInOptions(timeout=timedelta(seconds=1)), timeout=timedelta(microseconds=1))
self.assertEqual(['wibble', 'gronk'],sdresult_2.content_as[list](0))
def test_scenarioF_subdoc(self):
item = Scenarios.AddressedUser("fred", 21, "45 Dupydaub Street")
self.coll.upsert("id", item)
subdoc = self.coll.get("id", project=("name", "age"))
user = subdoc.content_as[Scenarios.UserPartial]
altuser = self.coll.lookup_in("id", (SD.get("name"), SD.get("age")))
self.assertEqual("fred", altuser.content_as[str](0))
self.assertEqual(21, altuser.content_as[int](1))
changed = user.with_attr(age=25)
self.assertEqual(Scenarios.UserPartial("fred", 25), changed)
self.coll.mutate_in(subdoc.id, [MutateSpec().upsert("user", changed)])
def test_lookup_in_multiple_specs(self):
if self.is_mock:
raise SkipTest("mock doesn't support getting xattrs (like $document.expiry)")
cas = self.coll.upsert(self.KEY, {"a": "aaa", "b": {"c": {"d": "yo!"}}}).cas
self.try_n_times(10, 3, self._cas_matches, self.KEY, cas)
result = self.coll.lookup_in(self.KEY,
(SD.with_expiry(),
SD.get("a"),
SD.exists("b"),
SD.get("b.c")))
self.assertTrue(result.success)
self.assertIsNone(result.expiry)
self.assertEquals("aaa", result.content_as[str](1))
self.assertTrue(result.exists(2))
self.assertDictEqual({"d": "yo!"}, result.content_as[dict](3))
def registerPUT(studentId, quarterId, courseNum, offeringId):
sched_key = studentId + "-" + quarterId
# verify that offering is real
try:
offering = list(
offering_bucket.lookup_in(quarterId,
subdoc.get(courseNum + '.' +
offeringId)))[0]
except (SubdocPathNotFoundError, NotFoundError):
return log_make_response("Offering does not exist", 400)
# verify that student is real
try:
assert db.users.get_user(studentId)
except:
return log_make_response("User {} does not exist".format(studentId),
400)
# see if student is already enrolled
try:
sched = db.schedules.get_user_schedule(studentId, quarterId)
assert sched is None or courseNum not in sched['offerings']
print("User is not already enrolled in course")
except AssertionError:
return log_make_response("User already enrolled in course",
304) # type: Response
except NotFoundError:
pass
num_enrolled = offering['enrolled']
capacity = offering['capacity']
if num_enrolled >= capacity:
return log_make_response("Class is full/maximum capacity reached", 403)
# make user's schedule entry for that quarter if it doesn't exist
db.schedules.initialize_user_schedule(studentId, quarterId)
# try:
# sched_bucket.insert(sched_key, {"studentId": studentId, "quarterId": quarterId})
# except:
# pass
# should throw exception if already enrolled
db.schedules.add_course_to_sched(studentId, quarterId, courseNum,
offeringId)
data = {
'studentId': studentId,
'quarterId': quarterId,
'courseNum': courseNum,
'offeringId': offeringId
}
offerings.incr_enrollment_count(quarterId, courseNum, offeringId)
Neo4JPublisher().create_enrollment(data)
return log_make_response(
"Registered {} for {}: {}-{}".format(studentId, quarterId, courseNum,
offeringId), 201)
async def sub_doc_operations(collection):
key = "hotel_10025"
res = await collection.lookup_in(key, [SD.get("reviews[0].ratings")])
print("Review ratings: {}".format(res.content_as[dict](0)))
res = await collection.mutate_in(
key, [SD.replace("reviews[0].ratings.Rooms", 3.5)])
print("CAS: {}".format(res.cas))
def get_xattr(self, client_ip, sdk_conn, bucket_name):
try:
temp_conn, result = self.connection(client_ip, bucket_name, 'Administrator', 'password')
self.set_xattr(temp_conn)
k = 'sdk_1'
rv = sdk_conn.lookup_in(k, SD.get('my', xattr=True))
return True
except Exception as e:
log.info("Exception is from get_xattr function {0}".format(e))
return False
def get_xattr(self, client_ip, sdk_conn, bucket_name):
try:
temp_conn, result = self.connection(client_ip, bucket_name, 'Administrator', 'password')
self.set_xattr(temp_conn)
k = 'sdk_1'
rv = sdk_conn.lookup_in(k, SD.get('my', xattr=True))
return True
except Exception as e:
log.info("Exception is from get_xattr function {0}".format(e))
return False
def test_fulldoc(self):
cb = self.cb
key = self.gen_key('fulldoc')
cb.mutate_in(key,
SD.upsert_fulldoc({'val': True}),
SD.upsert('my.xattr',
'attrval',
create_parents=True,
xattr=True),
insert_doc=True)
self.assertEqual(True, cb.retrieve_in(key, 'val')[0])
self.assertEqual('attrval',
cb.lookup_in(key, SD.get('my.xattr', xattr=True))[0])
rv = cb.lookup_in(key, SD.get('my.xattr'))
self.assertFalse(rv.exists(0))
# Get the document back
rv = cb.lookup_in(key, SD.get_fulldoc())
self.assertEqual(True, rv[0]['val'])
def lookup_in(self, doc_id, path):
try:
result = self.bucket.lookup_in(doc_id, sd.get(path))
if result:
return result[0]
return None
except Exception as error:
print(
'Error performing KV lookupIn for docId: {0} & path: {1}. Error: {2}'
.format(doc_id, path, error))
raise
def test_eventing_does_not_use_xattrs(self):
body = self.create_save_function_body(self.function_name, HANDLER_CODE.BUCKET_OPS_WITH_TIMERS,
dcp_stream_boundary="from_now")
# deploy eventing function
self.deploy_function(body)
url = 'couchbase://{ip}/{name}'.format(ip=self.master.ip, name=self.src_bucket_name)
bucket = Bucket(url, username="******", password="******")
for docid in ['customer123', 'customer1234', 'customer12345']:
bucket.upsert(docid, {'a': 1})
self.verify_eventing_results(self.function_name, 3, skip_stats_validation=True)
# add new multiple xattrs , delete old xattrs and delete the documents
for docid in ['customer123', 'customer1234', 'customer12345']:
r = bucket.mutate_in(docid, SD.get('eventing', xattr=True))
log.info(r)
if "Could not execute one or more multi lookups or mutations" not in str(r):
self.fail("eventing is still using xattrs for timers")
r = bucket.mutate_in(docid, SD.get('_eventing', xattr=True))
log.info(r)
if "Could not execute one or more multi lookups or mutations" not in str(r):
self.fail("eventing is still using xattrs for timers")
self.undeploy_and_delete_function(body)
def test_lookup_in_simple_get_longer_path(self):
cas = self.coll.upsert(self.KEY, {
"a": "aaa",
"b": {
"c": {
"d": "yo!"
}
}
}).cas
self.try_n_times(10, 3, self._cas_matches, self.KEY, cas)
result = self.coll.lookup_in(self.KEY, (SD.get("b.c.d"), ))
self.assertEqual(result.cas, cas)
self.assertEqual("yo!", result.content_as[str](0))
def save_address(self, cust_id, path, address):
'''
Lab 5: K/V sub-document operation(s):
1. generate key: customer_<custId>
2. get customer addresses
3. create business logic to add new address
4. update customer address path
5. update customer modified date and modifiedBy
When updating, think about pros/cons to UPSERT v. REPLACE
'''
try:
#TODO - figure out what SD.exists returns False, although SD.get returns a doc
key = 'customer_{0}'.format(cust_id)
result = self.__collection.lookup_in(
key,
[
#SD.exists(path),
SD.get(path)
])
#if not result.content_as[bool](0) or result.content_as[dict](1) is None:
if result.content_as[dict](0) is None:
return {'success': False, 'error': None}
addresses = result.content_as[dict](0)
addresses[address['name']] = {
'address': address['address'],
'city': address['city'],
'state': address['state'],
'zipCode': address['zipCode'],
'country':
address['country'] if 'country' in address else 'US',
}
modified_date = int(datetime.datetime.now().timestamp())
self.__collection.mutate_in(key, [
SD.upsert(path, addresses),
SD.upsert('doc.modified', modified_date),
SD.upsert('doc.modifiedby', cust_id),
])
return {'success': True, 'error': None}
except Exception as ex:
output_message(ex, 'repository.py:save_address() - error:')
return {
'success': False,
'error': {
'message': repr(ex),
'stackTrace': traceback.format_exc()
}
}
def remove_item(bucket_name, doc_id, item):
while True:
cb_bucket = cluster.open_bucket(bucket_name)
document = cb_bucket.get(doc_id)
cur_cas = document.cas
ifEncoded = cb_bucket.lookup_in(doc_id, SD.get('isBase64'))
ifZiped = cb_bucket.lookup_in(doc_id, SD.get('isZip'))
if ifZiped[0] == False and ifEncoded[0] == False:
#read body
body = cb_bucket.lookup_in(doc_id, SD.get('body'))[0]
json_obj = json.loads(body)
#print(json.dumps(json_obj, indent=2))
iterate(json_obj, item)
try:
cb_bucket.mutate_in(doc_id,
SD.replace('body', json.dumps(json_obj)),
cas=cur_cas)
break
except KeyExistsError:
continue
else:
bytes_body = decompress(
base64.b64decode(
cb_bucket.lookup_in(doc_id, SD.get('body'))[0]))
json_obj = json.loads(bytes_body)
print(json.dumps(json_obj, indent=2))
iterate(json_obj, item)
json_obj = base64.b64encode(compress(
json.dumps(json_obj))).decode('utf-8')
try:
cb_bucket.mutate_in(doc_id,
SD.replace('body',
json.dumps(json_obj)[1:-1]),
cas=cur_cas)
break
except KeyExistsError:
continue
def test_lookup_in(self):
res = None
self.bucket.upsert(
'king_arthur', {
'name': 'Arthur',
'email': '*****@*****.**',
'interests': ['Holy Grail', 'African Swallows']
})
with tracer.start_active_span('test'):
res = self.bucket.lookup_in('king_arthur', SD.get('email'),
SD.get('interests'))
assert (res)
self.assertTrue(res.success)
spans = self.recorder.queued_spans()
self.assertEqual(2, len(spans))
test_span = get_first_span_by_name(spans, 'sdk')
assert (test_span)
self.assertEqual(test_span.data["sdk"]["name"], 'test')
cb_span = get_first_span_by_name(spans, 'couchbase')
assert (cb_span)
# Same traceId and parent relationship
self.assertEqual(test_span.t, cb_span.t)
self.assertEqual(cb_span.p, test_span.s)
assert (cb_span.stack)
self.assertIsNone(cb_span.ec)
self.assertEqual(cb_span.data["couchbase"]["hostname"],
"%s:8091" % testenv['couchdb_host'])
self.assertEqual(cb_span.data["couchbase"]["bucket"], 'travel-sample')
self.assertEqual(cb_span.data["couchbase"]["type"], 'lookup_in')
def get_string_and_verify_return(self, client, key = '', path = '', expected_value = None, xattr=None):
new_path = self.generate_path(self.nesting_level, path)
try:
if self.is_sdk_client:
if xattr:
data = self._fix_unicode(client.cb.lookup_in(key, SD.get(path, xattr=xattr))[path])
else:
data = self._fix_unicode(client.cb.retrieve_in(key, new_path).get(0)[1])
else:
opaque, cas, data = client.get_sd(key, new_path)
data = yaml.safe_load(data)
except Exception as e:
self.log.error(e)
msg = "Unable to get key {0} for path {1} after {2} tries".format(key, path, 1)
return False, msg
return (str(data).encode('utf-8') == str(expected_value)), str(data).encode('utf-8')
def get_string_and_verify_return(self, client, key = '', path = '', expected_value = None, xattr=None):
new_path = self.generate_path(self.nesting_level, path)
try:
if self.is_sdk_client:
if xattr:
data = self._fix_unicode(client.cb.lookup_in(key, SD.get(path, xattr=xattr))[path])
else:
data = self._fix_unicode(client.cb.retrieve_in(key, new_path).get(0)[1])
else:
opaque, cas, data = client.get_sd(key, new_path)
data = yaml.safe_load(data)
except Exception as e:
self.log.error(e)
msg = "Unable to get key {0} for path {1} after {2} tries".format(key, path, 1)
return False, msg
return (str(data) == str(expected_value)), str(data).encode('utf-8')
def get_and_verify_with_value(self, client, key = '', path = '', expected_value = None, xattr=None):
new_path = self.generate_path(self.nesting_level, path)
try:
if self.is_sdk_client:
if xattr:
data = self._fix_unicode(client.cb.lookup_in(key, SD.get(path, xattr=xattr))[path])
else:
data = client.cb.retrieve_in(key, new_path).get(0)[1]
else:
opaque, cas, data = client.get_sd(key, new_path)
data = json.loads(data)
except Exception as e:
self.log.error(e)
msg = "Unable to get key {0} for path {1} after {2} tries".format(key, path, 1)
return False, msg
msg = "expected {0}, actual {1}".format(str(data), expected_value)
return str(data) == expected_value, msg
def get_and_verify_with_value(self, client, key = '', path = '', expected_value = None, xattr=None):
new_path = self.generate_path(self.nesting_level, path)
try:
if self.is_sdk_client:
if xattr:
data = self._fix_unicode(client.cb.lookup_in(key, SD.get(path, xattr=xattr))[path])
else:
data = client.cb.retrieve_in(key, new_path).get(0)[1]
else:
opaque, cas, data = client.get_sd(key, new_path)
data = json.loads(data)
except Exception as e:
self.log.error(e)
msg = "Unable to get key {0} for path {1} after {2} tries".format(key, path, 1)
return False, msg
msg = "expected {0}, actual {1}".format(str(data), expected_value)
return str(data) == expected_value, msg
def test_xattrs_basic(self):
cb = self.cb
k = self.gen_key('xattrs')
cb.upsert(k, {})
# Try to upsert a single xattr
rv = cb.mutate_in(
k, SD.upsert('my.attr', 'value', xattr=True, create_parents=True))
self.assertTrue(rv.success)
body = cb.get(k)
self.assertFalse('my' in body.value)
self.assertFalse('my.attr' in body.value)
# Try using lookup_in
rv = cb.retrieve_in(k, 'my.attr')
self.assertFalse(rv.exists('my.attr'))
# Finally, use lookup_in with 'xattrs' attribute enabled
rv = cb.lookup_in(k, SD.get('my.attr', xattr=True))
self.assertTrue(rv.exists('my.attr'))
self.assertEqual('value', rv['my.attr'])
def test_xattrs_basic(self):
cb = self.cb
k = self.gen_key('xattrs')
cb.upsert(k, {})
# Try to upsert a single xattr
rv = cb.mutate_in(k, SD.upsert('my.attr', 'value',
xattr=True,
create_parents=True))
self.assertTrue(rv.success)
body = cb.get(k)
self.assertFalse('my' in body.value)
self.assertFalse('my.attr' in body.value)
# Try using lookup_in
rv = cb.retrieve_in(k, 'my.attr')
self.assertFalse(rv.exists('my.attr'))
# Finally, use lookup_in with 'xattrs' attribute enabled
rv = cb.lookup_in(k, SD.get('my.attr', xattr=True))
self.assertTrue(rv.exists('my.attr'))
self.assertEqual('value', rv['my.attr'])
def verify_results(self, skip_verify_data=[], skip_verify_revid=[], sg_run=False):
"""Verify data between each couchbase and remote clusters.
Run below steps for each source and destination cluster..
1. Run expiry pager.
2. Wait for disk queue size to 0 on each nodes.
3. Wait for Outbound mutations to 0.
4. Wait for Items counts equal to kv_store size of buckets.
5. Verify items value on each bucket.
6. Verify Revision id of each item.
"""
skip_key_validation = self._input.param("skip_key_validation", False)
self.__merge_all_buckets()
for cb_cluster in self.get_cb_clusters():
for remote_cluster_ref in cb_cluster.get_remote_clusters():
try:
src_cluster = remote_cluster_ref.get_src_cluster()
dest_cluster = remote_cluster_ref.get_dest_cluster()
if self._evict_with_compactor:
for b in src_cluster.get_buckets():
# only need to do compaction on the source cluster, evictions are propagated to the remote
# cluster
src_cluster.get_cluster().compact_bucket(src_cluster.get_master_node(), b)
else:
src_cluster.run_expiry_pager()
dest_cluster.run_expiry_pager()
src_cluster.wait_for_flusher_empty()
dest_cluster.wait_for_flusher_empty()
src_dcp_queue_drained = src_cluster.wait_for_dcp_queue_drain()
dest_dcp_queue_drained = dest_cluster.wait_for_dcp_queue_drain()
src_cluster.wait_for_outbound_mutations()
dest_cluster.wait_for_outbound_mutations()
except Exception as e:
# just log any exception thrown, do not fail test
self.log.error(e)
if not skip_key_validation:
try:
if not sg_run:
src_active_passed, src_replica_passed = \
src_cluster.verify_items_count(timeout=self._item_count_timeout)
dest_active_passed, dest_replica_passed = \
dest_cluster.verify_items_count(timeout=self._item_count_timeout)
src_cluster.verify_data(max_verify=self._max_verify, skip=skip_verify_data,
only_store_hash=self.only_store_hash)
dest_cluster.verify_data(max_verify=self._max_verify, skip=skip_verify_data,
only_store_hash=self.only_store_hash)
for _, cluster in enumerate(self.get_cb_clusters()):
for bucket in cluster.get_buckets():
h = httplib2.Http(".cache")
resp, content = h.request(
"http://{0}:4984/db/_all_docs".format(cluster.get_master_node().ip))
self.assertEqual(json.loads(content)['total_rows'], self._num_items)
client = SDKClient(scheme="couchbase", hosts=[cluster.get_master_node().ip],
bucket=bucket.name).cb
for i in xrange(self._num_items):
key = 'k_%s_%s' % (i, str(cluster).replace(' ', '_').
replace('.', '_').replace(',', '_').replace(':', '_'))
res = client.get(key)
for xk, xv in res.value.iteritems():
rv = client.mutate_in(key, SD.get(xk, xattr=True))
self.assertTrue(rv.exists(xk))
self.assertEqual(xv, rv[xk])
if sg_run:
resp, content = h.request("http://{0}:4984/db/{1}".format(cluster.get_master_node().ip, key))
self.assertEqual(json.loads(content)['_id'], key)
self.assertEqual(json.loads(content)[xk], xv)
self.assertTrue('2-' in json.loads(content)['_rev'])
except Exception as e:
self.log.error(e)
finally:
if not sg_run:
rev_err_count = self.verify_rev_ids(remote_cluster_ref.get_replications(),
skip=skip_verify_revid)
# we're done with the test, now report specific errors
if (not (src_active_passed and dest_active_passed)) and \
(not (src_dcp_queue_drained and dest_dcp_queue_drained)):
self.fail("Incomplete replication: Keys stuck in dcp queue")
if not (src_active_passed and dest_active_passed):
self.fail("Incomplete replication: Active key count is incorrect")
if not (src_replica_passed and dest_replica_passed):
self.fail("Incomplete intra-cluster replication: "
"replica count did not match active count")
if rev_err_count > 0:
self.fail("RevID verification failed for remote-cluster: {0}".
format(remote_cluster_ref))
# treat errors in self.__report_error_list as failures
if len(self.get_report_error_list()) > 0:
error_logger = self.check_errors_in_goxdcr_logs()
if error_logger:
self.fail("Errors found in logs : {0}".format(error_logger))
def test_lookup_in_simple_get(self):
cas = self.coll.upsert(self.KEY, {"a": "aaa", "b": [1, 2, 3, 4]}).cas
self.try_n_times(10, 3, self._cas_matches, self.KEY, cas)
result = self.coll.lookup_in(self.KEY, (SD.get("b"), ))
self.assertEqual(result.cas, cas)
self.assertEqual([1, 2, 3, 4], result.content_as[list](0))
def __offering_lookup_helper(qId, path):
ob_data = offering_bucket.lookup_in(qId,
subdoc.get(path)) # type: SubdocResult
return ob_data[0]
def verify_results(self,
skip_verify_data=[],
skip_verify_revid=[],
sg_run=False):
"""Verify data between each couchbase and remote clusters.
Run below steps for each source and destination cluster..
1. Run expiry pager.
2. Wait for disk queue size to 0 on each nodes.
3. Wait for Outbound mutations to 0.
4. Wait for Items counts equal to kv_store size of buckets.
5. Verify items value on each bucket.
6. Verify Revision id of each item.
"""
skip_key_validation = self._input.param("skip_key_validation", False)
self.__merge_all_buckets()
for cb_cluster in self.get_cb_clusters():
for remote_cluster_ref in cb_cluster.get_remote_clusters():
try:
src_cluster = remote_cluster_ref.get_src_cluster()
dest_cluster = remote_cluster_ref.get_dest_cluster()
if self._evict_with_compactor:
for b in src_cluster.get_buckets():
# only need to do compaction on the source cluster, evictions are propagated to the remote
# cluster
src_cluster.get_cluster().compact_bucket(
src_cluster.get_master_node(), b)
else:
src_cluster.run_expiry_pager()
dest_cluster.run_expiry_pager()
src_cluster.wait_for_flusher_empty()
dest_cluster.wait_for_flusher_empty()
src_dcp_queue_drained = src_cluster.wait_for_dcp_queue_drain(
)
dest_dcp_queue_drained = dest_cluster.wait_for_dcp_queue_drain(
)
src_cluster.wait_for_outbound_mutations()
dest_cluster.wait_for_outbound_mutations()
except Exception as e:
# just log any exception thrown, do not fail test
self.log.error(e)
if not skip_key_validation:
try:
if not sg_run:
src_active_passed, src_replica_passed = \
src_cluster.verify_items_count(timeout=self._item_count_timeout)
dest_active_passed, dest_replica_passed = \
dest_cluster.verify_items_count(timeout=self._item_count_timeout)
src_cluster.verify_data(
max_verify=self._max_verify,
skip=skip_verify_data,
only_store_hash=self.only_store_hash)
dest_cluster.verify_data(
max_verify=self._max_verify,
skip=skip_verify_data,
only_store_hash=self.only_store_hash)
for _, cluster in enumerate(self.get_cb_clusters()):
for bucket in cluster.get_buckets():
h = httplib2.Http(".cache")
resp, content = h.request(
"http://{0}:4984/db/_all_docs".format(
cluster.get_master_node().ip))
self.assertEqual(
json.loads(content)['total_rows'],
self._num_items)
client = SDKClient(
scheme="couchbase",
hosts=[cluster.get_master_node().ip],
bucket=bucket.name).cb
for i in range(self._num_items):
key = 'k_%s_%s' % (i, str(cluster).replace(
' ', '_').replace('.', '_').replace(
',', '_').replace(':', '_'))
res = client.get(key)
for xk, xv in res.value.items():
rv = client.mutate_in(
key, SD.get(xk, xattr=True))
self.assertTrue(rv.exists(xk))
self.assertEqual(xv, rv[xk])
if sg_run:
resp, content = h.request(
"http://{0}:4984/db/{1}".format(
cluster.get_master_node().ip,
key))
self.assertEqual(
json.loads(content)['_id'], key)
self.assertEqual(
json.loads(content)[xk], xv)
self.assertTrue('2-' in json.loads(
content)['_rev'])
except Exception as e:
self.log.error(e)
finally:
if not sg_run:
rev_err_count = self.verify_rev_ids(
remote_cluster_ref.get_replications(),
skip=skip_verify_revid)
# we're done with the test, now report specific errors
if (not (src_active_passed and dest_active_passed)) and \
(not (src_dcp_queue_drained and dest_dcp_queue_drained)):
self.fail(
"Incomplete replication: Keys stuck in dcp queue"
)
if not (src_active_passed and dest_active_passed):
self.fail(
"Incomplete replication: Active key count is incorrect"
)
if not (src_replica_passed
and dest_replica_passed):
self.fail(
"Incomplete intra-cluster replication: "
"replica count did not match active count")
if rev_err_count > 0:
self.fail(
"RevID verification failed for remote-cluster: {0}"
.format(remote_cluster_ref))
# treat errors in self.__report_error_list as failures
if len(self.get_report_error_list()) > 0:
error_logger = self.check_errors_in_goxdcr_logs()
if error_logger:
self.fail("Errors found in logs : {0}".format(error_logger))
def read(self, key: str, field: str):
self.client.lookup_in(key, subdocument.get(path=field))
def read_xattr(self, key: str, field: str):
self.client.lookup_in(key, subdocument.get(path=field, xattr=True))
def read(self, key, subdoc_fields):
for field in subdoc_fields.split(','):
self.client.lookup_in(key, SD.get(field))
def read(self, key, subdoc_fields):
for field in subdoc_fields.split(','):
self.client.lookup_in(key, SD.get(field))
def sd_get(self, key, path):
return self._cb.lookup_in(key, SD.get(path))
],
"abandoned": [
157, 42, 999
]
}
}
try:
collection.insert("customer123", json_doc)
except DocumentExistsException:
collection.remove("customer123")
collection.insert("customer123", json_doc)
# tag::lookup_in[]
result = collection.lookup_in("customer123",
[SD.get("addresses.delivery.country")])
country = result.content_as[str](0) # "United Kingdom"
# end::lookup_in[]
print(country)
# fixed in v. 3.1.0; prior to 3.1. result.exists(index)
# would throw an exception if the path did not exist
# tag::lookup_in_exists[]
result = collection.lookup_in(
"customer123", [
SD.exists("purchases.pending[-1]")])
print('Path exists: {}.'.format(result.exists(0)))
# Path exists: False.
# end::lookup_in_exists[]
# NOTE: result.content_as[bool](1) would return False
def read(self, key: str, field: str):
self.client.lookup_in(key, subdocument.get(path=field))
def read_xattr(self, key: str, field: str):
self.client.lookup_in(key, subdocument.get(path=field,
xattr=True))
def subdocument_get(bucket, path, document_id):
rv = bucket.lookup_in(
document_id, SD.get(path)
) # don't convert the dict to json. the method will automatically
return rv[0]
The latter saves even more bandwidth by not retrieving the contents of the path if it is not needed.
.Retrieve sub-document value
[source,csharp]
----
"""
#tag::content_as[]
import couchbase.collection
import couchbase.subdocument as SD
from couchbase.durability import Durability
collection = couchbase.collection.Collection()
result = collection.lookup_in("customer123",
[SD.get("addresses.delivery.country")])
country = result.content_as[str](0) # "United Kingdom"
#end::content_as[]
"""
----
.Check existence of sub-document path
[source,csharp]
----
"""
#tag::exists[]
result = collection.lookup_in("customer123",
[SD.exists("purchases.pending[-1]")])
print("Path exists? {}".format(result.content_as[bool](0)))
"""
# Path exists? false
import couchbase.subdocument as SD
cb = Bucket('couchbase://localhost/default')
cb.upsert('docid', {'name': 'Mark', 'email': '*****@*****.**', 'array': [1, 2, 3, 4]})
# Do it the simple way:
rv = cb.retrieve_in('docid', 'name', 'array[1]')
print('Name is: {0}, array[1] is: {1}'.format(rv[0], rv[1]))
# If all results are successful:
name, array_2ndelem = rv
print('Name is: {0}, Array[1] is: {1}'.format(name, array_2ndelem))
# Perform mixed-mode operations
rv = cb.lookup_in('docid', SD.get('name'), SD.get('array[1]'),
SD.exists('non-exist'))
print('Name is', rv[0])
print('Array[1] is', rv[1])
print('non-exist exists?', rv.exists(2))
# See what happens when we try to reference a failed path:
try:
rv[2]
except E.SubdocPathNotFoundError:
print('Using subscript access raises exception for missing item')
# If we try to get a non-existent document, it will fail as normal
try:
cb.retrieve_in('non-exist', 'pth1', 'pth2', 'pth3')
except E.NotFoundError:
