def test_eventing_processes_mutations_when_mutated_through_subdoc_api_and_set_expiry_through_sdk(self):
# set expiry pager interval
ClusterOperationHelper.flushctl_set(self.master, "exp_pager_stime", 1, bucket=self.src_bucket_name)
url = 'couchbase://{ip}/{name}'.format(ip=self.master.ip, name=self.src_bucket_name)
bucket = Bucket(url, username="******", password="******")
for docid in ['customer123', 'customer1234', 'customer12345']:
bucket.insert(docid, {'some': 'value'})
body = self.create_save_function_body(self.function_name, self.handler_code,
dcp_stream_boundary="from_now")
# deploy eventing function
self.deploy_function(body)
# upserting a new sub-document
bucket.mutate_in('customer123', SD.upsert('fax', '775-867-5309'))
# inserting a sub-document
bucket.mutate_in('customer1234', SD.insert('purchases.complete', [42, True, None], create_parents=True))
# Creating and populating an array document
bucket.mutate_in('customer12345', SD.array_append('purchases.complete', ['Hello'], create_parents=True))
self.verify_eventing_results(self.function_name, 3, skip_stats_validation=True)
for docid in ['customer123', 'customer1234', 'customer12345']:
# set expiry on all the docs created using sub doc API
bucket.touch(docid, ttl=5)
self.sleep(10, "wait for expiry of the documents")
# Wait for eventing to catch up with all the expiry mutations and verify results
self.verify_eventing_results(self.function_name, 0, skip_stats_validation=True)
self.undeploy_and_delete_function(body)
def test_mutatein(self):
count = 0
for durability in Durability:
somecontents = {'some': {'path': 'keith'}}
key="somekey_{}".format(count)
try:
self.coll.remove(key)
except:
pass
self.coll.insert(key, somecontents)
inserted_value = "inserted_{}".format(count)
replacement_value = "replacement_{}".format(count)
count += 1
try:
self.coll.mutate_in(key, (
SD.replace('some.path', replacement_value),
SD.insert('some.other.path', inserted_value, create_parents=True),
), durability_level=durability)
somecontents['some']['path'] = replacement_value
somecontents['some'].update({'other': {'path': inserted_value}})
self.assertEqual(somecontents, self.coll.get(key).content)
except NotSupportedError as e:
if not self.is_mock:
raise
else:
logging.error("Assuming failure is due to mock not supporting durability")
def test_eventing_processes_mutations_when_mutated_through_subdoc_api_and_set_expiry_through_sdk(self):
# set expiry pager interval
ClusterOperationHelper.flushctl_set(self.master, "exp_pager_stime", 1, bucket=self.src_bucket_name)
url = 'couchbase://{ip}/{name}'.format(ip=self.master.ip, name=self.src_bucket_name)
bucket = Bucket(url, username="******", password="******")
for docid in ['customer123', 'customer1234', 'customer12345']:
bucket.insert(docid, {'some': 'value'})
body = self.create_save_function_body(self.function_name, self.handler_code,
dcp_stream_boundary="from_now")
# deploy eventing function
self.deploy_function(body)
# upserting a new sub-document
bucket.mutate_in('customer123', SD.upsert('fax', '775-867-5309'))
# inserting a sub-document
bucket.mutate_in('customer1234', SD.insert('purchases.complete', [42, True, None], create_parents=True))
# Creating and populating an array document
bucket.mutate_in('customer12345', SD.array_append('purchases.complete', ['Hello'], create_parents=True))
self.verify_eventing_results(self.function_name, 3, skip_stats_validation=True)
for docid in ['customer123', 'customer1234', 'customer12345']:
# set expiry on all the docs created using sub doc API
bucket.touch(docid, ttl=5)
self.sleep(10, "wait for expiry of the documents")
# Wait for eventing to catch up with all the expiry mutations and verify results
self.verify_eventing_results(self.function_name, 0, skip_stats_validation=True)
self.undeploy_and_delete_function(body)
def dict_add(self, client, key='', path='', value=None):
try:
if self.verbose_func_usage:
self.log.info(" dict_add ----> {0} :: {1}".format(path, value))
if self.use_sdk_client:
client.mutate_in(key, SD.insert(path, value, xattr=self.xattr))
else:
client.dict_add_sd(key, path, value)
except Exception:
raise
def dict_add(self, client, key='', path='', value=None):
try:
if self.verbose_func_usage:
self.log.info(" dict_add ----> {0} :: {1}".format(path, value))
if self.use_sdk_client:
client.mutate_in(key, SD.insert(path, value, xattr=self.xattr))
else:
client.dict_add_sd(key, path, value)
except Exception:
raise
def test_multi_lookup(self):
cb = self.cb
key = self.gen_key('sdmlookup')
cb.upsert(
key, {
'field1': 'value1',
'field2': 'value2',
'array': [1, 2, 3],
'boolean': False
})
rvs = cb.lookup_in(key,
SD.get('field1'),
SD.exists('field2'),
SD.exists('field3'),
quiet=True)
self.assertFalse(rvs.success)
self.assertEqual(3, rvs.result_count)
self.assertEqual((0, 'value1'), rvs.get(0))
self.assertEqual((0, 'value1'), rvs.get('field1'))
self.assertEqual('value1', rvs[0])
self.assertEqual('value1', rvs['field1'])
self.assertEqual((0, None), rvs.get(1))
self.assertEqual((0, None), rvs.get('field2'))
self.assertEqual(None, rvs[1])
self.assertEqual(None, rvs['field2'])
self.assertTrue(rvs.exists('field2'))
self.assertTrue(rvs.exists(1))
self.assertTrue(1 in rvs)
self.assertTrue('field2' in rvs)
self.assertEqual((E.SubdocPathNotFoundError.CODE, None),
rvs.get('field3'))
self.assertEqual((E.SubdocPathNotFoundError.CODE, None), rvs.get(2))
self.assertFalse(rvs.exists('field3'))
self.assertFalse(rvs.exists(2))
def _getix(rv_, ix):
return rv_[ix]
self.assertRaises(E.SubdocPathNotFoundError, _getix, rvs, 2)
self.assertRaises(E.SubdocPathNotFoundError, _getix, rvs, 'field3')
self.assertFalse(rvs.exists('field3'))
# See what happens when we mix operations
self.assertRaises(E.CouchbaseError, cb.lookup_in, key,
SD.get('field1'), SD.insert('a', 'b'))
# Empty path (invalid)
self.assertRaises(E.CouchbaseError, cb.lookup_in, SD.get(''))
def dict_add(self, client, key='', path='', value = None, xattr=None, create_parents=None):
try:
new_path = self.generate_path(self.nesting_level, path)
if self.is_sdk_client:
print path, ":", value
rv = client.mutate_in(key, SD.insert(path, value, xattr=xattr, create_parents=create_parents))
self.log.info("xattr '%s' inserted successfully" % path)
else:
client.dict_add_sd(key, new_path, json.dumps(value))
except Exception as e:
self.log.info(e)
self.fail("Unable to add key {0} for path {1} after {2} tries".format(key, path, 1))
def dict_add(self, client, key='', path='', value = None, xattr=None, create_parents=None):
try:
new_path = self.generate_path(self.nesting_level, path)
if self.is_sdk_client:
print(path, ":", value)
rv = client.mutate_in(key, SD.insert(path, value, xattr=xattr, create_parents=create_parents))
self.log.info("xattr '%s' inserted successfully" % path)
else:
client.dict_add_sd(key, new_path, json.dumps(value))
except Exception as e:
self.log.info(e)
self.fail("Unable to add key {0} for path {1} after {2} tries".format(key, path, 1))
def test_multi_lookup(self):
cb = self.cb
key = self.gen_key('sdmlookup')
cb.upsert(key, {
'field1': 'value1',
'field2': 'value2',
'array': [1, 2, 3],
'boolean': False
})
rvs = cb.lookup_in(
key, SD.get('field1'), SD.exists('field2'), SD.exists('field3'),
quiet=True
)
self.assertFalse(rvs.success)
self.assertEqual(3, rvs.result_count)
self.assertEqual((0, 'value1'), rvs.get(0))
self.assertEqual((0, 'value1'), rvs.get('field1'))
self.assertEqual('value1', rvs[0])
self.assertEqual('value1', rvs['field1'])
self.assertEqual((0, None), rvs.get(1))
self.assertEqual((0, None), rvs.get('field2'))
self.assertEqual(None, rvs[1])
self.assertEqual(None, rvs['field2'])
self.assertTrue(rvs.exists('field2'))
self.assertTrue(rvs.exists(1))
self.assertTrue(1 in rvs)
self.assertTrue('field2' in rvs)
self.assertEqual((E.SubdocPathNotFoundError.CODE, None),
rvs.get('field3'))
self.assertEqual((E.SubdocPathNotFoundError.CODE, None),
rvs.get(2))
self.assertFalse(rvs.exists('field3'))
self.assertFalse(rvs.exists(2))
def _getix(rv_, ix):
return rv_[ix]
self.assertRaises(E.SubdocPathNotFoundError, _getix, rvs, 2)
self.assertRaises(E.SubdocPathNotFoundError, _getix, rvs, 'field3')
self.assertFalse(rvs.exists('field3'))
# See what happens when we mix operations
self.assertRaises(E.CouchbaseError, cb.lookup_in, key,
SD.get('field1'), SD.insert('a', 'b'))
# Empty path (invalid)
self.assertRaises(E.CouchbaseError, cb.lookup_in, SD.get(''))
def offeringPUT(quarter, courseNum, sectionId):
# TODO: make redis call
# TODO: should only be able to do this if course exists
if not (quarter and courseNum and sectionId):
return make_response(
"Missing a parameter. Need quarter, courseNum, sectionId", 400)
upsert_quarter(quarter)
data = request.get_json()
data['enrolled'] = 0
data['capacity'] = int(data['capacity']) if 'capacity' in data else 0
offering_bucket.mutate_in(
quarter,
subdoc.insert(courseNum + "." + str(sectionId),
data,
create_parents=True))
return make_response(
"Created offering {}/{}-{}".format(quarter, courseNum, sectionId), 201)
def test_mutatein(self, # type: Scenarios
dur_name):
durability=Durability[dur_name]
dur_option = DurabilityOptionBlock(durability=ServerDurability(level=durability))
count = 0
replica_count = self.bucket._bucket.configured_replica_count
if dur_name != Durability.NONE and (replica_count == 0 or self.is_mock):
raise SkipTest("cluster will not support {}".format(dur_name))
if not self.supports_sync_durability():
dur_option = self.sdk3_to_sdk2_durability(dur_name, replica_count)
somecontents = {'some': {'path': 'keith'}}
key="{}_{}".format("somekey_{}", count)
try:
self.coll.remove(key)
except:
pass
self.coll.insert(key, somecontents)
inserted_value = "inserted_{}".format(count)
replacement_value = "replacement_{}".format(count)
count += 1
try:
self.coll.mutate_in(key, (
SD.replace('some.path', replacement_value),
SD.insert('some.other.path', inserted_value, create_parents=True),
), dur_option)
somecontents['some']['path'] = replacement_value
somecontents['some'].update({'other': {'path': inserted_value}})
self.assertEqual(somecontents, self.coll.get(key).content)
except NotSupportedException as e:
if not self.is_mock:
raise
else:
logging.error("Assuming failure is due to mock not supporting durability")
except couchbase.exceptions.TimeoutException as e:
self.assertIn("Operational",e.message)
raise SkipTest("Raised {}, skipped pending further verification".format(e.message))
"""
#tag::upsert[]
collection.mutate_in("customer123", [SD.upsert("fax", "311-555-0151")])
#end::upsert[]
"""
----
Likewise, the _subdoc-insert_ operation will only add the new value to the path if it does not exist:
.Inserting a sub-document
[source,csharp]
----
"""
#tag::insert[]
collection.mutate_in("customer123",
[SD.insert("purchases.complete", [42, True, "None"])])
# SubdocPathExistsError
#end::insert[]
"""
----
Dictionary values can also be replaced or removed, and you may combine any number of mutation operations within the same general _mutate-in_ API.
Here's an example of one which replaces one path and removes another.
[source,csharp]
----
"""
#tag::combine_dict[]
collection.mutate_in("customer123",
(SD.remove("addresses.billing"),
"customer123",
[SD.get("addresses.delivery.country"),
SD.exists("purchases.complete[-1]")])
print("{0}".format(result.content_as[str](0)))
print("Path exists: {}.".format(result.exists(1)))
# path exists: True.
# end::lookup_in_multi[]
# tag::mutate_in_upsert[]
collection.mutate_in("customer123", [SD.upsert("fax", "311-555-0151")])
# end::mutate_in_upsert[]
# tag::mutate_in_insert[]
collection.mutate_in(
"customer123", [SD.insert("purchases.pending", [42, True, "None"])])
try:
collection.mutate_in(
"customer123", [
SD.insert(
"purchases.complete",
[42, True, "None"])])
except PathExistsException:
print("Path exists, cannot use insert.")
# end::mutate_in_insert[]
# tag::combine_dict[]
collection.mutate_in(
"customer123",
(SD.remove("addresses.billing"),
def subdocument_insert(bucket, document_id, path, insertdoc):
rv = bucket.mutate_in(
document_id, SD.insert(path, insertdoc)
) # don't convert the dict to json. the method will automatically
return rv[0]
def test_mutate_in(self):
cb = self.cb
key = self.gen_key('sdstore_upsert')
cb.upsert(key, {})
cb.mutate_in(key, SD.upsert('newDict', ['hello']))
result = cb.retrieve_in(key, 'newDict')
self.assertEqual(['hello'], result[0])
# Create deep path without create_parents
self.assertRaises(E.SubdocPathNotFoundError,
cb.mutate_in, key,
SD.upsert('path.with.missing.parents', 'value'))
# Create deep path using create_parents
cb.mutate_in(key,
SD.upsert('new.parent.path', 'value', create_parents=True))
result = cb.retrieve_in(key, 'new.parent')
self.assertEqual('value', result[0]['path'])
# Test CAS operations
self.assertTrue(result.cas)
self.assertRaises(E.KeyExistsError, cb.mutate_in,
key, SD.upsert('newDict', None), cas=result.cas+1)
# Try it again, using the CAS
result2 = cb.mutate_in(key, SD.upsert('newDict', {}), cas=result.cas)
self.assertNotEqual(result.cas, result2.cas)
# Test insert, should fail
self.assertRaises(E.SubdocPathExistsError, cb.mutate_in,
key, SD.insert('newDict', {}))
# Test insert on new path, should succeed
cb.mutate_in(key, SD.insert('anotherDict', {}))
self.assertEqual({}, cb.retrieve_in(key, 'anotherDict')[0])
# Test replace, should not fail
cb.mutate_in(key, SD.replace('newDict', {'Hello': 'World'}))
self.assertEqual('World', cb.retrieve_in(key, 'newDict')[0]['Hello'])
# Test replace with missing value, should fail
self.assertRaises(E.SubdocPathNotFoundError,
cb.mutate_in, key, SD.replace('nonexist', {}))
# Test with empty string (should be OK)
cb.mutate_in(key, SD.upsert('empty', ''))
self.assertEqual('', cb.retrieve_in(key, 'empty')[0])
# Test with null (None). Should be OK
cb.mutate_in(key, SD.upsert('null', None))
self.assertEqual(None, cb.retrieve_in(key, 'null')[0])
# Test with empty path. Should throw some kind of error?
self.assertRaises(
(E.SubdocCantInsertValueError, E.SubdocEmptyPathError),
cb.mutate_in, key, SD.upsert('', {}))
cb.mutate_in(key, SD.upsert('array', [1, 2, 3]))
self.assertRaises(E.SubdocPathMismatchError, cb.mutate_in, key,
SD.upsert('array.newKey', 'newVal'))
self.assertRaises(E.SubdocPathInvalidError, cb.mutate_in, key,
SD.upsert('array[0]', 'newVal'))
self.assertRaises(E.SubdocPathNotFoundError, cb.mutate_in, key,
SD.upsert('array[3].bleh', 'newVal'))
def sd_insert(self, key, path, doc, cas=None):
if cas is not None:
return self._cb.mutate_in(key, SD.insert(path, doc), cas=cas)
else:
return self._cb.mutate_in(key, SD.insert(path, doc))
def test_mutate_in(self):
cb = self.cb
key = self.gen_key('sdstore_upsert')
cb.upsert(key, {})
cb.mutate_in(key, SD.upsert('newDict', ['hello']))
result = cb.retrieve_in(key, 'newDict')
self.assertEqual(['hello'], result[0])
# Create deep path without create_parents
self.assertRaises(E.SubdocPathNotFoundError, cb.mutate_in, key,
SD.upsert('path.with.missing.parents', 'value'))
# Create deep path using create_parents
cb.mutate_in(
key, SD.upsert('new.parent.path', 'value', create_parents=True))
result = cb.retrieve_in(key, 'new.parent')
self.assertEqual('value', result[0]['path'])
# Test CAS operations
self.assertTrue(result.cas)
self.assertRaises(E.KeyExistsError,
cb.mutate_in,
key,
SD.upsert('newDict', None),
cas=result.cas + 1)
# Try it again, using the CAS
result2 = cb.mutate_in(key, SD.upsert('newDict', {}), cas=result.cas)
self.assertNotEqual(result.cas, result2.cas)
# Test insert, should fail
self.assertRaises(E.SubdocPathExistsError, cb.mutate_in, key,
SD.insert('newDict', {}))
# Test insert on new path, should succeed
cb.mutate_in(key, SD.insert('anotherDict', {}))
self.assertEqual({}, cb.retrieve_in(key, 'anotherDict')[0])
# Test replace, should not fail
cb.mutate_in(key, SD.replace('newDict', {'Hello': 'World'}))
self.assertEqual('World', cb.retrieve_in(key, 'newDict')[0]['Hello'])
# Test replace with missing value, should fail
self.assertRaises(E.SubdocPathNotFoundError, cb.mutate_in, key,
SD.replace('nonexist', {}))
# Test with empty string (should be OK)
cb.mutate_in(key, SD.upsert('empty', ''))
self.assertEqual('', cb.retrieve_in(key, 'empty')[0])
# Test with null (None). Should be OK
cb.mutate_in(key, SD.upsert('null', None))
self.assertEqual(None, cb.retrieve_in(key, 'null')[0])
# Test with empty path. Should throw some kind of error?
self.assertRaises(
(E.SubdocCantInsertValueError, E.SubdocEmptyPathError),
cb.mutate_in, key, SD.upsert('', {}))
cb.mutate_in(key, SD.upsert('array', [1, 2, 3]))
self.assertRaises(E.SubdocPathMismatchError, cb.mutate_in, key,
SD.upsert('array.newKey', 'newVal'))
self.assertRaises(E.SubdocPathInvalidError, cb.mutate_in, key,
SD.upsert('array[0]', 'newVal'))
self.assertRaises(E.SubdocPathNotFoundError, cb.mutate_in, key,
SD.upsert('array[3].bleh', 'newVal'))
