def test_scenario_c_server_side_durability(self):
# Use a helper wrapper to retry our operation in the face of durability failures
# remove is idempotent iff the app guarantees that the doc's id won't be reused (e.g. if it's a UUID). This seems
# a reasonable restriction.
for durability_type in Durability:
self.coll.upsert("id","fred",durability=ServerDurability(Durability.NONE))
self.retry_idempotent_remove_server_side(
lambda: self.coll.remove("id", RemoveOptions(durability=ServerDurability(durability_type))))
async def _mset(self,
key,
doc,
ttl=0,
persist_to=0,
replicate_to=0,
durability_level=Durability.NONE):
try:
if persist_to != 0 or replicate_to != 0:
durability = ClientDurability(
replicate_to=Cardinal(replicate_to),
persist_to=Cardinal(persist_to))
else:
durability = ServerDurability(durability_level)
if ttl != 0:
timedelta = datetime.timedelta(ttl)
upsert_options = UpsertOptions(expiry=timedelta,
durability=durability)
else:
upsert_options = UpsertOptions(durability=durability)
await self.cb.upsert(key, doc, options=upsert_options)
except TemporaryFailException:
logging.warn("temp failure during mset - cluster may be unstable")
except TimeoutException:
logging.warn(f"[{self.name}] cluster timed trying to handle mset")
except NetworkException as nx:
logging.error("network error")
logging.error(nx)
except Exception as ex:
logging.error(ex)
def test_server_durable_insert(self):
if not self.supports_sync_durability():
raise SkipTest("ServerDurability not supported")
durability = ServerDurability(level=Durability.PERSIST_TO_MAJORITY)
self.cb.insert(self.NOKEY, self.CONTENT,
InsertOptions(durability=durability))
result = self.cb.get(self.NOKEY)
self.assertEqual(self.CONTENT, result.content_as[dict])
def get_cluster():
opts = ClusterOptions(
authenticator=PasswordAuthenticator("Administrator", "password"),
transaction_config=TransactionConfig(
durability=ServerDurability(DurabilityLevel.PERSIST_TO_MAJORITY)))
example_cluster = Cluster.connect('couchbase://localhost', opts)
return example_cluster
def test_server_durable_replace(self):
content = {"new": "content"}
if not self.supports_sync_durability():
raise SkipTest("ServerDurability not supported")
durability = ServerDurability(level=Durability.PERSIST_TO_MAJORITY)
self.cb.replace(self.KEY, content,
ReplaceOptions(durability=durability))
result = self.cb.get(self.KEY)
self.assertEqual(content, result.content_as[dict])
async def _mdelete(self, key, durability_level=Durability.NONE):
try:
await self.cb.remove(
key,
RemoveOptions(durability=ServerDurability(durability_level)))
except DocumentNotFoundException as nf:
logging.warn(f"[{self.name}] get key not found! %s: " % nf.key)
except TimeoutException:
logging.warn(
"cluster timed out trying to handle mdelete - cluster may be unstable"
)
except NetworkException as nx:
logging.error("network error")
logging.error(nx)
except Exception as ex:
logging.error(ex)
def test_mutatein(self, # type: Scenarios
dur_name):
durability=Durability[dur_name]
dur_option = DurabilityOptionBlock(durability=ServerDurability(level=durability))
count = 0
replica_count = self.bucket._bucket.configured_replica_count
if dur_name != Durability.NONE and (replica_count == 0 or self.is_mock):
raise SkipTest("cluster will not support {}".format(dur_name))
if not self.supports_sync_durability():
dur_option = self.sdk3_to_sdk2_durability(dur_name, replica_count)
somecontents = {'some': {'path': 'keith'}}
key="{}_{}".format("somekey_{}", count)
try:
self.coll.remove(key)
except:
pass
self.coll.insert(key, somecontents)
inserted_value = "inserted_{}".format(count)
replacement_value = "replacement_{}".format(count)
count += 1
try:
self.coll.mutate_in(key, (
SD.replace('some.path', replacement_value),
SD.insert('some.other.path', inserted_value, create_parents=True),
), dur_option)
somecontents['some']['path'] = replacement_value
somecontents['some'].update({'other': {'path': inserted_value}})
self.assertEqual(somecontents, self.coll.get(key).content)
except NotSupportedException as e:
if not self.is_mock:
raise
else:
logging.error("Assuming failure is due to mock not supporting durability")
except couchbase.exceptions.TimeoutException as e:
self.assertIn("Operational",e.message)
raise SkipTest("Raised {}, skipped pending further verification".format(e.message))
def test_server_durable_remove(self):
if not self.supports_sync_durability():
raise SkipTest("ServerDurability not supported")
durability = ServerDurability(level=Durability.PERSIST_TO_MAJORITY)
self.cb.remove(self.KEY, RemoveOptions(durability=durability))
self.assertRaises(DocumentNotFoundException, self.cb.get, self.KEY)
print("doc with key: {} already exists".format(key))
# end::DocumentExistsException[]
# tag::CASMismatchException[]
try:
result = collection.get("hotel_10026")
collection.replace("hotel_10026", {}, cas=result.cas)
except CASMismatchException:
# the CAS value has changed
pass
# end::CASMismatchException[]
# tag::DurabilitySyncWriteAmbiguousException[]
for i in range(5):
try:
durability = ServerDurability(level=Durability.PERSIST_TO_MAJORITY)
collection.insert("my-key", {"title": "New Hotel"},
InsertOptions(durability=durability))
except (
DocumentExistsException,
DurabilitySyncWriteAmbiguousException,
) as ex:
# if previously retried and the document now exists,
# we can assume it was written successfully by a previous ambiguous exception
if isinstance(ex, DocumentExistsException) and i > 0:
continue
# simply retry the durable operation again
if isinstance(ex, DurabilitySyncWriteAmbiguousException):
continue
# tag::insert_w_opts[]
# Insert document with options
document = {"foo": "bar", "bar": "foo"}
opts = InsertOptions(timeout=timedelta(seconds=5))
result = collection.insert("document-key-opts",
document,
opts,
expiry=timedelta(seconds=30))
# end::insert_w_opts[]
try:
# tag::durability[]
# Upsert with Durability (Couchbase Server >= 6.5) level Majority
document = dict(foo="bar", bar="foo")
opts = UpsertOptions(durability=ServerDurability(Durability.MAJORITY))
result = collection.upsert("document-key", document, opts)
# end::durability[]
except CouchbaseException as ex:
# we expect an exception on local/test host, as Durability requirement
# requires appropriately configured cluster
pass
try:
# tag::obs_durability[]
# Upsert with observe based durability (Couchbase Server < 6.5)
document = {"foo": "bar", "bar": "foo"}
opts = UpsertOptions(
durability=ClientDurability(ReplicateTo.ONE, PersistTo.ONE))
result = collection.upsert("document-key", document, opts)
# end::obs_durability[]
Couchbase Data Platform 6.5 refines these two options, with xref:synchronous-replication.adoc[Synchronous Replication] -- although they remain essentially the same in use -- as well as adding the option of xref:transactions.adoc[atomic document transactions].
The following example demonstrates using the newer durability features available in Couchbase server 6.5 onwards.
[source,python]
----
"""
# tag::upsert_syncrep[]
from couchbase.durability import Durability
document = dict(foo="bar", bar="foo")
result = collection.upsert("document-key",
document,
cas=12345,
expiry=timedelta(minutes=1),
durability=ServerDurability(Durability.MAJORITY))
# end::upsert_syncrep[]
"""
----
[TIP]
.Sub-Document Operations
====
All of these operations involve fetching the complete document from the Cluster.
Where the number of operations or other circumstances make bandwidth a significant issue, the SDK can work on just a specific _path_ of the document with xref:subdocument-operations.adoc[Sub-Docunent Operations].
====
== Retrieving full documents
Using the `Get()` method with the document key can be done in a similar fashion to the other operations:
[source,python]
MutateInOptions(cas=1234))
# end::cas3[]
except (DocumentExistsException, CASMismatchException) as ex:
# we expect an exception here as the CAS value is chosen
# for example purposes
print(ex)
try:
# tag::obs_durability[]
collection.mutate_in(
"key", [SD.insert("username", "dreynholm")],
MutateInOptions(durability=ClientDurability(
ReplicateTo.ONE,
PersistTo.ONE)))
# end::obs_durability[]
except CouchbaseException as ex:
print('Need to have more than 1 node for durability')
print(ex)
try:
# tag::durability[]
collection.mutate_in(
"customer123", [SD.insert("username", "dreynholm")],
MutateInOptions(durability=ServerDurability(
Durability.MAJORITY)))
# end::durability[]
except CouchbaseException as ex:
print('Need to have more than 1 node for durability')
print(ex)
def main():
# tag::config[]
opts = ClusterOptions(
authenticator=PasswordAuthenticator("Administrator", "password"),
transaction_config=TransactionConfig(
durability=ServerDurability(DurabilityLevel.PERSIST_TO_MAJORITY)))
cluster = Cluster.connect('couchbase://localhost', opts)
# end::config[]
test_doc = "foo"
# tag::ts-bucket[]
# get a reference to our bucket
bucket = cluster.bucket("travel-sample")
# end::ts-bucket[]
# tag::ts-collection[]
# get a reference to our collection
collection = bucket.scope("inventory").collection("airline")
# end::ts-collection[]
# tag::ts-default-collection[]
# get a reference to the default collection, required for older Couchbase server versions
collection_default = bucket.default_collection()
# tag::ts-default-collection[]
# Set up for what we'll do below
remove_or_warn(collection, 'doc-a')
remove_or_warn(collection, 'doc-b')
remove_or_warn(collection, 'doc-c')
remove_or_warn(collection, test_doc)
remove_or_warn(collection, 'docId')
# await collection.upsert("doc-a", {})
collection.upsert('doc-b', {})
collection.upsert('doc-c', {})
collection.upsert('doc-id', {})
collection.upsert('a-doc', {})
def txn_insert(ctx):
ctx.insert(collection, test_doc, 'hello')
try:
cluster.transactions.run(txn_insert)
except TransactionFailed as ex:
print(f'Transaction did not reach commit point. Error: {ex}')
except TransactionCommitAmbiguous as ex:
print(f'Transaction possibly committed. Error: {ex}')
# tag::create[]
def txn_logic_ex(ctx # type: AttemptContext
):
"""
… Your transaction logic here …
"""
try:
"""
'txn_logic_ex' is a Python closure that takes an AttemptContext. The
AttemptContext permits getting, inserting, removing and replacing documents,
performing N1QL queries, etc.
Committing is implicit at the end of the closure.
"""
cluster.transactions.run(txn_logic_ex)
except TransactionFailed as ex:
print(f'Transaction did not reach commit point. Error: {ex}')
except TransactionCommitAmbiguous as ex:
print(f'Transaction possibly committed. Error: {ex}')
# end::create[]
# tag::examples[]
inventory = cluster.bucket("travel-sample").scope("inventory")
def txn_example(ctx):
# insert doc
ctx.insert(collection, 'doc-a', {})
# get a doc
doc_a = ctx.get(collection, 'doc-a')
# replace a doc
doc_b = ctx.get(collection, 'doc-b')
content = doc_b.content_as[dict]
content['transactions'] = 'are awesome!'
ctx.replace(doc_b, content)
# remove a doc
doc_c = ctx.get(collection, 'doc-c')
ctx.remove(doc_c)
# tag::scope-example[]
# Added the above tag (scope-example) to ignore this section in the docs for now.
# Once the below TODO is addressed we can remove the tag completely.
# N1QL query
# @TODO: clean up txns query options, scope, pos args and named args won't work
# query_str = 'SELECT * FROM hotel WHERE country = $1 LIMIT 2'
# res = ctx.query(query_str,
# TransactionQueryOptions(scope=inventory,
# positional_args = ['United Kingdom']))
# end::scope-example[]
query_str = 'SELECT * FROM `travel-sample`.inventory.hotel WHERE country = "United Kingdom" LIMIT 2;'
res = ctx.query(query_str)
rows = [r for r in res.rows()]
query_str = 'UPDATE `travel-sample`.inventory.route SET airlineid = "airline_137" WHERE airline = "AF"'
res = ctx.query(query_str)
rows = [r for r in res.rows()]
try:
cluster.transactions.run(txn_example)
except TransactionFailed as ex:
print(f'Transaction did not reach commit point. Error: {ex}')
except TransactionCommitAmbiguous as ex:
print(f'Transaction possibly committed. Error: {ex}')
# end::examples[]
# execute other examples
try:
print('transaction - get')
get(cluster, collection, 'doc-a')
# be sure to use a new key here...
print('transaction - get w/ read own writes')
get_read_own_writes(cluster, collection, 'doc-id2',
{'some': 'content'})
print('transaction - replace')
replace(cluster, collection, 'doc-id')
print('transaction - remove')
remove(cluster, collection, 'doc-id')
print('transaction - insert')
insert(cluster, collection, 'doc-id', {'some': 'content'})
print("transaction - query_examples")
query_examples(cluster)
except TransactionFailed as ex:
print(f'Txn did not reach commit point. Error: {ex}')
except TransactionCommitAmbiguous as ex:
print(f'Txn possibly committed. Error: {ex}')
