def submit_single(
self, c_key, c_len, value, item, key_options, global_options, mres):
value_format = get_option('format', key_options, global_options)
if item:
if self.OPTYPE in (C.LCB_APPEND, C.LCB_PREPEND):
try:
value = key_options['fragment']
except (KeyError, TypeError):
pycbc_exc_args('append/prepend must provide `fragment`')
try:
v_enc, flags = self.parent._tc.encode_value(value, value_format)
if not isinstance(v_enc, bytes):
raise ValueFormatError.pyexc("Value was not bytes", obj=v_enc)
s_val = ffi.new('char[]', v_enc)
except CouchbaseError:
raise
except Exception as ex:
raise ValueFormatError.pyexc(str(value_format), inner=ex)
C._Cb_set_key(self.c_command, c_key, c_len)
C._Cb_set_val(self.c_command, s_val, len(v_enc))
if self.OPTYPE not in (C.LCB_APPEND, C.LCB_PREPEND):
try:
self.c_command.flags = flags
except OverflowError:
pycbc_exc_enc('Invalid flags value', obj=flags)
self.c_command.exptime = get_ttl(key_options, global_options, item)
self.c_command.cas = get_cas(key_options, global_options, item)
self.c_command.operation = self.OPTYPE
return C.lcb_store3(self.instance, mres._cdata, self.c_command)
def submit_single(self, c_key, c_len, value, item, key_options,
global_options, mres):
value_format = get_option('format', key_options, global_options)
if item:
if self.OPTYPE in (C.LCB_APPEND, C.LCB_PREPEND):
try:
value = key_options['fragment']
except (KeyError, TypeError):
pycbc_exc_args('append/prepend must provide `fragment`')
try:
v_enc, flags = self.parent._tc.encode_value(value, value_format)
if not isinstance(v_enc, bytes):
raise ValueFormatError.pyexc("Value was not bytes", obj=v_enc)
s_val = ffi.new('char[]', v_enc)
except CouchbaseError:
raise
except Exception as ex:
raise ValueFormatError.pyexc(str(value_format), inner=ex)
C._Cb_set_key(self.c_command, c_key, c_len)
C._Cb_set_val(self.c_command, s_val, len(v_enc))
if self.OPTYPE not in (C.LCB_APPEND, C.LCB_PREPEND):
try:
self.c_command.flags = flags
except OverflowError:
pycbc_exc_enc('Invalid flags value', obj=flags)
self.c_command.exptime = get_ttl(key_options, global_options, item)
self.c_command.cas = get_cas(key_options, global_options, item)
self.c_command.operation = self.OPTYPE
return C.lcb_store3(self.instance, mres._cdata, self.c_command)
def default_format(self, arg):
if arg is PyCBC.fmt_auto:
self.__default_format = arg
return
if isinstance(arg, bool):
raise pycbc_exc_args('Must be a number', obj=arg)
if not isinstance(arg, int):
raise pycbc_exc_args('Must be a number', obj=arg)
self.__default_format = arg
def _invoke_submit(self, iterobj, is_dict, is_itmcoll, mres, global_kw):
"""
Internal function to invoke the actual submit_single function
:param iterobj: The raw object returned as the next item of the iterator
:param is_dict: True if iterator is a dictionary
:param is_itmcoll: True if the iterator contains Item objects
:param mres: The multi result object
:param global_kw: The global settings
:return: The return value of :meth:`submit_single`
"""
if is_itmcoll:
item, key_options = next(iterobj)
if not isinstance(item, Item):
pycbc_exc_args('Expected item object')
key = item.key
value = item.value
result = item
else:
if is_dict:
key, value = next(iterobj)
if not self.VALUES_ALLOWED and not is_itmcoll:
raise ArgumentError.pyexc(
'Values not allowed for this command', obj=value)
else:
key = next(iterobj)
value = None
key_options = {}
item = None
result = self.make_result(key, value)
result.rc = -1
# Attempt to get the encoded key:
key, value, key_options = self.make_entry_params(key, value, key_options)
c_key, c_len = create_key(self.parent._tc, key)
rc = self.submit_single(c_key, c_len, value, item, key_options, global_kw, mres)
if rc:
raise pycbc_exc_lcb(rc)
try:
if key in mres and not self.DUPKEY_OK:
# For tests:
self.parent._warn_dupkey(key)
mres[key] = result
except TypeError:
raise pycbc_exc_enc(obj=key)
def _cntl(self, op, value=None, value_type=None):
if value_type is None and op in self.OLD_CNTL_MAP:
value_type = self.OLD_CNTL_MAP[op]
try:
handler = CNTL_VTYPE_MAP[value_type]
except KeyError:
raise pycbc_exc_args('Invalid value type', obj=value_type)
return handler.execute(self._lcbh, op, value)
def __run_stat(self, k, mres):
bm = BufManager(ffi)
if k:
if not isinstance(k, basestring):
raise pycbc_exc_args('Stats arguments must be strings only!')
c_key, c_len = bm.new_cbuf(k)
else:
c_key, c_len = ffi.NULL, 0
C._Cb_set_key(self.c_command, c_key, c_len)
rc = C.lcb_stats3(self.instance, mres._cdata, self.c_command)
if rc:
raise pycbc_exc_lcb(rc)
def __run_stat(self, k, mres):
bm = BufManager(ffi)
if k:
if not isinstance(k, basestring):
raise pycbc_exc_args('Stats arguments must be strings only!')
c_key, c_len = bm.new_cbuf(k)
else:
c_key, c_len = ffi.NULL, 0
C._Cb_set_key(self.c_command, c_key, c_len)
rc = C.lcb_stats3(self.instance, mres._cdata, self.c_command)
if rc:
raise pycbc_exc_lcb(rc)
def submit_single(self, c_key, c_len, value, item, key_options, global_options, mres):
ctx = global_options['_MCTX']
cas = get_cas(key_options, global_options, item)
if not cas:
if isinstance(value, OperationResult):
cas = value.cas
elif not isinstance(value, (int, long)):
raise pycbc_exc_args('Bad CAS value', obj=cas)
self.c_command.cas = cas
C._Cb_set_key(self.c_command, c_key, c_len)
# Add to the current context
return ctx.addcmd(ctx, ffi.cast('lcb_CMDBASE*', self.c_command))
def submit_single(self, c_key, c_len, value, item, key_options,
global_options, mres):
ctx = global_options['_MCTX']
cas = get_cas(key_options, global_options, item)
if not cas:
if isinstance(value, OperationResult):
cas = value.cas
elif not isinstance(value, (int, long)):
raise pycbc_exc_args('Bad CAS value', obj=cas)
self.c_command.cas = cas
C._Cb_set_key(self.c_command, c_key, c_len)
# Add to the current context
return ctx.addcmd(ctx, ffi.cast('lcb_CMDBASE*', self.c_command))
def execute(self, lcbh, op, value):
if value is not None:
mode = C.LCB_CNTL_SET
else:
mode = C.LCB_CNTL_GET
c_data = self.allocate(mode)
if mode == C.LCB_CNTL_SET:
try:
self.convert_input(value, c_data)
except:
raise pycbc_exc_args(obj=value)
rc = C.lcb_cntl(lcbh, mode, op, c_data)
else:
rc = C.lcb_cntl(lcbh, mode, op, c_data)
if not rc:
return self.convert_output(c_data)
if rc:
raise pycbc_exc_lcb(rc)
def create_context(self, **kwargs):
C.memset(self.c_options, 0, ffi.sizeof(self.c_options[0]))
ok, persist_to, replicate_to = handle_durability(self.parent, **kwargs)
if not ok:
persist_to = -1
replicate_to = -1
self.__opt.persist_to = self._mk_criteria(self.__opt, persist_to)
self.__opt.replicate_to = self._mk_criteria(self.__opt, replicate_to)
if kwargs.get('check_removed'):
self.__opt.check_delete = 1
tmo = kwargs.get('timeout')
if tmo:
# print "Found timeout in options!"
try:
tmo = int(tmo * 1000000)
except:
raise pycbc_exc_args('Invalid timeout', obj=tmo)
else:
tmo = self.parent._dur_timeout
if tmo:
self.__opt.timeout = tmo
# print "OPTIONS: "
# print "PersistTo:", self.__opt.persist_to
# print "ReplicateTo:", self.__opt.replicate_to
# print "CapMax:", self.__opt.cap_max
# print "Timeout:", self.__opt.timeout
ctx = C.lcb_endure3_ctxnew(self.instance, self.c_options, self.__errp)
if not ctx:
raise pycbc_exc_lcb(self.__errp[0])
return ctx
def handle_durability(parent, **kwargs):
"""
Handle durability requirements passed inside options
:param parent: The parent bucket import
:param kwargs: The arguments
:return: A tuple of `(found, persist_to, replicate_to)`. The first
element is True if either persistence or replication has been
found
"""
persist_to = kwargs.get('persist_to')
replicate_to = kwargs.get('replicate_to')
if not persist_to and not replicate_to:
persist_to = parent._dur_persist_to
replicate_to = parent._dur_replicate_to
if not persist_to and not replicate_to:
return False, 0, 0
n_replicas = C.lcb_get_num_replicas(parent._lcbh)
if replicate_to > n_replicas or persist_to > n_replicas+1:
raise pycbc_exc_args('Durability requirements will never be satisfied')
return True, persist_to, replicate_to
def create_context(self, **kwargs):
C.memset(self.c_options, 0, ffi.sizeof(self.c_options[0]))
ok, persist_to, replicate_to = handle_durability(self.parent, **kwargs)
if not ok:
persist_to = -1
replicate_to = -1
self.__opt.persist_to = self._mk_criteria(self.__opt, persist_to)
self.__opt.replicate_to = self._mk_criteria(self.__opt, replicate_to)
if kwargs.get('check_removed'):
self.__opt.check_delete = 1
tmo = kwargs.get('timeout')
if tmo:
# print "Found timeout in options!"
try:
tmo = int(tmo * 1000000)
except:
raise pycbc_exc_args('Invalid timeout', obj=tmo)
else:
tmo = self.parent._dur_timeout
if tmo:
self.__opt.timeout = tmo
# print "OPTIONS: "
# print "PersistTo:", self.__opt.persist_to
# print "ReplicateTo:", self.__opt.replicate_to
# print "CapMax:", self.__opt.cap_max
# print "Timeout:", self.__opt.timeout
ctx = C.lcb_endure3_ctxnew(self.instance, self.c_options, self.__errp)
if not ctx:
raise pycbc_exc_lcb(self.__errp[0])
return ctx
def handle_durability(parent, **kwargs):
"""
Handle durability requirements passed inside options
:param parent: The parent bucket import
:param kwargs: The arguments
:return: A tuple of `(found, persist_to, replicate_to)`. The first
element is True if either persistence or replication has been
found
"""
persist_to = kwargs.get('persist_to')
replicate_to = kwargs.get('replicate_to')
if not persist_to and not replicate_to:
persist_to = parent._dur_persist_to
replicate_to = parent._dur_replicate_to
if not persist_to and not replicate_to:
return False, 0, 0
n_replicas = C.lcb_get_num_replicas(parent._lcbh)
if replicate_to > n_replicas or persist_to > n_replicas + 1:
raise pycbc_exc_args('Durability requirements will never be satisfied')
return True, persist_to, replicate_to
def quiet(self, arg):
if isinstance(arg, bool):
self.__quiet = arg
else:
raise pycbc_exc_args("'quiet' must be bool")
def __init__(self, connection_string=None, connstr=None,
username=None, password=None, quiet=False,
transcoder=None, default_format=FMT_JSON,
lockmode=LOCKMODE_EXC, unlock_gil=True,
_iops=None, _conntype=C.LCB_TYPE_BUCKET,
_flags=0):
crst = ffi.new('struct lcb_create_st*')
bm = BufManager(ffi)
if not connstr and not connection_string:
raise pycbc_exc_args('Must have connection string')
if connstr and connection_string:
raise pycbc_exc_args(
"Cannot specify both 'connstr' and 'connection_string'")
if not connstr:
connstr = connection_string
crst.version = 3
crst.v.v3.connstr = bm.new_cstr(connstr)
crst.v.v3.passwd = bm.new_cstr(password)
crst.v.v3.username = bm.new_cstr(username)
crst.v.v3.type = _conntype
if _iops:
procs = _iops
self._iowrap = IOPSWrapper(procs)
crst.v.v3.io = self._iowrap.get_lcb_iops()
else:
crst.v.v3.io = ffi.NULL
self._handles = set()
lcb_pp = ffi.new('lcb_t*')
rc = C.lcb_create(lcb_pp, crst)
if rc != C.LCB_SUCCESS:
raise pycbc_exc_lcb(rc)
self._lcbh = lcb_pp[0]
self._bound_cb = {
'store': ffi.callback(CALLBACK_DECL, self._storage_callback),
'get': ffi.callback(CALLBACK_DECL, self._get_callback),
'remove': ffi.callback(CALLBACK_DECL, self._remove_callback),
'counter': ffi.callback(CALLBACK_DECL, self._counter_callback),
'observe': ffi.callback(CALLBACK_DECL, self._observe_callback),
'stats': ffi.callback(CALLBACK_DECL, self._stats_callback),
'http': ffi.callback(CALLBACK_DECL, self._http_callback),
'_default': ffi.callback(CALLBACK_DECL, self._default_callback),
'_bootstrap': ffi.callback('void(lcb_t,lcb_error_t)',
self._bootstrap_callback),
'_dtor': ffi.callback('void(const void*)',
self._instance_destroyed)
}
self._executors = {
'upsert': executors.UpsertExecutor(self),
'insert': executors.InsertExecutor(self),
'replace': executors.ReplaceExecutor(self),
'append': executors.AppendExecutor(self),
'prepend': executors.PrependExecutor(self),
'get': executors.GetExecutor(self),
'lock': executors.LockExecutor(self),
'_unlock': executors.UnlockExecutor(self),
'touch': executors.TouchExecutor(self),
'remove': executors.RemoveExecutor(self),
'counter': executors.CounterExecutor(self),
'endure': executors.DurabilityExecutor(self),
'_chained_endure': executors.DurabilityChainExecutor(self),
'observe': executors.ObserveExecutor(self),
'stats': executors.StatsExecutor(self),
'_rget': executors.GetReplicaExecutor(self)
}
self._install_cb(C.LCB_CALLBACK_DEFAULT, '_default')
self._install_cb(C.LCB_CALLBACK_STORE, 'store')
self._install_cb(C.LCB_CALLBACK_GET, 'get')
self._install_cb(C.LCB_CALLBACK_GETREPLICA, 'get')
self._install_cb(C.LCB_CALLBACK_REMOVE, 'remove')
self._install_cb(C.LCB_CALLBACK_COUNTER, 'counter')
self._install_cb(C.LCB_CALLBACK_OBSERVE, 'observe')
self._install_cb(C.LCB_CALLBACK_STATS, 'stats')
self._install_cb(C.LCB_CALLBACK_HTTP, 'http')
C.lcb_set_bootstrap_callback(self._lcbh, self._bound_cb['_bootstrap'])
# Set our properties
self.data_passthrough = False
self.transcoder = transcoder
self.default_format = default_format
self.quiet = quiet
self.unlock_gil = unlock_gil
self._dur_persist_to = 0
self._dur_replicate_to = 0
self._dur_timeout = 0
self._dur_testhook = None
self._privflags = _flags
self._conncb = None
self.__bucket = self._cntl(C.LCB_CNTL_BUCKETNAME, value_type='string')
self.__connected = False
self._lockmode = lockmode if unlock_gil else LOCKMODE_NONE
self._lock = Lock()
self._pipeline_queue = None
self._embedref = None
InstanceReference.addref(self)
