def _add_creds(self, user, password):
bm = BufManager(ffi)
c_user = bm.new_cstr(user)
c_pass = bm.new_cstr(password)
pp = ffi.new('char*[2]', (c_user, c_pass))
rc = C.lcb_cntl(self._lcbh, C.LCB_CNTL_SET, C.LCB_CNTL_BUCKET_CRED, pp)
if rc:
pycbc_exc_lcb(rc)
def _connect(self):
rc = C.lcb_connect(self._lcbh)
if rc != C.LCB_SUCCESS:
raise pycbc_exc_lcb(rc)
if self._is_async:
return
C.lcb_wait(self._lcbh)
rc = C.lcb_get_bootstrap_status(self._lcbh)
if rc != C.LCB_SUCCESS:
raise pycbc_exc_lcb(rc)
self.__connected = True
def __init__(self, parent, ddoc, view, options, _flags=0, mres=None):
super(ViewResult, self).__init__(parent)
self._ddoc = ddoc
self._view = view
self._bound_cb = ffi.callback(ROWCB_DECL, self._on_single_row)
bm = BufManager(ffi)
urlopts = ffi.NULL
pypost = None
cmd = ffi.new('lcb_CMDVIEWQUERY*')
if options:
in_uri, in_post = options._long_query_encoded
# Note, encoded means URI/JSON encoded; not charset
urlopts = bm.new_cstr(in_uri)
if in_post and in_post != '{}':
pypost = in_post
C.lcb_view_query_initcmd(
cmd, bm.new_cstr(self._ddoc), bm.new_cstr(self._view),
urlopts, self._bound_cb)
if pypost:
cmd.postdata, cmd.npostdata = bm.new_cbuf(pypost)
cmd.cmdflags |= _flags
rc = C.lcb_view_query(self._instance, mres._cdata, cmd)
if rc:
raise pycbc_exc_lcb(rc)
def set_namedarg(self, arg, value):
bm = BufManager(ffi)
arg = bm.new_cbuf(arg)
value = bm.new_cbuf(value)
rc = C.lcb_n1p_namedparam(self._lp, arg, len(arg), value, len(value))
if rc:
raise pycbc_exc_lcb(rc)
def setoption(self, option, value):
bm = BufManager(ffi)
option = bm.new_cbuf(option)
value = bm.new_cbuf(value)
rc = C.lcb_n1p_setopt(self._lp, option, len(option), value, len(value))
if rc:
raise pycbc_exc_lcb(rc)
def _schedule(self, parent, mres):
bm = BufManager(ffi)
urlopts = ffi.NULL
pypost = None
cmd = self._c_command
if self._options:
in_uri, in_post = self._options._long_query_encoded
# Note, encoded means URI/JSON encoded; not charset
urlopts = bm.new_cstr(in_uri)
if in_post and in_post != '{}':
pypost = in_post
C.lcb_view_query_initcmd(cmd, bm.new_cstr(self._ddoc),
bm.new_cstr(self._view), urlopts,
self._bound_cb)
if pypost:
cmd.postdata, cmd.npostdata = bm.new_cbuf(pypost)
if self._include_docs:
cmd.cmdflags |= C.LCB_CMDVIEWQUERY_F_INCLUDE_DOCS
self._c_command.handle = self._c_handle
self._parent = parent
rc = C.lcb_view_query(parent._lcbh, mres._cdata, self._c_command)
if rc:
raise pycbc_exc_lcb(rc)
def _schedule(self, parent, mres):
bm = BufManager(ffi)
urlopts = ffi.NULL
pypost = None
cmd = self._c_command
if self._options:
in_uri, in_post = self._options._long_query_encoded
# Note, encoded means URI/JSON encoded; not charset
urlopts = bm.new_cstr(in_uri)
if in_post and in_post != '{}':
pypost = in_post
C.lcb_view_query_initcmd(
cmd, bm.new_cstr(self._ddoc), bm.new_cstr(self._view),
urlopts, self._bound_cb)
if pypost:
cmd.postdata, cmd.npostdata = bm.new_cbuf(pypost)
if self._include_docs:
cmd.cmdflags |= C.LCB_CMDVIEWQUERY_F_INCLUDE_DOCS
self._c_command.handle = self._c_handle
self._parent = parent
rc = C.lcb_view_query(parent._lcbh, mres._cdata, self._c_command)
if rc:
raise pycbc_exc_lcb(rc)
def setoption(self, option, value):
bm = BufManager(ffi)
option = bm.new_cbuf(option)
value = bm.new_cbuf(value)
rc = C.lcb_n1p_setopt(self._lp, option, len(option), value, len(value))
if rc:
raise pycbc_exc_lcb(rc)
def set_namedarg(self, arg, value):
bm = BufManager(ffi)
arg = bm.new_cbuf(arg)
value = bm.new_cbuf(value)
rc = C.lcb_n1p_namedparam(self._lp, arg, len(arg), value, len(value))
if rc:
raise pycbc_exc_lcb(rc)
def _vbmap(self, key):
bm = BufManager(ffi)
info_obj = ffi.new('lcb_cntl_vbinfo_t*')
info_obj.v.v0.key, info_obj.v.v0.nkey = bm.new_cbuf(key)
rc = C.lcb_cntl(self._lcbh, C.LCB_CNTL_GET, C.LCB_CNTL_VBMAP, info_obj)
if rc:
raise pycbc_exc_lcb(rc)
return info_obj.v.v0.vbucket, info_obj.v.v0.server_index
def _subdoc_callback(self, instance, cbtype, rb):
resp = ffi.cast('lcb_RESPSUBDOC*', rb)
mres = ffi.from_handle(resp.cookie)
buf = bytes(ffi.buffer(resp.key, resp.nkey))
try:
key = self._tc.decode_key(buf)
result = mres[key] # type: _SDResult
except:
raise pycbc_exc_enc(buf)
result.rc = resp.rc
if resp.rc not in (C.LCB_SUCCESS, C.LCB_SUBDOC_MULTI_FAILURE):
mres._add_bad_rc(resp.rc, result)
self._chk_op_done(mres)
return
else:
result.cas = resp.cas
# naming conventions as in the Couchbase C extension
cur = self._sd_entry
vii = self._sd_iterpos
vii[0] = 0 # Reset iterator
oix = 0
is_mutate = cbtype == C.LCB_CALLBACK_SDMUTATE
while C.lcb_sdresult_next(resp, cur, vii) != 0:
if is_mutate:
cur_index = cur.index
else:
cur_index = oix
oix += 1
if cur.status == 0 and cur.nvalue:
buf = bytes(ffi.buffer(cur.value, cur.nvalue))
try:
value = self._tc.decode_value(buf, FMT_JSON)
except:
raise pycbc_exc_enc(obj=buf)
else:
value = None
cur_tuple = (cur.status, value)
if cur.status:
if is_mutate or cur.status != C.LCB_SUBDOC_PATH_ENOENT:
spec = result._specs[cur_index]
try:
raise pycbc_exc_lcb(cur.status, 'Subcommand failure', obj=spec)
except PyCBC.default_exception:
mres._add_err(sys.exc_info())
result._add_result(cur_index, cur_tuple)
if is_mutate and result.success:
self._set_mutinfo(result, cbtype, rb)
if mres._dur and self._chain_endure(cbtype, mres, result, mres._dur):
return
self._chk_op_done(mres)
def _mutinfo(self):
pp = ffi.new('void**')
kb = ffi.new('lcb_KEYBUF*')
rc = C.lcb_cntl(self._lcbh, C.LCB_CNTL_GET, C.LCB_CNTL_VBCONFIG, pp)
if rc:
pycbc_exc_lcb(rc, "Couldn't get vBucket config")
nvb = C.vbucket_config_get_num_vbuckets(pp[0])
ll = []
kb.type = C.LCB_KV_VBID
for vbid in xrange(nvb):
kb.contig.nbytes = vbid
mt = C.lcb_get_mutation_token(self._lcbh, kb, ffi.NULL)
if not mt:
continue
ll.append((C.CBFFI_mt_vb(mt), C.CBFFI_mt_uuid(mt), C.CBFFI_mt_seq(mt)))
return ll
def __run_stat(self, k, mres):
bm = BufManager(ffi)
if k:
if not isinstance(k, basestring):
raise pycbc_exc_args('Stats arguments must be strings only!')
c_key, c_len = bm.new_cbuf(k)
else:
c_key, c_len = ffi.NULL, 0
C._Cb_set_key(self.c_command, c_key, c_len)
rc = C.lcb_stats3(self.instance, mres._cdata, self.c_command)
if rc:
raise pycbc_exc_lcb(rc)
def __run_stat(self, k, mres):
bm = BufManager(ffi)
if k:
if not isinstance(k, basestring):
raise pycbc_exc_args('Stats arguments must be strings only!')
c_key, c_len = bm.new_cbuf(k)
else:
c_key, c_len = ffi.NULL, 0
C._Cb_set_key(self.c_command, c_key, c_len)
rc = C.lcb_stats3(self.instance, mres._cdata, self.c_command)
if rc:
raise pycbc_exc_lcb(rc)
def _bootstrap_callback(self, _, status):
arg = None
self.__connected = True
if status != C.LCB_SUCCESS:
try:
raise pycbc_exc_lcb(status)
except Exception as e:
# Note _libcouchbase.so does this as well!
arg = e
if self._conncb:
cb = self._conncb
self._conncb = None
cb(arg)
def __init__(self, parent, params, prepare, mres=None):
super(N1qlResult, self).__init__(parent)
self._bound_cb = ffi.callback(
'void(lcb_t,int,lcb_RESPN1QL*)', self._on_single_row)
bm = BufManager(ffi)
cmd = ffi.new('lcb_CMDN1QL*')
cmd.query, cmd.nquery = bm.new_cbuf(params)
cmd.callback = self._bound_cb
if prepare:
cmd.cmdflags |= C.LCB_CMDN1QL_F_PREPCACHE
rc = C.lcb_n1ql_query(parent._lcbh, mres._cdata, cmd)
if rc:
raise pycbc_exc_lcb(rc)
def _invoke_submit(self, iterobj, is_dict, is_itmcoll, mres, global_kw):
"""
Internal function to invoke the actual submit_single function
:param iterobj: The raw object returned as the next item of the iterator
:param is_dict: True if iterator is a dictionary
:param is_itmcoll: True if the iterator contains Item objects
:param mres: The multi result object
:param global_kw: The global settings
:return: The return value of :meth:`submit_single`
"""
if is_itmcoll:
item, key_options = next(iterobj)
key = item.key
value = item.value
result = item
else:
if is_dict:
key, value = next(iterobj)
if not self.VALUES_ALLOWED and not is_itmcoll:
raise ArgumentError.pyexc(
'Values not allowed for this command', obj=value)
else:
key = next(iterobj)
value = None
key_options = {}
item = None
result = self.make_result(key, value)
result.rc = -1
# Attempt to get the encoded key:
key, value, key_options = self.make_entry_params(
key, value, key_options)
c_key, c_len = create_key(self.parent._tc, key)
rc = self.submit_single(c_key, c_len, value, item, key_options,
global_kw, mres)
if rc:
raise pycbc_exc_lcb(rc)
try:
if key in mres and not self.DUPKEY_OK:
# For tests:
self.parent._warn_dupkey(key)
mres[key] = result
except TypeError:
raise pycbc_exc_enc(obj=key)
def _invoke_submit(self, iterobj, is_dict, is_itmcoll, mres, global_kw):
"""
Internal function to invoke the actual submit_single function
:param iterobj: The raw object returned as the next item of the iterator
:param is_dict: True if iterator is a dictionary
:param is_itmcoll: True if the iterator contains Item objects
:param mres: The multi result object
:param global_kw: The global settings
:return: The return value of :meth:`submit_single`
"""
if is_itmcoll:
item, key_options = next(iterobj)
if not isinstance(item, Item):
pycbc_exc_args('Expected item object')
key = item.key
value = item.value
result = item
else:
if is_dict:
key, value = next(iterobj)
if not self.VALUES_ALLOWED and not is_itmcoll:
raise ArgumentError.pyexc(
'Values not allowed for this command', obj=value)
else:
key = next(iterobj)
value = None
key_options = {}
item = None
result = self.make_result(key, value)
result.rc = -1
# Attempt to get the encoded key:
key, value, key_options = self.make_entry_params(key, value, key_options)
c_key, c_len = create_key(self.parent._tc, key)
rc = self.submit_single(c_key, c_len, value, item, key_options, global_kw, mres)
if rc:
raise pycbc_exc_lcb(rc)
try:
if key in mres and not self.DUPKEY_OK:
# For tests:
self.parent._warn_dupkey(key)
mres[key] = result
except TypeError:
raise pycbc_exc_enc(obj=key)
def execute(self, kv, **kwargs):
mctx = self.create_context(**kwargs)
kwargs['_MCTX'] = mctx
try:
rv = super(MultiContextExecutor, self).execute(kv, **kwargs)
C.lcb_sched_enter(self.instance)
rc = mctx.done(mctx, rv._cdata)
if rc:
raise pycbc_exc_lcb(rc)
C.lcb_sched_leave(self.instance)
mctx = None
return rv
finally:
if mctx is not None:
mctx.fail(mctx)
C.lcb_sched_fail(self.instance)
def execute(self, kv, **kwargs):
mctx = self.create_context(**kwargs)
kwargs['_MCTX'] = mctx
try:
rv = super(MultiContextExecutor, self).execute(kv, **kwargs)
C.lcb_sched_enter(self.instance)
rc = mctx.done(mctx, rv._cdata)
if rc:
raise pycbc_exc_lcb(rc)
C.lcb_sched_leave(self.instance)
mctx = None
return rv
finally:
if mctx is not None:
mctx.fail(mctx)
C.lcb_sched_fail(self.instance)
def execute(self, lcbh, op, value):
if value is not None:
mode = C.LCB_CNTL_SET
else:
mode = C.LCB_CNTL_GET
c_data = self.allocate(mode)
if mode == C.LCB_CNTL_SET:
try:
self.convert_input(value, c_data)
except:
raise pycbc_exc_args(obj=value)
rc = C.lcb_cntl(lcbh, mode, op, c_data)
else:
rc = C.lcb_cntl(lcbh, mode, op, c_data)
if not rc:
return self.convert_output(c_data)
if rc:
raise pycbc_exc_lcb(rc)
def _add_bad_rc(self, rc, result):
"""
Sets an error with a bad return code. Handles 'quiet' logic
:param rc: The error code
"""
if not rc:
return
self.all_ok = False
if rc == C.LCB_KEY_ENOENT and self._quiet:
return
try:
raise pycbc_exc_lcb(rc)
except PyCBC.default_exception as e:
e.all_results = self
e.key = result.key
e.result = result
self._add_err(sys.exc_info())
def _add_bad_rc(self, rc, result):
"""
Sets an error with a bad return code. Handles 'quiet' logic
:param rc: The error code
"""
if not rc:
return
self.all_ok = False
if rc == C.LCB_KEY_ENOENT and self._quiet:
return
try:
raise pycbc_exc_lcb(rc)
except PyCBC.default_exception as e:
e.all_results = self
e.key = result.key
e.result = result
self._add_err(sys.exc_info())
def create_context(self, **kwargs):
C.memset(self.c_options, 0, ffi.sizeof(self.c_options[0]))
ok, persist_to, replicate_to = handle_durability(self.parent, **kwargs)
if not ok:
persist_to = -1
replicate_to = -1
self.__opt.persist_to = self._mk_criteria(self.__opt, persist_to)
self.__opt.replicate_to = self._mk_criteria(self.__opt, replicate_to)
if kwargs.get('check_removed'):
self.__opt.check_delete = 1
tmo = kwargs.get('timeout')
if tmo:
# print "Found timeout in options!"
try:
tmo = int(tmo * 1000000)
except:
raise pycbc_exc_args('Invalid timeout', obj=tmo)
else:
tmo = self.parent._dur_timeout
if tmo:
self.__opt.timeout = tmo
# print "OPTIONS: "
# print "PersistTo:", self.__opt.persist_to
# print "ReplicateTo:", self.__opt.replicate_to
# print "CapMax:", self.__opt.cap_max
# print "Timeout:", self.__opt.timeout
ctx = C.lcb_endure3_ctxnew(self.instance, self.c_options, self.__errp)
if not ctx:
raise pycbc_exc_lcb(self.__errp[0])
return ctx
def create_context(self, **kwargs):
C.memset(self.c_options, 0, ffi.sizeof(self.c_options[0]))
ok, persist_to, replicate_to = handle_durability(self.parent, **kwargs)
if not ok:
persist_to = -1
replicate_to = -1
self.__opt.persist_to = self._mk_criteria(self.__opt, persist_to)
self.__opt.replicate_to = self._mk_criteria(self.__opt, replicate_to)
if kwargs.get('check_removed'):
self.__opt.check_delete = 1
tmo = kwargs.get('timeout')
if tmo:
# print "Found timeout in options!"
try:
tmo = int(tmo * 1000000)
except:
raise pycbc_exc_args('Invalid timeout', obj=tmo)
else:
tmo = self.parent._dur_timeout
if tmo:
self.__opt.timeout = tmo
# print "OPTIONS: "
# print "PersistTo:", self.__opt.persist_to
# print "ReplicateTo:", self.__opt.replicate_to
# print "CapMax:", self.__opt.cap_max
# print "Timeout:", self.__opt.timeout
ctx = C.lcb_endure3_ctxnew(self.instance, self.c_options, self.__errp)
if not ctx:
raise pycbc_exc_lcb(self.__errp[0])
return ctx
def add_posarg(self, arg):
bm = BufManager(ffi)
arg = bm.new_cbuf(arg)
rc = C.lcb_n1p_posparam(self._lp, arg, len(arg))
if rc:
raise pycbc_exc_lcb(rc)
def setquery(self, query=None, type=C.LCB_N1P_QUERY_STATEMENT):
bm = BufManager(ffi)
query = bm.new_cbuf(query)
rc = C.lcb_n1p_setquery(self._lp, bm.new_cstr(query), len(query), type)
if rc:
raise pycbc_exc_lcb(rc)
def setquery(self, query=None, type=C.LCB_N1P_QUERY_STATEMENT):
bm = BufManager(ffi)
query = bm.new_cbuf(query)
rc = C.lcb_n1p_setquery(self._lp, bm.new_cstr(query), len(query), type)
if rc:
raise pycbc_exc_lcb(rc)
def _cntlstr(self, cntl_key, cntl_value):
rc = C.lcb_cntl_string(self._lcbh,
cntl_key.encode('utf-8'),
cntl_value.encode('utf-8'))
if rc:
raise pycbc_exc_lcb(rc)
def add_posarg(self, arg):
bm = BufManager(ffi)
arg = bm.new_cbuf(arg)
rc = C.lcb_n1p_posparam(self._lp, arg, len(arg))
if rc:
raise pycbc_exc_lcb(rc)
def __init__(self, connection_string=None, connstr=None,
username=None, password=None, quiet=False,
transcoder=None, default_format=FMT_JSON,
lockmode=LOCKMODE_EXC, unlock_gil=True,
_iops=None, _conntype=C.LCB_TYPE_BUCKET,
_flags=0):
crst = ffi.new('struct lcb_create_st*')
bm = BufManager(ffi)
if not connstr and not connection_string:
raise pycbc_exc_args('Must have connection string')
if connstr and connection_string:
raise pycbc_exc_args(
"Cannot specify both 'connstr' and 'connection_string'")
if not connstr:
connstr = connection_string
crst.version = 3
crst.v.v3.connstr = bm.new_cstr(connstr)
crst.v.v3.passwd = bm.new_cstr(password)
crst.v.v3.username = bm.new_cstr(username)
crst.v.v3.type = _conntype
if _iops:
procs = _iops
self._iowrap = IOPSWrapper(procs)
crst.v.v3.io = self._iowrap.get_lcb_iops()
else:
crst.v.v3.io = ffi.NULL
self._handles = set()
lcb_pp = ffi.new('lcb_t*')
rc = C.lcb_create(lcb_pp, crst)
if rc != C.LCB_SUCCESS:
raise pycbc_exc_lcb(rc)
self._lcbh = lcb_pp[0]
self._bound_cb = {
'store': ffi.callback(CALLBACK_DECL, self._storage_callback),
'get': ffi.callback(CALLBACK_DECL, self._get_callback),
'remove': ffi.callback(CALLBACK_DECL, self._remove_callback),
'counter': ffi.callback(CALLBACK_DECL, self._counter_callback),
'observe': ffi.callback(CALLBACK_DECL, self._observe_callback),
'stats': ffi.callback(CALLBACK_DECL, self._stats_callback),
'http': ffi.callback(CALLBACK_DECL, self._http_callback),
'_default': ffi.callback(CALLBACK_DECL, self._default_callback),
'_bootstrap': ffi.callback('void(lcb_t,lcb_error_t)',
self._bootstrap_callback),
'_dtor': ffi.callback('void(const void*)',
self._instance_destroyed)
}
self._executors = {
'upsert': executors.UpsertExecutor(self),
'insert': executors.InsertExecutor(self),
'replace': executors.ReplaceExecutor(self),
'append': executors.AppendExecutor(self),
'prepend': executors.PrependExecutor(self),
'get': executors.GetExecutor(self),
'lock': executors.LockExecutor(self),
'_unlock': executors.UnlockExecutor(self),
'touch': executors.TouchExecutor(self),
'remove': executors.RemoveExecutor(self),
'counter': executors.CounterExecutor(self),
'endure': executors.DurabilityExecutor(self),
'_chained_endure': executors.DurabilityChainExecutor(self),
'observe': executors.ObserveExecutor(self),
'stats': executors.StatsExecutor(self),
'_rget': executors.GetReplicaExecutor(self)
}
self._install_cb(C.LCB_CALLBACK_DEFAULT, '_default')
self._install_cb(C.LCB_CALLBACK_STORE, 'store')
self._install_cb(C.LCB_CALLBACK_GET, 'get')
self._install_cb(C.LCB_CALLBACK_GETREPLICA, 'get')
self._install_cb(C.LCB_CALLBACK_REMOVE, 'remove')
self._install_cb(C.LCB_CALLBACK_COUNTER, 'counter')
self._install_cb(C.LCB_CALLBACK_OBSERVE, 'observe')
self._install_cb(C.LCB_CALLBACK_STATS, 'stats')
self._install_cb(C.LCB_CALLBACK_HTTP, 'http')
C.lcb_set_bootstrap_callback(self._lcbh, self._bound_cb['_bootstrap'])
# Set our properties
self.data_passthrough = False
self.transcoder = transcoder
self.default_format = default_format
self.quiet = quiet
self.unlock_gil = unlock_gil
self._dur_persist_to = 0
self._dur_replicate_to = 0
self._dur_timeout = 0
self._dur_testhook = None
self._privflags = _flags
self._conncb = None
self.__bucket = self._cntl(C.LCB_CNTL_BUCKETNAME, value_type='string')
self.__connected = False
self._lockmode = lockmode if unlock_gil else LOCKMODE_NONE
self._lock = Lock()
self._pipeline_queue = None
self._embedref = None
InstanceReference.addref(self)