def __init__(self, connection_string=None, connstr=None,
username=None, password=None, quiet=False,
transcoder=None, default_format=FMT_JSON,
lockmode=LOCKMODE_EXC, unlock_gil=True,
_iops=None, _conntype=C.LCB_TYPE_BUCKET,
_flags=0):
crst = ffi.new('struct lcb_create_st*')
bm = BufManager(ffi)
if not connstr and not connection_string:
raise pycbc_exc_args('Must have connection string')
if connstr and connection_string:
raise pycbc_exc_args(
"Cannot specify both 'connstr' and 'connection_string'")
if not connstr:
connstr = connection_string
crst.version = 3
crst.v.v3.connstr = bm.new_cstr(connstr)
crst.v.v3.passwd = bm.new_cstr(password)
crst.v.v3.username = bm.new_cstr(username)
crst.v.v3.type = _conntype
if _iops:
procs = _iops
self._iowrap = IOPSWrapper(procs)
crst.v.v3.io = self._iowrap.get_lcb_iops()
else:
crst.v.v3.io = ffi.NULL
self._handles = set()
lcb_pp = ffi.new('lcb_t*')
rc = C.lcb_create(lcb_pp, crst)
if rc != C.LCB_SUCCESS:
raise pycbc_exc_lcb(rc)
self._lcbh = lcb_pp[0]
self._bound_cb = {
'store': ffi.callback(CALLBACK_DECL, self._storage_callback),
'get': ffi.callback(CALLBACK_DECL, self._get_callback),
'remove': ffi.callback(CALLBACK_DECL, self._remove_callback),
'counter': ffi.callback(CALLBACK_DECL, self._counter_callback),
'observe': ffi.callback(CALLBACK_DECL, self._observe_callback),
'stats': ffi.callback(CALLBACK_DECL, self._stats_callback),
'http': ffi.callback(CALLBACK_DECL, self._http_callback),
'_default': ffi.callback(CALLBACK_DECL, self._default_callback),
'_bootstrap': ffi.callback('void(lcb_t,lcb_error_t)',
self._bootstrap_callback),
'_dtor': ffi.callback('void(const void*)',
self._instance_destroyed)
}
self._executors = {
'upsert': executors.UpsertExecutor(self),
'insert': executors.InsertExecutor(self),
'replace': executors.ReplaceExecutor(self),
'append': executors.AppendExecutor(self),
'prepend': executors.PrependExecutor(self),
'get': executors.GetExecutor(self),
'lock': executors.LockExecutor(self),
'_unlock': executors.UnlockExecutor(self),
'touch': executors.TouchExecutor(self),
'remove': executors.RemoveExecutor(self),
'counter': executors.CounterExecutor(self),
'endure': executors.DurabilityExecutor(self),
'_chained_endure': executors.DurabilityChainExecutor(self),
'observe': executors.ObserveExecutor(self),
'stats': executors.StatsExecutor(self),
'_rget': executors.GetReplicaExecutor(self)
}
self._install_cb(C.LCB_CALLBACK_DEFAULT, '_default')
self._install_cb(C.LCB_CALLBACK_STORE, 'store')
self._install_cb(C.LCB_CALLBACK_GET, 'get')
self._install_cb(C.LCB_CALLBACK_GETREPLICA, 'get')
self._install_cb(C.LCB_CALLBACK_REMOVE, 'remove')
self._install_cb(C.LCB_CALLBACK_COUNTER, 'counter')
self._install_cb(C.LCB_CALLBACK_OBSERVE, 'observe')
self._install_cb(C.LCB_CALLBACK_STATS, 'stats')
self._install_cb(C.LCB_CALLBACK_HTTP, 'http')
C.lcb_set_bootstrap_callback(self._lcbh, self._bound_cb['_bootstrap'])
# Set our properties
self.data_passthrough = False
self.transcoder = transcoder
self.default_format = default_format
self.quiet = quiet
self.unlock_gil = unlock_gil
self._dur_persist_to = 0
self._dur_replicate_to = 0
self._dur_timeout = 0
self._dur_testhook = None
self._privflags = _flags
self._conncb = None
self.__bucket = self._cntl(C.LCB_CNTL_BUCKETNAME, value_type='string')
self.__connected = False
self._lockmode = lockmode if unlock_gil else LOCKMODE_NONE
self._lock = Lock()
self._pipeline_queue = None
self._embedref = None
InstanceReference.addref(self)
class Bucket(object):
# I'm using slots here because i have a LOT of attributes and I want to
# make sure i'm not assigning to anything silly:
__slots__ = [
# Private to cffi
'_handles', '_lcbh', '_bound_cb', '_executors', '_iowrap',
'__bucket', '_lock', '_lockmode', '_pipeline_queue',
'_embedref',
# Property holders
'__default_format', '__quiet', '__connected', '__dtor_handle',
# User-facing
'__transcoder', '__is_default_tc', 'data_passthrough', 'unlock_gil',
# Internal (used by couchbase and couchbase_ffi
'_dur_persist_to', '_dur_replicate_to', '_dur_timeout', '_dur_testhook',
'_privflags', '_conncb'
]
@property
def _tc(self):
return self.__transcoder
def __init__(self, connection_string=None, connstr=None,
username=None, password=None, quiet=False,
transcoder=None, default_format=FMT_JSON,
lockmode=LOCKMODE_EXC, unlock_gil=True,
_iops=None, _conntype=C.LCB_TYPE_BUCKET,
_flags=0):
crst = ffi.new('struct lcb_create_st*')
bm = BufManager(ffi)
if not connstr and not connection_string:
raise pycbc_exc_args('Must have connection string')
if connstr and connection_string:
raise pycbc_exc_args(
"Cannot specify both 'connstr' and 'connection_string'")
if not connstr:
connstr = connection_string
crst.version = 3
crst.v.v3.connstr = bm.new_cstr(connstr)
crst.v.v3.passwd = bm.new_cstr(password)
crst.v.v3.username = bm.new_cstr(username)
crst.v.v3.type = _conntype
if _iops:
procs = _iops
self._iowrap = IOPSWrapper(procs)
crst.v.v3.io = self._iowrap.get_lcb_iops()
else:
crst.v.v3.io = ffi.NULL
self._handles = set()
lcb_pp = ffi.new('lcb_t*')
rc = C.lcb_create(lcb_pp, crst)
if rc != C.LCB_SUCCESS:
raise pycbc_exc_lcb(rc)
self._lcbh = lcb_pp[0]
self._bound_cb = {
'store': ffi.callback(CALLBACK_DECL, self._storage_callback),
'get': ffi.callback(CALLBACK_DECL, self._get_callback),
'remove': ffi.callback(CALLBACK_DECL, self._remove_callback),
'counter': ffi.callback(CALLBACK_DECL, self._counter_callback),
'observe': ffi.callback(CALLBACK_DECL, self._observe_callback),
'stats': ffi.callback(CALLBACK_DECL, self._stats_callback),
'http': ffi.callback(CALLBACK_DECL, self._http_callback),
'_default': ffi.callback(CALLBACK_DECL, self._default_callback),
'_bootstrap': ffi.callback('void(lcb_t,lcb_error_t)',
self._bootstrap_callback),
'_dtor': ffi.callback('void(const void*)',
self._instance_destroyed)
}
self._executors = {
'upsert': executors.UpsertExecutor(self),
'insert': executors.InsertExecutor(self),
'replace': executors.ReplaceExecutor(self),
'append': executors.AppendExecutor(self),
'prepend': executors.PrependExecutor(self),
'get': executors.GetExecutor(self),
'lock': executors.LockExecutor(self),
'_unlock': executors.UnlockExecutor(self),
'touch': executors.TouchExecutor(self),
'remove': executors.RemoveExecutor(self),
'counter': executors.CounterExecutor(self),
'endure': executors.DurabilityExecutor(self),
'_chained_endure': executors.DurabilityChainExecutor(self),
'observe': executors.ObserveExecutor(self),
'stats': executors.StatsExecutor(self),
'_rget': executors.GetReplicaExecutor(self)
}
self._install_cb(C.LCB_CALLBACK_DEFAULT, '_default')
self._install_cb(C.LCB_CALLBACK_STORE, 'store')
self._install_cb(C.LCB_CALLBACK_GET, 'get')
self._install_cb(C.LCB_CALLBACK_GETREPLICA, 'get')
self._install_cb(C.LCB_CALLBACK_REMOVE, 'remove')
self._install_cb(C.LCB_CALLBACK_COUNTER, 'counter')
self._install_cb(C.LCB_CALLBACK_OBSERVE, 'observe')
self._install_cb(C.LCB_CALLBACK_STATS, 'stats')
self._install_cb(C.LCB_CALLBACK_HTTP, 'http')
C.lcb_set_bootstrap_callback(self._lcbh, self._bound_cb['_bootstrap'])
# Set our properties
self.data_passthrough = False
self.transcoder = transcoder
self.default_format = default_format
self.quiet = quiet
self.unlock_gil = unlock_gil
self._dur_persist_to = 0
self._dur_replicate_to = 0
self._dur_timeout = 0
self._dur_testhook = None
self._privflags = _flags
self._conncb = None
self.__bucket = self._cntl(C.LCB_CNTL_BUCKETNAME, value_type='string')
self.__connected = False
self._lockmode = lockmode if unlock_gil else LOCKMODE_NONE
self._lock = Lock()
self._pipeline_queue = None
self._embedref = None
InstanceReference.addref(self)
@property
def default_format(self):
return self.__default_format
@default_format.setter
def default_format(self, arg):
if arg is PyCBC.fmt_auto:
self.__default_format = arg
return
if isinstance(arg, bool):
raise pycbc_exc_args('Must be a number', obj=arg)
if not isinstance(arg, int):
raise pycbc_exc_args('Must be a number', obj=arg)
self.__default_format = arg
@property
def quiet(self):
return self.__quiet
@quiet.setter
def quiet(self, arg):
if isinstance(arg, bool):
self.__quiet = arg
else:
raise pycbc_exc_args("'quiet' must be bool")
@property
def bucket(self):
return self.__bucket
@property
def lockmode(self):
return self._lockmode
@property
def transcoder(self):
return None if self.__is_default_tc else self.__transcoder
@transcoder.setter
def transcoder(self, arg):
if arg in (None, False):
self.__transcoder = _make_transcoder()
self.__is_default_tc = True
else:
self.__transcoder = arg
self.__is_default_tc = True
@property
def _is_async(self):
return self._privflags & PYCBC_CONN_F_ASYNC
@property
def connected(self):
return self.__connected
@property
def _wref(self):
return self._embedref if self._embedref \
else InstanceReference.getref(self)
@property
def _dtorcb(self):
return self._wref.dtorhook
@_dtorcb.setter
def _dtorcb(self, newval):
self._wref.dtorhook = newval
def _make_mres(self):
if self._is_async:
return AsyncResult()
else:
return MultiResult()
def _install_cb(self, cbtype, name):
C.lcb_install_callback3(self._lcbh, cbtype, self._bound_cb[name])
def _connect(self):
rc = C.lcb_connect(self._lcbh)
if rc != C.LCB_SUCCESS:
raise pycbc_exc_lcb(rc)
if self._is_async:
return
C.lcb_wait(self._lcbh)
rc = C.lcb_get_bootstrap_status(self._lcbh)
if rc != C.LCB_SUCCESS:
raise pycbc_exc_lcb(rc)
self.__connected = True
def _bootstrap_callback(self, _, status):
arg = None
self.__connected = True
if status != C.LCB_SUCCESS:
try:
raise pycbc_exc_lcb(status)
except Exception as e:
# Note _libcouchbase.so does this as well!
arg = e
if self._conncb:
cb = self._conncb
self._conncb = None
cb(arg)
def _do_lock(self):
if self._lockmode == LOCKMODE_NONE:
return
elif self._lockmode == LOCKMODE_EXC:
if not self._lock.acquire(False):
raise PyCBC.exc_lock()
else:
self._lock.acquire(True)
def _do_unlock(self):
if self._lockmode != LOCKMODE_NONE:
self._lock.release()
def _thr_lockop(self, arg):
# Used by tests
if not self._lockmode:
raise PyCBC.exc_lock()
if not arg:
self._lock.acquire(True)
else:
self._lock.release()
def _chk_no_pipeline(self, msg='Pipeline active'):
if self._pipeline_queue is not None:
PyCBC.exc_pipeline(msg)
def _pipeline_begin(self):
self._chk_no_pipeline('Pipeline is already active')
self._pipeline_queue = []
def _pipeline_end(self):
if self._pipeline_queue is None:
PyCBC.exc_pipeline('No pipeline in progress!')
C.lcb_wait(self._lcbh)
results = self._pipeline_queue
self._pipeline_queue = None
rv = []
for mres in results:
mres._maybe_throw()
if mres._is_single:
rv.append(mres.unwrap_single())
else:
rv.append(mres)
return rv
def _run_sync(self, mres):
self._handles.add(mres)
if self._pipeline_queue is None:
C.lcb_wait(self._lcbh)
mres._maybe_throw()
else:
self._pipeline_queue.append(mres)
return mres
def _run_async(self, mres):
self._handles.add(mres)
return mres
def _run_sync_single(self, mres):
return self._run_sync(mres).unwrap_single()
def _run_single(self, mres):
if not self._is_async:
return self._run_sync_single(mres)
else:
return self._run_async(mres)
def _run_multi(self, mres):
mres._is_single = False
if not self._is_async:
return self._run_sync(mres)
else:
return self._run_async(mres)
def _execute_single_k(self, name, key, **kwargs):
self._do_lock()
try:
proc = self._executors[name]
mres = proc.execute((key,), **kwargs)
return self._run_single(mres)
finally:
self._do_unlock()
def _execute_single_kv(self, name, key, value, **kwargs):
try:
kv = {key: value}
except TypeError:
raise pycbc_exc_enc('Bad key-value', obj=(key,value))
self._do_lock()
try:
proc = self._executors[name]
mres = proc.execute(kv, **kwargs)
return self._run_single(mres)
finally:
self._do_unlock()
def _execute_multi(self, name, kv, **kwargs):
self._do_lock()
try:
proc = self._executors[name]
mres = proc.execute(kv, **kwargs)
return self._run_multi(mres)
finally:
self._do_unlock()
_VALUE_METHS = ['upsert', 'insert', 'replace', 'append', 'prepend']
_KEY_METHS = ['get', 'lock', 'touch', 'remove', 'counter',
'observe', 'endure', '_rget', '_unlock']
for name in _VALUE_METHS + _KEY_METHS:
n_single = name
n_multi = name + '_multi'
if name in _VALUE_METHS:
m_single, m_multi = _gen_valmeth(name)
else:
m_single, m_multi = _gen_keymeth(name)
locals().update({n_single: m_single, n_multi: m_multi})
# This name is used by couchbase.bucket.Bucket
def _stats(self, kv):
return self._execute_multi('stats', kv)
# Unlock is special since we document the bare second arg as meaning
# the CAS
def unlock(self, key, cas, **kwargs):
kwargs['cas'] = cas
return self._unlock(key, **kwargs)
# noinspection PyUnresolvedReferences
unlock_multi = _unlock_multi
# noinspection PyUnresolvedReferences
_rgetix = _rget
# noinspection PyUnresolvedReferences
_rgetix_multi = _rget_multi
def _view_request(self, design, view, options, include_docs):
self._chk_no_pipeline('View requests not valid in pipeline mode')
res = ViewResult(design, view, options, include_docs)
mres = self._make_mres()
mres[None] = res
res._schedule(self, mres)
return mres
def _http_request(self, path, **kwargs):
self._chk_no_pipeline('HTTP requests not valid in pipeline mode')
htreq = HttpRequest(path, **kwargs)
mres = self._make_mres()
htreq._schedule(self, mres)
self._handles.add(mres)
return self._run_single(mres)
def _cntlstr(self, cntl_key, cntl_value):
rc = C.lcb_cntl_string(self._lcbh,
cntl_key.encode('utf-8'),
cntl_value.encode('utf-8'))
if rc:
raise pycbc_exc_lcb(rc)
OLD_CNTL_MAP = {
0x00: 'uint32_t',
0x01: 'uint32_t'
}
def _cntl(self, op, value=None, value_type=None):
if value_type is None and op in self.OLD_CNTL_MAP:
value_type = self.OLD_CNTL_MAP[op]
try:
handler = CNTL_VTYPE_MAP[value_type]
except KeyError:
raise pycbc_exc_args('Invalid value type', obj=value_type)
return handler.execute(self._lcbh, op, value)
def _vbmap(self, key):
bm = BufManager(ffi)
info_obj = ffi.new('lcb_cntl_vbinfo_t*')
info_obj.v.v0.key, info_obj.v.v0.nkey = bm.new_cbuf(key)
rc = C.lcb_cntl(self._lcbh, C.LCB_CNTL_GET, C.LCB_CNTL_VBMAP, info_obj)
if rc:
raise pycbc_exc_lcb(rc)
return info_obj.v.v0.vbucket, info_obj.v.v0.server_index
@property
def _closed(self):
return self._privflags & PYCBC_CONN_F_CLOSED
def _close(self, is_async=False):
if self._closed or not self._lcbh:
return
# Delete the reference to ourselves
wref = InstanceReference.getref(self)
InstanceReference.delref(self)
self._embedref = wref
if not is_async:
C.lcb_destroy(self._lcbh)
else:
C.lcb_set_destroy_callback(self._lcbh, self._bound_cb['_dtor'])
C.lcb_destroy_async(self._lcbh, ffi.NULL)
lcb_pp = ffi.new('lcb_t*')
C.lcb_create(lcb_pp, ffi.NULL)
self._lcbh = lcb_pp[0]
self._privflags |= PYCBC_CONN_F_CLOSED
def _instance_destroyed(self, _):
if self._dtorcb:
self._dtorcb()
self._dtorcb = None
def _async_shutdown(self):
self._close(is_async=True)
@property
def server_nodes(self):
nodelist = C.lcb_get_server_list(self._lcbh)
s_list = []
ix = 0
while True:
cur_str = nodelist[ix]
if not cur_str:
break
s_list.append(from_cstring(cur_str))
ix += 1
return s_list
def _chk_op_done(self, mres):
if not mres:
return
if not mres._decr_remaining():
return
# So the result is complete
self._handles.remove(mres)
if self._is_async:
mres.invoke()
def _chain_endure(self, optype, mres, result, dur):
persist_to, replicate_to = dur
proc = self._executors['_chained_endure']
try:
proc.execute(kv={result.key: result},
persist_to=persist_to, replicate_to=replicate_to,
check_removed=optype == C.LCB_CALLBACK_REMOVE,
_MRES=mres)
return True
except:
mres._add_err(sys.exc_info())
return False
def _default_callback(self, *args):
_, mres = self._callback_common(*args)
self._chk_op_done(mres)
def _callback_common(self, _, cbtype, resp):
mres = ffi.from_handle(resp.cookie)
buf = bytes(ffi.buffer(resp.key, resp.nkey))
try:
key = self._tc.decode_key(buf)
result = mres[key]
except:
raise pycbc_exc_enc(buf)
result.rc = resp.rc
if resp.rc:
mres._add_bad_rc(resp.rc, result)
else:
result.cas = resp.cas
if self._dur_testhook:
self._dur_testhook(result)
if cbtype != C.LCB_CALLBACK_ENDURE and mres._dur:
if self._chain_endure(cbtype, mres, result, mres._dur):
return None, None
return result, mres
def _storage_callback(self, instance, cbtype, resp):
_, mres = self._callback_common(instance, cbtype, resp)
self._chk_op_done(mres)
def _get_callback(self, instance, cbtype, resp):
result, mres = self._callback_common(instance, cbtype, resp)
resp = ffi.cast('lcb_RESPGET*', resp)
result.flags = resp.itmflags
if not resp.rc:
buf = bytes(ffi.buffer(resp.value, resp.nvalue))
if not self.data_passthrough and not mres._no_format:
try:
result.value = self._tc.decode_value(buf, resp.itmflags)
except:
result.value = buf[::]
try:
raise pycbc_exc_enc(obj=buf)
except PyCBC.default_exception:
mres._add_err(sys.exc_info())
else:
result.value = buf[::]
self._chk_op_done(mres)
def _remove_callback(self, instance, cbtype, resp):
_, mres = self._callback_common(instance, cbtype, resp)
self._chk_op_done(mres)
def _counter_callback(self, instance, cbtype, resp):
result, mres = self._callback_common(instance, cbtype, resp)
if not resp.rc:
resp = ffi.cast('lcb_RESPCOUNTER*', resp)
result.value = resp.value
self._chk_op_done(mres)
def _observe_callback(self, instance, cbtype, resp):
resp = ffi.cast('lcb_RESPOBSERVE*', resp)
if resp.rflags & C.LCB_RESP_F_FINAL:
mres = ffi.from_handle(resp.cookie)
self._chk_op_done(mres)
return
rc = resp.rc
result, mres = self._callback_common(instance, cbtype, resp)
if rc:
if not result.rc:
result.rc = rc
return
oi = ObserveInfo()
oi.cas = resp.cas
oi.flags = resp.status
oi.from_master = resp.ismaster
result.value.append(oi)
# noinspection PyUnusedLocal
def _stats_callback(self, instance, cbtype, resp):
resp = ffi.cast('lcb_RESPSTATS*', resp)
mres = ffi.from_handle(resp.cookie)
if resp.rc:
r = ValueResult()
r.key = '__dummy__'
mres._add_bad_rc(resp.rc, r)
if resp.key == ffi.NULL and resp.server == ffi.NULL:
self._chk_op_done(mres)
return
if resp.rc:
return
kbuf = bytes(ffi.buffer(resp.key, resp.nkey))
key = self._tc.decode_key(kbuf)
if resp.nvalue:
value = from_cstring(resp.value, resp.nvalue)
try:
value = int(value)
except ValueError:
pass
else:
value = None
server = from_cstring(resp.server)
mres.setdefault(key, {})[server] = value
def _http_callback(self, instance, cbtype, resp):
mres = ffi.from_handle(resp.cookie)
resp = ffi.cast('lcb_RESPHTTP*', resp)
htres = mres[None]
self._handles.remove(mres)
htres._handle_response(mres, resp)
def _warn_dupkey(self, k):
"""
Really odd function - used to help ensure we actually warn for duplicate
keys.
"""
if self._privflags & PYCBC_CONN_F_WARNEXPLICIT:
warnings.warn_explicit(
'Found duplicate keys! {0}'.format(k), RuntimeWarning,
__file__, -1, module='couchbase_ffi.bucket', registry={})
else:
warnings.warn('Found duplicate keys!', RuntimeWarning)