def __init__(self,
keystore,
public_key_name,
hmac_key_name,
iv=None,
block_size=32):
"""
Create a new instance of the AES-256-HMAC-SHA1 encryption provider.
:param keystore: The keystore used to encrypt / decrypt
:param hmac_key_name: The HMAC key name used to sign and verify
:param iv:
:param block_size:
"""
super(AES256CryptoProvider, self).__init__()
if not keystore:
raise ArgumentError.pyexc("KeyStore must be provided.")
if not public_key_name:
raise ArgumentError.pyexc("Public key name must be provided.")
if not hmac_key_name:
raise ArgumentError.pyexc("HMAC key name must be provided.")
self.keystore = keystore
self.public_key_name = public_key_name
self.hmac_key_name = hmac_key_name
self.iv = iv
self.block_size = block_size
def _bool_param_handler(input):
if isinstance(input, bool):
if input:
return "true"
else:
return "false"
if isinstance(input, basestring):
if input not in ("true", "false"):
raise ArgumentError.pyexc(
"String for boolean must be "
"'true' or 'false'", input)
return input
try:
input + 0
if input:
return "true"
else:
return "false"
except TypeError:
raise ArgumentError.pyexc(
"Boolean value must be boolean, "
"numeric, or a string of 'true' "
"or 'false'", input)
def build(self, kv):
self._verify_iter(kv)
if not len(kv):
raise ArgumentError.pyexc(obj=kv, message="No items in container")
self.make_struct_pointers(len(kv))
curix = 0
if isinstance(kv, dict):
kviter = kv.iteritems()
else:
kviter = iter(kv)
while True:
try:
self._build_single(kv, kviter, curix)
curix += 1
except StopIteration:
break
if curix != self._ncmds:
raise ArgumentError.pyexc(message="Bad iterator", obj=kv)
return self._cmdpp
def _num_param_handler(input):
# Don't allow booleans:
if isinstance(input, bool):
raise ArgumentError.pyexc("Cannot use booleans as numeric values", input)
try:
return str(int(input))
except Exception as e:
raise ArgumentError.pyexc("Expected a numeric argument", input, e)
def _num_param_handler(input):
# Don't allow booleans:
if isinstance(input, bool):
raise ArgumentError.pyexc("Cannot use booleans as numeric values",
input)
try:
return str(int(input))
except Exception as e:
raise ArgumentError.pyexc("Expected a numeric argument", input, e)
def extract_ttl(req, options):
try:
ttl = options.get('ttl', 0)
if ttl < 0:
raise ArgumentError.pyexc("TTL cannot be negative", obj=ttl)
req.exptime = ttl
return ttl
except CouchbaseError:
raise
except Exception as e:
raise ArgumentError.pyexc(inner=e, obj=options.get('ttl', 0))
def handle_durability(self, parent, **kwargs):
self.persist_to = kwargs.get('persist_to', parent._dur_persist_to)
self.replicate_to = kwargs.get('replicate_to',
parent._dur_replicate_to)
if not self.persist_to or self.replicate_to:
return
n_replicas = C.lcb_get_num_replicas(parent._instance)
if self.replicate_to > n_replicas or self.persist_to + (n_replicas +
1):
ArgumentError.pyexc("Durability requirements will never "
"be satisfied")
def extract_ttl(req, options):
try:
ttl = options.get('ttl', 0)
if ttl < 0:
raise ArgumentError.pyexc("TTL cannot be negative", obj=ttl)
req.exptime = ttl
return ttl
except CouchbaseError:
raise
except Exception as e:
raise ArgumentError.pyexc(inner=e, obj=options.get('ttl', 0))
def handle_durability(self, parent, **kwargs):
self.persist_to = kwargs.get('persist_to',
parent._dur_persist_to)
self.replicate_to = kwargs.get('replicate_to',
parent._dur_replicate_to)
if not self.persist_to or self.replicate_to:
return
n_replicas = C.lcb_get_num_replicas(parent._instance)
if self.replicate_to > n_replicas or self.persist_to + (n_replicas+1):
ArgumentError.pyexc("Durability requirements will never "
"be satisfied")
def _verify_iter(cls, kv):
if isinstance(kv, (dict, list, tuple)):
return
if isinstance(kv, cls.FORBIDDEN_ITER):
raise ArgumentError.pyexc(obj=kv, message="Bad sequence type")
if not hasattr(kv, '__iter__'):
raise ArgumentError.pyexc(obj=kv, message="Object is not iterable")
try:
len(kv)
except TypeError:
raise ArgumentError.pyexc(obj=kv,
message="Iterator must have length")
def _verify_iter(self, kv):
if isinstance(kv, (dict, list, tuple)):
return
if isinstance(kv, self._disallowed_types):
raise ArgumentError.pyexc(obj=kv, message="Bad sequence type")
if not hasattr(kv, '__iter__'):
raise ArgumentError.pyexc(obj=kv, message="Object is not iterable")
try:
len(kv)
except TypeError:
raise ArgumentError.pyexc(obj=kv,
message="Iterator must have length")
def _verify_iter(cls, kv):
if isinstance(kv, (dict, list, tuple)):
return
if isinstance(kv, cls.FORBIDDEN_ITER):
raise ArgumentError.pyexc(obj=kv, message="Bad sequence type")
if not hasattr(kv, '__iter__'):
raise ArgumentError.pyexc(obj=kv, message="Object is not iterable")
try:
len(kv)
except TypeError:
raise ArgumentError.pyexc(obj=kv,
message="Iterator must have length")
def _verify_iter(self, kv):
if isinstance(kv, (dict, list, tuple)):
return
if isinstance(kv, self._disallowed_types):
raise ArgumentError.pyexc(obj=kv, message="Bad sequence type")
if not hasattr(kv, '__iter__'):
raise ArgumentError.pyexc(obj=kv, message="Object is not iterable")
try:
len(kv)
except TypeError:
raise ArgumentError.pyexc(obj=kv,
message="Iterator must have length")
def raise_include_docs(self):
"""
Raise an error on include docs
"""
raise ArgumentError.pyexc(
"Include docs not supported with async views. If you "
"must gather docs, you should do so manually")
def _jarry_param_handler(input):
ret = _jval_param_handler(input)
if not ret.startswith('['):
raise ArgumentError.pyexc(
"Value must be converted to JSON array", input)
return ret
def raise_include_docs(self):
"""
Raise an error on include docs
"""
raise ArgumentError.pyexc(
"Include docs not supported with async views. If you "
"must gather docs, you should do so manually")
def _set_common(self, param, value, set_user=True):
# Invalidate encoded string
self._encoded = None
if value is UNSPEC:
self._real_options.pop(param, None)
if set_user:
self._user_options.pop(param, None)
return
handler = _HANDLER_MAP.get(param)
if not handler:
if not self.unrecognized_ok:
raise ArgumentError.pyexc(
"Unrecognized parameter. To use unrecognized parameters, "
"set 'unrecognized_ok' to True")
if not handler:
self._extra_options[param] = _string_param_handler(value)
return
if self.passthrough:
handler = _string_param_handler
self._real_options[param] = handler(value)
if set_user:
self._user_options[param] = value
def _set_common(self, param, value, set_user=True):
# Invalidate encoded string
self._encoded = None
if value is UNSPEC:
self._real_options.pop(param, None)
if set_user:
self._user_options.pop(param, None)
return
handler = _HANDLER_MAP.get(param)
if not handler:
if not self.unrecognized_ok:
raise ArgumentError.pyexc(
"Unrecognized parameter. To use unrecognized parameters, "
"set 'unrecognized_ok' to True")
if not handler:
self._extra_options[param] = _string_param_handler(value)
return
if self.passthrough:
handler = _string_param_handler
self._real_options[param] = handler(value)
if set_user:
self._user_options[param] = value
def _jarry_param_handler(input):
ret = _jval_param_handler(input)
if not ret.startswith('['):
raise ArgumentError.pyexc("Value must be converted to JSON array",
input)
return ret
def submit_single(self, c_key, c_len, value, item, key_options, global_options, mres):
cas = get_cas(key_options, global_options, item)
if not cas:
raise ArgumentError.pyexc("Must have CAS")
C._Cb_set_key(self.c_command, c_key, c_len)
self.c_command.cas = cas
return C.lcb_unlock3(self.instance, mres._cdata, self.c_command)
def process_single_command(self, req, koptions):
options = self._koptions.copy()
options.update(koptions)
if not options.get('cas', 0):
raise ArgumentError.pyexc("Must have CAS for unlock")
req.cas = options['cas']
def process_single_command(self, req, koptions):
options = self._koptions.copy()
options.update(koptions)
if not options.get('cas', 0):
raise ArgumentError.pyexc("Must have CAS for unlock")
req.cas = options['cas']
def _set_range_common(self, k_sugar, k_start, k_end, value):
"""
Checks to see if the client-side convenience key is present, and if so
converts the sugar convenience key into its real server-side
equivalents.
:param string k_sugar: The client-side convenience key
:param string k_start: The server-side key specifying the beginning of
the range
:param string k_end: The server-side key specifying the end of the
range
"""
if not isinstance(value, (list, tuple, _Unspec)):
raise ArgumentError.pyexc(
"Range specification for {0} must be a list, tuple or UNSPEC".
format(k_sugar))
if self._user_options.get(
k_start, UNSPEC) is not UNSPEC or (self._user_options.get(
k_end, UNSPEC) is not UNSPEC):
raise ArgumentError.pyexc(
"Cannot specify {0} with either {1} or {2}".format(
k_sugar, k_start, k_end))
if not value:
self._set_common(k_start, UNSPEC, set_user=False)
self._set_common(k_end, UNSPEC, set_user=False)
self._user_options[k_sugar] = UNSPEC
return
if len(value) not in (1, 2):
raise ArgumentError.pyexc(
"Range specification "
"must have one or two elements", value)
value = value[::]
if len(value) == 1:
value.append(UNSPEC)
for p, ix in ((k_start, 0), (k_end, 1)):
self._set_common(p, value[ix], set_user=False)
self._user_options[k_sugar] = value
def submit_single(self, c_key, c_len, value, item, key_options,
global_options, mres):
cas = get_cas(key_options, global_options, item)
if not cas:
raise ArgumentError.pyexc("Must have CAS")
C._Cb_set_key(self.c_command, c_key, c_len)
self.c_command.cas = cas
return C.lcb_unlock3(self.instance, mres._cdata, self.c_command)
def submit_single(self, c_key, c_len, value, item, key_options, global_options, mres):
ttl = get_ttl(key_options, global_options, item)
if self.IS_LOCK and not ttl:
raise ArgumentError.pyexc("Lock must have TTL")
C._Cb_set_key(self.c_command, c_key, c_len)
self.c_command.exptime = ttl
self.c_command.lock = self.IS_LOCK
return C.lcb_get3(self.instance, mres._cdata, self.c_command)
def get_key(self, key_id):
"""
Retrieves a key from the key store using the given key id.
:param key_id: The name for the key in the key store.
"""
if not key_id:
raise ArgumentError.pyexc("key_id must not be empty.")
return self.keystore.secret_keys[key_id]
def _set_range_common(self, k_sugar, k_start, k_end, value):
"""
Checks to see if the client-side convenience key is present, and if so
converts the sugar convenience key into its real server-side
equivalents.
:param string k_sugar: The client-side convenience key
:param string k_start: The server-side key specifying the beginning of
the range
:param string k_end: The server-side key specifying the end of the
range
"""
if not isinstance(value, (list, tuple, _Unspec)):
raise ArgumentError.pyexc(
"Range specification for {0} must be a list, tuple or UNSPEC"
.format(k_sugar))
if self._user_options.get(k_start, UNSPEC) is not UNSPEC or (
self._user_options.get(k_end, UNSPEC) is not UNSPEC):
raise ArgumentError.pyexc(
"Cannot specify {0} with either {1} or {2}"
.format(k_sugar, k_start, k_end))
if not value:
self._set_common(k_start, UNSPEC, set_user=False)
self._set_common(k_end, UNSPEC, set_user=False)
self._user_options[k_sugar] = UNSPEC
return
if len(value) not in (1, 2):
raise ArgumentError.pyexc("Range specification "
"must have one or two elements",
value)
value = value[::]
if len(value) == 1:
value.append(UNSPEC)
for p, ix in ((k_start, 0), (k_end, 1)):
self._set_common(p, value[ix], set_user=False)
self._user_options[k_sugar] = value
def process_single_command(self, req, koptions):
options = koptions.copy()
options.update(self._koptions)
ttl = self.extract_ttl(req, options)
if self._is_lock:
req.lock = 1
if not ttl:
raise ArgumentError.pyexc("Lock must have TTL")
def process_single_command(self, req, koptions):
options = koptions.copy()
options.update(self._koptions)
ttl = self.extract_ttl(req, options)
if self._is_lock:
req.lock = 1
if not ttl:
raise ArgumentError.pyexc("Lock must have TTL")
def __init__(self, file_path, passphrase):
"""
Creates a new instance of a Key Store that can retrieve keys from a Java Key Store
:param file_path: Path to the JKS file.
:param passphrase: The passphrase used to access the key store.
"""
if not file_path:
raise ArgumentError.pyexc("file_path cannot be empty.")
self.keystore = jks.KeyStore.load(file_path, passphrase)
def submit_single(self, c_key, c_len, value, item, key_options,
global_options, mres):
ttl = get_ttl(key_options, global_options, item)
if self.IS_LOCK and not ttl:
raise ArgumentError.pyexc("Lock must have TTL")
C._Cb_set_key(self.c_command, c_key, c_len)
self.c_command.exptime = ttl
self.c_command.lock = self.IS_LOCK
return C.lcb_get3(self.instance, mres._cdata, self.c_command)
def _string_param_common(input, do_quote=False):
# TODO, if we pass this to urlencode, do we ever need to quote?
# For the moment, i'm always forcing non-quote behavior
do_quote = False
s = None
if isinstance(input, basestring):
s = input
elif isinstance(input, bool):
raise ArgumentError.pyexc("Can't use boolean as string", input)
elif isinstance(input, (int, long, float)):
# Basic numeric types:
s = str(input)
else:
raise ArgumentError.pyexc("Expected simple numeric type or string ", input)
if do_quote:
s = ulp.quote(s)
return s
def get_ttl(key_options, global_options, *_):
"""
Quick function to extract the expiration time from options. Throws
appropriate exceptions when necessary
:param key_options:
:param global_options:
:return:
"""
ttl = get_option('ttl', key_options, global_options)
if ttl is None or ttl == 0:
return 0
try:
ttl = int(ttl)
if ttl < 0:
raise ArgumentError.pyexc("TTL cannot be negative", obj=ttl)
elif ttl > 1 << 31:
raise ArgumentError.pyexc("TTL Value too large", obj=ttl)
return ttl
except CouchbaseError:
raise
except Exception as e:
raise ArgumentError.pyexc(inner=e, obj=ttl)
def _bool_param_handler(input):
if isinstance(input, bool):
if input:
return "true"
else:
return "false"
if isinstance(input, basestring):
if input not in ("true", "false"):
raise ArgumentError.pyexc("String for boolean must be " "'true' or 'false'", input)
return input
try:
input + 0
if input:
return "true"
else:
return "false"
except TypeError:
raise ArgumentError.pyexc(
"Boolean value must be boolean, " "numeric, or a string of 'true' " "or 'false'", input
)
def _string_param_common(input, do_quote=False):
# TODO, if we pass this to urlencode, do we ever need to quote?
# For the moment, i'm always forcing non-quote behavior
do_quote = False
s = None
if isinstance(input, basestring):
s = input
elif isinstance(input, bool):
raise ArgumentError.pyexc("Can't use boolean as string", input)
elif isinstance(input, (int, long, float)):
# Basic numeric types:
s = str(input)
else:
raise ArgumentError.pyexc("Expected simple numeric type or string ",
input)
if do_quote:
s = ulp.quote(s)
return s
def get_ttl(key_options, global_options, *_):
"""
Quick function to extract the expiration time from options. Throws
appropriate exceptions when necessary
:param key_options:
:param global_options:
:return:
"""
ttl = get_option('ttl', key_options, global_options)
if ttl is None or ttl == 0:
return 0
try:
ttl = int(ttl)
if ttl < 0:
raise ArgumentError.pyexc("TTL cannot be negative", obj=ttl)
elif ttl > 1 << 31:
raise ArgumentError.pyexc("TTL Value too large", obj=ttl)
return ttl
except CouchbaseError:
raise
except Exception as e:
raise ArgumentError.pyexc(inner=e, obj=ttl)
def _design_poll(self, name, mode, oldres, timeout=5, use_devmode=False):
"""
Poll for an 'async' action to be complete.
:param string name: The name of the design document
:param string mode: One of ``add`` or ``del`` to indicate whether
we should check for addition or deletion of the document
:param oldres: The old result from the document's previous state, if
any
:param float timeout: How long to poll for. If this is 0 then this
function returns immediately
:type oldres: :class:`~couchbase.result.HttpResult`
"""
if not timeout:
return True
if timeout < 0:
raise ArgumentError.pyexc("Interval must not be negative")
t_end = time.time() + timeout
old_rev = None
if oldres:
old_rev = self._doc_rev(oldres)
while time.time() < t_end:
try:
cur_resp = self.design_get(name, use_devmode=use_devmode)
if old_rev and self._doc_rev(cur_resp) == old_rev:
continue
# Try to execute a view..
vname = list(cur_resp.value['views'].keys())[0]
try:
self._view(name, vname, use_devmode=use_devmode,
params={'limit': 1, 'stale': 'ok'})
# We're able to query it? whoopie!
return True
except CouchbaseError:
continue
except CouchbaseError as e:
if mode == 'del':
# Deleted, whopee!
return True
cur_resp = e.objextra
raise exceptions.TimeoutError.pyexc(
"Wait time for design action completion exceeded")
def submit_single(self, c_key, c_len, value, item, key_options, global_options, mres):
if item:
raise ArgumentError.pyexc('Item not allowed for subdoc')
# Allocate the subdoc array
if not isinstance(value, tuple):
raise ArgumentError.pyexc('Value must be a tuple!')
if not len(value):
raise ArgumentError.pyexc('Need one or more commands')
speclist = ffi.new('lcb_SDSPEC[{0}]'.format(len(value)))
self.c_command.specs = speclist
self.c_command.nspecs = len(value)
self.c_command.cas = get_cas(key_options, global_options, item)
C.CBFFI_set_key(self.c_command, c_key, c_len)
bm = BufManager(ffi)
for i in xrange(len(speclist)):
pyspec = value[i]
cspec = speclist + i
self._process_spec(pyspec, cspec, bm)
return C.lcb_subdoc3(self.instance, mres._cdata, self.c_command)
def build(self, kv):
self._verify_iter(kv)
if not len(kv):
raise ArgumentError.pyexc(obj=kv, message="No items in container")
self.make_struct_pointers(len(kv))
curix = 0
if isinstance(kv, dict):
kviter = kv.iteritems()
else:
kviter = iter(kv)
while True:
try:
self._build_single(kv, kviter, curix)
curix += 1
except StopIteration:
break
if curix != self._ncmds:
raise ArgumentError.pyexc(message="Bad iterator", obj=kv)
return self._cmdpp
def __init__(self,
id_generator=None,
data_converter=lambda x: x,
operation=BucketOperators.UPSERT):
# type: (IdGenerator, DataConverter, BucketOperator) -> None
"""
Initialise ingester.
:param DataConverter data_converter: Single parameter Callable which takes a JSON
input and returns a transformed JSON output.
:param IdGenerator id_generator: Callable that takes a JSON input and
returns an ID string
:param BucketOperator operation: Callable that takes a bucket object, a key and a
value and applies the key and value to the bucket (e.g. upsert/insert/replace)
"""
if not isinstance(operation, BucketOperator):
raise ArgumentError("Operation is not a BucketOperator")
if operation == BucketOperators.REPLACE and not id_generator:
raise ArgumentError("Replace cannot use default ID generator.")
self.id_generator = id_generator or (lambda x: str(uuid.uuid4()))
self.data_converter = data_converter
self.operation = operation
def _invoke_submit(self, iterobj, is_dict, is_itmcoll, mres, global_kw):
"""
Internal function to invoke the actual submit_single function
:param iterobj: The raw object returned as the next item of the iterator
:param is_dict: True if iterator is a dictionary
:param is_itmcoll: True if the iterator contains Item objects
:param mres: The multi result object
:param global_kw: The global settings
:return: The return value of :meth:`submit_single`
"""
if is_itmcoll:
item, key_options = next(iterobj)
key = item.key
value = item.value
result = item
else:
if is_dict:
key, value = next(iterobj)
if not self.VALUES_ALLOWED and not is_itmcoll:
raise ArgumentError.pyexc(
'Values not allowed for this command', obj=value)
else:
key = next(iterobj)
value = None
key_options = {}
item = None
result = self.make_result(key, value)
result.rc = -1
# Attempt to get the encoded key:
key, value, key_options = self.make_entry_params(
key, value, key_options)
c_key, c_len = create_key(self.parent._tc, key)
rc = self.submit_single(c_key, c_len, value, item, key_options,
global_kw, mres)
if rc:
raise pycbc_exc_lcb(rc)
try:
if key in mres and not self.DUPKEY_OK:
# For tests:
self.parent._warn_dupkey(key)
mres[key] = result
except TypeError:
raise pycbc_exc_enc(obj=key)
def execute(self, kv, **kwargs):
"""
Execute the operation scheduling items as needed
:param kv: An iterable of keys (or key-values, or Items)
:param kwargs: Settings for the operation
:return: A MultiResult object
"""
self._verify_iter(kv)
if not len(kv):
raise ArgumentError.pyexc(obj=kv, message="No items in container")
if isinstance(kv, dict):
is_dict = True
try:
kviter = kv.iteritems()
except AttributeError:
kviter = iter(kv.items())
else:
is_dict = False
kviter = iter(kv)
is_itmcoll = isinstance(kv, ItemCollection)
mres = kwargs.get('_MRES')
if mres is None:
mres = self.parent._make_mres()
self.set_mres_flags(mres, kwargs)
C.lcb_sched_enter(self.instance)
num_items = 0
while True:
# Clear the previous command object
C.memset(self.c_command, 0, ffi.sizeof(self.c_command[0]))
try:
self._invoke_submit(kviter, is_dict, is_itmcoll, mres, kwargs)
num_items += 1
except StopIteration:
break
except:
C.lcb_sched_fail(self.instance)
raise
C.lcb_sched_leave(self.instance)
mres._remaining += num_items
# print "Execute(): mres:", mres
return mres
def execute(self, kv, **kwargs):
"""
Execute the operation scheduling items as needed
:param kv: An iterable of keys (or key-values, or Items)
:param kwargs: Settings for the operation
:return: A MultiResult object
"""
self._verify_iter(kv)
if not len(kv):
raise ArgumentError.pyexc(obj=kv, message="No items in container")
if isinstance(kv, dict):
is_dict = True
try:
kviter = kv.iteritems()
except AttributeError:
kviter = iter(kv.items())
else:
is_dict = False
kviter = iter(kv)
is_itmcoll = isinstance(kv, ItemCollection)
mres = kwargs.get('_MRES')
if mres is None:
mres = self.parent._make_mres()
self.set_mres_flags(mres, kwargs)
C.lcb_sched_enter(self.instance)
num_items = 0
while True:
# Clear the previous command object
C.memset(self.c_command, 0, ffi.sizeof(self.c_command[0]))
try:
self._invoke_submit(kviter, is_dict, is_itmcoll, mres, kwargs)
num_items += 1
except StopIteration:
break
except:
C.lcb_sched_fail(self.instance)
raise
C.lcb_sched_leave(self.instance)
mres._remaining += num_items
# print "Execute(): mres:", mres
return mres
def _invoke_submit(self, iterobj, is_dict, is_itmcoll, mres, global_kw):
"""
Internal function to invoke the actual submit_single function
:param iterobj: The raw object returned as the next item of the iterator
:param is_dict: True if iterator is a dictionary
:param is_itmcoll: True if the iterator contains Item objects
:param mres: The multi result object
:param global_kw: The global settings
:return: The return value of :meth:`submit_single`
"""
if is_itmcoll:
item, key_options = next(iterobj)
if not isinstance(item, Item):
pycbc_exc_args('Expected item object')
key = item.key
value = item.value
result = item
else:
if is_dict:
key, value = next(iterobj)
if not self.VALUES_ALLOWED and not is_itmcoll:
raise ArgumentError.pyexc(
'Values not allowed for this command', obj=value)
else:
key = next(iterobj)
value = None
key_options = {}
item = None
result = self.make_result(key, value)
result.rc = -1
# Attempt to get the encoded key:
key, value, key_options = self.make_entry_params(key, value, key_options)
c_key, c_len = create_key(self.parent._tc, key)
rc = self.submit_single(c_key, c_len, value, item, key_options, global_kw, mres)
if rc:
raise pycbc_exc_lcb(rc)
try:
if key in mres and not self.DUPKEY_OK:
# For tests:
self.parent._warn_dupkey(key)
mres[key] = result
except TypeError:
raise pycbc_exc_enc(obj=key)
def _design_poll(self, name, mode, oldres, timeout=5, use_devmode=False):
"""
Poll for an 'async' action to be complete.
:param string name: The name of the design document
:param string mode: One of ``add`` or ``del`` to indicate whether
we should check for addition or deletion of the document
:param oldres: The old result from the document's previous state, if
any
:param float timeout: How long to poll for. If this is 0 then this
function returns immediately
:type oldres: :class:`~couchbase.result.HttpResult`
"""
if not timeout:
return True
if timeout < 0:
raise ArgumentError.pyexc("Interval must not be negative")
t_end = time.time() + timeout
old_rev = None
if oldres:
old_rev = self._doc_rev(oldres)
while time.time() < t_end:
try:
cur_resp = self.design_get(name, use_devmode=use_devmode)
if old_rev and self._doc_rev(cur_resp) == old_rev:
continue
try:
if not self._poll_vq_single(name, use_devmode,
cur_resp.value):
continue
return True
except CouchbaseError:
continue
except CouchbaseError:
if mode == 'del':
# Deleted, whopee!
return True
raise exceptions.TimeoutError.pyexc(
"Wait time for design action completion exceeded")
def query(self, *args, **kwargs):
"""
Reimplemented from base class. This method does not add additional
functionality of the base class`
:meth:`~couchbase.connection.Connection.query` method (all the
functionality is encapsulated in the view class anyway). However it
does require one additional keyword argument
:param class itercls: A class used for instantiating the view
object. This should be a subclass of
:class:`~couchbase.async.view.AsyncViewBase`.
"""
if not issubclass(kwargs.get('itercls', None), AsyncViewBase):
raise ArgumentError.pyexc("itercls must be defined "
"and must be derived from AsyncViewBase")
return super(AsyncBucket, self).query(*args, **kwargs)
def _design_poll(self, name, mode, oldres, timeout=5, use_devmode=False):
"""
Poll for an 'async' action to be complete.
:param string name: The name of the design document
:param string mode: One of ``add`` or ``del`` to indicate whether
we should check for addition or deletion of the document
:param oldres: The old result from the document's previous state, if
any
:param float timeout: How long to poll for. If this is 0 then this
function returns immediately
:type oldres: :class:`~couchbase.result.HttpResult`
"""
if not timeout:
return True
if timeout < 0:
raise ArgumentError.pyexc("Interval must not be negative")
t_end = time.time() + timeout
old_rev = None
if oldres:
old_rev = self._doc_rev(oldres)
while time.time() < t_end:
try:
cur_resp = self.design_get(name, use_devmode=use_devmode)
if old_rev and self._doc_rev(cur_resp) == old_rev:
continue
try:
if not self._poll_vq_single(
name, use_devmode, cur_resp.value):
continue
return True
except CouchbaseError:
continue
except CouchbaseError:
if mode == 'del':
# Deleted, whopee!
return True
raise exceptions.TimeoutError.pyexc(
"Wait time for design action completion exceeded")
def query(self, *args, **kwargs):
"""
Reimplemented from base class. This method does not add additional
functionality of the base class`
:meth:`~couchbase.connection.Connection.query` method (all the
functionality is encapsulated in the view class anyway). However it
does require one additional keyword argument
:param class itercls: A class used for instantiating the view
object. This should be a subclass of
:class:`~couchbase.async.view.AsyncViewBase`.
"""
if not issubclass(kwargs.get('itercls', None), AsyncViewBase):
raise ArgumentError.pyexc("itercls must be defined "
"and must be derived from AsyncViewBase")
return super(AsyncBucket, self).query(*args, **kwargs)
def __init__(self, passthrough=False, unrecognized_ok=False, **params):
self.passthrough = passthrough
self.unrecognized_ok = unrecognized_ok
self._real_options = {}
self._user_options = {}
self._extra_options = {}
self._encoded = None
if params:
for k, v in params.items():
if not hasattr(self, k):
if not unrecognized_ok:
raise ArgumentError.pyexc("Unknown option", k)
self._set_common(k, v)
else:
setattr(self, k, v)
def from_any(cls, params, **ctor_opts):
"""
Creates a new Query object from input.
:param params: Parameter to convert to query
:type params: dict, string, or :class:`Query`
If ``params`` is a :class:`Query` object already, a deep copy is made
and a new :class:`Query` object is returned.
If ``params`` is a string, then a :class:`Query` object is contructed
from it. The string itself is not parsed, but rather prepended to
any additional parameters (defined via the object's methods)
with an additional ``&`` characted.
If ``params`` is a dictionary, it is passed to the :class:`Query`
constructor.
:return: a new :class:`Query` object
:raise: :exc:`ArgumentError` if the input is none of the acceptable
types mentioned above. Also raises any exceptions possibly thrown
by the constructor.
"""
if isinstance(params, cls):
return deepcopy(params)
elif isinstance(params, dict):
ctor_opts.update(**params)
if cls is QueryBase:
if ('bbox' in params or 'start_range' in params or
'end_range' in params):
return SpatialQuery(**ctor_opts)
else:
return ViewQuery(**ctor_opts)
elif isinstance(params, basestring):
ret = cls()
ret._base_str = params
return ret
else:
raise ArgumentError.pyexc("Params must be Query, dict, or string")
def from_any(cls, params, **ctor_opts):
"""
Creates a new Query object from input.
:param params: Parameter to convert to query
:type params: dict, string, or :class:`Query`
If ``params`` is a :class:`Query` object already, a deep copy is made
and a new :class:`Query` object is returned.
If ``params`` is a string, then a :class:`Query` object is contructed
from it. The string itself is not parsed, but rather prepended to
any additional parameters (defined via the object's methods)
with an additional ``&`` characted.
If ``params`` is a dictionary, it is passed to the :class:`Query`
constructor.
:return: a new :class:`Query` object
:raise: :exc:`ArgumentError` if the input is none of the acceptable
types mentioned above. Also raises any exceptions possibly thrown
by the constructor.
"""
if isinstance(params, cls):
return deepcopy(params)
elif isinstance(params, dict):
ctor_opts.update(**params)
if cls is QueryBase:
if ('bbox' in params or 'start_range' in params
or 'end_range' in params):
return SpatialQuery(**ctor_opts)
else:
return ViewQuery(**ctor_opts)
elif isinstance(params, basestring):
ret = cls()
ret._base_str = params
return ret
else:
raise ArgumentError.pyexc("Params must be Query, dict, or string")
def __init__(self, parent, kv, **kwargs):
kwargs.pop('quiet', None)
super(DurabilityCommandContext, self).__init__(parent, **kwargs)
kwargs.setdefault('persist_to', -1)
kwargs.setdefault('replicate_to', -1)
self.handle_durability(parent, **kwargs)
self.cas = kwargs.get('cas', 0)
self.options = ffi.new('lcb_durability_opts_t*')
self.options.version = 0
optreq = self.options.v.v0
try:
self._init_options(optreq, kwargs)
except Exception as e:
raise ArgumentError.pyexc(inner=e)
self.build(kv)
def __init__(self, parent, kv, **kwargs):
kwargs.pop('quiet', None)
super(DurabilityCommandContext, self).__init__(parent, **kwargs)
kwargs.setdefault('persist_to', -1)
kwargs.setdefault('replicate_to', -1)
self.handle_durability(parent, **kwargs)
self.cas = kwargs.get('cas', 0)
self.options = ffi.new('lcb_durability_opts_t*')
self.options.version = 0
optreq = self.options.v.v0
try:
self._init_options(optreq, kwargs)
except Exception as e:
raise ArgumentError.pyexc(inner=e)
self.build(kv)
def update(self, copy=False, **params):
"""
Chained assignment operator.
This may be used to quickly assign extra parameters to the
:class:`Query` object.
Example::
q = Query(reduce=True, full_sec=True)
# Someplace later
v = View(design, view, query=q.update(mapkey_range=["foo"]))
Its primary use is to easily modify the query object (in-place).
:param boolean copy:
If set to true, the original object is copied before new attributes
are added to it
:param params: Extra arguments. These must be valid query options.
:return: A :class:`Query` object. If ``copy`` was set to true, this
will be a new instance, otherwise it is the same instance on which
this method was called
"""
if copy:
self = deepcopy(self)
for k, v in params.items():
if not hasattr(self, k):
if not self.unrecognized_ok:
raise ArgumentError.pyexc("Unknown option", k)
self._set_common(k, v)
else:
setattr(self, k, v)
return self
def update(self, copy=False, **params):
"""
Chained assignment operator.
This may be used to quickly assign extra parameters to the
:class:`Query` object.
Example::
q = Query(reduce=True, full_sec=True)
# Someplace later
v = View(design, view, query=q.update(mapkey_range=["foo"]))
Its primary use is to easily modify the query object (in-place).
:param boolean copy:
If set to true, the original object is copied before new attributes
are added to it
:param params: Extra arguments. These must be valid query options.
:return: A :class:`Query` object. If ``copy`` was set to true, this
will be a new instance, otherwise it is the same instance on which
this method was called
"""
if copy:
self = deepcopy(self)
for k, v in params.items():
if not hasattr(self, k):
if not self.unrecognized_ok:
raise ArgumentError.pyexc("Unknown option", k)
self._set_common(k, v)
else:
setattr(self, k, v)
return self
def __init__(self, parent, kv, **kwargs):
super(StatsCommandContext, self).__init__(parent, **kwargs)
if not kv:
kv = tuple([""])
ncmds = 1
else:
ncmds = len(kv)
if not ncmds:
raise ArgumentError.pyexc("Empty sequence", obj=kv)
self.make_struct_pointers(ncmds)
ix = 0
for k_orig in kv:
k = ffi.new('char[]', k_orig)
self._cdata.append(k)
cur_cmd = self._cmdlist[ix]
self._cmdpp[ix] = ffi.addressof(cur_cmd)
cur_cmd.version = 0
cur_cmd.v.v0.name = k
cur_cmd.v.v0.nname = len(k_orig)
ix += 1
def __init__(self, parent, kv, **kwargs):
super(StatsCommandContext, self).__init__(parent, **kwargs)
if not kv:
kv = tuple([""])
ncmds = 1
else:
ncmds = len(kv)
if not ncmds:
raise ArgumentError.pyexc("Empty sequence", obj=kv)
self.make_struct_pointers(ncmds)
ix = 0
for k_orig in kv:
k = ffi.new('char[]', k_orig)
self._cdata.append(k)
cur_cmd = self._cmdlist[ix]
self._cmdpp[ix] = ffi.addressof(cur_cmd)
cur_cmd.version = 0
cur_cmd.v.v0.name = k
cur_cmd.v.v0.nname = len(k_orig)
ix += 1
def convert_to_koptions(self, input):
try:
input + 0
return Options(ttl=input)
except:
raise ArgumentError.pyexc("Invalid TTL", obj=input)
def __init__(self,
parent,
design,
view,
row_processor=None,
streaming=0,
include_docs=False,
query=None,
**params):
"""
Construct a iterable which can be used to iterate over view query
results.
:param parent: The parent Connection object
:type parent: :class:`~couchbase.connection.Connection`
:param string design: The design document
:param string view: The name of the view within the design document
:param callable row_processor: See :attr:`row_processor` for more
details.
:param boolean include_docs: If set, the document itself will be
retrieved for each row in the result. The default algorithm
uses :meth:`~couchbase.connection.Connection.get_multi` for each
page (i.e. every :attr:`streaming` results).
The :attr:`~couchbase.views.params.Query.reduce`
family of attributes must not be active, as results fro
``reduce`` views do not have corresponding
doc IDs (as these are aggregation functions).
:param bool streaming:
Whether a streaming chunked request should be used. This is
helpful for handling the view results in small chunks rather
than loading the entire resultset into memory at once. By default,
a single request is made and the response is decoded at once. With
streaming enabled, rows are decoded incrementally.
:param query: If set, is a :class:`~couchbase.views.params.Query`
object. It is illegal to use this in conjunction with
additional ``params``
:param params: Extra view options. This may be used to pass view
arguments (as defined in :class:`~couchbase.views.params.Query`)
without explicitly constructing a
:class:`~couchbase.views.params.Query` object.
It is illegal to use this together with the ``query`` argument.
If you wish to 'inline' additional arguments to the provided
``query`` object, use the
query's :meth:`~couchbase.views.params.Query.update` method
instead.
This object is an iterator - it does not send out the request until
the first item from the iterator is request. See :meth:`__iter__` for
more details on what this object returns.
Simple view query, with no extra options::
# c is the Connection object.
for result in View(c, "beer", "brewery_beers"):
print("emitted key: {0}, doc_id: {1}"
.format(result.key, result.docid))
Execute a view with extra query options::
# Implicitly creates a Query object
view = View(c, "beer", "by_location",
limit=4,
reduce=True,
group_level=2)
Pass a Query object::
q = Query(
stale=False,
inclusive_end=True,
mapkey_range=[
["21st_ammendment_brewery_cafe"],
["21st_ammendment_brewery_cafe", Query.STRING_RANGE_END]
]
)
view = View(c, "beer", "brewery_beer", query=q)
Add extra parameters to query object for single call::
view = View(c, "beer", "brewery_beer",
query=q.update(debug=True, copy=True))
Include documents with query::
view = View(c, "beer", "brewery_beer",
query=q, include_docs=True)
for result in view:
print("Emitted key: {0}, Document: {1}".format(
result.key, result.doc.value))
"""
self._parent = parent
self.design = design
self.view = view
self.errors = []
self.raw = None
self.rows_returned = 0
self.include_docs = include_docs
self.indexed_rows = 0
if not row_processor:
row_processor = RowProcessor()
self.row_processor = row_processor
self._rp_iter = None
if query and params:
raise ArgumentError.pyexc(
"Extra parameters are mutually exclusive with the "
"'query' argument. Use query.update() to add extra arguments")
if query:
self._query = deepcopy(query)
else:
self._query = Query.from_any(params)
if include_docs:
if (self._query.reduce or self._query.group
or self._query.group_level):
raise ArgumentError.pyexc(
"include_docs is only applicable "
"for map-only views, but 'reduce', "
"'group', or 'group_level' "
"was specified", self._query)
# The original 'limit' parameter, passed to the query.
self._streaming = streaming
self._do_iter = True
def convert_to_koptions(self, options):
raise ArgumentError.pyexc(obj=options, message="Options expected")