def __init__(self,
                 keystore,
                 public_key_name,
                 hmac_key_name,
                 iv=None,
                 block_size=32):
        """
        Create a new instance of the AES-256-HMAC-SHA1 encryption provider.
        :param keystore: The keystore used to encrypt / decrypt
        :param hmac_key_name: The HMAC key name used to sign and verify
        :param iv:
        :param block_size:
        """
        super(AES256CryptoProvider, self).__init__()

        if not keystore:
            raise ArgumentError.pyexc("KeyStore must be provided.")
        if not public_key_name:
            raise ArgumentError.pyexc("Public key name must be provided.")
        if not hmac_key_name:
            raise ArgumentError.pyexc("HMAC key name must be provided.")

        self.keystore = keystore
        self.public_key_name = public_key_name
        self.hmac_key_name = hmac_key_name
        self.iv = iv
        self.block_size = block_size
Example #2
0
def _bool_param_handler(input):
    if isinstance(input, bool):
        if input:
            return "true"
        else:
            return "false"

    if isinstance(input, basestring):
        if input not in ("true", "false"):
            raise ArgumentError.pyexc(
                "String for boolean must be "
                "'true' or 'false'", input)
        return input

    try:
        input + 0
        if input:
            return "true"
        else:
            return "false"

    except TypeError:
        raise ArgumentError.pyexc(
            "Boolean value must be boolean, "
            "numeric, or a string of 'true' "
            "or 'false'", input)
    def build(self, kv):
        self._verify_iter(kv)
        if not len(kv):
            raise ArgumentError.pyexc(obj=kv, message="No items in container")

        self.make_struct_pointers(len(kv))
        curix = 0


        if isinstance(kv, dict):
            kviter = kv.iteritems()
        else:
            kviter = iter(kv)

        while True:
            try:
                self._build_single(kv, kviter, curix)
                curix += 1

            except StopIteration:
                break

        if curix != self._ncmds:
            raise ArgumentError.pyexc(message="Bad iterator", obj=kv)
        return self._cmdpp
Example #4
0
def _num_param_handler(input):
    # Don't allow booleans:
    if isinstance(input, bool):
        raise ArgumentError.pyexc("Cannot use booleans as numeric values", input)
    try:
        return str(int(input))
    except Exception as e:
        raise ArgumentError.pyexc("Expected a numeric argument", input, e)
Example #5
0
def _num_param_handler(input):
    # Don't allow booleans:
    if isinstance(input, bool):
        raise ArgumentError.pyexc("Cannot use booleans as numeric values",
                                  input)
    try:
        return str(int(input))
    except Exception as e:
        raise ArgumentError.pyexc("Expected a numeric argument", input, e)
Example #6
0
    def extract_ttl(req, options):
        try:
            ttl = options.get('ttl', 0)
            if ttl < 0:
                raise ArgumentError.pyexc("TTL cannot be negative", obj=ttl)

            req.exptime = ttl
            return ttl

        except CouchbaseError:
            raise

        except Exception as e:
            raise ArgumentError.pyexc(inner=e, obj=options.get('ttl', 0))
Example #7
0
    def handle_durability(self, parent, **kwargs):
        self.persist_to = kwargs.get('persist_to', parent._dur_persist_to)

        self.replicate_to = kwargs.get('replicate_to',
                                       parent._dur_replicate_to)

        if not self.persist_to or self.replicate_to:
            return

        n_replicas = C.lcb_get_num_replicas(parent._instance)
        if self.replicate_to > n_replicas or self.persist_to + (n_replicas +
                                                                1):
            ArgumentError.pyexc("Durability requirements will never "
                                "be satisfied")
    def extract_ttl(req, options):
        try:
            ttl = options.get('ttl', 0)
            if ttl < 0:
                raise ArgumentError.pyexc("TTL cannot be negative", obj=ttl)

            req.exptime = ttl
            return ttl

        except CouchbaseError:
            raise

        except Exception as e:
            raise ArgumentError.pyexc(inner=e, obj=options.get('ttl', 0))
    def handle_durability(self, parent, **kwargs):
        self.persist_to = kwargs.get('persist_to',
                                     parent._dur_persist_to)

        self.replicate_to = kwargs.get('replicate_to',
                                       parent._dur_replicate_to)

        if not self.persist_to or self.replicate_to:
            return

        n_replicas = C.lcb_get_num_replicas(parent._instance)
        if self.replicate_to > n_replicas or self.persist_to + (n_replicas+1):
            ArgumentError.pyexc("Durability requirements will never "
                                "be satisfied")
    def _verify_iter(cls, kv):
        if isinstance(kv, (dict, list, tuple)):
            return

        if isinstance(kv, cls.FORBIDDEN_ITER):
            raise ArgumentError.pyexc(obj=kv, message="Bad sequence type")

        if not hasattr(kv, '__iter__'):
            raise ArgumentError.pyexc(obj=kv, message="Object is not iterable")

        try:
            len(kv)
        except TypeError:
            raise ArgumentError.pyexc(obj=kv,
                                      message="Iterator must have length")
Example #11
0
    def _verify_iter(self, kv):
        if isinstance(kv, (dict, list, tuple)):
            return

        if isinstance(kv, self._disallowed_types):
            raise ArgumentError.pyexc(obj=kv, message="Bad sequence type")

        if not hasattr(kv, '__iter__'):
            raise ArgumentError.pyexc(obj=kv, message="Object is not iterable")

        try:
            len(kv)
        except TypeError:
            raise ArgumentError.pyexc(obj=kv,
                                      message="Iterator must have length")
Example #12
0
    def _verify_iter(cls, kv):
        if isinstance(kv, (dict, list, tuple)):
            return

        if isinstance(kv, cls.FORBIDDEN_ITER):
            raise ArgumentError.pyexc(obj=kv, message="Bad sequence type")

        if not hasattr(kv, '__iter__'):
            raise ArgumentError.pyexc(obj=kv, message="Object is not iterable")

        try:
            len(kv)
        except TypeError:
            raise ArgumentError.pyexc(obj=kv,
                                      message="Iterator must have length")
    def _verify_iter(self, kv):
        if isinstance(kv, (dict, list, tuple)):
            return

        if isinstance(kv, self._disallowed_types):
            raise ArgumentError.pyexc(obj=kv, message="Bad sequence type")

        if not hasattr(kv, '__iter__'):
            raise ArgumentError.pyexc(obj=kv, message="Object is not iterable")

        try:
            len(kv)
        except TypeError:
            raise ArgumentError.pyexc(obj=kv,
                                      message="Iterator must have length")
 def raise_include_docs(self):
     """
     Raise an error on include docs
     """
     raise ArgumentError.pyexc(
         "Include docs not supported with async views. If you "
         "must gather docs, you should do so manually")
def _jarry_param_handler(input):
    ret = _jval_param_handler(input)
    if not ret.startswith('['):
        raise ArgumentError.pyexc(
            "Value must be converted to JSON array", input)

    return ret
Example #16
0
 def raise_include_docs(self):
     """
     Raise an error on include docs
     """
     raise ArgumentError.pyexc(
         "Include docs not supported with async views. If you "
         "must gather docs, you should do so manually")
Example #17
0
    def _set_common(self, param, value, set_user=True):
        # Invalidate encoded string
        self._encoded = None

        if value is UNSPEC:
            self._real_options.pop(param, None)
            if set_user:
                self._user_options.pop(param, None)
            return

        handler = _HANDLER_MAP.get(param)
        if not handler:
            if not self.unrecognized_ok:
                raise ArgumentError.pyexc(
                    "Unrecognized parameter. To use unrecognized parameters, "
                    "set 'unrecognized_ok' to True")

        if not handler:
            self._extra_options[param] = _string_param_handler(value)
            return

        if self.passthrough:
            handler = _string_param_handler

        self._real_options[param] = handler(value)
        if set_user:
            self._user_options[param] = value
    def _set_common(self, param, value, set_user=True):
        # Invalidate encoded string
        self._encoded = None

        if value is UNSPEC:
            self._real_options.pop(param, None)
            if set_user:
                self._user_options.pop(param, None)
            return

        handler = _HANDLER_MAP.get(param)
        if not handler:
            if not self.unrecognized_ok:
                raise ArgumentError.pyexc(
                    "Unrecognized parameter. To use unrecognized parameters, "
                    "set 'unrecognized_ok' to True")

        if not handler:
            self._extra_options[param] = _string_param_handler(value)
            return

        if self.passthrough:
            handler = _string_param_handler

        self._real_options[param] = handler(value)
        if set_user:
            self._user_options[param] = value
Example #19
0
def _jarry_param_handler(input):
    ret = _jval_param_handler(input)
    if not ret.startswith('['):
        raise ArgumentError.pyexc("Value must be converted to JSON array",
                                  input)

    return ret
    def submit_single(self, c_key, c_len, value, item, key_options, global_options, mres):
        cas = get_cas(key_options, global_options, item)
        if not cas:
            raise ArgumentError.pyexc("Must have CAS")

        C._Cb_set_key(self.c_command, c_key, c_len)
        self.c_command.cas = cas
        return C.lcb_unlock3(self.instance, mres._cdata, self.c_command)
Example #21
0
    def process_single_command(self, req, koptions):
        options = self._koptions.copy()
        options.update(koptions)

        if not options.get('cas', 0):
            raise ArgumentError.pyexc("Must have CAS for unlock")

        req.cas = options['cas']
    def process_single_command(self, req, koptions):
        options = self._koptions.copy()
        options.update(koptions)

        if not options.get('cas', 0):
            raise ArgumentError.pyexc("Must have CAS for unlock")

        req.cas = options['cas']
Example #23
0
    def _set_range_common(self, k_sugar, k_start, k_end, value):
        """
        Checks to see if the client-side convenience key is present, and if so
        converts the sugar convenience key into its real server-side
        equivalents.

        :param string k_sugar: The client-side convenience key
        :param string k_start: The server-side key specifying the beginning of
            the range
        :param string k_end: The server-side key specifying the end of the
            range
        """

        if not isinstance(value, (list, tuple, _Unspec)):
            raise ArgumentError.pyexc(
                "Range specification for {0} must be a list, tuple or UNSPEC".
                format(k_sugar))

        if self._user_options.get(
                k_start, UNSPEC) is not UNSPEC or (self._user_options.get(
                    k_end, UNSPEC) is not UNSPEC):

            raise ArgumentError.pyexc(
                "Cannot specify {0} with either {1} or {2}".format(
                    k_sugar, k_start, k_end))

        if not value:
            self._set_common(k_start, UNSPEC, set_user=False)
            self._set_common(k_end, UNSPEC, set_user=False)
            self._user_options[k_sugar] = UNSPEC
            return

        if len(value) not in (1, 2):
            raise ArgumentError.pyexc(
                "Range specification "
                "must have one or two elements", value)

        value = value[::]
        if len(value) == 1:
            value.append(UNSPEC)

        for p, ix in ((k_start, 0), (k_end, 1)):
            self._set_common(p, value[ix], set_user=False)

        self._user_options[k_sugar] = value
Example #24
0
    def submit_single(self, c_key, c_len, value, item, key_options,
                      global_options, mres):
        cas = get_cas(key_options, global_options, item)
        if not cas:
            raise ArgumentError.pyexc("Must have CAS")

        C._Cb_set_key(self.c_command, c_key, c_len)
        self.c_command.cas = cas
        return C.lcb_unlock3(self.instance, mres._cdata, self.c_command)
    def submit_single(self, c_key, c_len, value, item, key_options, global_options, mres):
        ttl = get_ttl(key_options, global_options, item)
        if self.IS_LOCK and not ttl:
            raise ArgumentError.pyexc("Lock must have TTL")

        C._Cb_set_key(self.c_command, c_key, c_len)
        self.c_command.exptime = ttl
        self.c_command.lock = self.IS_LOCK
        return C.lcb_get3(self.instance, mres._cdata, self.c_command)
Example #26
0
    def get_key(self, key_id):
        """
        Retrieves a key from the key store using the given key id.
        :param key_id: The name for the key in the key store.
        """
        if not key_id:
            raise ArgumentError.pyexc("key_id must not be empty.")

        return self.keystore.secret_keys[key_id]
    def _set_range_common(self, k_sugar, k_start, k_end, value):
        """
        Checks to see if the client-side convenience key is present, and if so
        converts the sugar convenience key into its real server-side
        equivalents.

        :param string k_sugar: The client-side convenience key
        :param string k_start: The server-side key specifying the beginning of
            the range
        :param string k_end: The server-side key specifying the end of the
            range
        """

        if not isinstance(value, (list, tuple, _Unspec)):
            raise ArgumentError.pyexc(
                "Range specification for {0} must be a list, tuple or UNSPEC"
                .format(k_sugar))

        if self._user_options.get(k_start, UNSPEC) is not UNSPEC or (
                self._user_options.get(k_end, UNSPEC) is not UNSPEC):

            raise ArgumentError.pyexc(
                "Cannot specify {0} with either {1} or {2}"
                .format(k_sugar, k_start, k_end))

        if not value:
            self._set_common(k_start, UNSPEC, set_user=False)
            self._set_common(k_end, UNSPEC, set_user=False)
            self._user_options[k_sugar] = UNSPEC
            return

        if len(value) not in (1, 2):
            raise ArgumentError.pyexc("Range specification "
                                      "must have one or two elements",
                                      value)

        value = value[::]
        if len(value) == 1:
            value.append(UNSPEC)

        for p, ix in ((k_start, 0), (k_end, 1)):
            self._set_common(p, value[ix], set_user=False)

        self._user_options[k_sugar] = value
Example #28
0
    def process_single_command(self, req, koptions):
        options = koptions.copy()
        options.update(self._koptions)

        ttl = self.extract_ttl(req, options)

        if self._is_lock:
            req.lock = 1
            if not ttl:
                raise ArgumentError.pyexc("Lock must have TTL")
    def process_single_command(self, req, koptions):
        options = koptions.copy()
        options.update(self._koptions)

        ttl = self.extract_ttl(req, options)

        if self._is_lock:
            req.lock = 1
            if not ttl:
                raise ArgumentError.pyexc("Lock must have TTL")
Example #30
0
    def __init__(self, file_path, passphrase):
        """
        Creates a new instance of a Key Store that can retrieve keys from a Java Key Store
        :param file_path: Path to the JKS file.
        :param passphrase: The passphrase used to access the key store.
        """
        if not file_path:
            raise ArgumentError.pyexc("file_path cannot be empty.")

        self.keystore = jks.KeyStore.load(file_path, passphrase)
Example #31
0
    def submit_single(self, c_key, c_len, value, item, key_options,
                      global_options, mres):
        ttl = get_ttl(key_options, global_options, item)
        if self.IS_LOCK and not ttl:
            raise ArgumentError.pyexc("Lock must have TTL")

        C._Cb_set_key(self.c_command, c_key, c_len)
        self.c_command.exptime = ttl
        self.c_command.lock = self.IS_LOCK
        return C.lcb_get3(self.instance, mres._cdata, self.c_command)
Example #32
0
def _string_param_common(input, do_quote=False):
    # TODO, if we pass this to urlencode, do we ever need to quote?
    # For the moment, i'm always forcing non-quote behavior
    do_quote = False

    s = None
    if isinstance(input, basestring):
        s = input

    elif isinstance(input, bool):
        raise ArgumentError.pyexc("Can't use boolean as string", input)

    elif isinstance(input, (int, long, float)):
        # Basic numeric types:
        s = str(input)

    else:
        raise ArgumentError.pyexc("Expected simple numeric type or string ", input)
    if do_quote:
        s = ulp.quote(s)

    return s
def get_ttl(key_options, global_options, *_):
    """
    Quick function to extract the expiration time from options. Throws
    appropriate exceptions when necessary
    :param key_options:
    :param global_options:
    :return:
    """
    ttl = get_option('ttl', key_options, global_options)
    if ttl is None or ttl == 0:
        return 0

    try:
        ttl = int(ttl)
        if ttl < 0:
            raise ArgumentError.pyexc("TTL cannot be negative", obj=ttl)
        elif ttl > 1 << 31:
            raise ArgumentError.pyexc("TTL Value too large", obj=ttl)
        return ttl
    except CouchbaseError:
        raise
    except Exception as e:
        raise ArgumentError.pyexc(inner=e, obj=ttl)
Example #34
0
def _bool_param_handler(input):
    if isinstance(input, bool):
        if input:
            return "true"
        else:
            return "false"

    if isinstance(input, basestring):
        if input not in ("true", "false"):
            raise ArgumentError.pyexc("String for boolean must be " "'true' or 'false'", input)
        return input

    try:
        input + 0
        if input:
            return "true"
        else:
            return "false"

    except TypeError:
        raise ArgumentError.pyexc(
            "Boolean value must be boolean, " "numeric, or a string of 'true' " "or 'false'", input
        )
Example #35
0
def _string_param_common(input, do_quote=False):
    # TODO, if we pass this to urlencode, do we ever need to quote?
    # For the moment, i'm always forcing non-quote behavior
    do_quote = False

    s = None
    if isinstance(input, basestring):
        s = input

    elif isinstance(input, bool):
        raise ArgumentError.pyexc("Can't use boolean as string", input)

    elif isinstance(input, (int, long, float)):
        # Basic numeric types:
        s = str(input)

    else:
        raise ArgumentError.pyexc("Expected simple numeric type or string ",
                                  input)
    if do_quote:
        s = ulp.quote(s)

    return s
Example #36
0
def get_ttl(key_options, global_options, *_):
    """
    Quick function to extract the expiration time from options. Throws
    appropriate exceptions when necessary
    :param key_options:
    :param global_options:
    :return:
    """
    ttl = get_option('ttl', key_options, global_options)
    if ttl is None or ttl == 0:
        return 0

    try:
        ttl = int(ttl)
        if ttl < 0:
            raise ArgumentError.pyexc("TTL cannot be negative", obj=ttl)
        elif ttl > 1 << 31:
            raise ArgumentError.pyexc("TTL Value too large", obj=ttl)
        return ttl
    except CouchbaseError:
        raise
    except Exception as e:
        raise ArgumentError.pyexc(inner=e, obj=ttl)
    def _design_poll(self, name, mode, oldres, timeout=5, use_devmode=False):
        """
        Poll for an 'async' action to be complete.
        :param string name: The name of the design document
        :param string mode: One of ``add`` or ``del`` to indicate whether
            we should check for addition or deletion of the document
        :param oldres: The old result from the document's previous state, if
            any
        :param float timeout: How long to poll for. If this is 0 then this
            function returns immediately
        :type oldres: :class:`~couchbase.result.HttpResult`
        """
        if not timeout:
            return True

        if timeout < 0:
            raise ArgumentError.pyexc("Interval must not be negative")

        t_end = time.time() + timeout
        old_rev = None

        if oldres:
            old_rev = self._doc_rev(oldres)

        while time.time() < t_end:
            try:
                cur_resp = self.design_get(name, use_devmode=use_devmode)
                if old_rev and self._doc_rev(cur_resp) == old_rev:
                    continue

                # Try to execute a view..
                vname = list(cur_resp.value['views'].keys())[0]
                try:
                    self._view(name, vname, use_devmode=use_devmode,
                               params={'limit': 1, 'stale': 'ok'})
                    # We're able to query it? whoopie!
                    return True

                except CouchbaseError:
                    continue

            except CouchbaseError as e:
                if mode == 'del':
                    # Deleted, whopee!
                    return True
                cur_resp = e.objextra

        raise exceptions.TimeoutError.pyexc(
            "Wait time for design action completion exceeded")
    def submit_single(self, c_key, c_len, value, item, key_options, global_options, mres):
        if item:
            raise ArgumentError.pyexc('Item not allowed for subdoc')

        # Allocate the subdoc array
        if not isinstance(value, tuple):
            raise ArgumentError.pyexc('Value must be a tuple!')
        if not len(value):
            raise ArgumentError.pyexc('Need one or more commands')

        speclist = ffi.new('lcb_SDSPEC[{0}]'.format(len(value)))

        self.c_command.specs = speclist
        self.c_command.nspecs = len(value)
        self.c_command.cas = get_cas(key_options, global_options, item)
        C.CBFFI_set_key(self.c_command, c_key, c_len)

        bm = BufManager(ffi)

        for i in xrange(len(speclist)):
            pyspec = value[i]
            cspec = speclist + i
            self._process_spec(pyspec, cspec, bm)
        return C.lcb_subdoc3(self.instance, mres._cdata, self.c_command)
Example #39
0
    def build(self, kv):
        self._verify_iter(kv)
        if not len(kv):
            raise ArgumentError.pyexc(obj=kv, message="No items in container")

        self.make_struct_pointers(len(kv))
        curix = 0

        if isinstance(kv, dict):
            kviter = kv.iteritems()
        else:
            kviter = iter(kv)

        while True:
            try:
                self._build_single(kv, kviter, curix)
                curix += 1

            except StopIteration:
                break

        if curix != self._ncmds:
            raise ArgumentError.pyexc(message="Bad iterator", obj=kv)
        return self._cmdpp
Example #40
0
    def __init__(self,
                 id_generator=None,
                 data_converter=lambda x: x,
                 operation=BucketOperators.UPSERT):
        # type: (IdGenerator, DataConverter, BucketOperator) -> None
        """
        Initialise ingester.

        :param DataConverter data_converter: Single parameter Callable which takes a JSON
            input and returns a transformed JSON output.
        :param IdGenerator id_generator: Callable that takes a JSON input and
            returns an ID string
        :param BucketOperator operation: Callable that takes a bucket object, a key and a
            value and applies the key and value to the bucket (e.g. upsert/insert/replace)
        """
        if not isinstance(operation, BucketOperator):
            raise ArgumentError("Operation is not a BucketOperator")

        if operation == BucketOperators.REPLACE and not id_generator:
            raise ArgumentError("Replace cannot use default ID generator.")

        self.id_generator = id_generator or (lambda x: str(uuid.uuid4()))
        self.data_converter = data_converter
        self.operation = operation
Example #41
0
    def _invoke_submit(self, iterobj, is_dict, is_itmcoll, mres, global_kw):
        """
        Internal function to invoke the actual submit_single function
        :param iterobj: The raw object returned as the next item of the iterator
        :param is_dict: True if iterator is a dictionary
        :param is_itmcoll: True if the iterator contains Item objects
        :param mres: The multi result object
        :param global_kw: The global settings
        :return: The return value of :meth:`submit_single`
        """
        if is_itmcoll:
            item, key_options = next(iterobj)
            key = item.key
            value = item.value
            result = item
        else:
            if is_dict:
                key, value = next(iterobj)
                if not self.VALUES_ALLOWED and not is_itmcoll:
                    raise ArgumentError.pyexc(
                        'Values not allowed for this command', obj=value)
            else:
                key = next(iterobj)
                value = None

            key_options = {}
            item = None
            result = self.make_result(key, value)

        result.rc = -1

        # Attempt to get the encoded key:
        key, value, key_options = self.make_entry_params(
            key, value, key_options)
        c_key, c_len = create_key(self.parent._tc, key)

        rc = self.submit_single(c_key, c_len, value, item, key_options,
                                global_kw, mres)
        if rc:
            raise pycbc_exc_lcb(rc)
        try:
            if key in mres and not self.DUPKEY_OK:
                # For tests:
                self.parent._warn_dupkey(key)

            mres[key] = result
        except TypeError:
            raise pycbc_exc_enc(obj=key)
Example #42
0
    def execute(self, kv, **kwargs):
        """
        Execute the operation scheduling items as needed
        :param kv: An iterable of keys (or key-values, or Items)
        :param kwargs: Settings for the operation
        :return: A MultiResult object
        """
        self._verify_iter(kv)
        if not len(kv):
            raise ArgumentError.pyexc(obj=kv, message="No items in container")

        if isinstance(kv, dict):
            is_dict = True
            try:
                kviter = kv.iteritems()
            except AttributeError:
                kviter = iter(kv.items())
        else:
            is_dict = False
            kviter = iter(kv)

        is_itmcoll = isinstance(kv, ItemCollection)

        mres = kwargs.get('_MRES')
        if mres is None:
            mres = self.parent._make_mres()

        self.set_mres_flags(mres, kwargs)

        C.lcb_sched_enter(self.instance)
        num_items = 0
        while True:
            # Clear the previous command object
            C.memset(self.c_command, 0, ffi.sizeof(self.c_command[0]))

            try:
                self._invoke_submit(kviter, is_dict, is_itmcoll, mres, kwargs)
                num_items += 1
            except StopIteration:
                break
            except:
                C.lcb_sched_fail(self.instance)
                raise

        C.lcb_sched_leave(self.instance)
        mres._remaining += num_items
        # print "Execute(): mres:", mres
        return mres
    def execute(self, kv, **kwargs):
        """
        Execute the operation scheduling items as needed
        :param kv: An iterable of keys (or key-values, or Items)
        :param kwargs: Settings for the operation
        :return: A MultiResult object
        """
        self._verify_iter(kv)
        if not len(kv):
            raise ArgumentError.pyexc(obj=kv, message="No items in container")

        if isinstance(kv, dict):
            is_dict = True
            try:
                kviter = kv.iteritems()
            except AttributeError:
                kviter = iter(kv.items())
        else:
            is_dict = False
            kviter = iter(kv)

        is_itmcoll = isinstance(kv, ItemCollection)

        mres = kwargs.get('_MRES')
        if mres is None:
            mres = self.parent._make_mres()

        self.set_mres_flags(mres, kwargs)

        C.lcb_sched_enter(self.instance)
        num_items = 0
        while True:
            # Clear the previous command object
            C.memset(self.c_command, 0, ffi.sizeof(self.c_command[0]))

            try:
                self._invoke_submit(kviter, is_dict, is_itmcoll, mres, kwargs)
                num_items += 1
            except StopIteration:
                break
            except:
                C.lcb_sched_fail(self.instance)
                raise

        C.lcb_sched_leave(self.instance)
        mres._remaining += num_items
        # print "Execute(): mres:", mres
        return mres
    def _invoke_submit(self, iterobj, is_dict, is_itmcoll, mres, global_kw):
        """
        Internal function to invoke the actual submit_single function
        :param iterobj: The raw object returned as the next item of the iterator
        :param is_dict: True if iterator is a dictionary
        :param is_itmcoll: True if the iterator contains Item objects
        :param mres: The multi result object
        :param global_kw: The global settings
        :return: The return value of :meth:`submit_single`
        """
        if is_itmcoll:
            item, key_options = next(iterobj)
            if not isinstance(item, Item):
                pycbc_exc_args('Expected item object')
            key = item.key
            value = item.value
            result = item
        else:
            if is_dict:
                key, value = next(iterobj)
                if not self.VALUES_ALLOWED and not is_itmcoll:
                    raise ArgumentError.pyexc(
                        'Values not allowed for this command', obj=value)
            else:
                key = next(iterobj)
                value = None

            key_options = {}
            item = None
            result = self.make_result(key, value)

        result.rc = -1

        # Attempt to get the encoded key:
        key, value, key_options = self.make_entry_params(key, value, key_options)
        c_key, c_len = create_key(self.parent._tc, key)

        rc = self.submit_single(c_key, c_len, value, item, key_options, global_kw, mres)
        if rc:
            raise pycbc_exc_lcb(rc)
        try:
            if key in mres and not self.DUPKEY_OK:
                # For tests:
                self.parent._warn_dupkey(key)

            mres[key] = result
        except TypeError:
            raise pycbc_exc_enc(obj=key)
    def _design_poll(self, name, mode, oldres, timeout=5, use_devmode=False):
        """
        Poll for an 'async' action to be complete.
        :param string name: The name of the design document
        :param string mode: One of ``add`` or ``del`` to indicate whether
            we should check for addition or deletion of the document
        :param oldres: The old result from the document's previous state, if
            any
        :param float timeout: How long to poll for. If this is 0 then this
            function returns immediately
        :type oldres: :class:`~couchbase.result.HttpResult`
        """
        if not timeout:
            return True

        if timeout < 0:
            raise ArgumentError.pyexc("Interval must not be negative")

        t_end = time.time() + timeout
        old_rev = None

        if oldres:
            old_rev = self._doc_rev(oldres)

        while time.time() < t_end:
            try:
                cur_resp = self.design_get(name, use_devmode=use_devmode)
                if old_rev and self._doc_rev(cur_resp) == old_rev:
                    continue

                try:
                    if not self._poll_vq_single(name, use_devmode,
                                                cur_resp.value):
                        continue
                    return True

                except CouchbaseError:
                    continue

            except CouchbaseError:
                if mode == 'del':
                    # Deleted, whopee!
                    return True

        raise exceptions.TimeoutError.pyexc(
            "Wait time for design action completion exceeded")
    def query(self, *args, **kwargs):
        """
        Reimplemented from base class. This method does not add additional
        functionality of the base class`
        :meth:`~couchbase.connection.Connection.query` method (all the
        functionality is encapsulated in the view class anyway). However it
        does require one additional keyword argument

        :param class itercls: A class used for instantiating the view
          object. This should be a subclass of
          :class:`~couchbase.async.view.AsyncViewBase`.
        """
        if not issubclass(kwargs.get('itercls', None), AsyncViewBase):
            raise ArgumentError.pyexc("itercls must be defined "
                                      "and must be derived from AsyncViewBase")

        return super(AsyncBucket, self).query(*args, **kwargs)
Example #47
0
    def _design_poll(self, name, mode, oldres, timeout=5, use_devmode=False):
        """
        Poll for an 'async' action to be complete.
        :param string name: The name of the design document
        :param string mode: One of ``add`` or ``del`` to indicate whether
            we should check for addition or deletion of the document
        :param oldres: The old result from the document's previous state, if
            any
        :param float timeout: How long to poll for. If this is 0 then this
            function returns immediately
        :type oldres: :class:`~couchbase.result.HttpResult`
        """
        if not timeout:
            return True

        if timeout < 0:
            raise ArgumentError.pyexc("Interval must not be negative")

        t_end = time.time() + timeout
        old_rev = None

        if oldres:
            old_rev = self._doc_rev(oldres)

        while time.time() < t_end:
            try:
                cur_resp = self.design_get(name, use_devmode=use_devmode)
                if old_rev and self._doc_rev(cur_resp) == old_rev:
                    continue

                try:
                    if not self._poll_vq_single(
                            name, use_devmode, cur_resp.value):
                        continue
                    return True

                except CouchbaseError:
                    continue

            except CouchbaseError:
                if mode == 'del':
                    # Deleted, whopee!
                    return True

        raise exceptions.TimeoutError.pyexc(
            "Wait time for design action completion exceeded")
Example #48
0
    def query(self, *args, **kwargs):
        """
        Reimplemented from base class. This method does not add additional
        functionality of the base class`
        :meth:`~couchbase.connection.Connection.query` method (all the
        functionality is encapsulated in the view class anyway). However it
        does require one additional keyword argument

        :param class itercls: A class used for instantiating the view
          object. This should be a subclass of
          :class:`~couchbase.async.view.AsyncViewBase`.
        """
        if not issubclass(kwargs.get('itercls', None), AsyncViewBase):
            raise ArgumentError.pyexc("itercls must be defined "
                                      "and must be derived from AsyncViewBase")

        return super(AsyncBucket, self).query(*args, **kwargs)
Example #49
0
    def __init__(self, passthrough=False, unrecognized_ok=False, **params):
        self.passthrough = passthrough
        self.unrecognized_ok = unrecognized_ok
        self._real_options = {}
        self._user_options = {}
        self._extra_options = {}

        self._encoded = None

        if params:
            for k, v in params.items():
                if not hasattr(self, k):
                    if not unrecognized_ok:
                        raise ArgumentError.pyexc("Unknown option", k)
                    self._set_common(k, v)

                else:
                    setattr(self, k, v)
Example #50
0
    def from_any(cls, params, **ctor_opts):
        """
        Creates a new Query object from input.

        :param params: Parameter to convert to query
        :type params: dict, string, or :class:`Query`

        If ``params`` is a :class:`Query` object already, a deep copy is made
        and a new :class:`Query` object is returned.

        If ``params`` is a string, then a :class:`Query` object is contructed
        from it. The string itself is not parsed, but rather prepended to
        any additional parameters (defined via the object's methods)
        with an additional ``&`` characted.

        If ``params`` is a dictionary, it is passed to the :class:`Query`
        constructor.

        :return: a new :class:`Query` object
        :raise: :exc:`ArgumentError` if the input is none of the acceptable
            types mentioned above. Also raises any exceptions possibly thrown
            by the constructor.

        """
        if isinstance(params, cls):
            return deepcopy(params)

        elif isinstance(params, dict):
            ctor_opts.update(**params)
            if cls is QueryBase:
                if ('bbox' in params or 'start_range' in params or
                            'end_range' in params):
                    return SpatialQuery(**ctor_opts)
                else:
                    return ViewQuery(**ctor_opts)

        elif isinstance(params, basestring):
            ret = cls()
            ret._base_str = params
            return ret

        else:
            raise ArgumentError.pyexc("Params must be Query, dict, or string")
Example #51
0
    def from_any(cls, params, **ctor_opts):
        """
        Creates a new Query object from input.

        :param params: Parameter to convert to query
        :type params: dict, string, or :class:`Query`

        If ``params`` is a :class:`Query` object already, a deep copy is made
        and a new :class:`Query` object is returned.

        If ``params`` is a string, then a :class:`Query` object is contructed
        from it. The string itself is not parsed, but rather prepended to
        any additional parameters (defined via the object's methods)
        with an additional ``&`` characted.

        If ``params`` is a dictionary, it is passed to the :class:`Query`
        constructor.

        :return: a new :class:`Query` object
        :raise: :exc:`ArgumentError` if the input is none of the acceptable
            types mentioned above. Also raises any exceptions possibly thrown
            by the constructor.

        """
        if isinstance(params, cls):
            return deepcopy(params)

        elif isinstance(params, dict):
            ctor_opts.update(**params)
            if cls is QueryBase:
                if ('bbox' in params or 'start_range' in params
                        or 'end_range' in params):
                    return SpatialQuery(**ctor_opts)
                else:
                    return ViewQuery(**ctor_opts)

        elif isinstance(params, basestring):
            ret = cls()
            ret._base_str = params
            return ret

        else:
            raise ArgumentError.pyexc("Params must be Query, dict, or string")
Example #52
0
    def __init__(self, parent, kv, **kwargs):
        kwargs.pop('quiet', None)

        super(DurabilityCommandContext, self).__init__(parent, **kwargs)

        kwargs.setdefault('persist_to', -1)
        kwargs.setdefault('replicate_to', -1)
        self.handle_durability(parent, **kwargs)

        self.cas = kwargs.get('cas', 0)

        self.options = ffi.new('lcb_durability_opts_t*')
        self.options.version = 0
        optreq = self.options.v.v0
        try:
            self._init_options(optreq, kwargs)
        except Exception as e:
            raise ArgumentError.pyexc(inner=e)

        self.build(kv)
    def __init__(self, parent, kv, **kwargs):
        kwargs.pop('quiet', None)

        super(DurabilityCommandContext, self).__init__(parent, **kwargs)

        kwargs.setdefault('persist_to', -1)
        kwargs.setdefault('replicate_to', -1)
        self.handle_durability(parent, **kwargs)

        self.cas = kwargs.get('cas', 0)

        self.options = ffi.new('lcb_durability_opts_t*')
        self.options.version = 0
        optreq = self.options.v.v0
        try:
            self._init_options(optreq, kwargs)
        except Exception as e:
            raise ArgumentError.pyexc(inner=e)

        self.build(kv)
    def update(self, copy=False, **params):
        """
        Chained assignment operator.

        This may be used to quickly assign extra parameters to the
        :class:`Query` object.

        Example::

            q = Query(reduce=True, full_sec=True)

            # Someplace later

            v = View(design, view, query=q.update(mapkey_range=["foo"]))

        Its primary use is to easily modify the query object (in-place).

        :param boolean copy:
            If set to true, the original object is copied before new attributes
            are added to it
        :param params: Extra arguments. These must be valid query options.

        :return: A :class:`Query` object. If ``copy`` was set to true, this
            will be a new instance, otherwise it is the same instance on which
            this method was called

        """
        if copy:
            self = deepcopy(self)

        for k, v in params.items():
            if not hasattr(self, k):
                if not self.unrecognized_ok:
                    raise ArgumentError.pyexc("Unknown option", k)
                self._set_common(k, v)

            else:
                setattr(self, k, v)

        return self
Example #55
0
    def update(self, copy=False, **params):
        """
        Chained assignment operator.

        This may be used to quickly assign extra parameters to the
        :class:`Query` object.

        Example::

            q = Query(reduce=True, full_sec=True)

            # Someplace later

            v = View(design, view, query=q.update(mapkey_range=["foo"]))

        Its primary use is to easily modify the query object (in-place).

        :param boolean copy:
            If set to true, the original object is copied before new attributes
            are added to it
        :param params: Extra arguments. These must be valid query options.

        :return: A :class:`Query` object. If ``copy`` was set to true, this
            will be a new instance, otherwise it is the same instance on which
            this method was called

        """
        if copy:
            self = deepcopy(self)

        for k, v in params.items():
            if not hasattr(self, k):
                if not self.unrecognized_ok:
                    raise ArgumentError.pyexc("Unknown option", k)
                self._set_common(k, v)

            else:
                setattr(self, k, v)

        return self
Example #56
0
    def __init__(self, parent, kv, **kwargs):
        super(StatsCommandContext, self).__init__(parent, **kwargs)
        if not kv:
            kv = tuple([""])
            ncmds = 1

        else:
            ncmds = len(kv)
            if not ncmds:
                raise ArgumentError.pyexc("Empty sequence", obj=kv)

        self.make_struct_pointers(ncmds)

        ix = 0
        for k_orig in kv:
            k = ffi.new('char[]', k_orig)
            self._cdata.append(k)
            cur_cmd = self._cmdlist[ix]
            self._cmdpp[ix] = ffi.addressof(cur_cmd)
            cur_cmd.version = 0
            cur_cmd.v.v0.name = k
            cur_cmd.v.v0.nname = len(k_orig)
            ix += 1
    def __init__(self, parent, kv, **kwargs):
        super(StatsCommandContext, self).__init__(parent, **kwargs)
        if not kv:
            kv = tuple([""])
            ncmds = 1

        else:
            ncmds = len(kv)
            if not ncmds:
                raise ArgumentError.pyexc("Empty sequence", obj=kv)

        self.make_struct_pointers(ncmds)

        ix = 0
        for k_orig in kv:
            k = ffi.new('char[]', k_orig)
            self._cdata.append(k)
            cur_cmd = self._cmdlist[ix]
            self._cmdpp[ix] = ffi.addressof(cur_cmd)
            cur_cmd.version = 0
            cur_cmd.v.v0.name = k
            cur_cmd.v.v0.nname = len(k_orig)
            ix += 1
 def convert_to_koptions(self, input):
     try:
         input + 0
         return Options(ttl=input)
     except:
         raise ArgumentError.pyexc("Invalid TTL", obj=input)
Example #59
0
    def __init__(self,
                 parent,
                 design,
                 view,
                 row_processor=None,
                 streaming=0,
                 include_docs=False,
                 query=None,
                 **params):
        """
        Construct a iterable which can be used to iterate over view query
        results.

        :param parent: The parent Connection object
        :type parent: :class:`~couchbase.connection.Connection`
        :param string design: The design document
        :param string view: The name of the view within the design document
        :param callable row_processor: See :attr:`row_processor` for more
            details.

        :param boolean include_docs: If set, the document itself will be
            retrieved for each row in the result. The default algorithm
            uses :meth:`~couchbase.connection.Connection.get_multi` for each
            page (i.e. every :attr:`streaming` results).

            The :attr:`~couchbase.views.params.Query.reduce`
            family of attributes must not be active, as results fro
            ``reduce`` views do not have corresponding
            doc IDs (as these are aggregation functions).

        :param bool streaming:
            Whether a streaming chunked request should be used. This is
            helpful for handling the view results in small chunks rather
            than loading the entire resultset into memory at once. By default,
            a single request is made and the response is decoded at once. With
            streaming enabled, rows are decoded incrementally.

        :param query: If set, is a :class:`~couchbase.views.params.Query`
            object. It is illegal to use this in conjunction with
            additional ``params``

        :param params: Extra view options. This may be used to pass view
            arguments (as defined in :class:`~couchbase.views.params.Query`)
            without explicitly constructing a
            :class:`~couchbase.views.params.Query` object.
            It is illegal to use this together with the ``query`` argument.
            If you wish to 'inline' additional arguments to the provided
            ``query`` object, use the
            query's :meth:`~couchbase.views.params.Query.update` method
            instead.

        This object is an iterator - it does not send out the request until
        the first item from the iterator is request. See :meth:`__iter__` for
        more details on what this object returns.


        Simple view query, with no extra options::

            # c is the Connection object.

            for result in View(c, "beer", "brewery_beers"):
                print("emitted key: {0}, doc_id: {1}"
                        .format(result.key, result.docid))


        Execute a view with extra query options::

            # Implicitly creates a Query object

            view = View(c, "beer", "by_location",
                        limit=4,
                        reduce=True,
                        group_level=2)

        Pass a Query object::

            q = Query(
                stale=False,
                inclusive_end=True,
                mapkey_range=[
                    ["21st_ammendment_brewery_cafe"],
                    ["21st_ammendment_brewery_cafe", Query.STRING_RANGE_END]
                ]
            )

            view = View(c, "beer", "brewery_beer", query=q)

        Add extra parameters to query object for single call::

            view = View(c, "beer", "brewery_beer",
                        query=q.update(debug=True, copy=True))


        Include documents with query::

            view = View(c, "beer", "brewery_beer",
                        query=q, include_docs=True)

            for result in view:
                print("Emitted key: {0}, Document: {1}".format(
                    result.key, result.doc.value))
        """

        self._parent = parent
        self.design = design
        self.view = view
        self.errors = []
        self.raw = None
        self.rows_returned = 0

        self.include_docs = include_docs
        self.indexed_rows = 0

        if not row_processor:
            row_processor = RowProcessor()
        self.row_processor = row_processor
        self._rp_iter = None

        if query and params:
            raise ArgumentError.pyexc(
                "Extra parameters are mutually exclusive with the "
                "'query' argument. Use query.update() to add extra arguments")

        if query:
            self._query = deepcopy(query)
        else:
            self._query = Query.from_any(params)

        if include_docs:
            if (self._query.reduce or self._query.group
                    or self._query.group_level):

                raise ArgumentError.pyexc(
                    "include_docs is only applicable "
                    "for map-only views, but 'reduce', "
                    "'group', or 'group_level' "
                    "was specified", self._query)

        # The original 'limit' parameter, passed to the query.
        self._streaming = streaming
        self._do_iter = True
 def convert_to_koptions(self, options):
     raise ArgumentError.pyexc(obj=options, message="Options expected")