def __init__(self, protocol='pbc', transport_options={}, nodes=None, credentials=None, multiget_pool_size=None, multiput_pool_size=None, **kwargs): """ Construct a new ``RiakClient`` object. :param protocol: the preferred protocol, defaults to 'pbc' :type protocol: string :param nodes: a list of node configurations, where each configuration is a dict containing the keys 'host', 'http_port', and 'pb_port' :type nodes: list :param transport_options: Optional key-value args to pass to the transport constructor :type transport_options: dict :param credentials: optional object of security info :type credentials: :class:`~riak.security.SecurityCreds` or dict :param multiget_pool_size: the number of threads to use in :meth:`multiget` operations. Defaults to a factor of the number of CPUs in the system :type multiget_pool_size: int :param multiput_pool_size: the number of threads to use in :meth:`multiput` operations. Defaults to a factor of the number of CPUs in the system :type multiput_pool_size: int """ kwargs = kwargs.copy() if nodes is None: self.nodes = [ self._create_node(kwargs), ] else: self.nodes = [self._create_node(n) for n in nodes] self._multiget_pool_size = multiget_pool_size self._multiput_pool_size = multiput_pool_size self.protocol = protocol or 'pbc' self._resolver = None self._credentials = self._create_credentials(credentials) self._http_pool = HttpPool(self, **transport_options) self._tcp_pool = TcpPool(self, **transport_options) self._closed = False if PY2: self._encoders = { 'application/json': default_encoder, 'text/json': default_encoder, 'text/plain': str } self._decoders = { 'application/json': json.loads, 'text/json': json.loads, 'text/plain': str } else: self._encoders = { 'application/json': binary_json_encoder, 'text/json': binary_json_encoder, 'text/plain': str_to_bytes, 'binary/octet-stream': binary_encoder_decoder } self._decoders = { 'application/json': binary_json_decoder, 'text/json': binary_json_decoder, 'text/plain': bytes_to_str, 'binary/octet-stream': binary_encoder_decoder } self._buckets = WeakValueDictionary() self._bucket_types = WeakValueDictionary() self._tables = WeakValueDictionary()
def __init__(self, protocol='pbc', transport_options={}, nodes=None, credentials=None, multiget_pool_size=None, multiput_pool_size=None, **kwargs): """ Construct a new ``RiakClient`` object. :param protocol: the preferred protocol, defaults to 'pbc' :type protocol: string :param nodes: a list of node configurations, where each configuration is a dict containing the keys 'host', 'http_port', and 'pb_port' :type nodes: list :param transport_options: Optional key-value args to pass to the transport constructor :type transport_options: dict :param credentials: optional object of security info :type credentials: :class:`~riak.security.SecurityCreds` or dict :param multiget_pool_size: the number of threads to use in :meth:`multiget` operations. Defaults to a factor of the number of CPUs in the system :type multiget_pool_size: int :param multiput_pool_size: the number of threads to use in :meth:`multiput` operations. Defaults to a factor of the number of CPUs in the system :type multiput_pool_size: int """ kwargs = kwargs.copy() if nodes is None: self.nodes = [self._create_node(kwargs), ] else: self.nodes = [self._create_node(n) for n in nodes] self._multiget_pool_size = multiget_pool_size self._multiput_pool_size = multiput_pool_size self.protocol = protocol or 'pbc' self._resolver = None self._credentials = self._create_credentials(credentials) self._http_pool = HttpPool(self, **transport_options) self._tcp_pool = TcpPool(self, **transport_options) if PY2: self._encoders = {'application/json': default_encoder, 'text/json': default_encoder, 'text/plain': str} self._decoders = {'application/json': json.loads, 'text/json': json.loads, 'text/plain': str} else: self._encoders = {'application/json': binary_json_encoder, 'text/json': binary_json_encoder, 'text/plain': str_to_bytes, 'binary/octet-stream': binary_encoder_decoder} self._decoders = {'application/json': binary_json_decoder, 'text/json': binary_json_decoder, 'text/plain': bytes_to_str, 'binary/octet-stream': binary_encoder_decoder} self._buckets = WeakValueDictionary() self._bucket_types = WeakValueDictionary() self._tables = WeakValueDictionary()
class RiakClient(RiakMapReduceChain, RiakClientOperations): """ The ``RiakClient`` object holds information necessary to connect to Riak. Requests can be made to Riak directly through the client or by using the methods on related objects. """ #: The supported protocols PROTOCOLS = ['http', 'pbc'] def __init__(self, protocol='pbc', transport_options={}, nodes=None, credentials=None, multiget_pool_size=None, multiput_pool_size=None, **kwargs): """ Construct a new ``RiakClient`` object. :param protocol: the preferred protocol, defaults to 'pbc' :type protocol: string :param nodes: a list of node configurations, where each configuration is a dict containing the keys 'host', 'http_port', and 'pb_port' :type nodes: list :param transport_options: Optional key-value args to pass to the transport constructor :type transport_options: dict :param credentials: optional object of security info :type credentials: :class:`~riak.security.SecurityCreds` or dict :param multiget_pool_size: the number of threads to use in :meth:`multiget` operations. Defaults to a factor of the number of CPUs in the system :type multiget_pool_size: int :param multiput_pool_size: the number of threads to use in :meth:`multiput` operations. Defaults to a factor of the number of CPUs in the system :type multiput_pool_size: int """ kwargs = kwargs.copy() if nodes is None: self.nodes = [ self._create_node(kwargs), ] else: self.nodes = [self._create_node(n) for n in nodes] self._multiget_pool_size = multiget_pool_size self._multiput_pool_size = multiput_pool_size self.protocol = protocol or 'pbc' self._resolver = None self._credentials = self._create_credentials(credentials) self._http_pool = HttpPool(self, **transport_options) self._tcp_pool = TcpPool(self, **transport_options) self._closed = False if PY2: self._encoders = { 'application/json': default_encoder, 'text/json': default_encoder, 'text/plain': str } self._decoders = { 'application/json': json.loads, 'text/json': json.loads, 'text/plain': str } else: self._encoders = { 'application/json': binary_json_encoder, 'text/json': binary_json_encoder, 'text/plain': str_to_bytes, 'binary/octet-stream': binary_encoder_decoder } self._decoders = { 'application/json': binary_json_decoder, 'text/json': binary_json_decoder, 'text/plain': bytes_to_str, 'binary/octet-stream': binary_encoder_decoder } self._buckets = WeakValueDictionary() self._bucket_types = WeakValueDictionary() self._tables = WeakValueDictionary() def __del__(self): self.close() def _get_protocol(self): return self._protocol def _set_protocol(self, value): if value not in self.PROTOCOLS: raise ValueError("protocol option is invalid, must be one of %s" % repr(self.PROTOCOLS)) self._protocol = value protocol = property(_get_protocol, _set_protocol, doc=""" Which protocol to prefer, one of :attr:`PROTOCOLS <riak.client.RiakClient.PROTOCOLS>`. Please note that when one protocol is selected, the other protocols MAY NOT attempt to connect. Changing to another protocol will cause a connection on the next request. Some requests are only valid over ``'http'``, and will always be sent via those transports, regardless of which protocol is preferred. """) def _get_resolver(self): return self._resolver or default_resolver def _set_resolver(self, value): if value is None or callable(value): self._resolver = value else: raise TypeError("resolver is not a function") resolver = property( _get_resolver, _set_resolver, doc=""" The sibling-resolution function for this client. Defaults to :func:`riak.resolver.default_resolver`.""") def _get_client_id(self): with self._transport() as transport: return transport.client_id def _set_client_id(self, client_id): for http in self._http_pool: http.client_id = client_id for pb in self._tcp_pool: pb.client_id = client_id client_id = property(_get_client_id, _set_client_id, doc="""The client ID for this client instance""") def get_encoder(self, content_type): """ Get the encoding function for the provided content type. :param content_type: the requested media type :type content_type: str :rtype: function """ return self._encoders.get(content_type) def set_encoder(self, content_type, encoder): """ Set the encoding function for the provided content type. :param content_type: the requested media type :type content_type: str :param encoder: an encoding function, takes a single object argument and returns encoded data :type encoder: function """ self._encoders[content_type] = encoder def get_decoder(self, content_type): """ Get the decoding function for the provided content type. :param content_type: the requested media type :type content_type: str :rtype: function """ return self._decoders.get(content_type) def set_decoder(self, content_type, decoder): """ Set the decoding function for the provided content type. :param content_type: the requested media type :type content_type: str :param decoder: a decoding function, takes encoded data and returns a Python type :type decoder: function """ self._decoders[content_type] = decoder def bucket(self, name, bucket_type='default'): """ Get the bucket by the specified name. Since buckets always exist, this will always return a :class:`RiakBucket <riak.bucket.RiakBucket>`. If you are using a bucket that is contained in a bucket type, it is preferable to access it from the bucket type object:: # Preferred: client.bucket_type("foo").bucket("bar") # Equivalent, but not preferred: client.bucket("bar", bucket_type="foo") :param name: the bucket name :type name: str :param bucket_type: the parent bucket-type :type bucket_type: :class:`BucketType <riak.bucket.BucketType>` or str :rtype: :class:`RiakBucket <riak.bucket.RiakBucket>` """ if not isinstance(name, string_types): raise TypeError('Bucket name must be a string') if isinstance(bucket_type, string_types): bucket_type = self.bucket_type(bucket_type) elif not isinstance(bucket_type, BucketType): raise TypeError('bucket_type must be a string ' 'or riak.bucket.BucketType') return self._buckets.setdefault((bucket_type, name), RiakBucket(self, name, bucket_type)) def bucket_type(self, name): """ Gets the bucket-type by the specified name. Bucket-types do not always exist (unlike buckets), but this will always return a :class:`BucketType <riak.bucket.BucketType>` object. :param name: the bucket-type name :type name: str :rtype: :class:`BucketType <riak.bucket.BucketType>` """ if not isinstance(name, string_types): raise TypeError('BucketType name must be a string') if name in self._bucket_types: return self._bucket_types[name] else: btype = BucketType(self, name) self._bucket_types[name] = btype return btype def table(self, name): """ Gets the table by the specified name. Tables do not always exist (unlike buckets), but this will always return a :class:`Table <riak.table.Table>` object. :param name: the table name :type name: str :rtype: :class:`Table <riak.table.Table>` """ if not isinstance(name, string_types): raise TypeError('Table name must be a string') if name in self._tables: return self._tables[name] else: table = Table(self, name) self._tables[name] = table return table def close(self): """ Iterate through all of the connections and close each one. """ if not self._closed: self._closed = True self._stop_multi_pools() if self._http_pool is not None: self._http_pool.clear() self._http_pool = None if self._tcp_pool is not None: self._tcp_pool.clear() self._tcp_pool = None def _stop_multi_pools(self): if self._multiget_pool: self._multiget_pool.stop() self._multiget_pool = None if self._multiput_pool: self._multiput_pool.stop() self._multiput_pool = None def _create_node(self, n): if isinstance(n, RiakNode): return n elif isinstance(n, tuple) and len(n) is 3: host, http_port, pb_port = n return RiakNode(host=host, http_port=http_port, pb_port=pb_port) elif isinstance(n, dict): return RiakNode(**n) else: raise TypeError("%s is not a valid node configuration" % repr(n)) def _create_credentials(self, n): """ Create security credentials, if necessary. """ if not n: return n elif isinstance(n, SecurityCreds): return n elif isinstance(n, dict): return SecurityCreds(**n) else: raise TypeError("%s is not a valid security configuration" % repr(n)) def _choose_node(self, nodes=None): """ Chooses a random node from the list of nodes in the client, taking into account each node's recent error rate. :rtype RiakNode """ if not nodes: nodes = self.nodes # Prefer nodes which have gone a reasonable time without # errors def _error_rate(node): return node.error_rate.value() good = [n for n in nodes if _error_rate(n) < 0.1] if len(good) is 0: # Fall back to a minimally broken node return min(nodes, key=_error_rate) else: return random.choice(good) @lazy_property def _multiget_pool(self): if self._multiget_pool_size: return MultiGetPool(self._multiget_pool_size) else: return None @lazy_property def _multiput_pool(self): if self._multiput_pool_size: return MultiPutPool(self._multiput_pool_size) else: return None def __hash__(self): return hash( frozenset([(n.host, n.http_port, n.pb_port) for n in self.nodes])) def __eq__(self, other): if isinstance(other, self.__class__): return hash(self) == hash(other) else: return False def __ne__(self, other): if isinstance(other, self.__class__): return hash(self) != hash(other) else: return True
class RiakClient(RiakMapReduceChain, RiakClientOperations): """ The ``RiakClient`` object holds information necessary to connect to Riak. Requests can be made to Riak directly through the client or by using the methods on related objects. """ #: The supported protocols PROTOCOLS = ['http', 'pbc'] def __init__(self, protocol='pbc', transport_options={}, nodes=None, credentials=None, multiget_pool_size=None, multiput_pool_size=None, **kwargs): """ Construct a new ``RiakClient`` object. :param protocol: the preferred protocol, defaults to 'pbc' :type protocol: string :param nodes: a list of node configurations, where each configuration is a dict containing the keys 'host', 'http_port', and 'pb_port' :type nodes: list :param transport_options: Optional key-value args to pass to the transport constructor :type transport_options: dict :param credentials: optional object of security info :type credentials: :class:`~riak.security.SecurityCreds` or dict :param multiget_pool_size: the number of threads to use in :meth:`multiget` operations. Defaults to a factor of the number of CPUs in the system :type multiget_pool_size: int :param multiput_pool_size: the number of threads to use in :meth:`multiput` operations. Defaults to a factor of the number of CPUs in the system :type multiput_pool_size: int """ kwargs = kwargs.copy() if nodes is None: self.nodes = [self._create_node(kwargs), ] else: self.nodes = [self._create_node(n) for n in nodes] self._multiget_pool_size = multiget_pool_size self._multiput_pool_size = multiput_pool_size self.protocol = protocol or 'pbc' self._resolver = None self._credentials = self._create_credentials(credentials) self._http_pool = HttpPool(self, **transport_options) self._tcp_pool = TcpPool(self, **transport_options) if PY2: self._encoders = {'application/json': default_encoder, 'text/json': default_encoder, 'text/plain': str} self._decoders = {'application/json': json.loads, 'text/json': json.loads, 'text/plain': str} else: self._encoders = {'application/json': binary_json_encoder, 'text/json': binary_json_encoder, 'text/plain': str_to_bytes, 'binary/octet-stream': binary_encoder_decoder} self._decoders = {'application/json': binary_json_decoder, 'text/json': binary_json_decoder, 'text/plain': bytes_to_str, 'binary/octet-stream': binary_encoder_decoder} self._buckets = WeakValueDictionary() self._bucket_types = WeakValueDictionary() self._tables = WeakValueDictionary() def _get_protocol(self): return self._protocol def _set_protocol(self, value): if value not in self.PROTOCOLS: raise ValueError("protocol option is invalid, must be one of %s" % repr(self.PROTOCOLS)) self._protocol = value protocol = property(_get_protocol, _set_protocol, doc=""" Which protocol to prefer, one of :attr:`PROTOCOLS <riak.client.RiakClient.PROTOCOLS>`. Please note that when one protocol is selected, the other protocols MAY NOT attempt to connect. Changing to another protocol will cause a connection on the next request. Some requests are only valid over ``'http'``, and will always be sent via those transports, regardless of which protocol is preferred. """) def _get_resolver(self): return self._resolver or default_resolver def _set_resolver(self, value): if value is None or callable(value): self._resolver = value else: raise TypeError("resolver is not a function") resolver = property(_get_resolver, _set_resolver, doc=""" The sibling-resolution function for this client. Defaults to :func:`riak.resolver.default_resolver`.""") def _get_client_id(self): with self._transport() as transport: return transport.client_id def _set_client_id(self, client_id): for http in self._http_pool: http.client_id = client_id for pb in self._tcp_pool: pb.client_id = client_id client_id = property(_get_client_id, _set_client_id, doc="""The client ID for this client instance""") def get_encoder(self, content_type): """ Get the encoding function for the provided content type. :param content_type: the requested media type :type content_type: str :rtype: function """ return self._encoders.get(content_type) def set_encoder(self, content_type, encoder): """ Set the encoding function for the provided content type. :param content_type: the requested media type :type content_type: str :param encoder: an encoding function, takes a single object argument and returns encoded data :type encoder: function """ self._encoders[content_type] = encoder def get_decoder(self, content_type): """ Get the decoding function for the provided content type. :param content_type: the requested media type :type content_type: str :rtype: function """ return self._decoders.get(content_type) def set_decoder(self, content_type, decoder): """ Set the decoding function for the provided content type. :param content_type: the requested media type :type content_type: str :param decoder: a decoding function, takes encoded data and returns a Python type :type decoder: function """ self._decoders[content_type] = decoder def bucket(self, name, bucket_type='default'): """ Get the bucket by the specified name. Since buckets always exist, this will always return a :class:`RiakBucket <riak.bucket.RiakBucket>`. If you are using a bucket that is contained in a bucket type, it is preferable to access it from the bucket type object:: # Preferred: client.bucket_type("foo").bucket("bar") # Equivalent, but not preferred: client.bucket("bar", bucket_type="foo") :param name: the bucket name :type name: str :param bucket_type: the parent bucket-type :type bucket_type: :class:`BucketType <riak.bucket.BucketType>` or str :rtype: :class:`RiakBucket <riak.bucket.RiakBucket>` """ if not isinstance(name, string_types): raise TypeError('Bucket name must be a string') if isinstance(bucket_type, string_types): bucket_type = self.bucket_type(bucket_type) elif not isinstance(bucket_type, BucketType): raise TypeError('bucket_type must be a string ' 'or riak.bucket.BucketType') return self._buckets.setdefault((bucket_type, name), RiakBucket(self, name, bucket_type)) def bucket_type(self, name): """ Gets the bucket-type by the specified name. Bucket-types do not always exist (unlike buckets), but this will always return a :class:`BucketType <riak.bucket.BucketType>` object. :param name: the bucket-type name :type name: str :rtype: :class:`BucketType <riak.bucket.BucketType>` """ if not isinstance(name, string_types): raise TypeError('BucketType name must be a string') if name in self._bucket_types: return self._bucket_types[name] else: btype = BucketType(self, name) self._bucket_types[name] = btype return btype def table(self, name): """ Gets the table by the specified name. Tables do not always exist (unlike buckets), but this will always return a :class:`Table <riak.table.Table>` object. :param name: the table name :type name: str :rtype: :class:`Table <riak.table.Table>` """ if not isinstance(name, string_types): raise TypeError('Table name must be a string') if name in self._tables: return self._tables[name] else: table = Table(self, name) self._tables[name] = table return table def close(self): """ Iterate through all of the connections and close each one. """ if self._http_pool is not None: self._http_pool.clear() if self._tcp_pool is not None: self._tcp_pool.clear() def _create_node(self, n): if isinstance(n, RiakNode): return n elif isinstance(n, tuple) and len(n) is 3: host, http_port, pb_port = n return RiakNode(host=host, http_port=http_port, pb_port=pb_port) elif isinstance(n, dict): return RiakNode(**n) else: raise TypeError("%s is not a valid node configuration" % repr(n)) def _create_credentials(self, n): """ Create security credentials, if necessary. """ if not n: return n elif isinstance(n, SecurityCreds): return n elif isinstance(n, dict): return SecurityCreds(**n) else: raise TypeError("%s is not a valid security configuration" % repr(n)) def _choose_node(self, nodes=None): """ Chooses a random node from the list of nodes in the client, taking into account each node's recent error rate. :rtype RiakNode """ if not nodes: nodes = self.nodes # Prefer nodes which have gone a reasonable time without # errors def _error_rate(node): return node.error_rate.value() good = [n for n in nodes if _error_rate(n) < 0.1] if len(good) is 0: # Fall back to a minimally broken node return min(nodes, key=_error_rate) else: return random.choice(good) @lazy_property def _multiget_pool(self): if self._multiget_pool_size: return MultiGetPool(self._multiget_pool_size) else: return None @lazy_property def _multiput_pool(self): if self._multiput_pool_size: return MultiPutPool(self._multiput_pool_size) else: return None def __hash__(self): return hash(frozenset([(n.host, n.http_port, n.pb_port) for n in self.nodes])) def __eq__(self, other): if isinstance(other, self.__class__): return hash(self) == hash(other) else: return False def __ne__(self, other): if isinstance(other, self.__class__): return hash(self) != hash(other) else: return True