class PQ(object): """Convenient queue manager.""" table = 'queue' template_path = os.path.dirname(__file__) def __init__(self, *args, **kwargs): self.queue_class = kwargs.pop('queue_class', Queue) self.params = args, kwargs self.queues = WeakValueDictionary() def __getitem__(self, name): try: return self.queues[name] except KeyError: return self.queues.setdefault( name, self.queue_class(name, *self.params[0], **self.params[1]) ) def close(self): self[''].close() def create(self): queue = self[''] with open(os.path.join(self.template_path, 'create.sql'), 'r') as f: sql = f.read() with queue._transaction() as cursor: cursor.execute(sql, {'name': Literal(queue.table)})
class PQ(object): """Convenient queue manager.""" table = 'queue' queue_class = Queue template_path = os.path.dirname(__file__) def __init__(self, *args, **kwargs): self.queue_class = kwargs.pop('queue_class', self.queue_class) self.params = args, kwargs self.queues = WeakValueDictionary() def __getitem__(self, name): try: return self.queues[name] except KeyError: return self.queues.setdefault( name, self.queue_class(name, *self.params[0], **self.params[1])) def close(self): self[''].close() def create(self): queue = self[''] with open(os.path.join(self.template_path, 'create.sql'), 'r') as f: sql = f.read() with queue._transaction() as cursor: cursor.execute(sql, {'name': Literal(queue.table)})
class PQ(object): """Convenient queue manager.""" table = 'queue' def __init__(self, *args, **kwargs): self.params = args, kwargs self.queues = WeakValueDictionary() def __getitem__(self, name): try: return self.queues[name] except KeyError: return self.queues.setdefault( name, Queue(name, *self.params[0], **self.params[1]) ) def close(self): self[''].close() def create(self): q = self[''] conn = q._conn() sql = _read_sql('create') with transaction(conn) as cursor: cursor.execute(sql, {'name': Literal(q.table)})
class Throttle: def __init__(self, concurrency: int, concurrency_per_host: int, loop: asyncio.AbstractEventLoop) -> None: self._concurrency_semaphore = asyncio.Semaphore(concurrency, loop=loop) self._host_semaphore_factory = partial(asyncio.Semaphore, concurrency_per_host, loop=loop) self._hosts = WeakValueDictionary() @asynccontextmanager async def request(self, host: str) -> None: semaphore = self._host_semaphore_factory() host_semaphore = self._hosts.setdefault(host, semaphore) async with self._concurrency_semaphore, host_semaphore: yield
class RiakClient(RiakMapReduceChain, RiakClientOperations): """ The ``RiakClient`` object holds information necessary to connect to Riak. Requests can be made to Riak directly through the client or by using the methods on related objects. """ #: The supported protocols PROTOCOLS = ['http', 'pbc'] def __init__(self, protocol='pbc', transport_options={}, nodes=None, credentials=None, multiget_pool_size=None, multiput_pool_size=None, **kwargs): """ Construct a new ``RiakClient`` object. :param protocol: the preferred protocol, defaults to 'pbc' :type protocol: string :param nodes: a list of node configurations, where each configuration is a dict containing the keys 'host', 'http_port', and 'pb_port' :type nodes: list :param transport_options: Optional key-value args to pass to the transport constructor :type transport_options: dict :param credentials: optional object of security info :type credentials: :class:`~riak.security.SecurityCreds` or dict :param multiget_pool_size: the number of threads to use in :meth:`multiget` operations. Defaults to a factor of the number of CPUs in the system :type multiget_pool_size: int :param multiput_pool_size: the number of threads to use in :meth:`multiput` operations. Defaults to a factor of the number of CPUs in the system :type multiput_pool_size: int """ kwargs = kwargs.copy() if nodes is None: self.nodes = [ self._create_node(kwargs), ] else: self.nodes = [self._create_node(n) for n in nodes] self._multiget_pool_size = multiget_pool_size self._multiput_pool_size = multiput_pool_size self.protocol = protocol or 'pbc' self._resolver = None self._credentials = self._create_credentials(credentials) self._http_pool = HttpPool(self, **transport_options) self._tcp_pool = TcpPool(self, **transport_options) self._closed = False if PY2: self._encoders = { 'application/json': default_encoder, 'text/json': default_encoder, 'text/plain': str } self._decoders = { 'application/json': json.loads, 'text/json': json.loads, 'text/plain': str } else: self._encoders = { 'application/json': binary_json_encoder, 'text/json': binary_json_encoder, 'text/plain': str_to_bytes, 'binary/octet-stream': binary_encoder_decoder } self._decoders = { 'application/json': binary_json_decoder, 'text/json': binary_json_decoder, 'text/plain': bytes_to_str, 'binary/octet-stream': binary_encoder_decoder } self._buckets = WeakValueDictionary() self._bucket_types = WeakValueDictionary() self._tables = WeakValueDictionary() def __del__(self): self.close() def _get_protocol(self): return self._protocol def _set_protocol(self, value): if value not in self.PROTOCOLS: raise ValueError("protocol option is invalid, must be one of %s" % repr(self.PROTOCOLS)) self._protocol = value protocol = property(_get_protocol, _set_protocol, doc=""" Which protocol to prefer, one of :attr:`PROTOCOLS <riak.client.RiakClient.PROTOCOLS>`. Please note that when one protocol is selected, the other protocols MAY NOT attempt to connect. Changing to another protocol will cause a connection on the next request. Some requests are only valid over ``'http'``, and will always be sent via those transports, regardless of which protocol is preferred. """) def _get_resolver(self): return self._resolver or default_resolver def _set_resolver(self, value): if value is None or callable(value): self._resolver = value else: raise TypeError("resolver is not a function") resolver = property( _get_resolver, _set_resolver, doc=""" The sibling-resolution function for this client. Defaults to :func:`riak.resolver.default_resolver`.""") def _get_client_id(self): with self._transport() as transport: return transport.client_id def _set_client_id(self, client_id): for http in self._http_pool: http.client_id = client_id for pb in self._tcp_pool: pb.client_id = client_id client_id = property(_get_client_id, _set_client_id, doc="""The client ID for this client instance""") def get_encoder(self, content_type): """ Get the encoding function for the provided content type. :param content_type: the requested media type :type content_type: str :rtype: function """ return self._encoders.get(content_type) def set_encoder(self, content_type, encoder): """ Set the encoding function for the provided content type. :param content_type: the requested media type :type content_type: str :param encoder: an encoding function, takes a single object argument and returns encoded data :type encoder: function """ self._encoders[content_type] = encoder def get_decoder(self, content_type): """ Get the decoding function for the provided content type. :param content_type: the requested media type :type content_type: str :rtype: function """ return self._decoders.get(content_type) def set_decoder(self, content_type, decoder): """ Set the decoding function for the provided content type. :param content_type: the requested media type :type content_type: str :param decoder: a decoding function, takes encoded data and returns a Python type :type decoder: function """ self._decoders[content_type] = decoder def bucket(self, name, bucket_type='default'): """ Get the bucket by the specified name. Since buckets always exist, this will always return a :class:`RiakBucket <riak.bucket.RiakBucket>`. If you are using a bucket that is contained in a bucket type, it is preferable to access it from the bucket type object:: # Preferred: client.bucket_type("foo").bucket("bar") # Equivalent, but not preferred: client.bucket("bar", bucket_type="foo") :param name: the bucket name :type name: str :param bucket_type: the parent bucket-type :type bucket_type: :class:`BucketType <riak.bucket.BucketType>` or str :rtype: :class:`RiakBucket <riak.bucket.RiakBucket>` """ if not isinstance(name, string_types): raise TypeError('Bucket name must be a string') if isinstance(bucket_type, string_types): bucket_type = self.bucket_type(bucket_type) elif not isinstance(bucket_type, BucketType): raise TypeError('bucket_type must be a string ' 'or riak.bucket.BucketType') return self._buckets.setdefault((bucket_type, name), RiakBucket(self, name, bucket_type)) def bucket_type(self, name): """ Gets the bucket-type by the specified name. Bucket-types do not always exist (unlike buckets), but this will always return a :class:`BucketType <riak.bucket.BucketType>` object. :param name: the bucket-type name :type name: str :rtype: :class:`BucketType <riak.bucket.BucketType>` """ if not isinstance(name, string_types): raise TypeError('BucketType name must be a string') if name in self._bucket_types: return self._bucket_types[name] else: btype = BucketType(self, name) self._bucket_types[name] = btype return btype def table(self, name): """ Gets the table by the specified name. Tables do not always exist (unlike buckets), but this will always return a :class:`Table <riak.table.Table>` object. :param name: the table name :type name: str :rtype: :class:`Table <riak.table.Table>` """ if not isinstance(name, string_types): raise TypeError('Table name must be a string') if name in self._tables: return self._tables[name] else: table = Table(self, name) self._tables[name] = table return table def close(self): """ Iterate through all of the connections and close each one. """ if not self._closed: self._closed = True self._stop_multi_pools() if self._http_pool is not None: self._http_pool.clear() self._http_pool = None if self._tcp_pool is not None: self._tcp_pool.clear() self._tcp_pool = None def _stop_multi_pools(self): if self._multiget_pool: self._multiget_pool.stop() self._multiget_pool = None if self._multiput_pool: self._multiput_pool.stop() self._multiput_pool = None def _create_node(self, n): if isinstance(n, RiakNode): return n elif isinstance(n, tuple) and len(n) is 3: host, http_port, pb_port = n return RiakNode(host=host, http_port=http_port, pb_port=pb_port) elif isinstance(n, dict): return RiakNode(**n) else: raise TypeError("%s is not a valid node configuration" % repr(n)) def _create_credentials(self, n): """ Create security credentials, if necessary. """ if not n: return n elif isinstance(n, SecurityCreds): return n elif isinstance(n, dict): return SecurityCreds(**n) else: raise TypeError("%s is not a valid security configuration" % repr(n)) def _choose_node(self, nodes=None): """ Chooses a random node from the list of nodes in the client, taking into account each node's recent error rate. :rtype RiakNode """ if not nodes: nodes = self.nodes # Prefer nodes which have gone a reasonable time without # errors def _error_rate(node): return node.error_rate.value() good = [n for n in nodes if _error_rate(n) < 0.1] if len(good) is 0: # Fall back to a minimally broken node return min(nodes, key=_error_rate) else: return random.choice(good) @lazy_property def _multiget_pool(self): if self._multiget_pool_size: return MultiGetPool(self._multiget_pool_size) else: return None @lazy_property def _multiput_pool(self): if self._multiput_pool_size: return MultiPutPool(self._multiput_pool_size) else: return None def __hash__(self): return hash( frozenset([(n.host, n.http_port, n.pb_port) for n in self.nodes])) def __eq__(self, other): if isinstance(other, self.__class__): return hash(self) == hash(other) else: return False def __ne__(self, other): if isinstance(other, self.__class__): return hash(self) != hash(other) else: return True
class RiakClient(RiakMapReduceChain, RiakClientOperations): """ The ``RiakClient`` object holds information necessary to connect to Riak. Requests can be made to Riak directly through the client or by using the methods on related objects. """ #: The supported protocols PROTOCOLS = ['http', 'pbc'] def __init__(self, protocol='pbc', transport_options={}, nodes=None, credentials=None, multiget_pool_size=None, multiput_pool_size=None, **kwargs): """ Construct a new ``RiakClient`` object. :param protocol: the preferred protocol, defaults to 'pbc' :type protocol: string :param nodes: a list of node configurations, where each configuration is a dict containing the keys 'host', 'http_port', and 'pb_port' :type nodes: list :param transport_options: Optional key-value args to pass to the transport constructor :type transport_options: dict :param credentials: optional object of security info :type credentials: :class:`~riak.security.SecurityCreds` or dict :param multiget_pool_size: the number of threads to use in :meth:`multiget` operations. Defaults to a factor of the number of CPUs in the system :type multiget_pool_size: int :param multiput_pool_size: the number of threads to use in :meth:`multiput` operations. Defaults to a factor of the number of CPUs in the system :type multiput_pool_size: int """ kwargs = kwargs.copy() if nodes is None: self.nodes = [self._create_node(kwargs), ] else: self.nodes = [self._create_node(n) for n in nodes] self._multiget_pool_size = multiget_pool_size self._multiput_pool_size = multiput_pool_size self.protocol = protocol or 'pbc' self._resolver = None self._credentials = self._create_credentials(credentials) self._http_pool = HttpPool(self, **transport_options) self._tcp_pool = TcpPool(self, **transport_options) if PY2: self._encoders = {'application/json': default_encoder, 'text/json': default_encoder, 'text/plain': str} self._decoders = {'application/json': json.loads, 'text/json': json.loads, 'text/plain': str} else: self._encoders = {'application/json': binary_json_encoder, 'text/json': binary_json_encoder, 'text/plain': str_to_bytes, 'binary/octet-stream': binary_encoder_decoder} self._decoders = {'application/json': binary_json_decoder, 'text/json': binary_json_decoder, 'text/plain': bytes_to_str, 'binary/octet-stream': binary_encoder_decoder} self._buckets = WeakValueDictionary() self._bucket_types = WeakValueDictionary() self._tables = WeakValueDictionary() def _get_protocol(self): return self._protocol def _set_protocol(self, value): if value not in self.PROTOCOLS: raise ValueError("protocol option is invalid, must be one of %s" % repr(self.PROTOCOLS)) self._protocol = value protocol = property(_get_protocol, _set_protocol, doc=""" Which protocol to prefer, one of :attr:`PROTOCOLS <riak.client.RiakClient.PROTOCOLS>`. Please note that when one protocol is selected, the other protocols MAY NOT attempt to connect. Changing to another protocol will cause a connection on the next request. Some requests are only valid over ``'http'``, and will always be sent via those transports, regardless of which protocol is preferred. """) def _get_resolver(self): return self._resolver or default_resolver def _set_resolver(self, value): if value is None or callable(value): self._resolver = value else: raise TypeError("resolver is not a function") resolver = property(_get_resolver, _set_resolver, doc=""" The sibling-resolution function for this client. Defaults to :func:`riak.resolver.default_resolver`.""") def _get_client_id(self): with self._transport() as transport: return transport.client_id def _set_client_id(self, client_id): for http in self._http_pool: http.client_id = client_id for pb in self._tcp_pool: pb.client_id = client_id client_id = property(_get_client_id, _set_client_id, doc="""The client ID for this client instance""") def get_encoder(self, content_type): """ Get the encoding function for the provided content type. :param content_type: the requested media type :type content_type: str :rtype: function """ return self._encoders.get(content_type) def set_encoder(self, content_type, encoder): """ Set the encoding function for the provided content type. :param content_type: the requested media type :type content_type: str :param encoder: an encoding function, takes a single object argument and returns encoded data :type encoder: function """ self._encoders[content_type] = encoder def get_decoder(self, content_type): """ Get the decoding function for the provided content type. :param content_type: the requested media type :type content_type: str :rtype: function """ return self._decoders.get(content_type) def set_decoder(self, content_type, decoder): """ Set the decoding function for the provided content type. :param content_type: the requested media type :type content_type: str :param decoder: a decoding function, takes encoded data and returns a Python type :type decoder: function """ self._decoders[content_type] = decoder def bucket(self, name, bucket_type='default'): """ Get the bucket by the specified name. Since buckets always exist, this will always return a :class:`RiakBucket <riak.bucket.RiakBucket>`. If you are using a bucket that is contained in a bucket type, it is preferable to access it from the bucket type object:: # Preferred: client.bucket_type("foo").bucket("bar") # Equivalent, but not preferred: client.bucket("bar", bucket_type="foo") :param name: the bucket name :type name: str :param bucket_type: the parent bucket-type :type bucket_type: :class:`BucketType <riak.bucket.BucketType>` or str :rtype: :class:`RiakBucket <riak.bucket.RiakBucket>` """ if not isinstance(name, string_types): raise TypeError('Bucket name must be a string') if isinstance(bucket_type, string_types): bucket_type = self.bucket_type(bucket_type) elif not isinstance(bucket_type, BucketType): raise TypeError('bucket_type must be a string ' 'or riak.bucket.BucketType') return self._buckets.setdefault((bucket_type, name), RiakBucket(self, name, bucket_type)) def bucket_type(self, name): """ Gets the bucket-type by the specified name. Bucket-types do not always exist (unlike buckets), but this will always return a :class:`BucketType <riak.bucket.BucketType>` object. :param name: the bucket-type name :type name: str :rtype: :class:`BucketType <riak.bucket.BucketType>` """ if not isinstance(name, string_types): raise TypeError('BucketType name must be a string') if name in self._bucket_types: return self._bucket_types[name] else: btype = BucketType(self, name) self._bucket_types[name] = btype return btype def table(self, name): """ Gets the table by the specified name. Tables do not always exist (unlike buckets), but this will always return a :class:`Table <riak.table.Table>` object. :param name: the table name :type name: str :rtype: :class:`Table <riak.table.Table>` """ if not isinstance(name, string_types): raise TypeError('Table name must be a string') if name in self._tables: return self._tables[name] else: table = Table(self, name) self._tables[name] = table return table def close(self): """ Iterate through all of the connections and close each one. """ if self._http_pool is not None: self._http_pool.clear() if self._tcp_pool is not None: self._tcp_pool.clear() def _create_node(self, n): if isinstance(n, RiakNode): return n elif isinstance(n, tuple) and len(n) is 3: host, http_port, pb_port = n return RiakNode(host=host, http_port=http_port, pb_port=pb_port) elif isinstance(n, dict): return RiakNode(**n) else: raise TypeError("%s is not a valid node configuration" % repr(n)) def _create_credentials(self, n): """ Create security credentials, if necessary. """ if not n: return n elif isinstance(n, SecurityCreds): return n elif isinstance(n, dict): return SecurityCreds(**n) else: raise TypeError("%s is not a valid security configuration" % repr(n)) def _choose_node(self, nodes=None): """ Chooses a random node from the list of nodes in the client, taking into account each node's recent error rate. :rtype RiakNode """ if not nodes: nodes = self.nodes # Prefer nodes which have gone a reasonable time without # errors def _error_rate(node): return node.error_rate.value() good = [n for n in nodes if _error_rate(n) < 0.1] if len(good) is 0: # Fall back to a minimally broken node return min(nodes, key=_error_rate) else: return random.choice(good) @lazy_property def _multiget_pool(self): if self._multiget_pool_size: return MultiGetPool(self._multiget_pool_size) else: return None @lazy_property def _multiput_pool(self): if self._multiput_pool_size: return MultiPutPool(self._multiput_pool_size) else: return None def __hash__(self): return hash(frozenset([(n.host, n.http_port, n.pb_port) for n in self.nodes])) def __eq__(self, other): if isinstance(other, self.__class__): return hash(self) == hash(other) else: return False def __ne__(self, other): if isinstance(other, self.__class__): return hash(self) != hash(other) else: return True
class RiakClient: ''' The ``RiakClient`` object holds information necessary to connect to Riak. Requests can be made to Riak directly through the client or by using the methods on related objects. ''' def __init__(self, host='localhost', port=8087, loop=None): self._host = host self._port = port self._loop = loop self._bucket_types = WeakValueDictionary() self._buckets = WeakValueDictionary() self._resolver = None self._decoders = { 'application/json': binary_json_decoder, 'text/json': binary_json_decoder, 'text/plain': bytes_to_str, 'binary/octet-stream': binary_encoder_decoder } self._encoders = { 'application/json': binary_json_encoder, 'text/json': binary_json_encoder, 'text/plain': str_to_bytes, 'binary/octet-stream': binary_encoder_decoder } def get_decoder(self, content_type): ''' Get the decoding function for the provided content type. :param content_type: the requested media type :type content_type: str :rtype: function ''' return self._decoders.get(content_type) def get_encoder(self, content_type): ''' Get the encoding function for the provided content type. :param content_type: the requested media type :type content_type: str :rtype: function ''' return self._encoders.get(content_type) def set_encoder(self, content_type, encoder): ''' Set the encoding function for the provided content type. :param content_type: the requested media type :type content_type: str :param encoder: an encoding function, takes a single object argument and returns encoded data :type encoder: function ''' self._encoders[content_type] = encoder def set_decoder(self, content_type, decoder): ''' Set the decoding function for the provided content type. :param content_type: the requested media type :type content_type: str :param decoder: a decoding function, takes encoded data and returns a Python type :type decoder: function ''' self._decoders[content_type] = decoder def _get_resolver(self): return self._resolver or default_resolver def _set_resolver(self, value): if value is None or callable(value): self._resolver = value else: raise TypeError('resolver is not a function') resolver = property(_get_resolver, _set_resolver, doc='''The sibling-resolution function for this client. Defaults to :func:`riak.resolver.default_resolver`.''') def close(self): self._transport.close() async def _create_transport(self): self._transport = await create_transport(self._host, self._port, self._loop) @classmethod async def create(cls, host='localhost', port=8087, loop=None): ''' Return initialized instance of RiakClient since RiakClient.__init__() can't be async. :Example: .. code-block:: python import asyncio from aioriak import RiakClient loop = asyncio.get_event_loop() async def go(): client = await RiakClient.create('localhost', 8087, loop) loop.run_until_complete(go()) :param host: Hostname or ip address of Riak instance :type host: str :param port: Port of riak instance :type port: int :param loop: asyncio event loop :rtype: :class:`~aioriak.client.RiakClient` ''' client = cls(host, port, loop) await client._create_transport() return client async def fetch_datatype(self, bucket, key): ''' Fetches the value of a Riak Datatype. :param bucket: the bucket of the datatype, which must belong to a :class:`~aioriak.bucket.BucketType` :type bucket: :class:`~aioriak.bucket.Bucket` :param key: the key of the datatype :type key: string :rtype: :class:`~aioriak.datatypes.Datatype` ''' dtype, value, context = await self._fetch_datatype(bucket, key) return TYPES[dtype](bucket=bucket, key=key, value=value, context=context) async def _fetch_datatype(self, bucket, key): ''' _fetch_datatype(bucket, key) Fetches the value of a Riak Datatype as raw data. This is used internally to update already reified Datatype objects. Use the public version to fetch a reified type. :param bucket: the bucket of the datatype, which must belong to a :class:`~aioriak.BucketType` :type bucket: RiakBucket :param key: the key of the datatype :type key: string, None :rtype: tuple of type, value and context ''' return await self._transport.fetch_datatype(bucket, key) async def ping(self): ''' Check if the Riak server for this ``RiakClient`` instance is alive. :rtype: boolean ''' return await self._transport.ping() is_alive = ping async def get_client_id(self): ''' Get client ID for this RiakClient instance :rtype: bytes ''' return await self._transport.get_client_id() async def set_client_id(self, id): ''' Set Client ID for this RiakClient instance' ''' return await self._transport.set_client_id(id) async def get_buckets(self, bucket_type=None): ''' Get the list of buckets as :class:`Bucket <aioriak.bucket.Bucket>` instances. .. warning:: Do not use this in production, as it requires traversing through all keys stored in a cluster. :param bucket_type: the optional containing bucket type :type bucket_type: :class:`~aioriak.bucket.BucketType` :rtype: list of :class:`Bucket <aioriak.bucket.Bucket>` instances ''' if bucket_type: maker = bucket_type.bucket else: maker = self.bucket return [ maker(name.decode()) for name in await self._transport.get_buckets( bucket_type=bucket_type) ] def bucket_type(self, name): ''' Gets the bucket-type by the specified name. Bucket-types do not always exist (unlike buckets), but this will always return a :class:`BucketType <aioriak.bucket.BucketType>` object. :param name: the bucket name :type name: str :rtype: :class:`BucketType <aioriak.bucket.BucketType>` ''' if not isinstance(name, str): raise TypeError('Bucket name must be a string') if name in self._bucket_types: return self._bucket_types[name] else: btype = BucketType(self, name) self._bucket_types[name] = btype return btype def bucket(self, name, bucket_type='default'): ''' Get the bucket by the specified name. Since buckets always exist, this will always return a :class:`Bucket <aioriak.bucket.Bucket>`. If you are using a bucket that is contained in a bucket type, it is preferable to access it from the bucket type object:: # Preferred: client.bucket_type("foo").bucket("bar") # Equivalent, but not preferred: client.bucket("bar", bucket_type="foo") :param name: the bucket name :type name: str :param bucket_type: the parent bucket-type :type bucket_type: :class:`BucketType <aioriak.bucket.BucketType>` or str :rtype: :class:`Bucket <aioriak.bucket.Bucket>` ''' if not isinstance(name, str): raise TypeError('Bucket name must be a string') if isinstance(bucket_type, str): bucket_type = self.bucket_type(bucket_type) elif not isinstance(bucket_type, BucketType): raise TypeError('bucket_type must be a string ' 'or aioriak.bucket.BucketType') return self._buckets.setdefault((bucket_type, name), Bucket(self, name, bucket_type)) async def get_bucket_type_props(self, bucket_type): ''' Fetches properties for the given bucket-type. :param bucket_type: the bucket-type whose properties will be fetched :type bucket_type: BucketType :rtype: dict ''' return await self._transport.get_bucket_type_props(bucket_type) async def set_bucket_type_props(self, bucket_type, props): ''' Sets properties for the given bucket-type. :param bucket_type: the bucket-type whose properties will be set :type bucket_type: BucketType :param props: the properties to set :type props: dict ''' return await self._transport.set_bucket_type_props(bucket_type, props) async def get_bucket_props(self, bucket): ''' Fetches bucket properties for the given bucket. :param bucket: the bucket whose properties will be fetched :type bucket: Bucket :rtype: dict ''' return await self._transport.get_bucket_props(bucket) async def set_bucket_props(self, bucket, props): ''' Sets bucket properties for the given bucket. :param bucket: the bucket whose properties will be set :type bucket: Bucket :param props: the properties to set :type props: dict ''' return await self._transport.set_bucket_props(bucket, props) async def get_keys(self, bucket): ''' Lists all keys in a bucket. .. warning:: Do not use this in production, as it requires traversing through all keys stored in a cluster. :param bucket: the bucket whose keys are fetched :type bucket: Bucket :rtype: list ''' return await self._transport.get_keys(bucket) async def get(self, robj): ''' Fetches the contents of a Riak object. :param robj: the object to fetch :type robj: RiakObject ''' if not isinstance(robj.key, str): raise TypeError('key must be a string, instead got {0}'.format( repr(robj.key))) return await self._transport.get(robj) async def put(self, robj, return_body): ''' Stores an object in the Riak cluster. :param return_body: whether to return the resulting object after the write :type return_body: boolean :param robj: the object to store :type robj: RiakObject ''' return await self._transport.put(robj, return_body=return_body) async def delete(self, robj): ''' Deletes an object from Riak. :param robj: the object to delete :type robj: RiakObject ''' return await self._transport.delete(robj) async def update_datatype(self, datatype, **params): ''' Sends an update to a Riak Datatype to the server. :param datatype: the datatype with pending updates :type datatype: :class:`~aioriak.datatypes.Datatype` :rtype: tuple of datatype, opaque value and opaque context ''' return await self._transport.update_datatype(datatype, **params)
class BuildRequestMerger(config.ReconfigurableServiceMixin, service.Service): # Basic list of properties that must match for a buildrequest to be merged # BuilderConfigs can define additional properties (see _propertiesMatch) BASE_MERGE_PROPERTIES = ['force_rebuild', 'force_chain_rebuild'] def __init__(self, master): self.master = master self.properties_cache = AsyncLRUCache( miss_fn=None, # Cache contents are handled manually max_size=20000) # Locks to indicate that merged builds are being added self.build_merging_locks = WeakValueDictionary() def getMergingLocks(self, build_request_ids): return [ self.build_merging_locks.setdefault(brid, defer.DeferredLock()) for brid in build_request_ids ] @defer.inlineCallbacks def addBuildset(self, sourcestampsetid, reason, properties, triggeredbybrid=None, builderNames=None, external_idstring=None, _reactor=reactor): """ Wrapper around buildsets.addBuildset that does merging before buildrequests hit the db. ..seealso:: self._getMergeBrDict For more documentation on merge conditions ..seealso:: buildsets.addBuildset For parameter details """ builderNames = sorted(builderNames) start = time.time() buildsetLog = { 'name': 'addBuildset', 'description': 'Log merges within a chain while adding new buildsets', 'sourcestampsetid': sourcestampsetid, 'builderNames': builderNames, } # For every builderName in this buildset, check which ones can be merged brDictsToMerge = {} # Don't read sourcestamp information yet, since we might not need it sourcestamps = None for builderName in builderNames: builderMergeStart = time.time() if 'selected_slave' in properties: # Never merge if a build request has a selected_slave # This might happen when a user wants to test the same build in different # slaves to look for instabilities mergeBrDict = None else: # Look for a builder that matches the configured properties mergeProperties = self.master.botmaster.builders[ builderName].config.mergeProperties # And sourcestamps (only need to read them once) if sourcestamps is None: sourcestamps = yield self.master.db.sourcestamps.getSimpleSourceStamps( sourcestampsetid) mergeBrDict = yield self._getMergeBrDict( builderName, sourcestamps, properties, mergeProperties) # If we found one, add it to our merge map if mergeBrDict: brDictsToMerge[builderName] = mergeBrDict buildsetLog[builderName] = { 'elapsed': time.time() - builderMergeStart, 'mergeBrid': brDictsToMerge.get(builderName, {}).get('brid', None) } buildsetLog['elapsed_merge'] = time.time() - start # Finally add the buildset passing the map of `brDictsToMerge` # This method will make sure that all new breqs will enter the db # marked as merged, and will not run. _master_objectid = yield self.master.getObjectId() # Create a lock on every build we can merge into. acquiring_locks_start = time.time() build_merging_locks = { builderName : self.getMergingLocks([brDict['brid']])[0] for builderName, brDict in brDictsToMerge.iteritems() } for builderName, lock in build_merging_locks.iteritems(): yield lock.acquire() buildsetLog[builderName]['elapsed_acquiring_lock'] = \ time.time() - acquiring_locks_start buildsetLog['elapsed_acquiring_locks'] = time.time() - acquiring_locks_start using_locks_start = time.time() # Some builds might have finished before we locked into them, so release those. finishedBrDicts = yield self.master.db.buildrequests.getBuildRequests( brids=[brDict['brid'] for brDict in brDictsToMerge.values()], complete=True) for brDict in finishedBrDicts: brDictsToMerge.pop(brDict['buildername']) build_merging_locks.pop(brDict['buildername']).release() # Add buildset try: result = yield self.master.db.buildsets.addBuildset( sourcestampsetid=sourcestampsetid, reason=reason, properties=properties, triggeredbybrid=triggeredbybrid, builderNames=builderNames, brDictsToMerge=brDictsToMerge, external_idstring=external_idstring, _reactor=_reactor, _master_objectid=_master_objectid) finally: for lock in build_merging_locks.itervalues(): lock.release() buildsetLog['elapsed_using_locks'] = time.time() - using_locks_start # Log more ids (bsid, brids) = result buildsetLog['buildsetid'] = bsid for builderName, brid in brids.iteritems(): buildsetLog[builderName]['brid'] = brid buildsetLog['elapsed_total'] = time.time() - start log.msg(json.dumps(buildsetLog)) defer.returnValue(result) @defer.inlineCallbacks def _getMergeBrDict(self, builderName, sourcestamps, properties, mergeProperties): """ Looks for a buildrequest we can merge into. It must match `builderName`, `sourcestamps` and all properties defined in `mergeProperties`. This will only merge against builds that have already been claimed and are currently running (unfinished builds). :return BrDict or None: Buildrequest dictionary for a request that can be merged into. `None` if no match was found. """ matchingBrDicts = yield self.master.db.buildrequests.getBuildRequests( buildername=builderName, complete=False, claimed=True, sourcestamps=sourcestamps, mergebrids="exclude") # Get properties for matching breqs (done in a single query for optimization) otherProperties = yield self._getBuildsetsProperties( [brdict['buildsetid'] for brdict in matchingBrDicts]) # Check if relevant properties match # Sort list of build requests to ensure we always merge against smallest id possible for brdict in sorted(matchingBrDicts, key=lambda b: int(b['brid'])): if self._propertiesMatch(properties, otherProperties[brdict['buildsetid']], mergeProperties): # If they match, fetch the build number and merge against this buildrequest brdict[ 'build_number'] = yield self.master.db.builds.getBuildNumberForRequest( brdict['brid']) defer.returnValue(brdict) return # If we can't find any match, return None defer.returnValue(None) def _propertiesMatch(self, properties, otherProperties, mergeProperties): """ :param dict(str,str) properties: :param dict(str,str) otherProperties: :param list(str) mergeProperties: List of properties that must match :return bool: True if `properties` and `otherProperties` match for a list of `mergeProperties` to be checked """ if 'selected_slave' in otherProperties: return False for propertyName in self.BASE_MERGE_PROPERTIES + mergeProperties: if properties.get(propertyName, None) != otherProperties.get( propertyName, None): return False return True @defer.inlineCallbacks def _getBuildsetsProperties(self, buildsetids): properties = {} # Fill in properties from cache for buildsetid in buildsetids: if buildsetid in self.properties_cache.cache: properties[buildsetid] = yield self.properties_cache.get( buildsetid) # Read missing properties in a single query missing_buildsetids = set(buildsetids).difference(properties.keys()) if missing_buildsetids: db_properties = yield self.master.db.buildsets.getBuildsetsProperties( missing_buildsetids) properties.update(db_properties) # Populate cache with new values for buildsetid in missing_buildsetids: bs_properties = PropertiesDict(properties[buildsetid]) self.properties_cache.put_new(buildsetid, bs_properties) defer.returnValue(properties)
class DiskStorage(BaseStorage): INFO_FILE = 'data.bin' class _Info(object): _int_fmt = '<Q' _int_length = 8 def __init__(self, p): self._info_file = make_direct_open(p, 'r+b', buffering=0) self._total_size = 0 self._total_count = 0 self._lock = threading.RLock() @classmethod def load_file(cls, file_path): obj = cls(file_path) with obj._lock: obj._info_file.seek(0) raw = obj._info_file.read(cls._int_length * 2) try: raw_size = raw[:cls._int_length] raw_count = raw[cls._int_length:] obj._total_size = struct.unpack(cls._int_fmt, raw_size)[0] obj._total_count = struct.unpack(cls._int_fmt, raw_count)[0] except struct.error: raise StandardError(repr(raw)) return obj def _write_num(self, i, n): with self._lock: self._info_file.seek(i * self._int_length) bin_str = struct.pack(self._int_fmt, n) self._info_file.write(bin_str) def reset(self): with self._lock: self._total_count = 0 self._total_size = 0 self._info_file.flush() @property def size(self): return self._total_size @size.setter def size(self, v): self._write_num(0, v) self._total_size = v @property def count(self): return self._total_count @count.setter def count(self, v): self._write_num(1, v) self._total_count = v def __init__(self, path, path_func=None): self._storage_path = os.path.join(path, "cache") self._path_func = path_func or (lambda _: _) self._file_locks = WeakValueDictionary() self._load_info_file() def _load_info_file(self): info_path = os.path.join(self._storage_path, self.INFO_FILE) if os.path.isfile(info_path): try: self._info = self._Info.load_file(info_path) except StandardError: self._info = self._Info(info_path) else: self._info = self._Info(info_path) def _get_file_lock(self, path): return self._file_locks.setdefault(path, threading.RLock()) def full_path(self, key): return os.path.join(self._storage_path, self._path_func(key)) def has(self, key): return os.path.isfile(self.full_path(key)) def set(self, key, value): full_path = self.full_path(key) tmp_path = full_path + ".tmp" with self._get_file_lock(full_path): try: with make_direct_open(tmp_path, 'wb') as fp: fp.write(value) except IOError: return False else: self.delete(key) # 防止文件存在导致移动失败 os.rename(tmp_path, full_path) self._info.count += 1 self._info.size += len(value) return True def get(self, key, default=None): full_path = self.full_path(key) if os.path.isfile(full_path): with self._get_file_lock(full_path): if os.path.isfile(full_path): with open(full_path, 'rb') as fp: return fp.read() return default def clear(self): os.rmdir(self._storage_path) self._info.reset() def delete(self, key): full_path = self.full_path(key) if os.path.isfile(full_path): with self._get_file_lock(full_path): if os.path.isfile(full_path): self._info.size -= os.path.getsize(full_path) self._info.count -= 1 os.remove(full_path) @property def size(self): return self._info.size @property def count(self): return self._info.count
class RiakClient: ''' The ``RiakClient`` object holds information necessary to connect to Riak. Requests can be made to Riak directly through the client or by using the methods on related objects. ''' def __init__(self, host='localhost', port=8087, loop=None): self._host = host self._port = port self._loop = loop self._bucket_types = WeakValueDictionary() self._buckets = WeakValueDictionary() self._resolver = None self._decoders = {'application/json': binary_json_decoder, 'text/json': binary_json_decoder, 'text/plain': bytes_to_str, 'binary/octet-stream': binary_encoder_decoder} self._encoders = {'application/json': binary_json_encoder, 'text/json': binary_json_encoder, 'text/plain': str_to_bytes, 'binary/octet-stream': binary_encoder_decoder} def get_decoder(self, content_type): ''' Get the decoding function for the provided content type. :param content_type: the requested media type :type content_type: str :rtype: function ''' return self._decoders.get(content_type) def get_encoder(self, content_type): ''' Get the encoding function for the provided content type. :param content_type: the requested media type :type content_type: str :rtype: function ''' return self._encoders.get(content_type) def set_encoder(self, content_type, encoder): ''' Set the encoding function for the provided content type. :param content_type: the requested media type :type content_type: str :param encoder: an encoding function, takes a single object argument and returns encoded data :type encoder: function ''' self._encoders[content_type] = encoder def set_decoder(self, content_type, decoder): ''' Set the decoding function for the provided content type. :param content_type: the requested media type :type content_type: str :param decoder: a decoding function, takes encoded data and returns a Python type :type decoder: function ''' self._decoders[content_type] = decoder def _get_resolver(self): return self._resolver or default_resolver def _set_resolver(self, value): if value is None or callable(value): self._resolver = value else: raise TypeError('resolver is not a function') resolver = property(_get_resolver, _set_resolver, doc='''The sibling-resolution function for this client. Defaults to :func:`riak.resolver.default_resolver`.''') def close(self): self._transport.close() async def _create_transport(self): self._transport = await create_transport( self._host, self._port, self._loop) @classmethod async def create(cls, host='localhost', port=8087, loop=None): ''' Return initialized instance of RiakClient since RiakClient.__init__() can't be async. :Example: .. code-block:: python import asyncio from aioriak import RiakClient loop = asyncio.get_event_loop() async def go(): client = await RiakClient.create('localhost', 8087, loop) loop.run_until_complete(go()) :param host: Hostname or ip address of Riak instance :type host: str :param port: Port of riak instance :type port: int :param loop: asyncio event loop :rtype: :class:`~aioriak.client.RiakClient` ''' client = cls(host, port, loop) await client._create_transport() return client async def fetch_datatype(self, bucket, key): ''' Fetches the value of a Riak Datatype. :param bucket: the bucket of the datatype, which must belong to a :class:`~aioriak.bucket.BucketType` :type bucket: :class:`~aioriak.bucket.Bucket` :param key: the key of the datatype :type key: string :rtype: :class:`~aioriak.datatypes.Datatype` ''' dtype, value, context = await self._fetch_datatype(bucket, key) return TYPES[dtype](bucket=bucket, key=key, value=value, context=context) async def _fetch_datatype(self, bucket, key): ''' _fetch_datatype(bucket, key) Fetches the value of a Riak Datatype as raw data. This is used internally to update already reified Datatype objects. Use the public version to fetch a reified type. :param bucket: the bucket of the datatype, which must belong to a :class:`~aioriak.BucketType` :type bucket: RiakBucket :param key: the key of the datatype :type key: string, None :rtype: tuple of type, value and context ''' return await self._transport.fetch_datatype(bucket, key) async def ping(self): ''' Check if the Riak server for this ``RiakClient`` instance is alive. :rtype: boolean ''' return await self._transport.ping() is_alive = ping async def get_client_id(self): ''' Get client ID for this RiakClient instance :rtype: bytes ''' return await self._transport.get_client_id() async def set_client_id(self, id): ''' Set Client ID for this RiakClient instance' ''' return await self._transport.set_client_id(id) async def get_buckets(self, bucket_type=None): ''' Get the list of buckets as :class:`Bucket <aioriak.bucket.Bucket>` instances. .. warning:: Do not use this in production, as it requires traversing through all keys stored in a cluster. :param bucket_type: the optional containing bucket type :type bucket_type: :class:`~aioriak.bucket.BucketType` :rtype: list of :class:`Bucket <aioriak.bucket.Bucket>` instances ''' if bucket_type: maker = bucket_type.bucket else: maker = self.bucket return [maker(name.decode()) for name in await self._transport.get_buckets( bucket_type=bucket_type)] def bucket_type(self, name): ''' Gets the bucket-type by the specified name. Bucket-types do not always exist (unlike buckets), but this will always return a :class:`BucketType <aioriak.bucket.BucketType>` object. :param name: the bucket name :type name: str :rtype: :class:`BucketType <aioriak.bucket.BucketType>` ''' if not isinstance(name, str): raise TypeError('Bucket name must be a string') if name in self._bucket_types: return self._bucket_types[name] else: btype = BucketType(self, name) self._bucket_types[name] = btype return btype def bucket(self, name, bucket_type='default'): ''' Get the bucket by the specified name. Since buckets always exist, this will always return a :class:`Bucket <aioriak.bucket.Bucket>`. If you are using a bucket that is contained in a bucket type, it is preferable to access it from the bucket type object:: # Preferred: client.bucket_type("foo").bucket("bar") # Equivalent, but not preferred: client.bucket("bar", bucket_type="foo") :param name: the bucket name :type name: str :param bucket_type: the parent bucket-type :type bucket_type: :class:`BucketType <aioriak.bucket.BucketType>` or str :rtype: :class:`Bucket <aioriak.bucket.Bucket>` ''' if not isinstance(name, str): raise TypeError('Bucket name must be a string') if isinstance(bucket_type, str): bucket_type = self.bucket_type(bucket_type) elif not isinstance(bucket_type, BucketType): raise TypeError('bucket_type must be a string ' 'or aioriak.bucket.BucketType') return self._buckets.setdefault((bucket_type, name), Bucket(self, name, bucket_type)) async def get_bucket_type_props(self, bucket_type): ''' Fetches properties for the given bucket-type. :param bucket_type: the bucket-type whose properties will be fetched :type bucket_type: BucketType :rtype: dict ''' return await self._transport.get_bucket_type_props(bucket_type) async def set_bucket_type_props(self, bucket_type, props): ''' Sets properties for the given bucket-type. :param bucket_type: the bucket-type whose properties will be set :type bucket_type: BucketType :param props: the properties to set :type props: dict ''' return await self._transport.set_bucket_type_props(bucket_type, props) async def get_bucket_props(self, bucket): ''' Fetches bucket properties for the given bucket. :param bucket: the bucket whose properties will be fetched :type bucket: Bucket :rtype: dict ''' return await self._transport.get_bucket_props(bucket) async def set_bucket_props(self, bucket, props): ''' Sets bucket properties for the given bucket. :param bucket: the bucket whose properties will be set :type bucket: Bucket :param props: the properties to set :type props: dict ''' return await self._transport.set_bucket_props(bucket, props) async def get_keys(self, bucket): ''' Lists all keys in a bucket. .. warning:: Do not use this in production, as it requires traversing through all keys stored in a cluster. :param bucket: the bucket whose keys are fetched :type bucket: Bucket :rtype: list ''' return await self._transport.get_keys(bucket) async def get(self, robj): ''' Fetches the contents of a Riak object. :param robj: the object to fetch :type robj: RiakObject ''' if not isinstance(robj.key, str): raise TypeError( 'key must be a string, instead got {0}'.format(repr(robj.key))) return await self._transport.get(robj) async def put(self, robj, return_body): ''' Stores an object in the Riak cluster. :param return_body: whether to return the resulting object after the write :type return_body: boolean :param robj: the object to store :type robj: RiakObject ''' return await self._transport.put(robj, return_body=return_body) async def delete(self, robj): ''' Deletes an object from Riak. :param robj: the object to delete :type robj: RiakObject ''' return await self._transport.delete(robj) async def update_datatype(self, datatype, **params): ''' Sends an update to a Riak Datatype to the server. :param datatype: the datatype with pending updates :type datatype: :class:`~aioriak.datatypes.Datatype` :rtype: tuple of datatype, opaque value and opaque context ''' return await self._transport.update_datatype(datatype, **params)