class LocalClientTransportSchema(fields.Dictionary):
contents = {
# Server class can be an import path or a class object
'server_class':
fields.Any(
fields.TypePath(
description=
'The importable Python path to the `Server`-extending class.',
base_classes=Server,
),
fields.TypeReference(
description='A reference to the `Server`-extending class',
base_classes=Server,
),
description=
'The path to the `Server` class to use locally (as a library), or a reference to the '
'`Server`-extending class/type itself.',
),
# No deeper validation than "schemaless dictionary" because the Server will perform its own validation
'server_settings':
fields.Any(
fields.PythonPath(
value_schema=_server_settings,
description=
'The importable Python path to the settings dict, in the format "module.name:VARIABLE".',
),
_server_settings,
description=
'The settings to use when instantiating the `server_class`.',
),
}
description = 'The constructor kwargs for the local client transport.'
class LocalClientTransportSchema(BasicClassSchema):
contents = {
'path': fields.UnicodeString(
description='The path to the local client transport, in the format `module.name:ClassName`',
),
'kwargs': fields.Dictionary({
# server class can be an import path or a class object
'server_class': fields.Any(
fields.UnicodeString(
description='The path to the `Server` class, in the format `module.name:ClassName`',
),
fields.ObjectInstance(
six.class_types,
description='A reference to the `Server`-extending class/type',
),
description='The path to the `Server` class to use locally (as a library), or a reference to the '
'`Server`-extending class/type itself',
),
# No deeper validation because the Server will perform its own validation
'server_settings': fields.SchemalessDictionary(
key_type=fields.UnicodeString(),
description='The settings to use when instantiating the `server_class`'
),
}),
}
optional_keys = ()
description = 'The settings for the local client transport'
class SettingsToTest(settings.Settings):
schema: settings.SettingsSchema = {
'one': fields.Dictionary({
'a': fields.ClassConfigurationSchema(base_class=ClassUsingAttrs27HintsToTest, description='Nifty schema.'),
'b': fields.PythonPath(value_schema=fields.UnicodeString(), description='Must be a path, yo.'),
'c': fields.TypeReference(base_classes=ClassHoldingSigsToTest, description='Refer to that thing!'),
}),
'two': fields.SchemalessDictionary(key_type=fields.UnicodeString(), value_type=fields.Boolean()),
'three': fields.List(fields.Integer()),
'four': fields.Nullable(fields.Set(fields.ByteString())),
'five': fields.Any(fields.Integer(), fields.Float()),
'six': fields.ObjectInstance(valid_type=ClassUsingAttrs27HintsToTest, description='Y u no instance?'),
'seven': fields.Polymorph(
'thing',
{
'thing1': fields.Dictionary({'z': fields.Boolean()}, allow_extra_keys=True),
'thing2': fields.Dictionary({'y': fields.Boolean()}, allow_extra_keys=True, optional_keys=('y', )),
},
),
}
defaults: settings.SettingsData = {
'one': {
'b': 'foo.bar:Class',
},
'three': [1, 5, 7],
}
class StubClientTransportSchema(BasicClassSchema):
contents = {
'path':
fields.UnicodeString(
description=
'The path to the stub client transport, in the format `module.name:ClassName`',
),
'kwargs':
fields.Dictionary(
{
'action_map':
fields.SchemalessDictionary(
key_type=fields.UnicodeString(
description='The name of the action to stub', ),
value_type=fields.Dictionary(
{
'body':
fields.SchemalessDictionary(
description=
'The body with which the action should respond',
),
'errors':
fields.List(
fields.Any(
fields.ObjectInstance(Error),
fields.Dictionary(
{
'code':
fields.UnicodeString(),
'message':
fields.UnicodeString(),
'field':
fields.UnicodeString(),
'traceback':
fields.UnicodeString(),
'variables':
fields.SchemalessDictionary(),
'denied_permissions':
fields.List(
fields.UnicodeString()),
}, ),
),
description='The ',
),
},
description=
'A dictionary containing either a body dict or an errors list, providing an '
'instruction on how the stub action should respond to requests',
optional_keys=('body', 'errors'),
),
),
}, ),
}
optional_keys = ()
description = 'The settings for the local transport'
class WalkAction(Action):
request_schema = fields.Dictionary({'value': fields.Any(fields.Integer(), fields.Float())})
response_schema = request_schema
add = 1
def run(self, request):
return {'value': request.body['value'] + self.add}
class LocalTransportSchema(BasicClassSchema):
contents = {
'path':
fields.UnicodeString(),
'kwargs':
fields.Dictionary({
# server class can be an import path or a class object
'server_class':
fields.Any(fields.UnicodeString(),
fields.ObjectInstance(six.class_types)),
# No deeper validation because the Server will perform its own validation
'server_settings':
fields.SchemalessDictionary(key_type=fields.UnicodeString()),
}),
}
from pysoa.common.transport.redis_gateway.backend.base import BaseRedisClient
from pysoa.common.transport.redis_gateway.constants import ProtocolVersion
from pysoa.common.transport.redis_gateway.core import RedisTransportClientCore
from pysoa.common.transport.redis_gateway.settings import RedisTransportSchema
from pysoa.common.transport.redis_gateway.utils import make_redis_queue_name
@fields.ClassConfigurationSchema.provider(RedisTransportSchema().extend(
contents={
'protocol_version':
fields.Any(
fields.Integer(),
fields.ObjectInstance(valid_type=ProtocolVersion),
description=
'The default protocol version between clients and servers was Version 1 prior to PySOA '
'0.67.0, Version 2 as of 0.67.0, and will be Version 3 as of 1.0.0. The server can only '
'detect what protocol the client is speaking and respond with the same protocol. However, '
'the client cannot pre-determine what protocol the server is speaking. So, if you need to '
'differ from the default (currently Version 2), use this setting to tell the client which '
'protocol to speak.',
),
},
optional_keys=('protocol_version', ),
description='The constructor kwargs for the Redis client transport.',
))
class RedisClientTransport(ClientTransport):
def __init__(self, service_name, metrics, **kwargs):
# type: (six.text_type, MetricsRecorder, **Any) -> None
"""
In addition to the two named positional arguments, this constructor expects keyword arguments abiding by the
Redis transport settings schema.
class RedisTransportSchema(BasicClassSchema):
contents = {
'path': fields.UnicodeString(
description='The path to the Redis client or server transport, in the format `module.name:ClassName`',
),
'kwargs': fields.Dictionary(
{
'backend_layer_kwargs': fields.Dictionary(
{
'connection_kwargs': fields.SchemalessDictionary(
description='The arguments used when creating all Redis connections (see Redis-Py docs)',
),
'hosts': fields.List(
fields.Any(
fields.Tuple(fields.UnicodeString(), fields.Integer()),
fields.UnicodeString(),
),
description='The list of Redis hosts, where each is a tuple of `("address", port)` or the '
'simple string address.',
),
'redis_db': fields.Integer(
description='The Redis database, a shortcut for putting this in `connection_kwargs`.',
),
'redis_port': fields.Integer(
description='The port number, a shortcut for putting this on all hosts',
),
'sentinel_failover_retries': fields.Integer(
description='How many times to retry (with a delay) getting a connection from the Sentinel '
'when a master cannot be found (cluster is in the middle of a failover); '
'should only be used for Sentinel backend type'
),
'sentinel_services': fields.List(
fields.UnicodeString(),
description='A list of Sentinel services (will be discovered by default); should only be '
'used for Sentinel backend type',
),
},
optional_keys=[
'connection_kwargs',
'hosts',
'redis_db',
'redis_port',
'sentinel_failover_retries',
'sentinel_services',
],
allow_extra_keys=False,
description='The arguments passed to the Redis connection manager',
),
'backend_type': fields.Constant(
*REDIS_BACKEND_TYPES,
description='Which backend (standard or sentinel) should be used for this Redis transport'
),
'log_messages_larger_than_bytes': fields.Integer(
description='By default, messages larger than 100KB that do not trigger errors (see '
'`maximum_message_size_in_bytes`) will be logged with level WARNING to a logger named '
'`pysoa.transport.oversized_message`. To disable this behavior, set this setting to '
'0. Or, you can set it to some other number to change the threshold that triggers '
'logging.',
),
'maximum_message_size_in_bytes': fields.Integer(
description='The maximum message size, in bytes, that is permitted to be transmitted over this '
'transport (defaults to 100KB on the client and 250KB on the server)',
),
'message_expiry_in_seconds': fields.Integer(
description='How long after a message is sent that it is considered expired, dropped from queue',
),
'queue_capacity': fields.Integer(
description='The capacity of the message queue to which this transport will send messages',
),
'queue_full_retries': fields.Integer(
description='How many times to retry sending a message to a full queue before giving up',
),
'receive_timeout_in_seconds': fields.Integer(
description='How long to block waiting on a message to be received',
),
'serializer_config': BasicClassSchema(
object_type=BaseSerializer,
description='The configuration for the serializer this transport should use',
),
},
optional_keys=[
'backend_layer_kwargs',
'log_messages_larger_than_bytes',
'maximum_message_size_in_bytes',
'message_expiry_in_seconds',
'queue_capacity',
'queue_full_retries',
'receive_timeout_in_seconds',
'serializer_config',
],
allow_extra_keys=False,
),
}
optional_keys = ()
description = 'The settings for the Redis transport'
class RedisTransportSchema(BasicClassSchema):
contents = {
'path': fields.UnicodeString(),
'kwargs': fields.Dictionary(
{
'backend_layer_kwargs': fields.Dictionary(
{
'connection_kwargs': fields.SchemalessDictionary(
description='The arguments used when creating all Redis connections (see Redis-Py docs)',
),
'hosts': fields.List(
fields.Any(
fields.Tuple(fields.UnicodeString(), fields.Integer()),
fields.UnicodeString(),
),
description='The list of Redis hosts',
),
'redis_db': fields.Integer(
description='The Redis database, a shortcut for putting this in `connection_kwargs`.',
),
'redis_port': fields.Integer(
description='The port number, a shortcut for putting this on all hosts',
),
'sentinel_failover_retries': fields.Integer(
description='How many times to retry (with a delay) getting a connection from the Sentinel '
'when a master cannot be found (cluster is in the middle of a failover); '
'should only be used for Sentinel backend type'
),
'sentinel_refresh_interval': fields.Integer(
description='Deprecated; unused; to be removed before final release.',
),
'sentinel_services': fields.List(
fields.UnicodeString(),
description='A list of Sentinel services (will be discovered by default); should only be '
'used for Sentinel backend type',
),
},
optional_keys=[
'connection_kwargs',
'hosts',
'redis_db',
'redis_port',
'sentinel_failover_retries',
'sentinel_refresh_interval',
'sentinel_services',
],
allow_extra_keys=False,
description='The arguments passed to the Redis connection manager',
),
'backend_type': fields.Constant(
*REDIS_BACKEND_TYPES,
description='Which backend (standard or sentinel) should be used for this Redis transport'
),
'message_expiry_in_seconds': fields.Integer(
description='How long after a message is sent that it is considered expired, dropped from queue',
),
'queue_capacity': fields.Integer(
description='The capacity of the message queue to which this transport will send messages',
),
'queue_full_retries': fields.Integer(
description='How many times to retry sending a message to a full queue before giving up',
),
'receive_timeout_in_seconds': fields.Integer(
description='How long to block waiting on a message to be received',
),
'serializer_config': BasicClassSchema(
object_type=BaseSerializer,
description='The configuration for the serializer this transport should use',
),
},
optional_keys=[
'backend_layer_kwargs',
'message_expiry_in_seconds',
'queue_capacity',
'queue_full_retries',
'receive_timeout_in_seconds',
'serializer_config',
],
allow_extra_keys=False,
),
}
from pymetrics.publishers.base import MetricsPublisher
__all__ = ('LogPublisher', )
@fields.ClassConfigurationSchema.provider(
fields.Dictionary(
{
'log_name':
fields.UnicodeString(
description='The name of the logger to which to publish metrics'
),
'log_level':
fields.Any(
fields.Constant(10, 20, 30, 40, 50),
PythonLogLevel(),
description=
'The log level (name or int) for publishing metrics, defaults to logging.INFO',
),
},
optional_keys=('log_level', ),
))
class LogPublisher(MetricsPublisher):
def __init__(self,
log_name,
log_level=logging.INFO
): # type: (six.text_type, Union[int, six.text_type]) -> None
self.log_name = log_name
self.logger = logging.getLogger(self.log_name)
if isinstance(log_level, int):
self.log_level = log_level
MAX_GIG_E_MTU_BYTES = 9000
MAX_FAST_E_MTU_BYTES = 1518
MAX_IPV4_PAYLOAD_SIZE_BYTES = MAX_IPV4_PACKET_SIZE_BYTES - IP_HEADER_BYTES - UDP_HEADER_BYTES
MAX_GIG_E_PAYLOAD_SIZE_BYTES = MAX_GIG_E_MTU_BYTES - IP_HEADER_BYTES - UDP_HEADER_BYTES
MAX_FAST_E_PAYLOAD_SIZE_BYTES = MAX_FAST_E_MTU_BYTES - IP_HEADER_BYTES - UDP_HEADER_BYTES
@fields.ClassConfigurationSchema.provider(fields.Dictionary(
{
'host': fields.UnicodeString(description='The host name or IP address on which the Statsd server is listening'),
'port': fields.Integer(description='The port number on which the Statsd server is listening'),
'maximum_packet_size': fields.Integer(
description='The maximum packet size to send (packets will be fragmented above this limit), defaults to '
'65000 bytes.',
),
'network_timeout': fields.Any(fields.Float(gt=0.0), fields.Integer(gt=0), description='The network timeout'),
},
optional_keys=('maximum_packet_size', 'network_timeout'),
))
class StatsdPublisher(MetricsPublisher):
"""
A publisher that emits UDP metrics packets to a Statsd consumer over a network connection.
For Statsd metric type suffixes, see https://github.com/etsy/statsd/blob/master/docs/metric_types.md.
"""
METRIC_TYPE_COUNTER = b'c'
METRIC_TYPE_GAUGE = b'g'
METRIC_TYPE_HISTOGRAM = b'ms'
METRIC_TYPE_TIMER = b'ms'
from pymetrics.instruments import (
Counter,
Gauge,
Histogram,
Metric,
Tag,
Timer,
TimerResolution,
)
from pymetrics.publishers.statsd import StatsdPublisher
__all__ = ('DogStatsdPublisher', )
_datadog_tags_value_type = fields.Nullable(
fields.Any(fields.UnicodeString(), fields.ByteString(), fields.Integer(),
fields.Float(), fields.Boolean()), )
@fields.ClassConfigurationSchema.provider(
fields.Dictionary(
{
'host':
fields.UnicodeString(
description=
'The host name or IP address on which the Dogstatsd server is listening',
),
'port':
fields.Integer(
description=
'The port number on which the Dogstatsd server is listening'),
'maximum_packet_size':