class StubClientTransportSchema(BasicClassSchema):
contents = {
'path':
fields.UnicodeString(
description=
'The path to the stub client transport, in the format `module.name:ClassName`',
),
'kwargs':
fields.Dictionary(
{
'action_map':
fields.SchemalessDictionary(
key_type=fields.UnicodeString(
description='The name of the action to stub', ),
value_type=fields.Dictionary(
{
'body':
fields.SchemalessDictionary(
description=
'The body with which the action should respond',
),
'errors':
fields.List(
fields.Any(
fields.ObjectInstance(Error),
fields.Dictionary(
{
'code':
fields.UnicodeString(),
'message':
fields.UnicodeString(),
'field':
fields.UnicodeString(),
'traceback':
fields.UnicodeString(),
'variables':
fields.SchemalessDictionary(),
'denied_permissions':
fields.List(
fields.UnicodeString()),
}, ),
),
description='The ',
),
},
description=
'A dictionary containing either a body dict or an errors list, providing an '
'instruction on how the stub action should respond to requests',
optional_keys=('body', 'errors'),
),
),
}, ),
}
optional_keys = ()
description = 'The settings for the local transport'
def test_schema_correct(self):
assert SettingsThreeTwoOne.schema == {
'foo': fields.UnicodeString(),
'bar': fields.Integer(),
'baz': fields.List(fields.Float()),
'qux': fields.Float(),
}
def test_schema_correct(self):
assert SettingsOneTwoThree.schema == {
'foo':
fields.UnicodeString(),
'bar':
fields.Boolean(),
'baz':
fields.Dictionary(
{
'inner_foo':
fields.UnicodeString(),
'inner_bar':
fields.Boolean(),
'inner_baz':
fields.List(fields.Integer()),
'inner_qux':
fields.Dictionary(
{
'most_inner_foo': fields.Boolean(),
'most_inner_bar': fields.UnicodeString(),
}, ),
}, ),
'qux':
fields.Float(),
}
def test_schema_correct(self):
assert SettingsTwoFour.schema == {
'bar':
fields.Integer(),
'baz':
fields.Dictionary(
{
'inner_foo':
fields.UnicodeString(),
'inner_bar':
fields.Boolean(),
'inner_baz':
fields.List(fields.Integer()),
'inner_qux':
fields.Dictionary(
{
'most_inner_foo': fields.Boolean(),
'most_inner_bar': fields.UnicodeString(),
}, ),
}, ),
'qux':
fields.Decimal(),
'new':
fields.ByteString(),
'old':
fields.UnicodeString(),
}
def test_schema_correct(self):
assert SettingsOneThree.schema == {
'foo': fields.UnicodeString(),
'bar': fields.Boolean(),
'baz': fields.List(fields.Float()),
'qux': fields.Float(),
}
def test_schema_correct(self):
assert SettingsFour.schema == {
'baz': fields.List(fields.Float()),
'qux': fields.Decimal(),
'new': fields.ByteString(),
'old': fields.UnicodeString(),
}
class SettingsToTest(settings.Settings):
schema: settings.SettingsSchema = {
'one': fields.Dictionary({
'a': fields.ClassConfigurationSchema(base_class=ClassUsingAttrs27HintsToTest, description='Nifty schema.'),
'b': fields.PythonPath(value_schema=fields.UnicodeString(), description='Must be a path, yo.'),
'c': fields.TypeReference(base_classes=ClassHoldingSigsToTest, description='Refer to that thing!'),
}),
'two': fields.SchemalessDictionary(key_type=fields.UnicodeString(), value_type=fields.Boolean()),
'three': fields.List(fields.Integer()),
'four': fields.Nullable(fields.Set(fields.ByteString())),
'five': fields.Any(fields.Integer(), fields.Float()),
'six': fields.ObjectInstance(valid_type=ClassUsingAttrs27HintsToTest, description='Y u no instance?'),
'seven': fields.Polymorph(
'thing',
{
'thing1': fields.Dictionary({'z': fields.Boolean()}, allow_extra_keys=True),
'thing2': fields.Dictionary({'y': fields.Boolean()}, allow_extra_keys=True, optional_keys=('y', )),
},
),
}
defaults: settings.SettingsData = {
'one': {
'b': 'foo.bar:Class',
},
'three': [1, 5, 7],
}
class SettingsTwo(Settings):
schema = {
'bar':
fields.Integer(),
'baz':
fields.Dictionary(
{
'inner_foo':
fields.UnicodeString(),
'inner_bar':
fields.Boolean(),
'inner_baz':
fields.List(fields.Integer()),
'inner_qux':
fields.Dictionary(
{
'most_inner_foo': fields.Boolean(),
'most_inner_bar': fields.UnicodeString(),
}, ),
}, ),
} # type: SettingsSchema
defaults = {
'bar': 1,
'baz': {
'inner_foo': 'Default inner',
'inner_qux': {
'most_inner_bar': 'Default most inner'
}
},
} # type: SettingsData
class TypesEchoAction(Action):
request_schema = fields.Dictionary(
{
'an_int': fields.Integer(),
'a_float': fields.Float(),
'a_bool': fields.Boolean(),
'a_bytes': fields.ByteString(),
'a_string': fields.UnicodeString(),
'a_datetime': fields.DateTime(),
'a_date': fields.Date(),
'a_time': fields.Time(),
'a_list': fields.List(fields.Anything(), max_length=0),
'a_dict': fields.Nullable(fields.Dictionary({})),
},
optional_keys=(
'an_int', 'a_float', 'a_bool', 'a_bytes', 'a_string', 'a_datetime', 'a_date', 'a_time', 'a_list', 'a_dict',
),
)
response_schema = fields.Dictionary(
{'r_{}'.format(k): v for k, v in six.iteritems(request_schema.contents)},
optional_keys=('r_{}'.format(k) for k in request_schema.optional_keys),
)
def run(self, request):
return {'r_{}'.format(k): v for k, v in six.iteritems(request.body)}
class ClientSettings(SOASettings):
"""
Base settings class for all clients, whose `middleware` values are restricted to subclasses of `ClientMiddleware`
and whose `transport` values are restricted to subclasses of `BaseClientTransport`. Middleware and transport
configuration settings schemas will automatically switch based on the configuration settings schema for the `path`
for each.
"""
schema = {
'middleware':
fields.List(
fields.ClassConfigurationSchema(base_class=ClientMiddleware),
description=
'The list of all `ClientMiddleware` objects that should be applied to requests made from this '
'client to the associated service',
),
'transport':
fields.ClassConfigurationSchema(base_class=BaseClientTransport),
} # type: SettingsSchema
defaults = {
'transport': {
'path':
'pysoa.common.transport.redis_gateway.client:RedisClientTransport',
},
} # type: SettingsData
class SettingsThree(Settings):
schema = {
'baz': fields.List(fields.Float()),
'qux': fields.Float(),
} # type: SettingsSchema
defaults = {
'qux': 1.234,
} # type: SettingsData
class ServerSettings(SOASettings):
"""
Settings specific to servers
"""
schema = {
'transport':
BasicClassSchema(BaseServerTransport),
'middleware':
fields.List(BasicClassSchema(ServerMiddleware)),
'client_routing':
fields.SchemalessDictionary(),
'logging':
fields.SchemalessDictionary(),
'harakiri':
fields.Dictionary({
'timeout': fields.Integer(
gte=0
), # seconds of inactivity before harakiri is triggered, 0 to disable
'shutdown_grace': fields.Integer(
gte=0
), # seconds to gracefully shutdown after harakiri is triggered
}),
}
defaults = {
'client_routing': {},
'logging': {
'version': 1,
'formatters': {
'console': {
'format': '%(asctime)s %(levelname)7s: %(message)s'
},
},
'handlers': {
'console': {
'level': 'INFO',
'class': 'logging.StreamHandler',
'formatter': 'console',
},
},
'root': {
'handlers': ['console'],
'level': 'INFO',
},
},
'harakiri': {
'timeout': 300,
'shutdown_grace': 30,
},
}
class ClientSettings(SOASettings):
"""Generic settings for a Client."""
schema = {
'middleware': fields.List(BasicClassSchema(ClientMiddleware)),
'transport': BasicClassSchema(BaseClientTransport),
'transport_cache_time_in_seconds': fields.Integer(
gte=0,
description='If enabled, uses a per-service transport cache that is keyed off the service name and '
'transport settings, persists across all clients in memory, and expires after this number of '
'seconds. By default, a new transport is created for every new client.',
),
}
defaults = {
'transport_cache_time_in_seconds': 0,
}
class SOASettings(Settings):
"""
Settings shared between client and server.
"""
schema = {
# Paths to the classes to use and then kwargs to pass
'transport': BasicClassSchema(),
'middleware': fields.List(BasicClassSchema()),
'metrics': MetricsSchema(),
}
defaults = {
'middleware': [],
'metrics': {
'path': 'pysoa.common.metrics:NoOpMetricsRecorder'
},
}
class EchoAction(Action):
request_schema = fields.SchemalessDictionary()
response_schema = fields.Dictionary(
{
'request_body': fields.SchemalessDictionary(),
'request_context': fields.SchemalessDictionary(),
'request_switches': fields.List(fields.Integer()),
'request_control': fields.SchemalessDictionary(),
}, )
def run(self, request):
return {
'request_body': request.body,
'request_context': request.context,
'request_switches': sorted(list(request.switches)),
'request_control': request.control,
}
class SOASettings(Settings):
"""
Settings shared between client and server.
"""
schema = {
# Paths to the classes to use and then kwargs to pass
'transport': BasicClassSchema(),
'middleware': fields.List(
BasicClassSchema(),
description='The list of all middleware objects that should be applied to this server or client',
),
'metrics': MetricsSchema(),
}
defaults = {
'middleware': [],
'metrics': {'path': 'pysoa.common.metrics:NoOpMetricsRecorder'},
}
class SOASettings(ConformitySettings):
"""
Settings shared between client and server.
"""
schema = {
# Paths to the classes to use and then kwargs to pass
'transport': fields.ClassConfigurationSchema(),
'middleware': fields.List(
fields.ClassConfigurationSchema(),
description='The list of all middleware objects that should be applied to this server or client',
),
'metrics': fields.ClassConfigurationSchema(
base_class=MetricsRecorder,
description='Configuration for defining a usage and performance metrics recorder.',
),
} # type: SettingsSchema
defaults = {
'middleware': [],
'metrics': {'path': 'pymetrics.recorders.noop:NonOperationalMetricsRecorder'},
} # type: SettingsData
class ClientSettings(SOASettings):
"""Generic settings for a Client."""
schema = {
'middleware':
fields.List(
BasicClassSchema(ClientMiddleware),
description=
'The list of all `ClientMiddleware` objects that should be applied to requests made from this '
'client to the associated service',
),
'transport':
BasicClassSchema(BaseClientTransport),
'transport_cache_time_in_seconds':
fields.Anything(
description=
'This field is deprecated. The transport cache is no longer supported. This settings field '
'will remain in place until 2018-06-15 to give a safe period for people to remove it from '
'settings, but its value will always be ignored.', ),
}
defaults = {
'transport_cache_time_in_seconds': 0,
}
)
return errors
PYTHON_ROOT_LOGGER_SCHEMA = fields.Dictionary(
{
'level':
PythonLogLevel(
description=
'The logging level at or above which this logger will handle logging events and send them to '
'its configured handlers.', ),
'filters':
fields.List(
fields.UnicodeString(),
description=
'A list of references to keys from `filters` for assigning those filters to this logger.',
),
'handlers':
fields.List(
fields.UnicodeString(),
description=
'A list of references to keys from `handlers` for assigning those handlers to this logger.',
),
},
optional_keys=('level', 'filters', 'handlers'),
)
PYTHON_LOGGER_SCHEMA = PYTHON_ROOT_LOGGER_SCHEMA.extend(
contents={
'propagate':
def test_schema_correct(self):
assert SettingsThree.schema == {
'baz': fields.List(fields.Float()),
'qux': fields.Float(),
}
class RedisTransportSchema(BasicClassSchema):
contents = {
'path': fields.UnicodeString(
description='The path to the Redis client or server transport, in the format `module.name:ClassName`',
),
'kwargs': fields.Dictionary(
{
'backend_layer_kwargs': fields.Dictionary(
{
'connection_kwargs': fields.SchemalessDictionary(
description='The arguments used when creating all Redis connections (see Redis-Py docs)',
),
'hosts': fields.List(
fields.Any(
fields.Tuple(fields.UnicodeString(), fields.Integer()),
fields.UnicodeString(),
),
description='The list of Redis hosts, where each is a tuple of `("address", port)` or the '
'simple string address.',
),
'redis_db': fields.Integer(
description='The Redis database, a shortcut for putting this in `connection_kwargs`.',
),
'redis_port': fields.Integer(
description='The port number, a shortcut for putting this on all hosts',
),
'sentinel_failover_retries': fields.Integer(
description='How many times to retry (with a delay) getting a connection from the Sentinel '
'when a master cannot be found (cluster is in the middle of a failover); '
'should only be used for Sentinel backend type'
),
'sentinel_services': fields.List(
fields.UnicodeString(),
description='A list of Sentinel services (will be discovered by default); should only be '
'used for Sentinel backend type',
),
},
optional_keys=[
'connection_kwargs',
'hosts',
'redis_db',
'redis_port',
'sentinel_failover_retries',
'sentinel_services',
],
allow_extra_keys=False,
description='The arguments passed to the Redis connection manager',
),
'backend_type': fields.Constant(
*REDIS_BACKEND_TYPES,
description='Which backend (standard or sentinel) should be used for this Redis transport'
),
'log_messages_larger_than_bytes': fields.Integer(
description='By default, messages larger than 100KB that do not trigger errors (see '
'`maximum_message_size_in_bytes`) will be logged with level WARNING to a logger named '
'`pysoa.transport.oversized_message`. To disable this behavior, set this setting to '
'0. Or, you can set it to some other number to change the threshold that triggers '
'logging.',
),
'maximum_message_size_in_bytes': fields.Integer(
description='The maximum message size, in bytes, that is permitted to be transmitted over this '
'transport (defaults to 100KB on the client and 250KB on the server)',
),
'message_expiry_in_seconds': fields.Integer(
description='How long after a message is sent that it is considered expired, dropped from queue',
),
'queue_capacity': fields.Integer(
description='The capacity of the message queue to which this transport will send messages',
),
'queue_full_retries': fields.Integer(
description='How many times to retry sending a message to a full queue before giving up',
),
'receive_timeout_in_seconds': fields.Integer(
description='How long to block waiting on a message to be received',
),
'serializer_config': BasicClassSchema(
object_type=BaseSerializer,
description='The configuration for the serializer this transport should use',
),
},
optional_keys=[
'backend_layer_kwargs',
'log_messages_larger_than_bytes',
'maximum_message_size_in_bytes',
'message_expiry_in_seconds',
'queue_capacity',
'queue_full_retries',
'receive_timeout_in_seconds',
'serializer_config',
],
allow_extra_keys=False,
),
}
optional_keys = ()
description = 'The settings for the Redis transport'
class ServerSettings(SOASettings):
"""
Settings specific to servers
"""
schema = {
'transport':
BasicClassSchema(BaseServerTransport),
'middleware':
fields.List(
BasicClassSchema(ServerMiddleware),
description=
'The list of all `ServerMiddleware` objects that should be applied to requests processed by '
'this server',
),
'client_routing':
fields.SchemalessDictionary(
key_type=fields.UnicodeString(),
value_type=fields.SchemalessDictionary(),
description=
'Client settings for sending requests to other services; keys should be service names, and '
'values should be the corresponding configuration dicts, which will be validated using the '
'PolymorphicClientSettings schema',
),
'logging':
fields.Dictionary(
{
'version':
fields.Integer(gte=1, lte=1),
'formatters':
fields.SchemalessDictionary(
key_type=fields.UnicodeString(),
value_type=fields.Dictionary(
{
'format': fields.UnicodeString(),
'datefmt': fields.UnicodeString(),
},
optional_keys=('datefmt', ),
),
),
'filters':
fields.SchemalessDictionary(
key_type=fields.UnicodeString(),
value_type=fields.Dictionary(
{
'()':
fields.Anything(
description='The optional filter class'),
'name':
fields.UnicodeString(
description='The optional filter name'),
},
optional_keys=('()', 'name'),
),
),
'handlers':
fields.SchemalessDictionary(
key_type=fields.UnicodeString(),
value_type=fields.Dictionary(
{
'class': fields.UnicodeString(),
'level': fields.UnicodeString(),
'formatter': fields.UnicodeString(),
'filters': fields.List(fields.UnicodeString()),
},
optional_keys=('level', 'formatter', 'filters'),
allow_extra_keys=True,
),
),
'loggers':
fields.SchemalessDictionary(
key_type=fields.UnicodeString(),
value_type=_logger_schema,
),
'root':
_logger_schema,
'incremental':
fields.Boolean(),
'disable_existing_loggers':
fields.Boolean(),
},
optional_keys=(
'version',
'formatters',
'filters',
'handlers',
'root',
'loggers',
'incremental',
),
description=
'Settings for service logging, which should follow the standard Python logging configuration',
),
'harakiri':
fields.Dictionary(
{
'timeout':
fields.Integer(
gte=0,
description=
'Seconds of inactivity before harakiri is triggered; 0 to disable, defaults to 300',
),
'shutdown_grace':
fields.Integer(
gt=0,
description=
'Seconds to forcefully shutdown after harakiri is triggered if shutdown does not occur',
),
},
description=
'Instructions for automatically terminating a server process when request processing takes '
'longer than expected.',
),
'request_log_success_level':
log_level_schema(
description=
'The logging level at which full request and response contents will be logged for successful '
'requests', ),
'request_log_error_level':
log_level_schema(
description=
'The logging level at which full request and response contents will be logged for requests '
'whose responses contain errors (setting this to a more severe level than '
'`request_log_success_level` will allow you to easily filter for unsuccessful requests)',
),
'heartbeat_file':
fields.Nullable(
fields.UnicodeString(
description=
'If specified, the server will create a heartbeat file at the specified path on startup, '
'update the timestamp in that file after the processing of every request or every time '
'idle operations are processed, and delete the file when the server shuts down. The file name '
'can optionally contain the specifier {{pid}}, which will be replaced with the server process '
'PID.', )),
'extra_fields_to_redact':
fields.Set(
fields.UnicodeString(),
description=
'Use this field to supplement the set of fields that are automatically redacted/censored in '
'request and response fields with additional fields that your service needs redacted.',
),
}
defaults = {
'client_routing': {},
'logging': {
'version': 1,
'formatters': {
'console': {
'format':
'%(asctime)s %(levelname)7s %(correlation_id)s %(request_id)s: %(message)s'
},
'syslog': {
'format':
('%(service_name)s_service: %(name)s %(levelname)s %(module)s %(process)d '
'correlation_id %(correlation_id)s request_id %(request_id)s %(message)s'
),
},
},
'filters': {
'pysoa_logging_context_filter': {
'()': 'pysoa.common.logging.PySOALogContextFilter',
},
},
'handlers': {
'console': {
'level': 'INFO',
'class': 'logging.StreamHandler',
'formatter': 'console',
'filters': ['pysoa_logging_context_filter'],
},
'syslog': {
'level': 'INFO',
'class': 'logging.handlers.SysLogHandler',
'facility': SysLogHandler.LOG_LOCAL7,
'address': ('localhost', 514),
'formatter': 'syslog',
'filters': ['pysoa_logging_context_filter'],
},
},
'loggers': {},
'root': {
'handlers': ['console'],
'level': 'INFO',
},
'disable_existing_loggers': False,
},
'harakiri': {
'timeout': 300,
'shutdown_grace': 30,
},
'request_log_success_level': 'INFO',
'request_log_error_level': 'INFO',
'heartbeat_file': None,
'extra_fields_to_redact': set(),
}
from pysoa.common.schemas import (
BasicClassSchema,
PolymorphClassSchema,
)
from pysoa.common.settings import SOASettings
from pysoa.common.transport.base import ServerTransport as BaseServerTransport
from pysoa.common.transport.local import LocalServerTransportSchema
from pysoa.common.transport.redis_gateway.settings import RedisTransportSchema
from pysoa.server.middleware import ServerMiddleware
_logger_schema = fields.Dictionary(
{
'level': fields.UnicodeString(),
'propagate': fields.Boolean(),
'filters': fields.List(fields.UnicodeString()),
'handlers': fields.List(fields.UnicodeString()),
},
optional_keys=('level', 'propagate', 'filters', 'handlers'),
)
log_level_schema = functools.partial(fields.Constant, 'DEBUG', 'INFO',
'WARNING', 'ERROR', 'CRITICAL')
class ServerSettings(SOASettings):
"""
Settings specific to servers
"""
schema = {
class BaseStatusAction(Action):
"""
Standard base action for status checks.
Returns heath check and version information.
If you want to use default StatusAction use StatusActionFactory(version)
passing in the version of your service.
If you want to make a custom StatusAction, subclass this class,
make it get self._version from your service and add additional health check methods.
Health check methods must start with the word check_.
Health check methods must take no arguments, and return a list of tuples in the format:
(is_error, code, description).
is_error: True if this is an error, False if it is a warning.
code: Invariant string for this error, like "MYSQL_FAILURE"
description: Human-readable description of the problem, like "Could not connect to host on port 1234"
Health check methods can also write to the self.diagnostics dictionary to add additional
data which will be sent back with the response if they like. They are responsible for their
own key management in this situation.
"""
def __init__(self, *args, **kwargs):
if self.__class__ is BaseStatusAction:
raise RuntimeError('You cannot use BaseStatusAction directly; it must be subclassed')
super(BaseStatusAction, self).__init__(*args, **kwargs)
self.diagnostics = {}
@property
def _version(self):
raise NotImplementedError('version must be defined using StatusActionFactory')
@property
def _build(self):
return None
request_schema = fields.Dictionary(
{
'verbose': fields.Boolean(
description='If specified and False, this instructs the status action to return only the baseline '
'status information (Python, service, PySOA, and other library versions) and omit any of '
'the health check operations (no `healthcheck` attribute will be included in the '
'response). This provides a useful way to obtain the service version very quickly without '
'executing the often time-consuming code necessary for the full health check. It defaults '
'to True, which means "return everything."',
),
},
optional_keys=('verbose', ),
)
response_schema = fields.Dictionary(
{
'build': fields.UnicodeString(),
'conformity': fields.UnicodeString(),
'healthcheck': fields.Dictionary(
{
'warnings': fields.List(fields.Tuple(fields.UnicodeString(), fields.UnicodeString())),
'errors': fields.List(fields.Tuple(fields.UnicodeString(), fields.UnicodeString())),
'diagnostics': fields.SchemalessDictionary(key_type=fields.UnicodeString()),
},
optional_keys=('warnings', 'errors', 'diagnostics'),
),
'pysoa': fields.UnicodeString(),
'python': fields.UnicodeString(),
'version': fields.UnicodeString(),
},
optional_keys=('build', 'healthcheck', ),
)
def run(self, request):
"""
Scans the class for check_ methods and runs them.
"""
status = {
'conformity': six.text_type(conformity.__version__),
'pysoa': six.text_type(pysoa.__version__),
'python': six.text_type(platform.python_version()),
'version': self._version,
}
if self._build:
status['build'] = self._build
if request.body.get('verbose', True) is True:
errors = []
warnings = []
self.diagnostics = {}
# Find all things called "check_<something>" on this class.
# We can't just scan __dict__ because of class inheritance.
check_methods = [getattr(self, x) for x in dir(self) if x.startswith('check_')]
for check_method in check_methods:
# Call the check, and see if it returned anything
problems = check_method()
if problems:
for is_error, code, description in problems:
# Parcel out the values into the right return list
if is_error:
errors.append((code, description))
else:
warnings.append((code, description))
status['healthcheck'] = {
'errors': errors,
'warnings': warnings,
'diagnostics': self.diagnostics,
}
return status
self.loop.call_soon_threadsafe(self._loop_stop_callback)
self._done.wait()
else:
self._logger.warning('Async event loop is already not running!')
# noinspection PyCompatibility
super().join(timeout)
def run_coroutine(
self, coroutine): # type: (Coroutine) -> concurrent.futures.Future
for middleware_obj in self._coroutine_middleware:
middleware_obj.before_run_coroutine()
for middleware_obj in reversed(self._coroutine_middleware):
coroutine = middleware_obj.coroutine(coroutine)
return asyncio.run_coroutine_threadsafe(coroutine, self.loop)
coroutine_middleware_config = fields.List(
fields.ClassConfigurationSchema(base_class=CoroutineMiddleware),
description=
'The list of all `CoroutineMiddleware` classes that should be constructed and applied to '
'`request.run_coroutine` calls processed by this server. By default, '
'`pysoa.server.coroutine:DefaultCoroutineMiddleware` will be configured first. You can change and/or '
'add to this, but we recommend that you always configure `DefaultCoroutineMiddleware` as the first '
'middleware.',
)
coroutine_middleware_config.contents.initiate_cache_for(
'pysoa.server.coroutine:DefaultCoroutineMiddleware')
class ServerSettings(SOASettings):
"""
Settings specific to servers
"""
schema = {
'transport': BasicClassSchema(BaseServerTransport),
'middleware': fields.List(BasicClassSchema(ServerMiddleware)),
'client_routing': fields.SchemalessDictionary(
key_type=fields.UnicodeString(),
value_type=fields.SchemalessDictionary(),
description='Client settings for sending requests to other services; keys should be service names, and '
'values should be the corresponding configuration dicts, which will be validated using the '
'PolymorphicClientSettings schema',
),
'logging': fields.Dictionary(
{
'version': fields.Integer(gte=1, lte=1),
'formatters': fields.SchemalessDictionary(
key_type=fields.UnicodeString(),
value_type=fields.Dictionary(
{
'format': fields.UnicodeString(),
'datefmt': fields.UnicodeString(),
},
optional_keys=('datefmt', ),
),
),
'filters': fields.SchemalessDictionary(
key_type=fields.UnicodeString(),
value_type=fields.Dictionary({'name': fields.UnicodeString()}, optional_keys=('name', )),
),
'handlers': fields.SchemalessDictionary(
key_type=fields.UnicodeString(),
value_type=fields.Dictionary(
{
'class': fields.UnicodeString(),
'level': fields.UnicodeString(),
'formatter': fields.UnicodeString(),
'filters': fields.List(fields.UnicodeString()),
},
optional_keys=('level', 'formatter', 'filters'),
allow_extra_keys=True,
),
),
'loggers': fields.SchemalessDictionary(
key_type=fields.UnicodeString(),
value_type=_logger_schema,
),
'root': _logger_schema,
'incremental': fields.Boolean(),
'disable_existing_loggers': fields.Boolean(),
},
optional_keys=(
'version',
'formatters',
'filters',
'handlers',
'root',
'loggers',
'incremental',
'disable_existing_loggers',
),
description='Settings for service logging, which should follow the standard Python logging configuration',
),
'harakiri': fields.Dictionary(
{
'timeout': fields.Integer(
gte=0,
description='Seconds of inactivity before harakiri is triggered; 0 to disable, defaults to 300',
),
'shutdown_grace': fields.Integer(
gt=0,
description='Seconds to forcefully shutdown after harakiri is triggered if shutdown does not occur',
),
},
),
}
defaults = {
'client_routing': {},
'logging': {
'version': 1,
'formatters': {
'console': {
'format': '%(asctime)s %(levelname)7s: %(message)s'
},
},
'handlers': {
'console': {
'level': 'INFO',
'class': 'logging.StreamHandler',
'formatter': 'console',
},
},
'root': {
'handlers': ['console'],
'level': 'INFO',
},
},
'harakiri': {
'timeout': 300,
'shutdown_grace': 30,
},
}
class RedisTransportSchema(BasicClassSchema):
contents = {
'path': fields.UnicodeString(),
'kwargs': fields.Dictionary(
{
'backend_layer_kwargs': fields.Dictionary(
{
'connection_kwargs': fields.SchemalessDictionary(
description='The arguments used when creating all Redis connections (see Redis-Py docs)',
),
'hosts': fields.List(
fields.Any(
fields.Tuple(fields.UnicodeString(), fields.Integer()),
fields.UnicodeString(),
),
description='The list of Redis hosts',
),
'redis_db': fields.Integer(
description='The Redis database, a shortcut for putting this in `connection_kwargs`.',
),
'redis_port': fields.Integer(
description='The port number, a shortcut for putting this on all hosts',
),
'sentinel_failover_retries': fields.Integer(
description='How many times to retry (with a delay) getting a connection from the Sentinel '
'when a master cannot be found (cluster is in the middle of a failover); '
'should only be used for Sentinel backend type'
),
'sentinel_refresh_interval': fields.Integer(
description='Deprecated; unused; to be removed before final release.',
),
'sentinel_services': fields.List(
fields.UnicodeString(),
description='A list of Sentinel services (will be discovered by default); should only be '
'used for Sentinel backend type',
),
},
optional_keys=[
'connection_kwargs',
'hosts',
'redis_db',
'redis_port',
'sentinel_failover_retries',
'sentinel_refresh_interval',
'sentinel_services',
],
allow_extra_keys=False,
description='The arguments passed to the Redis connection manager',
),
'backend_type': fields.Constant(
*REDIS_BACKEND_TYPES,
description='Which backend (standard or sentinel) should be used for this Redis transport'
),
'message_expiry_in_seconds': fields.Integer(
description='How long after a message is sent that it is considered expired, dropped from queue',
),
'queue_capacity': fields.Integer(
description='The capacity of the message queue to which this transport will send messages',
),
'queue_full_retries': fields.Integer(
description='How many times to retry sending a message to a full queue before giving up',
),
'receive_timeout_in_seconds': fields.Integer(
description='How long to block waiting on a message to be received',
),
'serializer_config': BasicClassSchema(
object_type=BaseSerializer,
description='The configuration for the serializer this transport should use',
),
},
optional_keys=[
'backend_layer_kwargs',
'message_expiry_in_seconds',
'queue_capacity',
'queue_full_retries',
'receive_timeout_in_seconds',
'serializer_config',
],
allow_extra_keys=False,
),
}
class IntrospectionAction(Action):
"""
This action returns detailed information about the service's defined actions and the request and response schemas
for each action, along with any documentation defined for the action or for the service itself. It can be passed
a single action name to return information limited to that single action. Otherwise, it will return information for
all of the service's actions.
This action will be added to your service on your behalf if you do not define an action with name `introspect`.
Making your services and actions capable of being introspected is simple. If your server class has a `description`
attribute, that will be the service's documentation that introspection returns. If your server class does not have
this attribute but does have a docstring, introspection will use the docstring. The same rule applies to action
classes: Introspection first looks for a `description` attribute and then uses the docstring, if any. If neither of
these are found, the applicable service or action documentation will be done.
Introspection then looks at the `request_schema` and `response_schema` attributes for each of your actions, and
includes the details about these schemas in the returned information for each action. Be sure you include field
descriptions in your schema for the most effective documentation possible.
"""
description = (
"This action returns detailed information about the service's defined actions and the request and response "
"schemas for each action, along with any documentation defined for the action or for the service itself. It "
"can be passed a single action name to return information limited to that single action. Otherwise, it will "
"return information for all of the service's actions. If an action is a switched action (meaning the action "
"extends `SwitchedAction`, and which action code runs is controlled with SOA switches), multiple action "
"introspection results will be returned for that action, each with a name ending in either `[switch:N]` (where "
"`N` is the switch value) or `[DEFAULT]` for the default action."
)
request_schema = fields.Dictionary(
{
'action_name': fields.UnicodeString(
min_length=1,
allow_blank=False,
description='Specify this to limit your introspection to a single action. It will be the only action '
'present in the `actions` response attribute. If the requested action does not exist, an '
'error will be returned.',
),
},
optional_keys=('action_name', ),
)
response_schema = fields.Dictionary(
{
'documentation': fields.Nullable(fields.UnicodeString(
description='The documentation for the server, unless `action_name` is specified in the request body, '
'in which case this is omitted.',
)),
'action_names': fields.List(
fields.UnicodeString(description='The name of an action.'),
description='An alphabetized list of every action name included in `actions`.',
),
'actions': fields.SchemalessDictionary(
key_type=fields.UnicodeString(description='The name of the action.'),
value_type=fields.Dictionary(
{
'documentation': fields.Nullable(
fields.UnicodeString(description='The documentation for the action'),
),
'request_schema': fields.Nullable(fields.Anything(
description='A description of the expected request schema, including any documentation '
'specified in the schema definition.',
)),
'response_schema': fields.Nullable(fields.Anything(
description='A description of the guaranteed response schema, including any documentation '
'specified in the schema definition.',
)),
},
description='A introspection of a single action',
),
description='A dict mapping action names to action description dictionaries. This contains details '
'about every action in the service unless `action_name` is specified in the request body, '
'in which case it contains details only for that action.',
),
},
optional_keys=('documentation', ),
)
def __init__(self, server):
"""
Construct a new introspection action. Unlike its base class, which accepts a server settings object, this
must be passed a `Server` object, from which it will obtain a settings object. The `Server` code that calls
this action has special handling to address this requirement.
:param server: A PySOA server instance
:type server: Server
"""
if not isinstance(server, Server):
raise TypeError('First argument (server) must be a Server instance')
super(IntrospectionAction, self).__init__(server.settings)
self.server = server
def run(self, request):
"""
Introspects all of the actions on the server and returns their documentation.
:param request: The request object
:type request: EnrichedActionRequest
:return: The response
"""
if request.body.get('action_name'):
return self._get_response_for_single_action(request.body.get('action_name'))
return self._get_response_for_all_actions()
def _get_response_for_single_action(self, request_action_name):
action_name = request_action_name
switch = None
if SWITCHED_ACTION_RE.match(action_name):
match = SWITCHED_ACTION_RE.match(action_name)
action_name = match.group(str('action'))
if match.group(str('default')):
switch = SwitchedAction.DEFAULT_ACTION
else:
switch = int(match.group(str('switch')))
if action_name not in self.server.action_class_map and action_name not in ('status', 'introspect'):
raise ActionError(errors=[
Error(code=ERROR_CODE_INVALID, message='Action not defined in service', field='action_name'),
])
if action_name in self.server.action_class_map:
action_class = self.server.action_class_map[action_name]
if issubclass(action_class, SwitchedAction):
if switch:
if switch == SwitchedAction.DEFAULT_ACTION:
action_class = action_class.switch_to_action_map[-1][1]
else:
for matching_switch, action_class in action_class.switch_to_action_map:
if switch == matching_switch:
break
else:
response = {
'action_names': [],
'actions': {}
}
for sub_name, sub_class in self._iterate_switched_actions(action_name, action_class):
response['action_names'].append(sub_name)
response['actions'][sub_name] = self._introspect_action(sub_class)
response['action_names'] = list(sorted(response['action_names']))
return response
elif action_name == 'introspect':
action_class = self.__class__
else:
action_class = BaseStatusAction
return {
'action_names': [request_action_name],
'actions': {request_action_name: self._introspect_action(action_class)}
}
def _get_response_for_all_actions(self):
response = {
'actions': {},
'action_names': [],
'documentation': getattr(self.server.__class__, 'description', self.server.__class__.__doc__) or None,
}
if 'introspect' not in self.server.action_class_map:
response['action_names'].append('introspect')
response['actions']['introspect'] = self._introspect_action(self.__class__)
if 'status' not in self.server.action_class_map:
response['action_names'].append('status')
response['actions']['status'] = self._introspect_action(BaseStatusAction)
for action_name, action_class in six.iteritems(self.server.action_class_map):
if issubclass(action_class, SwitchedAction):
for sub_action_name, sub_action_class in self._iterate_switched_actions(action_name, action_class):
response['action_names'].append(sub_action_name)
response['actions'][sub_action_name] = self._introspect_action(sub_action_class)
else:
response['action_names'].append(action_name)
response['actions'][action_name] = self._introspect_action(action_class)
response['action_names'] = list(sorted(response['action_names']))
return response
@staticmethod
def _iterate_switched_actions(action_name, action_class):
found_default = False
last_index = len(action_class.switch_to_action_map) - 1
for i, (switch, sub_action_class) in enumerate(action_class.switch_to_action_map):
if switch == SwitchedAction.DEFAULT_ACTION:
sub_action_name = '{}[DEFAULT]'.format(action_name)
found_default = True
elif not found_default and i == last_index:
sub_action_name = '{}[DEFAULT]'.format(action_name)
else:
sub_action_name = '{}[switch:{}]'.format(action_name, get_switch(switch))
yield sub_action_name, sub_action_class
@staticmethod
def _introspect_action(action_class):
action = {
'documentation': getattr(action_class, 'description', action_class.__doc__) or None,
'request_schema': None,
'response_schema': None,
}
if getattr(action_class, 'request_schema', None):
action['request_schema'] = action_class.request_schema.introspect()
if getattr(action_class, 'response_schema', None):
action['response_schema'] = action_class.response_schema.introspect()
return action
class BaseStatusAction(Action):
"""
Standard base action for status checks. Returns health check and version information.
If you want to use the status action use `StatusActionFactory(version)`, passing in the version of your service
and, optionally, the build of your service. If you do not specify an action with name `status` in your server,
this will be done on your behalf.
If you want to make a custom status action, subclass this class, make `self._version` return your service's version
string, `self._build` optionally return your service's build string, and add any additional health check methods
you desire. Health check methods must start with `check_`.
Health check methods accept a single argument, the request object (an instance of `ActionRequest`), and return a
list of tuples in the format `(is_error, code, description)` (or a false-y value if there are no problems):
- `is_error`: `True` if this is an error, `False` if it is a warning.
- `code`: Invariant string for this error, like "MYSQL_FAILURE"
- `description`: Human-readable description of the problem, like "Could not connect to host on port 1234"
Health check methods can also write to the `self.diagnostics` dictionary to add additional data which will be sent
back with the response if they like. They are responsible for their own key management in this situation.
This base status action comes with a disabled-by-default health check method named `_check_client_settings` (the
leading underscore disables it), which calls `status` on all other services that this service is configured to call
(using `verbose: False`, which guarantees no further recursive status checking) and includes those responses in
this action's response. To enable this health check, simply reference it as a new, valid `check_` method name, like
so:
.. code:: python
class MyStatusAction(BaseStatusAction):
...
check_client_settings = BaseStatusAction._check_client_settings
"""
def __init__(self, *args, **kwargs):
"""
Constructs a new base status action. Concrete status actions can override this if they want, but must call
`super`.
:param settings: The server settings object
:type settings: dict
"""
super(BaseStatusAction, self).__init__(*args, **kwargs)
self.diagnostics = {}
@abc.abstractproperty
def _version(self):
raise NotImplementedError(
'version must be defined using StatusActionFactory')
@property
def _build(self):
return None
description = (
'Returns version info for the service, Python, PySOA, and Conformity. If the service has a build string, that '
'is also returned. If the service has defined additional health check behavior and the `verbose` request '
'attribute is not set to `False`, those additional health checks are performed and returned in the '
'`healthcheck` response attribute. If the `verbose` request attribute is set to `False`, the additional '
'health checks are not performed and `healthcheck` is not included in the response (importantly, the `check_` '
'methods are not invoked).')
request_schema = fields.Nullable(
fields.Dictionary(
{
'verbose':
fields.Boolean(
description=
'If specified and False, this instructs the status action to return only the baseline '
'status information (Python, service, PySOA, and other library versions) and omit any of '
'the health check operations (no `healthcheck` attribute will be included in the '
'response). This provides a useful way to obtain the service version very quickly without '
'executing the often time-consuming code necessary for the full health check. It defaults '
'to True, which means "return everything."', ),
},
optional_keys=('verbose', ),
))
response_schema = fields.Dictionary(
{
'build':
fields.UnicodeString(
description='The version build string, if applicable.'),
'conformity':
fields.UnicodeString(
description='The version of Conformity in use.'),
'healthcheck':
fields.Dictionary(
{
'warnings':
fields.List(
fields.Tuple(
fields.UnicodeString(
description='The invariant warning code'),
fields.UnicodeString(
description='The readable warning description'
),
),
description=
'A list of any warnings encountered during the health checks.',
),
'errors':
fields.List(
fields.Tuple(
fields.UnicodeString(
description='The invariant error code'),
fields.UnicodeString(
description='The readable error description'),
),
description=
'A list of any errors encountered during the health checks.',
),
'diagnostics':
fields.SchemalessDictionary(
key_type=fields.UnicodeString(),
description=
'A dictionary containing any additional diagnostic information output by the '
'health check operations.',
),
},
optional_keys=('warnings', 'errors', 'diagnostics'),
description=
'Information about any additional health check operations performed.',
),
'pysoa':
fields.UnicodeString(description='The version of PySOA in use.'),
'python':
fields.UnicodeString(description='The version of Python in use.'),
'version':
fields.UnicodeString(
description='The version of the responding service.'),
},
optional_keys=(
'build',
'healthcheck',
),
)
def run(self, request):
"""
Adds version information for Conformity, PySOA, Python, and the service to the response, then scans the class
for `check_` methods and runs them (unless `verbose` is `False`).
:param request: The request object
:type request: EnrichedActionRequest
:return: The response
"""
status = {
'conformity': six.text_type(conformity.__version__),
'pysoa': six.text_type(pysoa.__version__),
'python': six.text_type(platform.python_version()),
'version': self._version,
}
if self._build:
status['build'] = self._build
if not request.body or request.body.get('verbose', True) is True:
errors = []
warnings = []
self.diagnostics = {}
# Find all things called "check_<something>" on this class.
# We can't just scan __dict__ because of class inheritance.
check_methods = [
getattr(self, x) for x in dir(self) if x.startswith('check_')
]
for check_method in check_methods:
# Call the check, and see if it returned anything
try:
problems = check_method(request)
except TypeError as e:
raise RuntimeError(
'Status action check_* methods must accept a single argument of type ActionRequest',
e,
)
if problems:
for is_error, code, description in problems:
# Parcel out the values into the right return list
if is_error:
errors.append((code, description))
else:
warnings.append((code, description))
status['healthcheck'] = {
'errors': errors,
'warnings': warnings,
'diagnostics': self.diagnostics,
}
return status
def _check_client_settings(self, request):
"""
This method checks any client settings configured for this service to call other services, calls the `status`
action of each configured service with `verbose: False` (which guarantees no further recursive status checking),
adds that diagnostic information, and reports any problems. To include this check in your status action, define
`check_client_settings = BaseStatusAction._check_client_settings` in your status action class definition.
"""
if not request.client.settings:
# There's no need to even add diagnostic details if no client settings are configured
return
self.diagnostics['services'] = {}
service_names = list(six.iterkeys(request.client.settings))
try:
job_responses = request.client.call_jobs_parallel(
[{
'service_name': service_name,
'actions': [{
'action': 'status',
'body': {
'verbose': False
}
}]
} for service_name in service_names],
timeout=2,
catch_transport_errors=True,
raise_action_errors=False,
raise_job_errors=False,
)
except Exception as e:
return [(True, 'CHECK_SERVICES_UNKNOWN_ERROR', six.text_type(e))]
problems = []
for i, service_name in enumerate(service_names):
response = job_responses[i]
if isinstance(response, Exception):
problems.append((True, '{}_TRANSPORT_ERROR'.format(
service_name.upper()), six.text_type(response)), )
elif response.errors:
problems.append((True, '{}_CALL_ERROR'.format(
service_name.upper()), six.text_type(response.errors)), )
elif response.actions[0].errors:
problems.append(
(True, '{}_STATUS_ERROR'.format(service_name.upper()),
six.text_type(response.actions[0].errors)), )
else:
self.diagnostics['services'][service_name] = response.actions[
0].body
return problems
class ServerSettings(SOASettings):
"""
Base settings class for all servers, whose `middleware` values are restricted to subclasses of `ServerMiddleware`
and whose `transport` values are restricted to subclasses of `BaseServerTransport`. Middleware and transport
configuration settings schemas will automatically switch based on the configuration settings schema for the `path`
for each.
"""
schema = dict(
{
'transport':
fields.ClassConfigurationSchema(base_class=BaseServerTransport),
'middleware':
fields.List(
fields.ClassConfigurationSchema(base_class=ServerMiddleware),
description=
'The list of all `ServerMiddleware` objects that should be applied to requests processed '
'by this server',
),
'client_routing':
fields.SchemalessDictionary(
key_type=fields.UnicodeString(),
value_type=fields.SchemalessDictionary(),
description=
'Client settings for sending requests to other services; keys should be service names, and '
'values should be the corresponding configuration dicts, which will be validated using the '
'ClientSettings schema.',
),
'logging':
PYTHON_LOGGING_CONFIG_SCHEMA,
'harakiri':
fields.Dictionary(
{
'timeout':
fields.Integer(
gte=0,
description=
'Seconds of inactivity before harakiri is triggered; 0 to disable, defaults to 300',
),
'shutdown_grace':
fields.Integer(
gt=0,
description=
'Seconds to forcefully shutdown after harakiri is triggered if shutdown does not '
'occur',
),
},
description=
'Instructions for automatically terminating a server process when request processing takes '
'longer than expected.',
),
'request_log_success_level':
PythonLogLevel(
description=
'The logging level at which full request and response contents will be logged for '
'successful requests', ),
'request_log_error_level':
PythonLogLevel(
description=
'The logging level at which full request and response contents will be logged for requests '
'whose responses contain errors (setting this to a more severe level than '
'`request_log_success_level` will allow you to easily filter for unsuccessful requests)',
),
'heartbeat_file':
fields.Nullable(
fields.UnicodeString(
description=
'If specified, the server will create a heartbeat file at the specified path on startup, '
'update the timestamp in that file after the processing of every request or every time '
'idle operations are processed, and delete the file when the server shuts down. The file '
'name can optionally contain the specifier {{pid}}, which will be replaced with the '
'server process PID. Finally, the file name can optionally contain the specifier {{fid}}, '
'which will be replaced with the unique-and-deterministic forked process ID whenever the '
'server is started with the --fork option (the minimum value is always 1 and the maximum '
'value is always equal to the value of the --fork option).',
)),
'extra_fields_to_redact':
fields.Set(
fields.UnicodeString(),
description=
'Use this field to supplement the set of fields that are automatically redacted/censored '
'in request and response fields with additional fields that your service needs redacted.',
),
}, **extra_schema) # type: SettingsSchema
defaults = dict(
{
'client_routing': {},
'logging': {
'version': 1,
'formatters': {
'console': {
'format':
'%(asctime)s %(levelname)7s %(correlation_id)s %(request_id)s: %(message)s'
},
'syslog': {
'format':
('%(service_name)s_service: %(name)s %(levelname)s %(module)s %(process)d '
'correlation_id %(correlation_id)s request_id %(request_id)s %(message)s'
),
},
},
'filters': {
'pysoa_logging_context_filter': {
'()': 'pysoa.common.logging.PySOALogContextFilter',
},
},
'handlers': {
'console': {
'level': 'INFO',
'class': 'logging.StreamHandler',
'formatter': 'console',
'filters': ['pysoa_logging_context_filter'],
},
'syslog': {
'level': 'INFO',
'class': 'pysoa.common.logging.SyslogHandler',
'facility': SyslogHandler.LOG_LOCAL7,
'address': ('localhost', 514),
'formatter': 'syslog',
'filters': ['pysoa_logging_context_filter'],
},
},
'loggers': {},
'root': {
'handlers': ['console'],
'level': 'INFO',
},
'disable_existing_loggers': False,
},
'harakiri': {
'timeout': 300,
'shutdown_grace': 30,
},
'request_log_success_level': 'INFO',
'request_log_error_level': 'INFO',
'heartbeat_file': None,
'extra_fields_to_redact': set(),
'transport': {
'path':
'pysoa.common.transport.redis_gateway.server:RedisServerTransport',
}
}, **extra_defaults) # type: SettingsData