def test_comma_separated_list_empty(self): self.create_fixture(typed(comma_separated_list, mock_value="")) loader = load_from_dict() metadata = Metadata("test", testing=True) config = configure(self.registry.defaults, metadata, loader) assert_that(config, has_entries(foo=has_entries(value=[], ), ))
def test_typed_optional(self): self.create_fixture(typed(int)) loader = load_from_dict() config = configure(self.registry.defaults, self.metadata, loader) assert_that(config, has_entries( foo=has_entries( value=None, ), ))
def test_nullable_null_default(self): self.create_fixture(value=typed( int, default_value=None, nullable=True, )) loader = load_from_dict() config = configure(self.registry.defaults, self.metadata, loader) assert_that(config, has_entries(foo=has_entries(value=None, ), ))
def test_nullable(self): self.create_fixture(value=typed( int, default_value=0, nullable=True, mock_value=None, )) loader = load_from_dict() metadata = Metadata("test", testing=True) config = configure(self.registry.defaults, metadata, loader) assert_that(config, has_entries(foo=has_entries(value=None, ), ))
def test_typed_converted(self): self.create_fixture(typed(int)) loader = load_from_dict( foo=dict( value="1", ), ) config = configure(self.registry.defaults, self.metadata, loader) assert_that(config, has_entries( foo=has_entries( value=1, ), ))
def test_boolean_typed_converted(self): self.create_fixture( bar=typed(bool, default_value=None), baz=typed(bool, default_value=None), qux=typed(bool, default_value=None), kog=typed(bool, default_value=None), ) loader = load_from_dict(foo=dict( bar="False", baz="True", qux="false", kog="true", ), ) config = configure(self.registry.defaults, self.metadata, loader) assert_that( config, has_entries(foo=has_entries( bar=False, baz=True, qux=False, kog=True, ), ))
# expected behavior (e.g. 404) if not request_info.options.log_as_debug: logger.info(request_info.to_dict()) else: logger.debug(request_info.to_dict()) # Setting request state for future middleware functions request.state.request_info = request_info return response return partial(audit_request, graph, options) @defaults( include_request_body_status=typed( type=int, default_value=DEFAULT_INCLUDE_REQUEST_BODY_STATUS), include_response_body_status=typed( type=int, default_value=DEFAULT_INCLUDE_RESPONSE_BODY_STATUS), include_path=typed(type=boolean, default_value=False), include_query_string=typed(type=boolean, default_value=False), log_as_debug=typed(type=boolean, default_value=False), ) def configure_audit_middleware(graph): """ Configure audit middleware """ options = AuditOptions( include_request_body_status=graph.config.audit_middleware. include_request_body_status, include_response_body_status=graph.config.audit_middleware.
def test_false_default(self): self.create_fixture(value=typed(bool, default_value=False)) loader = load_from_dict() config = configure(self.registry.defaults, self.metadata, loader) assert_that(config, has_entries(foo=has_entries(value=False, ), ))
resource_dict = graph.config.sns_topic_arns.get(lifecycle_change, {}) for resource_name, topic in iter_topic_mappings(resource_dict): media_type = make_media_type(resource_name, lifecycle_change) sns_topic_arns[media_type] = topic return sns_topic_arns @defaults( profile_name=None, region_name=None, endpoint_url=None, mock_sns=True, skip=None, # the size used to determine batching in the deferred batch producer deferred_batch_size=typed(int, default_value=100), # SNS endpoint timeout configuration connect_timeout=typed(int, default_value=60), read_timeout=typed(int, default_value=60), ) def configure_sns_producer(graph): """ Configure an SNS producer. The SNS Producer requires the following collaborators: - Opaque from microcosm.opaque for capturing context information - an aws sns client, i.e. from boto. - pubsub message codecs: see tests for examples. - sns topic arns: see tests for examples. """
resource_dict = graph.config.sns_topic_arns.get(lifecycle_change, {}) for resource_name, topic in iter_topic_mappings(resource_dict): media_type = make_media_type(resource_name, lifecycle_change) sns_topic_arns[media_type] = topic return sns_topic_arns @defaults( profile_name=None, region_name=None, endpoint_url=None, mock_sns=True, skip=None, # the size used to determine batching in the deferred batch producer deferred_batch_size=typed(int, default_value=100), ) def configure_sns_producer(graph): """ Configure an SNS producer. The SNS Producer requires the following collaborators: - Opaque from microcosm.opaque for capturing context information - an aws sns client, i.e. from boto. - pubsub message codecs: see tests for examples. - sns topic arns: see tests for examples. """ if graph.metadata.testing: from unittest.mock import MagicMock
tags=tags + [ f"classifier:{normalize_status_code(request_info.status_code)}" ], ) if request_info.timing.get("elapsed_time"): elapsed_ms = request_info.timing["elapsed_time"] graph.metrics.histogram( name_for(key), elapsed_ms, tags=tags, ) return response return partial(route_metrics, graph) @defaults( enabled=typed(boolean, default_value=True), ) def configure_route_metrics(graph): """ Configure route metrics """ metrics = get_metrics(graph) enabled = bool(metrics and metrics.host != "localhost" and graph.config.route_metrics.enabled) if enabled: graph.app.middleware("http")(create_route_metrics(graph))
session = Session(profile_name=profile_name) return session.client( "sqs", endpoint_url=endpoint_url, region_name=region_name, ) @defaults( endpoint_url=None, profile_name=None, region_name=None, # backoff policy backoff_policy="NaiveBackoffPolicy", # SQS will not return more than ten messages at a time limit=typed(int, default_value=10), # SQS will only return a few messages at time unless long polling is enabled (>0) wait_seconds=typed(int, default_value=1), # On error, change the visibility timeout when nacking message_retry_visibility_timeout_seconds=typed(int, default_value=5), ) def configure_sqs_consumer(graph): """ Configure an SQS consumer. """ sqs_queue_url = graph.config.sqs_consumer.sqs_queue_url if graph.metadata.testing or sqs_queue_url == "test": from unittest.mock import MagicMock sqs_client = MagicMock()
""" Neo4J driver factory. """ from logging import getLogger, INFO from microcosm.api import defaults, typed from microcosm.config.types import boolean from neo4j.v1 import GraphDatabase @defaults( # NB: some features are not available unless enabled enterprise=typed(boolean, default_value=False), password="******", uri=None, username="******", ) def configure_neo4j_driver(graph): if graph.metadata.testing: logger = getLogger("neobolt") logger.level = INFO default_uri = "bolt://localhost:17687" else: default_uri = "bolt://localhost:7687" return GraphDatabase.driver( graph.config.neo4j.uri or default_uri, auth=( graph.config.neo4j.username,
Routing registration support. Intercepts Flask's normal route registration to inject conventions. """ from flask_cors import cross_origin from microcosm.api import defaults, typed from microcosm.config.types import boolean from microcosm_logging.decorators import context_logger @defaults( converters=[ "uuid", ], enable_audit=typed(boolean, default_value=True), enable_basic_auth=typed(boolean, default_value=False), enable_context_logger=typed(boolean, default_value=True), enable_cors=typed(boolean, default_value=True), ) def configure_route_decorator(graph): """ Configure a flask route decorator that operates on `Operation` and `Namespace` objects. By default, enables CORS support, assuming that service APIs are not exposed directly to browsers except when using API browsing tools. Usage: @graph.route(ns.collection_path, Operation.Search, ns) def search_foo():
from microcosm.api import defaults, typed from microcosm.config.types import boolean from microcosm.errors import NotBoundError from microcosm_pubsub.result import MessageHandlingResult, MessageHandlingResultType @defaults(enabled=typed(boolean, default_value=True)) class PubSubSendMetrics: """ Send metrics relating to a single MessageHandlingResult """ def __init__(self, graph): self.metrics = self.get_metrics(graph) self.enabled = bool(self.metrics and self.metrics.host != "localhost" and graph.config.pubsub_send_metrics.enabled) def get_metrics(self, graph): """ Fetch the metrics client from the graph. Metrics will be disabled if the not configured. """ try: return graph.metrics except NotBoundError: return None def __call__(self, result: MessageHandlingResult):
from microcosm.api import defaults, typed from microcosm.config.types import boolean, comma_separated_list from microcosm_caching.memcached import MemcachedCache @defaults( enabled=typed(boolean, default_value=False), servers=typed(comma_separated_list, default_value="localhost:11211"), connect_timeout=typed(float, default_value=3.0), read_timeout=typed(float, default_value=2.0), ignore_exc=typed(boolean, default_value=False), ) def configure_resource_cache(graph): """ Configure the resource cache which will be exposed via the microcosm application graph. """ if not graph.config.resource_cache.enabled: return None kwargs = dict( servers=parse_server_config(graph.config.resource_cache.servers), connect_timeout=graph.config.resource_cache.connect_timeout, read_timeout=graph.config.resource_cache.read_timeout, ignore_exc=graph.config.resource_cache.ignore_exc, ) if graph.metadata.testing: kwargs.update(dict(testing=True))
from microcosm.api import defaults, typed from microcosm_fastapi.conventions.health.models import Health from microcosm_fastapi.conventions.health.resources import HealthSchema @defaults( include_build_info=typed(bool, default=True), ) def configure_health(graph): """ Mount the health endpoint to the graph """ health_container = Health( graph, graph.config.health_convention.include_build_info) @graph.app.get("/api/health") def configure_health_endpoint(full: bool = False) -> HealthSchema: response_data = health_container.to_object(full=full) if not response_data.ok: raise return response_data return health_container
from datetime import datetime, timedelta from logging import Logger from tracemalloc import start, take_snapshot from microcosm.api import binding, defaults, typed from microcosm.config.types import boolean @binding("memory_profiler") @defaults( enabled=typed(boolean, False), report_size_lines=typed(int, 10), sampling_interval_min=typed(int, 10), ) class MemoryProfiler: logger: Logger def __init__(self, graph): self.enabled = graph.config.memory_profiler.enabled self.report_size_lines = graph.config.memory_profiler.report_size_lines self.logger = graph.logger self.sampling_interval_min = graph.config.memory_profiler.sampling_interval_min self.last_sampling_time_delta = timedelta( minutes=self.sampling_interval_min) self.last_sampling_time = datetime.now() if not self.enabled: self.logger.info( "Skipping initialization because memory profiling is not enabled!"
kwargs["response_model"] = fn.__annotations__["return"] except AttributeError: pass return kwargs def inject_default_response(self, kwargs): if kwargs.get("responses", None): kwargs["responses"]["default"] = {"model": ErrorSchema} else: kwargs["responses"] = {"default": {"model": ErrorSchema}} return kwargs @defaults( port=typed(int, default_value=5000), host="127.0.0.1", ) def configure_fastapi(graph): # Docs use 3rd party dependencies by default - if documentation # is desired by client callers, use the `graph.use("docs")` bundled # with microcosm-fastapi. This hook provides a mirror to the default # docs/redocs but while hosted locally. app = FastAPIWrapper( port=graph.config.app.port, debug=graph.metadata.debug, docs_url=None, redoc_url=None, ) # Request_context is used for logging purposes
from microcosm.api import defaults, typed from microcosm_logging.decorators import logger from microcosm_logging.timing import elapsed_time from microcosm_pubsub.dispatcher import SQSMessageDispatcher from microcosm_pubsub.result import MessageHandlingResultType from microcosm_fastapi.pubsub.result import MessageHandlingResultAsync PUBLISHED_KEY = "X-Request-Published" @logger @defaults( # Number of failed attempts after which the message stops being processed message_max_processing_attempts=typed(int, default_value=None), # Quantity of messages to parse within the same runloop message_max_concurrent_operations=typed(int, default_value=5), ) class SQSMessageDispatcherAsync(SQSMessageDispatcher): def __init__(self, graph): super().__init__(graph) self.max_processing_attempts = ( graph.config.sqs_message_dispatcher_async. message_max_processing_attempts) self.max_concurrent_operations = ( graph.config.sqs_message_dispatcher_async. message_max_concurrent_operations) def handle_batch(self, bound_handlers) -> List[MessageHandlingResultAsync]:
""" Message context. """ from typing import Dict from microcosm.api import defaults, typed from microcosm.config.types import boolean from microcosm_logging.decorators import logger from microcosm_pubsub.constants import TTL_KEY, URI_KEY from microcosm_pubsub.message import SQSMessage @defaults( enable_ttl=typed(boolean, default_value=True), initial_ttl=typed(int, default_value=32), ) @logger class SQSMessageContext: """ Factory for per-message contexts. """ def __init__(self, graph): self.enable_ttl = graph.config.sqs_message_context.enable_ttl self.initial_ttl = graph.config.sqs_message_context.initial_ttl def __call__(self, context: SQSMessage, **kwargs) -> Dict[str, str]: """ Create a new context from a message.
from typing import List from inflection import titleize from microcosm.api import defaults, typed from microcosm_logging.decorators import context_logger, logger from microcosm_logging.timing import elapsed_time from microcosm_pubsub.context import TTL_KEY from microcosm_pubsub.errors import IgnoreMessage, TTLExpired, SkipMessage from microcosm_pubsub.result import MessageHandlingResult @logger @defaults( # Number of failed attempts after which the message stops being processed message_max_processing_attempts=typed(int, default_value=None) ) class SQSMessageDispatcher: """ Dispatch batches of SQSMessages to handler functions. """ def __init__(self, graph): self.opaque = graph.opaque self.sqs_consumer = graph.sqs_consumer self.sqs_message_context = graph.sqs_message_context self.sqs_message_handler_registry = graph.sqs_message_handler_registry self.send_metrics = graph.pubsub_send_metrics self.max_processing_attempts = graph.config.sqs_message_dispatcher.message_max_processing_attempts def handle_batch(self, bound_handlers) -> List[MessageHandlingResult]:
""" Message context. """ from typing import Dict from microcosm.api import defaults, typed from microcosm.config.types import boolean from microcosm_logging.decorators import logger from microcosm_pubsub.constants import RECEIPT_HANDLE_KEY, TTL_KEY, URI_KEY from microcosm_pubsub.message import SQSMessage @defaults( enable_ttl=typed(boolean, default_value=True), initial_ttl=typed(int, default_value=32), ) @logger class SQSMessageContext: """ Factory for per-message contexts. """ def __init__(self, graph): self.enable_ttl = graph.config.sqs_message_context.enable_ttl self.initial_ttl = graph.config.sqs_message_context.initial_ttl def __call__(self, context: SQSMessage, **kwargs) -> Dict[str, str]: """ Create a new context from a message.
""" Store build information. """ from dataclasses import dataclass from typing import Optional from microcosm.api import defaults, typed @dataclass class BuildInfo: build_num: Optional[str] sha1: Optional[str] @defaults( build_num=typed(str, default_value=None), sha1=typed(str, default_value=None), ) def configure_build_info(graph): """ Configure build info """ return BuildInfo( build_num=graph.config.build_info.build_num, sha1=graph.config.build_info.sha1, )
from typing import List from inflection import titleize from microcosm.api import defaults, typed from microcosm_logging.decorators import context_logger, logger from microcosm_logging.timing import elapsed_time from microcosm_pubsub.context import TTL_KEY from microcosm_pubsub.errors import IgnoreMessage, TTLExpired, SkipMessage from microcosm_pubsub.result import MessageHandlingResult @logger @defaults( # Number of failed attempts after which the message stops being processed message_max_processing_attempts=typed(int, default_value=None)) class SQSMessageDispatcher: """ Dispatch batches of SQSMessages to handler functions. """ def __init__(self, graph): self.opaque = graph.opaque self.sqs_consumer = graph.sqs_consumer self.sqs_message_context = graph.sqs_message_context self.sqs_message_handler_registry = graph.sqs_message_handler_registry self.send_metrics = graph.pubsub_send_metrics self.max_processing_attempts = graph.config.sqs_message_dispatcher.message_max_processing_attempts def handle_batch(self, bound_handlers) -> List[MessageHandlingResult]: """
def test_valid_default_factory(self): self.create_fixture(value=typed(list, default_factory=list)) loader = load_from_dict() config = configure(self.registry.defaults, self.metadata, loader) assert_that(config, has_entries(foo=has_entries(value=empty(), ), ))
from jaeger_client.config import ( DEFAULT_REPORTING_HOST, DEFAULT_REPORTING_PORT, DEFAULT_SAMPLING_PORT, Config, ) from microcosm.api import binding, defaults, typed SPAN_NAME = "span_name" @binding("tracer") @defaults( sample_type="ratelimiting", sample_param=typed(int, 10), sampling_port=typed(int, DEFAULT_SAMPLING_PORT), reporting_port=typed(int, DEFAULT_REPORTING_PORT), reporting_host=DEFAULT_REPORTING_HOST, ) def configure_tracing(graph): """ See https://www.jaegertracing.io/docs/1.12/sampling/ for more info about available sampling strategies. """ config = Config( config={ "sampler": { "type": graph.config.tracer.sample_type, "param": graph.config.tracer.sample_param,
""" Metrics extensions for routes. """ from microcosm.api import defaults, typed from microcosm.config.types import boolean from microcosm.errors import NotBoundError @defaults( enabled=typed(boolean, default_value=True), ) class RouteMetrics: def __init__(self, graph): self.metrics = self.get_metrics(graph) self.enabled = bool( self.metrics and self.metrics.host != "localhost" and graph.config.route_metrics.enabled ) self.graph = graph def get_metrics(self, graph): """ Fetch the metrics client from the graph. Metrics will be disabled if the not configured. """ try: