def __init__( self, endpoint: config.EndpointConfiguration, size: int = 10, max_age: int = 120, timeout: int = 1, max_connection_attempts: Optional[int] = None, max_retries: Optional[int] = None, protocol_factory: TProtocolFactory = THeaderProtocol. THeaderProtocolFactory(), ): if max_connection_attempts and max_retries: raise Exception( "do not mix max_retries and max_connection_attempts") if max_retries: warn_deprecated( "ThriftConnectionPool's max_retries is now named max_connection_attempts" ) max_connection_attempts = max_retries elif not max_connection_attempts: max_connection_attempts = 3 self.endpoint = endpoint self.max_age = max_age self.retry_policy = RetryPolicy.new(attempts=max_connection_attempts) self.timeout = timeout self.protocol_factory = protocol_factory self.size = size self.pool: ProtocolPool = queue.LifoQueue() for _ in range(size): self.pool.put(None)
def configure_context(self, *args: Any, **kwargs: Any) -> None: # noqa: F811 """Add a number of objects to each request's context object. Configure and attach multiple clients to the :py:class:`~baseplate.RequestContext` in one place. This takes a full configuration spec like :py:func:`baseplate.lib.config.parse_config` and will attach the specified structure onto the context object each request. For example, a configuration like:: baseplate = Baseplate(app_config) baseplate.configure_context({ "cfg": { "doggo_is_good": config.Boolean, }, "cache": MemcachedClient(), "cassandra": { "foo": CassandraClient("foo_keyspace"), "bar": CassandraClient("bar_keyspace"), }, }) would build a context object that could be used like:: assert context.cfg.doggo_is_good == True context.cache.get("example") context.cassandra.foo.execute() :param app_config: The raw stringy configuration dictionary. :param context_spec: A specification of what the configuration should look like. """ if len(args) == 1: kwargs["context_spec"] = args[0] elif len(args) == 2: kwargs["app_config"] = args[0] kwargs["context_spec"] = args[1] else: raise Exception("bad parameters to configure_context") if "app_config" in kwargs: if self._app_config: raise Exception("pass app_config to the constructor or this method but not both") warn_deprecated( "Passing configuration to configure_context is deprecated in " "favor of passing it to the Baseplate constructor" ) app_config = kwargs["app_config"] else: app_config = self._app_config context_spec = kwargs["context_spec"] cfg = config.parse_config(app_config, context_spec) self._context_config.update(cfg)
def consume( baseplate: Baseplate, exchange: Exchange, connection: Connection, queue_name: str, routing_keys: Sequence[str], handler: Handler, ) -> NoReturn: """Create a long-running process to consume messages from a queue. A queue with name ``queue_name`` is created and bound to the ``routing_keys`` so messages published to the ``routing_keys`` are routed to the queue. Next, the process registers a consumer that receives messages from the queue and feeds them to the ``handler``. The ``handler`` function must take 3 arguments: * ``context``: a baseplate context * ``message_body``: the text body of the message * ``message``: :py:class:`kombu.message.Message` The consumer will automatically ``ack`` each message after the handler method exits. If there is an error in processing and the message must be retried the handler should raise an exception to crash the process. This will prevent the ``ack`` and the message will be re-queued at the head of the queue. :param baseplate: A baseplate instance for the service. :param exchange: :param connection: :param queue_name: The name of the queue. :param routing_keys: List of routing keys. :param handler: The handler method. """ warn_deprecated( "baseplate.frameworks.queue_consumer is deprecated and will be removed " "in the next major release. You should migrate your consumers to use " "baseplate.server.queue_consumer.\n" "https://baseplate.readthedocs.io/en/stable/api/baseplate/frameworks/queue_consumer/deprecated.html" ) queues = [] for routing_key in routing_keys: queues.append( Queue(name=queue_name, exchange=exchange, routing_key=routing_key)) logger.info("registering %s as a handler for %r", handler.__name__, queues) kombu_consumer = KombuConsumer.new(connection, queues) logger.info("waiting for messages") while True: context = baseplate.make_context_object() with baseplate.make_server_span(context, queue_name) as span: message = kombu_consumer.get_message(span) handler(context, message.body, message) message.ack()
def make_client( service_name: str, tracing_endpoint: Optional[config.EndpointConfiguration] = None, tracing_queue_name: Optional[str] = None, max_span_queue_size: int = 50000, num_span_workers: int = 5, span_batch_interval: float = 0.5, num_conns: int = 100, sample_rate: float = 0.1, log_if_unconfigured: bool = True, ) -> TracingClient: """Create and return a tracing client based on configuration options. This client can be used by the :py:class:`TraceBaseplateObserver`. :param service_name: The name for the service this observer is registered to. :param tracing_endpoint: Destination to record span data. :param tracing_queue_name: POSIX queue name for reporting spans. :param num_conns: pool size for remote recorder connection pool. :param max_span_queue_size: span processing queue limit. :param num_span_workers: number of worker threads for span processing. :param span_batch_interval: wait time for span processing in seconds. :param sample_rate: percentage of unsampled requests to record traces for. """ recorder: Recorder if tracing_queue_name: recorder = SidecarRecorder(tracing_queue_name) elif tracing_endpoint: warn_deprecated( "In-app trace publishing is deprecated in favor of the sidecar model." ) remote_addr = str(tracing_endpoint.address) recorder = RemoteRecorder( remote_addr, num_conns=num_conns, max_queue_size=max_span_queue_size, num_workers=num_span_workers, batch_wait_interval=span_batch_interval, ) elif log_if_unconfigured: recorder = LoggingRecorder( max_queue_size=max_span_queue_size, num_workers=num_span_workers, batch_wait_interval=span_batch_interval, ) else: recorder = NullRecorder( max_queue_size=max_span_queue_size, num_workers=num_span_workers, batch_wait_interval=span_batch_interval, ) return TracingClient(service_name, sample_rate, recorder)
def get_vault_token(self) -> str: """Return a Vault authentication token. The token will have policies attached based on the current EC2 server's Vault role. This is only necessary if talking directly to Vault. """ warn_deprecated( "get_vault_token is deprecated and will be removed in v3.0.") data, _ = self._get_data() return data["vault"]["token"]
def __init__( self, baseplate: Baseplate, trust_trace_headers: Optional[bool] = None, edge_context_factory: Optional[EdgeRequestContextFactory] = None, header_trust_handler: Optional[HeaderTrustHandler] = None, ): self.baseplate = baseplate self.trust_trace_headers = bool(trust_trace_headers) if trust_trace_headers is not None: warn_deprecated( "setting trust_trace_headers is deprecated in favor of using" " a header trust handler." ) self.edge_context_factory = edge_context_factory if header_trust_handler: self.header_trust_handler = header_trust_handler else: self.header_trust_handler = StaticTrustHandler(trust_headers=self.trust_trace_headers)
def _get_experiment(self, name: str) -> Optional[Experiment]: if name in self._global_cache: return self._global_cache[name] if self._cfg_data is None: warn_deprecated("config_watcher will be removed in Baseplate 2.0.") self._cfg_data = self._get_config() if name not in self._cfg_data: logger.info("Experiment <%r> not found in experiment config", name) return None try: experiment = parse_experiment(self._cfg_data[name]) self._global_cache[name] = experiment return experiment except Exception as err: logger.error("Invalid configuration for experiment %s: %s", name, err) return None
def from_config( cls, app_config: config.RawConfig) -> "TimeoutBaseplateObserver": cfg = config.parse_config( app_config, { "server_timeout": { "default": config.Optional(config.TimespanOrInfinite, default=None), "debug": config.Optional(config.Boolean, default=False), "by_endpoint": config.DictOf(config.TimespanOrInfinite), } }, ) if cfg.server_timeout.default is None: warn_deprecated( "No server_timeout.default configured. Defaulting to no timeout. " "Set the default timeout to 'infinite' or a timespan like '2 seconds'. " "This will become mandatory in Baseplate.py 2.0.") cfg.server_timeout.default = config.InfiniteTimespan return cls(cfg.server_timeout)
def __init__(self, secret_key: bytes): warn_deprecated( "MessageSigner is deprecated in favor of the top-level " "make_signature and validate_signature functions which " "accept versioned secrets from the secret store.") self.secret = VersionedSecret.from_simple_secret(secret_key)
def get_vault_url(self) -> str: """Return the URL for accessing Vault directly.""" warn_deprecated( "get_vault_url is deprecated and will be removed in v3.0.") data, _ = self._get_data() return data["vault"]["url"]
def parse_experiment(config: Dict[str, str]) -> Experiment: """Parse an experiment config dict and return an appropriate Experiment class. The config dict is expected to have the following values: * **id**: Integer experiment ID, should be unique for each experiment. * **name**: String experiment name, should be unique for each experiment. * **owner**: The group or individual that owns this experiment. * **version**: String to identify the specific version of the experiment. * **start_ts**: A float of seconds since the epoch of date and time when you want the experiment to start. If an experiment has not been started yet, it is considered disabled. * **stop_ts**: A float of seconds since the epoch of date and time when you want the experiment to stop. Once an experiment is stopped, it is considered disabled. * **type**: String specifying the type of experiment to run. If this value is not recognized, the experiment will be considered disabled. * **experiment**: The experiment config dict for the specific type of experiment. The format of this is determined by the specific experiment type. * **enabled**: (Optional) If set to False, the experiment will be disabled and calls to experiment.variant will always return None and will not log bucketing events to the event pipeline. Defaults to True. * **global_override**: (Optional) If this is set, calls to experiment.variant will always return the override value and will not log bucketing events to the event pipeline. :param config: Configuration dict for the experiment you wish to run. :return: A subclass of :py:class:`Experiment` for the given experiment type. """ experiment_type = config.get("type") if experiment_type: experiment_type = experiment_type.lower() experiment_id = config.get("id") if not isinstance(experiment_id, int): raise TypeError("Integer id must be provided for experiment.") name = config.get("name") owner = config.get("owner") start_ts = config.get("start_ts") stop_ts = config.get("stop_ts") if start_ts is None or stop_ts is None: if "expires" in config: warn_deprecated( "The 'expires' field in experiment %s is deprecated, you should " "use 'start_ts' and 'stop_ts'." % name ) start_ts = time.time() expires = datetime.strptime(config["expires"], ISO_DATE_FMT) epoch = datetime(1970, 1, 1) stop_ts = (expires - epoch).total_seconds() else: raise ValueError( f"Invalid config for experiment {name}, missing start_ts and/or stop_ts." ) if "version" in config: version = config["version"] else: warn_deprecated( "The 'version' field is not in experiment %s. This field will be " "required in the future." % name ) version = None now = time.time() enabled = config.get("enabled", True) if now < start_ts or now > stop_ts: enabled = False if not enabled and experiment_type in legacy_type_class_map: return ForcedVariantExperiment(None) experiment_config = config["experiment"] if "global_override" in config: # We want to check if "global_override" is in config rather than # checking config.get("global_override") because global_override = None # is a valid setting. override = config.get("global_override") return ForcedVariantExperiment(override) if experiment_type in legacy_type_class_map: experiment_class = legacy_type_class_map[experiment_type] return experiment_class.from_dict( id=experiment_id, name=name, owner=owner, version=version, config=experiment_config ) if experiment_type in simple_type_class_list: return SimpleExperiment.from_dict( id=experiment_id, name=name, owner=owner, start_ts=start_ts, stop_ts=stop_ts, enabled=enabled, config=experiment_config, variant_type=experiment_type, ) logger.warning( "Found an experiment <%s:%s> with an unknown experiment type <%s> " "that is owned by <%s>. Please clean up.", experiment_id, name, experiment_type, owner, ) return ForcedVariantExperiment(None)
def variant( self, experiment_name: Optional[str] = None, user: Optional[User] = None, bucketing_event_override: Optional[bool] = None, name: Optional[str] = None, # DEPRECATED **kwargs: str, ) -> Optional[str]: r"""Return which variant, if any, is active. If a variant is active, a bucketing event will be logged to the event pipeline unless any one of the following conditions are met: 1. bucketing_event_override is set to False. 2. The experiment specified by "name" explicitly disables bucketing events. 3. We have already logged a bucketing event for the value specified by ``experiment.get_unique_id(\*\*kwargs)`` within the current request. Since checking the status an experiment will fire a bucketing event, it is best to only check the variant when you are making the decision that will expose the experiment to the user. If you absolutely must check the status of an experiment before you are sure that the experiment will be exposed to the user, you can use `bucketing_event_override` to disabled bucketing events for that check. :param experiment_name: Name of the experiment you want to run. :param name: DEPRECATED - use experiment_name instead :param user: User object for the user you want to check the experiment variant for. If you set user, the experiment parameters for that user ("user_id", "logged_in", and "user_roles") will be extracted and added to the inputs to the call to Experiment.variant. The user's event_fields will also be extracted and added to the bucketing event if one is logged. It is recommended that you provide a value for user rather than setting the user parameters manually in ``kwargs``. :param bucketing_event_override: Set if you need to override the default behavior for sending bucketing events. This parameter should be set sparingly as it breaks the assumption that you will fire a bucketing event when you first check the state of an experiment. If set to False, will never send a bucketing event. If set to None, no override will be applied. Set to None by default. Note that setting bucketing_event_override to True has no effect, it will behave the same as when it is set to None. :param kwargs: Arguments that will be passed to experiment.variant to determine bucketing, targeting, and overrides. These values will also be passed to the logger. :return: Variant name if a variant is active, None otherwise. .. versionchanged:: 1.5 ``name`` was renamed to ``experiment_name`` """ experiment_name = experiment_name or name assert experiment_name if name: warn_deprecated( f"The 'name' parameter on 'variant' method is deprecated, use 'experiment_name' instead. (name={name})" ) experiment = self._get_experiment(experiment_name) if experiment is None: return None inputs = dict(kwargs) if user: inputs.update(user.event_fields()) variant = experiment.variant(**inputs) bucketing_id = experiment.get_unique_id(**inputs) do_log = True if not bucketing_id: do_log = False if variant is None: do_log = False if bucketing_event_override is False: do_log = False if bucketing_id and bucketing_id in self._already_bucketed: do_log = False do_log = do_log and experiment.should_log_bucketing() if do_log: assert bucketing_id self._event_logger.log( experiment=experiment, variant=variant, user_id=inputs.get("user_id"), logged_in=inputs.get("logged_in"), cookie_created_timestamp=inputs.get( "cookie_created_timestamp"), app_name=inputs.get("app_name"), event_type=EventType.BUCKET, inputs=inputs, span=self._span, ) self._already_bucketed.add(bucketing_id) return variant
def configure_observers( self, app_config: Optional[config.RawConfig] = None, module_name: Optional[str] = None ) -> None: """Configure diagnostics observers based on application configuration. This installs all the currently supported observers that have settings in the configuration file. See :py:mod:`baseplate.observers` for the configuration settings available for each observer. :param app_config: The application configuration which should have settings for the error reporter. If not specified, the config must be passed to the Baseplate() constructor. :param module_name: Name of the root package of the application. If not specified, will be guessed from the package calling this function. """ skipped = [] if app_config: if self._app_config: raise Exception("pass app_config to the constructor or this method but not both") warn_deprecated( "Passing configuration to configure_observers is deprecated in " "favor of passing it to the Baseplate constructor" ) else: app_config = self._app_config self.configure_logging() if gevent.monkey.is_module_patched("socket"): # pylint: disable=cyclic-import from baseplate.observers.timeout import TimeoutBaseplateObserver timeout_observer = TimeoutBaseplateObserver.from_config(app_config) self.register(timeout_observer) else: skipped.append("timeout") if "metrics.tagging" in app_config: if "metrics.namespace" in app_config: raise ValueError("metrics.namespace not allowed with metrics.tagging") from baseplate.lib.metrics import metrics_client_from_config metrics_client = metrics_client_from_config(app_config) self.configure_tagged_metrics(metrics_client) elif "metrics.namespace" in app_config: from baseplate.lib.metrics import metrics_client_from_config metrics_client = metrics_client_from_config(app_config) self.configure_metrics(metrics_client) else: skipped.append("metrics") if "tracing.service_name" in app_config: from baseplate.observers.tracing import tracing_client_from_config tracing_client = tracing_client_from_config(app_config) self.configure_tracing(tracing_client) else: skipped.append("tracing") if "sentry.dsn" in app_config: from baseplate.observers.sentry import error_reporter_from_config if module_name is None: module_name = get_calling_module_name() error_reporter = error_reporter_from_config(app_config, module_name) self.configure_error_reporting(error_reporter) else: skipped.append("error_reporter") if skipped: logger.debug( "The following observers are unconfigured and won't run: %s", ", ".join(skipped) )
def shutting_down(self) -> bool: warn_deprecated( "SERVER_STATE.shutting_down is deprecated in favor of SERVER_STATE.state" ) return self.state == ServerLifecycle.SHUTTING_DOWN