def handler_from_dsn(dsn: str = None, workers: int = 5, include_paths: Iterable[str] = None, loglevel: int = None, qsize: int = 1000, **kwargs: Any) -> Optional[logging.Handler]: if raven is None: raise ImproperlyConfigured( 'faust.contrib.sentry requires the `raven` library.') if raven_aiohttp is None: raise ImproperlyConfigured( 'faust.contrib.sentry requires the `raven_aiohttp` library.') level: int = loglevel if loglevel is not None else DEFAULT_LEVEL if dsn: client = raven.Client(dsn=dsn, include_paths=include_paths, transport=partial( raven_aiohttp.QueuedAioHttpTransport, workers=workers, qsize=qsize, ), disable_existing_loggers=False, **kwargs) handler = _build_sentry_handler()(client) handler.setLevel(level) return handler return None
def credentials_to_aiokafka_auth(credentials: CredentialsT = None, ssl_context: Any = None) -> Mapping: if credentials is not None: if isinstance(credentials, SSLCredentials): return { 'security_protocol': credentials.protocol.value, 'ssl_context': credentials.context, } elif isinstance(credentials, SASLCredentials): return { 'security_protocol': credentials.protocol.value, 'sasl_mechanism': credentials.mechanism.value, 'sasl_plain_username': credentials.username, 'sasl_plain_password': credentials.password, 'ssl_context': credentials.ssl_context, } elif isinstance(credentials, GSSAPICredentials): return { 'security_protocol': credentials.protocol.value, 'sasl_mechanism': credentials.mechanism.value, 'sasl_kerberos_service_name': credentials.kerberos_service_name, 'sasl_kerberos_domain_name': credentials.kerberos_domain_name, 'ssl_context': credentials.ssl_context, } else: raise ImproperlyConfigured( f'aiokafka does not support {credentials}') elif ssl_context is not None: return { 'security_protocol': 'SSL', 'ssl_context': ssl_context, } else: return {'security_protocol': 'PLAINTEXT'}
def setup_prometheus_sensors( app: AppT, pattern: str = "/metrics", registry: CollectorRegistry = REGISTRY, name_prefix: Optional[str] = None, ) -> None: """ A utility function which sets up prometheus and attaches the config to the app. @param app: the faust app instance @param pattern: the url pattern for prometheus @param registry: the prometheus registry @param name_prefix: the name prefix. Defaults to the app name @return: None """ if prometheus_client is None: raise ImproperlyConfigured( "prometheus_client requires `pip install prometheus_client`." ) if name_prefix is None: name_prefix = app.conf.name name_prefix = name_prefix.replace("-", "_").replace(".", "_") faust_metrics = FaustMetrics.create(registry, name_prefix) app.monitor = PrometheusMonitor(metrics=faust_metrics) @app.page(pattern) async def metrics_handler(self: _web.View, request: _web.Request) -> _web.Response: headers = {"Content-Type": CONTENT_TYPE_LATEST} return cast( _web.Response, Response(body=generate_latest(REGISTRY), headers=headers, status=200), )
def conf(self) -> Settings: if not self.finalized and STRICT: raise ImproperlyConfigured( 'App configuration accessed before app.finalize()') if self._conf is None: self._configure() return cast(Settings, self._conf)
def __init__( self, url: Union[str, URL], app: AppT, table: CollectionT, *, key_index_size: int = None, options: Mapping[str, Any] = None, **kwargs: Any, ) -> None: if rocksdb is None: error = ImproperlyConfigured( "RocksDB bindings not installed? pip install python-rocksdb") try: import rocksdb as _rocksdb # noqa: F401 except Exception as exc: # pragma: no cover raise error from exc else: # pragma: no cover raise error super().__init__(url, app, table, **kwargs) if not self.url.path: self.url /= self.table_name self.options = options or {} self.rocksdb_options = RocksDBOptions(**self.options) if key_index_size is None: key_index_size = app.conf.table_key_index_size self.key_index_size = key_index_size self._dbs = {} self._key_index = LRUCache(limit=self.key_index_size) self.db_lock = asyncio.Lock() self.rebalance_ack = False
class Store(base.SerializedStore): """RocksDB table storage.""" offset_key = b'__faust\0offset__' #: Decides the size of the K=>TopicPartition index (10_000). key_index_size: int #: Used to configure the RocksDB settings for table stores. options: RocksDBOptions _dbs: MutableMapping[int, DB] _key_index: LRUCache[bytes, int] def __init__(self, url: Union[str, URL], app: AppT, table: CollectionT, *, key_index_size: int = 10_000, options: Mapping = None, **kwargs: Any) -> None: if rocksdb is None: raise ImproperlyConfigured( 'RocksDB bindings not installed: pip install python-rocksdb') super().__init__(url, app, table, **kwargs) if not self.url.path: self.url /= self.table_name self.options = RocksDBOptions(**options or {}) self.key_index_size = key_index_size self._dbs = {} self._key_index = LRUCache(limit=self.key_index_size)
async def _prepare_actor(self, aref: ActorRefT, beacon: NodeT) -> ActorRefT: coro: Any if isinstance(aref, Awaitable): # agent does not yield coro = aref if self._sinks: raise ImproperlyConfigured("Agent must yield to use sinks") else: # agent yields and is an AsyncIterator so we have to consume it. coro = self._slurp(aref, aiter(aref)) req_version = (3, 8) cur_version = sys.version_info if cur_version >= req_version: if isinstance(self.channel, TopicT): name = self.channel.get_topic_name() else: name = "channel" task = asyncio.Task( self._execute_actor(coro, aref), loop=self.loop, name=f"{str(aref)}-{name}", ) else: task = asyncio.Task( self._execute_actor(coro, aref), loop=self.loop, ) task._beacon = beacon # type: ignore aref.actor_task = task self._actors.add(aref) return aref
def to_credentials(obj: CredentialsArg = None) -> Optional[CredentialsT]: if obj is not None: if isinstance(obj, ssl.SSLContext): return SSLCredentials(obj) if isinstance(obj, CredentialsT): return obj raise ImproperlyConfigured( f'Unknown credentials type {type(obj)}: {obj}') return None
async def on_first_start(self) -> None: if not self.app.agents: # XXX I can imagine use cases where an app is useful # without agents, but use this as more of an assertion # to make sure agents are registered correctly. [ask] raise ImproperlyConfigured( 'Attempting to start app that has no agents') self.app._create_directories() await self.app.on_first_start()
def to_credentials(obj: CredentialsArg = None) -> Optional[CredentialsT]: if obj is not None: from faust.auth import SSLCredentials # XXX :( if isinstance(obj, ssl.SSLContext): return SSLCredentials(obj) if isinstance(obj, CredentialsT): return obj from faust.exceptions import ImproperlyConfigured raise ImproperlyConfigured( f'Unknown credentials type {type(obj)}: {obj}') return None
def _relative_handler(self, relative_to: RelativeArg) -> Optional[RelativeHandler]: if relative_to is None: return None elif isinstance(relative_to, datetime): return self.table._relative_timestamp(relative_to.timestamp()) elif isinstance(relative_to, float): return self.table._relative_timestamp(relative_to) elif isinstance(relative_to, FieldDescriptorT): return self.table._relative_field(relative_to) elif callable(relative_to): return relative_to raise ImproperlyConfigured(f"Relative cannot be type {type(relative_to)}")
def __init__( self, url: Union[str, URL], app: AppT, table: CollectionT, *, key_index_size: Optional[int] = None, options: Optional[Mapping[str, Any]] = None, read_only: Optional[bool] = False, **kwargs: Any, ) -> None: if rocksdb is None: error = ImproperlyConfigured( "RocksDB bindings not installed? pip install python-rocksdb" ) try: import rocksdb as _rocksdb # noqa: F401 except Exception as exc: # pragma: no cover raise error from exc else: # pragma: no cover raise error super().__init__(url, app, table, **kwargs) if not self.url.path: self.url /= self.table_name self.options = options or {} self.read_only = self.options.pop("read_only", read_only) self.rocksdb_options = RocksDBOptions(**self.options) if key_index_size is None: key_index_size = app.conf.table_key_index_size self.key_index_size = key_index_size self._dbs = {} self._key_index = LRUCache(limit=self.key_index_size) self.db_lock = asyncio.Lock() self.rebalance_ack = False self._backup_path = os.path.join(self.path, f"{str(self.basename)}-backups") try: self._backup_engine = None if not os.path.isdir(self._backup_path): os.makedirs(self._backup_path, exist_ok=True) testfile = tempfile.TemporaryFile(dir=self._backup_path) testfile.close() except PermissionError: self.log.warning( f'Unable to make directory for path "{self._backup_path}",' f"disabling backups." ) except OSError: self.log.warning( f'Unable to create files in "{self._backup_path}",' f"disabling backups" ) else: self._backup_engine = rocksdb.BackupEngine(self._backup_path)
def finalize(self) -> None: # Finalization signals that the application have been configured # and is ready to use. # If you access configuration before an explicit call to # ``app.finalize()`` you will get an error. # The ``app.main`` entrypoint and the ``faust -A app`` command # both will automatically finalize the app for you. if not self.finalized: self.finalized = True id = self.conf.id if not id: raise ImproperlyConfigured('App requires an id!')
def __init__(self, app: AppT, pattern: str = '/metrics', **kwargs: Any) -> None: self.app = app self.pattern = pattern if prometheus_client is None: raise ImproperlyConfigured( 'prometheus_client requires `pip install prometheus_client`.') self._initialize_metrics() self.expose_metrics() super().__init__(**kwargs)
def _create_worker_consumer( self, transport: 'Transport', loop: asyncio.AbstractEventLoop) -> aiokafka.AIOKafkaConsumer: isolation_level: str = 'read_uncommitted' conf = self.app.conf if self.consumer.in_transaction: isolation_level = 'read_committed' self._assignor = self.app.assignor auth_settings = credentials_to_aiokafka_auth( conf.broker_credentials, conf.ssl_context) max_poll_interval = conf.broker_max_poll_interval or 0 request_timeout = conf.broker_request_timeout session_timeout = conf.broker_session_timeout rebalance_timeout = conf.broker_rebalance_timeout if session_timeout > request_timeout: raise ImproperlyConfigured( f'Setting broker_session_timeout={session_timeout} ' f'cannot be greater than ' f'broker_request_timeout={request_timeout}') return aiokafka.AIOKafkaConsumer( loop=loop, api_version=conf.consumer_api_version, client_id=conf.broker_client_id, group_id=conf.id, bootstrap_servers=server_list( transport.url, transport.default_port), partition_assignment_strategy=[self._assignor], enable_auto_commit=False, auto_offset_reset=conf.consumer_auto_offset_reset, max_poll_records=conf.broker_max_poll_records, max_poll_interval_ms=int(max_poll_interval * 1000.0), max_partition_fetch_bytes=conf.consumer_max_fetch_size, fetch_max_wait_ms=1500, request_timeout_ms=int(request_timeout * 1000.0), check_crcs=conf.broker_check_crcs, session_timeout_ms=int(session_timeout * 1000.0), rebalance_timeout_ms=int(rebalance_timeout * 1000.0), heartbeat_interval_ms=int(conf.broker_heartbeat_interval * 1000.0), isolation_level=isolation_level, traced_from_parent_span=self.traced_from_parent_span, start_rebalancing_span=self.start_rebalancing_span, start_coordinator_span=self.start_coordinator_span, on_generation_id_known=self.on_generation_id_known, flush_spans=self.flush_spans, **auth_settings, )
def __init__(self, host: str = 'localhost', port: int = 8125, prefix: str = 'faust-app', rate: float = 1.0, **kwargs: Any) -> None: self.host = host self.port = port self.prefix = prefix self.rate = rate if statsd is None: raise ImproperlyConfigured( 'StatsMonitor requires `pip install statsd`.') super().__init__(**kwargs)
def __init__(self, host: str = 'localhost', port: int = 8125, prefix: str = 'faust-app', rate: float = 1.0, **kwargs: Any) -> None: self.host = host self.port = port self.prefix = prefix self.rate = rate if datadog is None: raise ImproperlyConfigured( f'{type(self).__name__} requires `pip install datadog`.') super().__init__(**kwargs)
def __init__( self, fun: AgentFun, *, app: AppT, name: str = None, channel: Union[str, ChannelT] = None, concurrency: int = 1, sink: Iterable[SinkT] = None, on_error: AgentErrorHandler = None, supervisor_strategy: Type[SupervisorStrategyT] = None, help: str = None, schema: SchemaT = None, key_type: ModelArg = None, value_type: ModelArg = None, isolated_partitions: bool = False, use_reply_headers: bool = None, **kwargs: Any, ) -> None: self.app = app self.fun: AgentFun = fun self.name = name or canonshortname(self.fun) # key-type/value_type arguments only apply when a channel # is not set if schema is not None: assert channel is None or isinstance(channel, str) if key_type is not None: assert channel is None or isinstance(channel, str) self._key_type = key_type if value_type is not None: assert channel is None or isinstance(channel, str) self._schema = schema self._value_type = value_type self._channel_arg = channel self._channel_kwargs = kwargs self.concurrency = concurrency or 1 self.isolated_partitions = isolated_partitions self.help = help or "" self._sinks = list(sink) if sink is not None else [] self._on_error: Optional[AgentErrorHandler] = on_error self.supervisor_strategy = supervisor_strategy self._actors = WeakSet() self._actor_by_partition = WeakValueDictionary() if self.isolated_partitions and self.concurrency > 1: raise ImproperlyConfigured( "Agent concurrency must be 1 when using isolated partitions" ) self.use_reply_headers = use_reply_headers Service.__init__(self)
async def _prepare_actor(self, aref: ActorRefT, beacon: NodeT) -> ActorRefT: coro: Any if isinstance(aref, Awaitable): # agent does not yield coro = aref if self._sinks: raise ImproperlyConfigured("Agent must yield to use sinks") else: # agent yields and is an AsyncIterator so we have to consume it. coro = self._slurp(aref, aiter(aref)) task = asyncio.Task(self._execute_actor(coro, aref), loop=self.loop) task._beacon = beacon # type: ignore aref.actor_task = task self._actors.add(aref) return aref
def _discovery_modules(self) -> List[str]: modules: List[str] = [] autodiscover = self.conf.autodiscover if autodiscover: if isinstance(autodiscover, bool): if self.conf.origin is None: raise ImproperlyConfigured(E_NEED_ORIGIN) elif callable(autodiscover): modules.extend( cast(Callable[[], Iterator[str]], autodiscover)()) else: modules.extend(autodiscover) if self.conf.origin: modules.append(self.conf.origin) return modules
def setup_prometheus_sensors( app: AppT, pattern: str = "/metrics", registry: CollectorRegistry = REGISTRY, name_prefix: str = None, ) -> None: """ A utility function which sets up prometheus and attaches the config to the app. @param app: the faust app instance @param pattern: the url pattern for prometheus @param registry: the prometheus registry @param name_prefix: the name prefix. Defaults to the app name @return: None """ if prometheus_client is None: raise ImproperlyConfigured( "prometheus_client requires `pip install prometheus_client`." ) if name_prefix is None: app_conf_name = app.conf.name app.logger.info( "Name prefix is not supplied. Using the name %s from App config.", app_conf_name, ) if "-" in app_conf_name: name_prefix = app_conf_name.replace("-", "_") app.logger.warning( "App config name %s does not conform to" " Prometheus naming conventions." " Using %s as a name_prefix.", app_conf_name, name_prefix, ) faust_metrics = FaustMetrics.create(registry, name_prefix) app.monitor = PrometheusMonitor(metrics=faust_metrics) @app.page(pattern) async def metrics_handler(self: _web.View, request: _web.Request) -> _web.Response: headers = {"Content-Type": CONTENT_TYPE_LATEST} return cast( _web.Response, Response(body=generate_latest(REGISTRY), headers=headers, status=200), )
def __init__(self, app: AppT, pm_config: PrometheusMonitorConfig = None, **kwargs) -> None: self.app = app if pm_config is None: self.pm_config = PrometheusMonitorConfig else: self.pm_config = pm_config if prometheus_client is None: raise ImproperlyConfigured( 'prometheus_client requires `pip install prometheus_client`.') self._python_gc_metrics() self._metrics = PrometheusMetrics(self.pm_config) self.expose_metrics() super().__init__(**kwargs)
def __init__(self, url: Union[str, URL], app: AppT, table: CollectionT, *, key_index_size: int = None, options: Mapping[str, Any] = None, **kwargs: Any) -> None: if rocksdb is None: raise ImproperlyConfigured( 'RocksDB bindings not installed: pip install python-rocksdb') super().__init__(url, app, table, **kwargs) if not self.url.path: self.url /= self.table_name self.options = options or {} self.rocksdb_options = RocksDBOptions(**self.options) if key_index_size is None: key_index_size = app.conf.table_key_index_size self.key_index_size = key_index_size self._dbs = {} self._key_index = LRUCache(limit=self.key_index_size)
def setup_prometheus_sensors(app: AppT, pattern: str = "/metrics", registry: CollectorRegistry = REGISTRY) -> None: if prometheus_client is None: raise ImproperlyConfigured( "prometheus_client requires `pip install prometheus_client`.") faust_metrics = FaustMetrics.create(registry) app.monitor = PrometheusMonitor(metrics=faust_metrics) @app.page(pattern) async def metrics_handler(self: _web.View, request: _web.Request) -> _web.Response: headers = {"Content-Type": CONTENT_TYPE_LATEST} return cast( _web.Response, Response(body=generate_latest(REGISTRY), headers=headers, status=200), )
def setup(app: AppT, *, dsn: str = None, workers: int = 4, max_queue_size: int = 1000, loglevel: int = None) -> None: sentry_handler = handler_from_dsn( dsn=dsn, workers=workers, qsize=max_queue_size, loglevel=loglevel, ) if sentry_handler is not None: if sentry_sdk is None or _sdk_aiohttp is None: raise ImproperlyConfigured( 'faust.contrib.sentry requires the `sentry_sdk` library.') sentry_sdk.init( dsn=dsn, integrations=[_sdk_aiohttp.AioHttpIntegration()], ) app.conf.loghandlers.append(sentry_handler)
def _prepare_compat_settings(self, options: MutableMapping) -> Mapping: COMPAT_OPTIONS = { 'client_id': 'broker_client_id', 'commit_interval': 'broker_commit_interval', 'create_reply_topic': 'reply_create_topic', 'num_standby_replicas': 'table_standby_replicas', 'default_partitions': 'topic_partitions', 'replication_factor': 'topic_replication_factor', } for old, new in COMPAT_OPTIONS.items(): val = options.get(new) try: options[new] = options[old] except KeyError: pass else: if val is not None: raise ImproperlyConfigured( f'Cannot use both compat option {old!r} and {new!r}') warnings.warn( FutureWarning(W_OPTION_DEPRECATED.format(old=old, new=new))) return options
def version(self, version: int) -> None: if not version: raise ImproperlyConfigured( f'Version cannot be {version}, please start at 1') self._version = version
def _out_of_range(self, value: float) -> ImproperlyConfigured: return ImproperlyConfigured( f"Value {value} is out of range for {self.class_name} " f"(min={self.min_value} max={self.max_value})")
def _dumps(self, s: Any) -> bytes: if _yaml is None: raise ImproperlyConfigured('Missing yaml: pip install PyYAML') return want_bytes(_yaml.safe_dump(s))
def _loads(self, s: bytes) -> Any: if _yaml is None: raise ImproperlyConfigured('Missing yaml: pip install PyYAML') return _yaml.safe_load(want_str(s))