def handler_from_dsn(dsn: str = None,
workers: int = 5,
include_paths: Iterable[str] = None,
loglevel: int = None,
qsize: int = 1000,
**kwargs: Any) -> Optional[logging.Handler]:
if raven is None:
raise ImproperlyConfigured(
'faust.contrib.sentry requires the `raven` library.')
if raven_aiohttp is None:
raise ImproperlyConfigured(
'faust.contrib.sentry requires the `raven_aiohttp` library.')
level: int = loglevel if loglevel is not None else DEFAULT_LEVEL
if dsn:
client = raven.Client(dsn=dsn,
include_paths=include_paths,
transport=partial(
raven_aiohttp.QueuedAioHttpTransport,
workers=workers,
qsize=qsize,
),
disable_existing_loggers=False,
**kwargs)
handler = _build_sentry_handler()(client)
handler.setLevel(level)
return handler
return None
def credentials_to_aiokafka_auth(credentials: CredentialsT = None,
ssl_context: Any = None) -> Mapping:
if credentials is not None:
if isinstance(credentials, SSLCredentials):
return {
'security_protocol': credentials.protocol.value,
'ssl_context': credentials.context,
}
elif isinstance(credentials, SASLCredentials):
return {
'security_protocol': credentials.protocol.value,
'sasl_mechanism': credentials.mechanism.value,
'sasl_plain_username': credentials.username,
'sasl_plain_password': credentials.password,
'ssl_context': credentials.ssl_context,
}
elif isinstance(credentials, GSSAPICredentials):
return {
'security_protocol': credentials.protocol.value,
'sasl_mechanism': credentials.mechanism.value,
'sasl_kerberos_service_name':
credentials.kerberos_service_name,
'sasl_kerberos_domain_name': credentials.kerberos_domain_name,
'ssl_context': credentials.ssl_context,
}
else:
raise ImproperlyConfigured(
f'aiokafka does not support {credentials}')
elif ssl_context is not None:
return {
'security_protocol': 'SSL',
'ssl_context': ssl_context,
}
else:
return {'security_protocol': 'PLAINTEXT'}
def setup_prometheus_sensors(
app: AppT,
pattern: str = "/metrics",
registry: CollectorRegistry = REGISTRY,
name_prefix: Optional[str] = None,
) -> None:
"""
A utility function which sets up prometheus and attaches the config to the app.
@param app: the faust app instance
@param pattern: the url pattern for prometheus
@param registry: the prometheus registry
@param name_prefix: the name prefix. Defaults to the app name
@return: None
"""
if prometheus_client is None:
raise ImproperlyConfigured(
"prometheus_client requires `pip install prometheus_client`."
)
if name_prefix is None:
name_prefix = app.conf.name
name_prefix = name_prefix.replace("-", "_").replace(".", "_")
faust_metrics = FaustMetrics.create(registry, name_prefix)
app.monitor = PrometheusMonitor(metrics=faust_metrics)
@app.page(pattern)
async def metrics_handler(self: _web.View, request: _web.Request) -> _web.Response:
headers = {"Content-Type": CONTENT_TYPE_LATEST}
return cast(
_web.Response,
Response(body=generate_latest(REGISTRY), headers=headers, status=200),
)
def conf(self) -> Settings:
if not self.finalized and STRICT:
raise ImproperlyConfigured(
'App configuration accessed before app.finalize()')
if self._conf is None:
self._configure()
return cast(Settings, self._conf)
def __init__(
self,
url: Union[str, URL],
app: AppT,
table: CollectionT,
*,
key_index_size: int = None,
options: Mapping[str, Any] = None,
**kwargs: Any,
) -> None:
if rocksdb is None:
error = ImproperlyConfigured(
"RocksDB bindings not installed? pip install python-rocksdb")
try:
import rocksdb as _rocksdb # noqa: F401
except Exception as exc: # pragma: no cover
raise error from exc
else: # pragma: no cover
raise error
super().__init__(url, app, table, **kwargs)
if not self.url.path:
self.url /= self.table_name
self.options = options or {}
self.rocksdb_options = RocksDBOptions(**self.options)
if key_index_size is None:
key_index_size = app.conf.table_key_index_size
self.key_index_size = key_index_size
self._dbs = {}
self._key_index = LRUCache(limit=self.key_index_size)
self.db_lock = asyncio.Lock()
self.rebalance_ack = False
class Store(base.SerializedStore):
"""RocksDB table storage."""
offset_key = b'__faust\0offset__'
#: Decides the size of the K=>TopicPartition index (10_000).
key_index_size: int
#: Used to configure the RocksDB settings for table stores.
options: RocksDBOptions
_dbs: MutableMapping[int, DB]
_key_index: LRUCache[bytes, int]
def __init__(self,
url: Union[str, URL],
app: AppT,
table: CollectionT,
*,
key_index_size: int = 10_000,
options: Mapping = None,
**kwargs: Any) -> None:
if rocksdb is None:
raise ImproperlyConfigured(
'RocksDB bindings not installed: pip install python-rocksdb')
super().__init__(url, app, table, **kwargs)
if not self.url.path:
self.url /= self.table_name
self.options = RocksDBOptions(**options or {})
self.key_index_size = key_index_size
self._dbs = {}
self._key_index = LRUCache(limit=self.key_index_size)
async def _prepare_actor(self, aref: ActorRefT,
beacon: NodeT) -> ActorRefT:
coro: Any
if isinstance(aref, Awaitable):
# agent does not yield
coro = aref
if self._sinks:
raise ImproperlyConfigured("Agent must yield to use sinks")
else:
# agent yields and is an AsyncIterator so we have to consume it.
coro = self._slurp(aref, aiter(aref))
req_version = (3, 8)
cur_version = sys.version_info
if cur_version >= req_version:
if isinstance(self.channel, TopicT):
name = self.channel.get_topic_name()
else:
name = "channel"
task = asyncio.Task(
self._execute_actor(coro, aref),
loop=self.loop,
name=f"{str(aref)}-{name}",
)
else:
task = asyncio.Task(
self._execute_actor(coro, aref),
loop=self.loop,
)
task._beacon = beacon # type: ignore
aref.actor_task = task
self._actors.add(aref)
return aref
def to_credentials(obj: CredentialsArg = None) -> Optional[CredentialsT]:
if obj is not None:
if isinstance(obj, ssl.SSLContext):
return SSLCredentials(obj)
if isinstance(obj, CredentialsT):
return obj
raise ImproperlyConfigured(
f'Unknown credentials type {type(obj)}: {obj}')
return None
async def on_first_start(self) -> None:
if not self.app.agents:
# XXX I can imagine use cases where an app is useful
# without agents, but use this as more of an assertion
# to make sure agents are registered correctly. [ask]
raise ImproperlyConfigured(
'Attempting to start app that has no agents')
self.app._create_directories()
await self.app.on_first_start()
def to_credentials(obj: CredentialsArg = None) -> Optional[CredentialsT]:
if obj is not None:
from faust.auth import SSLCredentials # XXX :(
if isinstance(obj, ssl.SSLContext):
return SSLCredentials(obj)
if isinstance(obj, CredentialsT):
return obj
from faust.exceptions import ImproperlyConfigured
raise ImproperlyConfigured(
f'Unknown credentials type {type(obj)}: {obj}')
return None
def _relative_handler(self, relative_to: RelativeArg) -> Optional[RelativeHandler]:
if relative_to is None:
return None
elif isinstance(relative_to, datetime):
return self.table._relative_timestamp(relative_to.timestamp())
elif isinstance(relative_to, float):
return self.table._relative_timestamp(relative_to)
elif isinstance(relative_to, FieldDescriptorT):
return self.table._relative_field(relative_to)
elif callable(relative_to):
return relative_to
raise ImproperlyConfigured(f"Relative cannot be type {type(relative_to)}")
def __init__(
self,
url: Union[str, URL],
app: AppT,
table: CollectionT,
*,
key_index_size: Optional[int] = None,
options: Optional[Mapping[str, Any]] = None,
read_only: Optional[bool] = False,
**kwargs: Any,
) -> None:
if rocksdb is None:
error = ImproperlyConfigured(
"RocksDB bindings not installed? pip install python-rocksdb"
)
try:
import rocksdb as _rocksdb # noqa: F401
except Exception as exc: # pragma: no cover
raise error from exc
else: # pragma: no cover
raise error
super().__init__(url, app, table, **kwargs)
if not self.url.path:
self.url /= self.table_name
self.options = options or {}
self.read_only = self.options.pop("read_only", read_only)
self.rocksdb_options = RocksDBOptions(**self.options)
if key_index_size is None:
key_index_size = app.conf.table_key_index_size
self.key_index_size = key_index_size
self._dbs = {}
self._key_index = LRUCache(limit=self.key_index_size)
self.db_lock = asyncio.Lock()
self.rebalance_ack = False
self._backup_path = os.path.join(self.path, f"{str(self.basename)}-backups")
try:
self._backup_engine = None
if not os.path.isdir(self._backup_path):
os.makedirs(self._backup_path, exist_ok=True)
testfile = tempfile.TemporaryFile(dir=self._backup_path)
testfile.close()
except PermissionError:
self.log.warning(
f'Unable to make directory for path "{self._backup_path}",'
f"disabling backups."
)
except OSError:
self.log.warning(
f'Unable to create files in "{self._backup_path}",' f"disabling backups"
)
else:
self._backup_engine = rocksdb.BackupEngine(self._backup_path)
def finalize(self) -> None:
# Finalization signals that the application have been configured
# and is ready to use.
# If you access configuration before an explicit call to
# ``app.finalize()`` you will get an error.
# The ``app.main`` entrypoint and the ``faust -A app`` command
# both will automatically finalize the app for you.
if not self.finalized:
self.finalized = True
id = self.conf.id
if not id:
raise ImproperlyConfigured('App requires an id!')
def __init__(self,
app: AppT,
pattern: str = '/metrics',
**kwargs: Any) -> None:
self.app = app
self.pattern = pattern
if prometheus_client is None:
raise ImproperlyConfigured(
'prometheus_client requires `pip install prometheus_client`.')
self._initialize_metrics()
self.expose_metrics()
super().__init__(**kwargs)
def _create_worker_consumer(
self,
transport: 'Transport',
loop: asyncio.AbstractEventLoop) -> aiokafka.AIOKafkaConsumer:
isolation_level: str = 'read_uncommitted'
conf = self.app.conf
if self.consumer.in_transaction:
isolation_level = 'read_committed'
self._assignor = self.app.assignor
auth_settings = credentials_to_aiokafka_auth(
conf.broker_credentials, conf.ssl_context)
max_poll_interval = conf.broker_max_poll_interval or 0
request_timeout = conf.broker_request_timeout
session_timeout = conf.broker_session_timeout
rebalance_timeout = conf.broker_rebalance_timeout
if session_timeout > request_timeout:
raise ImproperlyConfigured(
f'Setting broker_session_timeout={session_timeout} '
f'cannot be greater than '
f'broker_request_timeout={request_timeout}')
return aiokafka.AIOKafkaConsumer(
loop=loop,
api_version=conf.consumer_api_version,
client_id=conf.broker_client_id,
group_id=conf.id,
bootstrap_servers=server_list(
transport.url, transport.default_port),
partition_assignment_strategy=[self._assignor],
enable_auto_commit=False,
auto_offset_reset=conf.consumer_auto_offset_reset,
max_poll_records=conf.broker_max_poll_records,
max_poll_interval_ms=int(max_poll_interval * 1000.0),
max_partition_fetch_bytes=conf.consumer_max_fetch_size,
fetch_max_wait_ms=1500,
request_timeout_ms=int(request_timeout * 1000.0),
check_crcs=conf.broker_check_crcs,
session_timeout_ms=int(session_timeout * 1000.0),
rebalance_timeout_ms=int(rebalance_timeout * 1000.0),
heartbeat_interval_ms=int(conf.broker_heartbeat_interval * 1000.0),
isolation_level=isolation_level,
traced_from_parent_span=self.traced_from_parent_span,
start_rebalancing_span=self.start_rebalancing_span,
start_coordinator_span=self.start_coordinator_span,
on_generation_id_known=self.on_generation_id_known,
flush_spans=self.flush_spans,
**auth_settings,
)
def __init__(self,
host: str = 'localhost',
port: int = 8125,
prefix: str = 'faust-app',
rate: float = 1.0,
**kwargs: Any) -> None:
self.host = host
self.port = port
self.prefix = prefix
self.rate = rate
if statsd is None:
raise ImproperlyConfigured(
'StatsMonitor requires `pip install statsd`.')
super().__init__(**kwargs)
def __init__(self,
host: str = 'localhost',
port: int = 8125,
prefix: str = 'faust-app',
rate: float = 1.0,
**kwargs: Any) -> None:
self.host = host
self.port = port
self.prefix = prefix
self.rate = rate
if datadog is None:
raise ImproperlyConfigured(
f'{type(self).__name__} requires `pip install datadog`.')
super().__init__(**kwargs)
def __init__(
self,
fun: AgentFun,
*,
app: AppT,
name: str = None,
channel: Union[str, ChannelT] = None,
concurrency: int = 1,
sink: Iterable[SinkT] = None,
on_error: AgentErrorHandler = None,
supervisor_strategy: Type[SupervisorStrategyT] = None,
help: str = None,
schema: SchemaT = None,
key_type: ModelArg = None,
value_type: ModelArg = None,
isolated_partitions: bool = False,
use_reply_headers: bool = None,
**kwargs: Any,
) -> None:
self.app = app
self.fun: AgentFun = fun
self.name = name or canonshortname(self.fun)
# key-type/value_type arguments only apply when a channel
# is not set
if schema is not None:
assert channel is None or isinstance(channel, str)
if key_type is not None:
assert channel is None or isinstance(channel, str)
self._key_type = key_type
if value_type is not None:
assert channel is None or isinstance(channel, str)
self._schema = schema
self._value_type = value_type
self._channel_arg = channel
self._channel_kwargs = kwargs
self.concurrency = concurrency or 1
self.isolated_partitions = isolated_partitions
self.help = help or ""
self._sinks = list(sink) if sink is not None else []
self._on_error: Optional[AgentErrorHandler] = on_error
self.supervisor_strategy = supervisor_strategy
self._actors = WeakSet()
self._actor_by_partition = WeakValueDictionary()
if self.isolated_partitions and self.concurrency > 1:
raise ImproperlyConfigured(
"Agent concurrency must be 1 when using isolated partitions"
)
self.use_reply_headers = use_reply_headers
Service.__init__(self)
async def _prepare_actor(self, aref: ActorRefT, beacon: NodeT) -> ActorRefT:
coro: Any
if isinstance(aref, Awaitable):
# agent does not yield
coro = aref
if self._sinks:
raise ImproperlyConfigured("Agent must yield to use sinks")
else:
# agent yields and is an AsyncIterator so we have to consume it.
coro = self._slurp(aref, aiter(aref))
task = asyncio.Task(self._execute_actor(coro, aref), loop=self.loop)
task._beacon = beacon # type: ignore
aref.actor_task = task
self._actors.add(aref)
return aref
def _discovery_modules(self) -> List[str]:
modules: List[str] = []
autodiscover = self.conf.autodiscover
if autodiscover:
if isinstance(autodiscover, bool):
if self.conf.origin is None:
raise ImproperlyConfigured(E_NEED_ORIGIN)
elif callable(autodiscover):
modules.extend(
cast(Callable[[], Iterator[str]], autodiscover)())
else:
modules.extend(autodiscover)
if self.conf.origin:
modules.append(self.conf.origin)
return modules
def setup_prometheus_sensors(
app: AppT,
pattern: str = "/metrics",
registry: CollectorRegistry = REGISTRY,
name_prefix: str = None,
) -> None:
"""
A utility function which sets up prometheus and attaches the config to the app.
@param app: the faust app instance
@param pattern: the url pattern for prometheus
@param registry: the prometheus registry
@param name_prefix: the name prefix. Defaults to the app name
@return: None
"""
if prometheus_client is None:
raise ImproperlyConfigured(
"prometheus_client requires `pip install prometheus_client`."
)
if name_prefix is None:
app_conf_name = app.conf.name
app.logger.info(
"Name prefix is not supplied. Using the name %s from App config.",
app_conf_name,
)
if "-" in app_conf_name:
name_prefix = app_conf_name.replace("-", "_")
app.logger.warning(
"App config name %s does not conform to"
" Prometheus naming conventions."
" Using %s as a name_prefix.",
app_conf_name,
name_prefix,
)
faust_metrics = FaustMetrics.create(registry, name_prefix)
app.monitor = PrometheusMonitor(metrics=faust_metrics)
@app.page(pattern)
async def metrics_handler(self: _web.View, request: _web.Request) -> _web.Response:
headers = {"Content-Type": CONTENT_TYPE_LATEST}
return cast(
_web.Response,
Response(body=generate_latest(REGISTRY), headers=headers, status=200),
)
def __init__(self,
app: AppT,
pm_config: PrometheusMonitorConfig = None,
**kwargs) -> None:
self.app = app
if pm_config is None:
self.pm_config = PrometheusMonitorConfig
else:
self.pm_config = pm_config
if prometheus_client is None:
raise ImproperlyConfigured(
'prometheus_client requires `pip install prometheus_client`.')
self._python_gc_metrics()
self._metrics = PrometheusMetrics(self.pm_config)
self.expose_metrics()
super().__init__(**kwargs)
def __init__(self,
url: Union[str, URL],
app: AppT,
table: CollectionT,
*,
key_index_size: int = None,
options: Mapping[str, Any] = None,
**kwargs: Any) -> None:
if rocksdb is None:
raise ImproperlyConfigured(
'RocksDB bindings not installed: pip install python-rocksdb')
super().__init__(url, app, table, **kwargs)
if not self.url.path:
self.url /= self.table_name
self.options = options or {}
self.rocksdb_options = RocksDBOptions(**self.options)
if key_index_size is None:
key_index_size = app.conf.table_key_index_size
self.key_index_size = key_index_size
self._dbs = {}
self._key_index = LRUCache(limit=self.key_index_size)
def setup_prometheus_sensors(app: AppT,
pattern: str = "/metrics",
registry: CollectorRegistry = REGISTRY) -> None:
if prometheus_client is None:
raise ImproperlyConfigured(
"prometheus_client requires `pip install prometheus_client`.")
faust_metrics = FaustMetrics.create(registry)
app.monitor = PrometheusMonitor(metrics=faust_metrics)
@app.page(pattern)
async def metrics_handler(self: _web.View,
request: _web.Request) -> _web.Response:
headers = {"Content-Type": CONTENT_TYPE_LATEST}
return cast(
_web.Response,
Response(body=generate_latest(REGISTRY),
headers=headers,
status=200),
)
def setup(app: AppT,
*,
dsn: str = None,
workers: int = 4,
max_queue_size: int = 1000,
loglevel: int = None) -> None:
sentry_handler = handler_from_dsn(
dsn=dsn,
workers=workers,
qsize=max_queue_size,
loglevel=loglevel,
)
if sentry_handler is not None:
if sentry_sdk is None or _sdk_aiohttp is None:
raise ImproperlyConfigured(
'faust.contrib.sentry requires the `sentry_sdk` library.')
sentry_sdk.init(
dsn=dsn,
integrations=[_sdk_aiohttp.AioHttpIntegration()],
)
app.conf.loghandlers.append(sentry_handler)
def _prepare_compat_settings(self, options: MutableMapping) -> Mapping:
COMPAT_OPTIONS = {
'client_id': 'broker_client_id',
'commit_interval': 'broker_commit_interval',
'create_reply_topic': 'reply_create_topic',
'num_standby_replicas': 'table_standby_replicas',
'default_partitions': 'topic_partitions',
'replication_factor': 'topic_replication_factor',
}
for old, new in COMPAT_OPTIONS.items():
val = options.get(new)
try:
options[new] = options[old]
except KeyError:
pass
else:
if val is not None:
raise ImproperlyConfigured(
f'Cannot use both compat option {old!r} and {new!r}')
warnings.warn(
FutureWarning(W_OPTION_DEPRECATED.format(old=old,
new=new)))
return options
def version(self, version: int) -> None:
if not version:
raise ImproperlyConfigured(
f'Version cannot be {version}, please start at 1')
self._version = version
def _out_of_range(self, value: float) -> ImproperlyConfigured:
return ImproperlyConfigured(
f"Value {value} is out of range for {self.class_name} "
f"(min={self.min_value} max={self.max_value})")
def _dumps(self, s: Any) -> bytes:
if _yaml is None:
raise ImproperlyConfigured('Missing yaml: pip install PyYAML')
return want_bytes(_yaml.safe_dump(s))
def _loads(self, s: bytes) -> Any:
if _yaml is None:
raise ImproperlyConfigured('Missing yaml: pip install PyYAML')
return _yaml.safe_load(want_str(s))
