def _finalize_concrete_app(self, app: AppT) -> AppT:
app.finalize()
# XXX How to find full argv[0] with click?
origin = app.conf.origin
if sys.argv:
origin = self._detect_main_package(sys.argv)
return prepare_app(app, origin)
def init_faust_crontabs(app: AppT, handlers_owner: object, on_handler_executed: Optional[Callable[[], None]]):
name_and_method_and_crontab_details = \
filter(lambda name_and_method_and_crontab_detail:
name_and_method_and_crontab_detail[2] is not None,
map(lambda name_and_method: (name_and_method[0], name_and_method[1],
getattr(name_and_method[1],
CrontabHandler.ATTR_CRONTAB_DETAIL,
None)),
inspect.getmembers(handlers_owner, inspect.ismethod)))
for name, method_obj, crontab_detail in name_and_method_and_crontab_details:
async def crontab_method(*args, **kwargs):
args_with_app_dropped = args
if len(args) >= 1 and isinstance(args[-1], App):
args_with_app_dropped = args[0:len(args) - 1]
res = await method_obj(*args_with_app_dropped, **kwargs)
await process_handler_sync_result(res)
if on_handler_executed is not None:
res = on_handler_executed()
if inspect.isawaitable(res):
await res
app.crontab(cron_format=crontab_detail.format, timezone=crontab_detail.timezone)(crontab_method)
def init_faust_timers(app: AppT, handlers_owner: object,
on_handler_executed: Optional[Callable[[], None]]):
name_and_method_and_timer_intervals = \
filter(lambda name_and_method_and_timer_interval:
name_and_method_and_timer_interval[2] is not None,
map(lambda name_and_method: (name_and_method[0], name_and_method[1],
getattr(name_and_method[1],
TimerHandler.ATTR_TIMER_INTERVAL,
None)),
inspect.getmembers(handlers_owner, inspect.ismethod)))
for name, method_obj, timer_interval in name_and_method_and_timer_intervals:
async def timer_method(*args, **kwargs):
args_with_app_dropped = args
if len(args) >= 1 and isinstance(args[-1], App):
args_with_app_dropped = args[0:len(args) - 1]
res = await method_obj(*args_with_app_dropped, **kwargs)
await process_handler_sync_result(res)
if on_handler_executed is not None:
res = on_handler_executed()
if inspect.isawaitable(res):
await res
app.timer(interval=timer_interval)(timer_method)
def _apply_route(self, app: AppT, route: FutureRoute,
url_prefix: Optional[str]) -> None:
uri = url_prefix + route.uri if url_prefix else route.uri
app.page(
path=uri[1:] if uri.startswith('//') else uri,
name=self._view_name(route.name),
)(route.handler)
def _apply_route(self, app: AppT, route: FutureRoute,
url_prefix: Optional[str]) -> None:
uri = self._url_with_prefix(route.uri, url_prefix)
# Create the actual view on the app (using app.page)
app.page(
path=uri[1:] if uri.startswith('//') else uri,
name=self._view_name(route.name),
)(route.handler)
def prepare_app(app: AppT, name: Optional[str]) -> AppT:
app.finalize()
if app.conf.origin is None:
app.conf.origin = name
if app.conf.autodiscover:
app.discover()
# Hack to fix cProfile support.
main = sys.modules.get('__main__')
if main is not None and 'cProfile.py' in getattr(main, '__file__', ''):
from ..models import registry
registry.update({(app.conf.origin or '') + k[8:]: v
for k, v in registry.items()
if k.startswith('cProfile.')})
return app
def setup_prometheus_sensors(
app: AppT,
pattern: str = "/metrics",
registry: CollectorRegistry = REGISTRY,
name_prefix: Optional[str] = None,
) -> None:
"""
A utility function which sets up prometheus and attaches the config to the app.
@param app: the faust app instance
@param pattern: the url pattern for prometheus
@param registry: the prometheus registry
@param name_prefix: the name prefix. Defaults to the app name
@return: None
"""
if prometheus_client is None:
raise ImproperlyConfigured(
"prometheus_client requires `pip install prometheus_client`."
)
if name_prefix is None:
name_prefix = app.conf.name
name_prefix = name_prefix.replace("-", "_").replace(".", "_")
faust_metrics = FaustMetrics.create(registry, name_prefix)
app.monitor = PrometheusMonitor(metrics=faust_metrics)
@app.page(pattern)
async def metrics_handler(self: _web.View, request: _web.Request) -> _web.Response:
headers = {"Content-Type": CONTENT_TYPE_LATEST}
return cast(
_web.Response,
Response(body=generate_latest(REGISTRY), headers=headers, status=200),
)
def _contribute_to_app(self, app: AppT) -> None:
from .patches.aiohttp import LiveCheckMiddleware
web_app = app.web.web_app # type: ignore
web_app.middlewares.append(LiveCheckMiddleware())
app.sensors.add(LiveCheckSensor())
app.livecheck = self # type: ignore
def create_topic(app: AppT, topic: TP) -> TopicT:
FaustUtilities.Admin.check_and_create_topic(topic)
return app.topic(topic.topic,
key_type=bytes,
value_type=bytes,
value_serializer="raw",
key_serializer="raw",
partitions=topic.partition)
def setup_prometheus_sensors(
app: AppT,
pattern: str = "/metrics",
registry: CollectorRegistry = REGISTRY,
name_prefix: str = None,
) -> None:
"""
A utility function which sets up prometheus and attaches the config to the app.
@param app: the faust app instance
@param pattern: the url pattern for prometheus
@param registry: the prometheus registry
@param name_prefix: the name prefix. Defaults to the app name
@return: None
"""
if prometheus_client is None:
raise ImproperlyConfigured(
"prometheus_client requires `pip install prometheus_client`."
)
if name_prefix is None:
app_conf_name = app.conf.name
app.logger.info(
"Name prefix is not supplied. Using the name %s from App config.",
app_conf_name,
)
if "-" in app_conf_name:
name_prefix = app_conf_name.replace("-", "_")
app.logger.warning(
"App config name %s does not conform to"
" Prometheus naming conventions."
" Using %s as a name_prefix.",
app_conf_name,
name_prefix,
)
faust_metrics = FaustMetrics.create(registry, name_prefix)
app.monitor = PrometheusMonitor(metrics=faust_metrics)
@app.page(pattern)
async def metrics_handler(self: _web.View, request: _web.Request) -> _web.Response:
headers = {"Content-Type": CONTENT_TYPE_LATEST}
return cast(
_web.Response,
Response(body=generate_latest(REGISTRY), headers=headers, status=200),
)
def prepare_app(app: AppT, name: Optional[str]) -> AppT:
app.finalize()
if app.conf._origin is None:
app.conf._origin = name
app.worker_init()
if app.conf.autodiscover:
app.discover()
app.worker_init_post_autodiscover()
# Hack to fix cProfile support.
if 1: # pragma: no cover
main = sys.modules.get("__main__")
if main is not None and "cProfile.py" in getattr(main, "__file__", ""):
from ..models import registry
registry.update({(app.conf.origin or "") + k[8:]: v
for k, v in registry.items()
if k.startswith("cProfile.")})
return app
def setup_prometheus_sensors(app: AppT,
pattern: str = "/metrics",
registry: CollectorRegistry = REGISTRY) -> None:
if prometheus_client is None:
raise ImproperlyConfigured(
"prometheus_client requires `pip install prometheus_client`.")
faust_metrics = FaustMetrics.create(registry)
app.monitor = PrometheusMonitor(metrics=faust_metrics)
@app.page(pattern)
async def metrics_handler(self: _web.View,
request: _web.Request) -> _web.Response:
headers = {"Content-Type": CONTENT_TYPE_LATEST}
return cast(
_web.Response,
Response(body=generate_latest(REGISTRY),
headers=headers,
status=200),
)
def __init__(self, app: AppT, name: str, num_of_partitions: int):
super().__init__()
table_name = f"table_of_storage_{name}"
table_help = table_name.replace('_', ' ')
self._table = app.Table(name=table_name, default=lambda: None, key_type=bytes, value_type=bytes,
help=table_help, partitions=num_of_partitions)
def initialize(self, app: AppT, storage_name: str, observer: STATE_OBSERVER):
state_storage = StateStorage(app, storage_name,
1 if self._forward_through_in_mem_channel
else self._stream_template.effective_topic_define.partition)
self._state_storage = state_storage
stateful_transformer = self._stateful_transformer
def do_transform(object_pk: Any, object_state_vars: Mapping[str, Any], headers: Dict[str, Any],
object_pk_bytes: bytes):
if stateful_transformer is None:
object_state_vars_2_save = object_state_vars_4_observer = object_state_vars
else:
transform_result: Mapping[str, StateStreamStorage.TransformedResult] = \
stateful_transformer(state_storage, object_pk, object_state_vars, headers, object_pk_bytes)
if transform_result is not None:
object_state_vars_2_save = dict(
map(lambda name_and_result: (name_and_result[0], name_and_result[1].value),
filter(lambda name_and_result: not name_and_result[1].memory_only,
transform_result.items()))
)
object_state_vars_4_observer = dict(
map(lambda name_and_result: (name_and_result[0], name_and_result[1].value),
transform_result.items())
)
else:
object_state_vars_2_save = object_state_vars_4_observer = None
return object_state_vars_2_save, object_state_vars_4_observer
async def process_transformed_results(object_pk: Any, object_state_vars_2_save: Mapping[str, Any],
object_state_vars_4_observer: Mapping[str, Any],
headers: Dict[str, Any], object_pk_bytes: bytes):
if object_state_vars_2_save:
state_storage(object_pk, object_state_vars_2_save, headers, object_pk_bytes)
if observer is not None and object_state_vars_4_observer:
res = observer(object_pk, object_state_vars_4_observer, headers, object_pk_bytes)
if inspect.isawaitable(res):
await res
if self._forward_through_in_mem_channel:
async def in_mem_channel_observer(object_pk: Any,
object_state_vars: Union[Dict[str, Any],
Tuple[Dict[str, Any], Dict[str, Any]]],
headers: Dict[str, Any], object_pk_bytes: bytes):
if isinstance(object_state_vars, tuple):
object_state_vars_2_save, object_state_vars_4_observer = object_state_vars
else:
object_state_vars_2_save, object_state_vars_4_observer = \
do_transform(object_pk, object_state_vars, headers, object_pk_bytes)
await process_transformed_results(object_pk, object_state_vars_2_save, object_state_vars_4_observer,
headers, object_pk_bytes)
in_mem_channel = app.channel()
in_mem_stream_transformer = StreamBinder()
in_mem_stream_transformer.bind(stream_as_template=None, topic_define=None)
in_mem_stream_transformer.initialize(in_mem_channel, storage_name, in_mem_channel_observer)
in_mem_channel_stream = self._in_mem_channel_stream = in_mem_stream_transformer.stream
def state_stream_observer(object_pk: Any, object_state_vars: Mapping[str, Any],
headers: Dict[str, Any], object_pk_bytes: bytes):
object_state_vars_2_save, object_state_vars_4_observer = \
do_transform(object_pk, object_state_vars, headers, object_pk_bytes)
# object_state_vars_4_observer should contain object_state_vars_2_save thus
# only need to check object_state_vars_4_observer
if object_state_vars_4_observer:
in_mem_channel_stream.upsert_object_state(
object_pk=object_pk,
object_state_vars=(object_state_vars_2_save, object_state_vars_4_observer))
else:
async def state_stream_observer(object_pk: Any, object_state_vars: Mapping[str, Any],
headers: Dict[str, Any], object_pk_bytes: bytes):
object_state_vars_2_save, object_state_vars_4_observer = \
do_transform(object_pk, object_state_vars, headers, object_pk_bytes)
await process_transformed_results(object_pk, object_state_vars_2_save, object_state_vars_4_observer,
headers, object_pk_bytes)
super().initialize(app, storage_name, state_stream_observer)