def test_metrics(mock_app_config): # noqa: F811 context = EventContext(app_config=mock_app_config, plugin_config=mock_app_config, event_name='mock_event', track_ids={}, auth_info={ 'auth_type': AuthType.UNSECURED, 'allowed': 'true' }) context.creation_ts = ZERO_TS metrics.datetime = MockDatetime MockDatetime.ts = 3.0 result = metrics.metrics(context) assert result['extra'] == 'metrics.duration=3000.000'
async def process_log_data(payload: LogRawBatch, context: EventContext) -> Optional[LogBatch]: """ Receives emitted batch of raw log lines from service handler, process and filter in order to emit processed batch to a stream. """ config = context.settings(datatype=LogReaderConfig) logger.info(context, "Processing batch of log entries...", extra=extra(batch_size=len(payload.data))) try: entries: List[LogEntry] = [] for entry in payload.data: processed_entry = await _process_log_entry(entry, context) if processed_entry is not None: entries.append(processed_entry) # type: ignore if len(entries) > 0: return LogBatch(entries=entries) logger.info("Filtered out all entries in batch.") return None except Exception as e: # pylint: disable=broad-except # pragma: no cover logger.error(context, e) return None finally: await asyncio.sleep(config.batch_wait_interval_secs)
async def __service__(context: EventContext) -> Spawn[LogRawBatch]: config = context.settings(datatype=LogReaderConfig) event_handler = LogFileHandler(config, context) logger.info(context, "Starting LogFileHandler...", extra=extra(logs_path=config.logs_path, checkpoint_path=config.checkpoint_path)) observer = start_observer(event_handler, config.logs_path) logger.info(context, "LogFileHandler started.") try: while True: batch = await event_handler.get_and_reset_batch() if len(batch) == 0: logger.info( context, "LogFileHandler returned empty batch. Sleeping...") await asyncio.sleep(config.batch_wait_interval_secs) else: for i in range(0, len(batch), config.batch_size): yield LogRawBatch(data=batch[i:i + config.batch_size + 1]) await asyncio.sleep(config.batch_wait_interval_secs) event_handler.close_inactive_files() except KeyboardInterrupt: # pragma: no cover pass except Exception as e: # pylint: disable=broad-except # pragma: no cover logger.error(context, e) finally: observer.stop() observer.join()
def authorize(context: EventContext, user_info: ContextUserInfo, now: datetime) -> AuthInfoExtended: """ Authorize user and returns auth info containing tokens for api access and authorization renewal :param context: event context from app requesting authorization or login happened :param user_info: already validated user info to be encoded in tokens: Notice this method wont check if user is valid, invoking app should ensure this. :param now: current datetime, fixed as start of authorization process :return: AuthInfoExtended, containing new access and refresh tokens """ cfg: AuthSettings = context.settings(key="auth", datatype=AuthSettings) renew_in = int(1000.0 * max( 1.0 * cfg.access_token_expiration - 1.0 * cfg.access_token_renew_window * (1.0 + 0.5 * random.random()), 0.5 * cfg.access_token_expiration * (0.5 * random.random() + 0.5))) token = _new_access_token(asdict(user_info), context, now, cfg.access_token_expiration, renew_in) refresh_token = _new_refresh_token(asdict(user_info), context, now, cfg.refresh_token_expiration) result = AuthInfoExtended( app=context.app_key, access_token=token, refresh_token=refresh_token, token_type=AuthType.BEARER.name, access_token_expiration=cfg.access_token_expiration, refresh_token_expiration=cfg.refresh_token_expiration, renew=renew_in, user_info=user_info) return result
async def invoke_login(context: EventContext): auth_info = await login.login(None, context) cfg = context.settings(key='auth', datatype=AuthSettings) assert auth_info.token_type == 'BEARER' access_token_info = auth.decode_token(auth_info.access_token) assert access_token_info['app'] == 'test_app.test' assert access_token_info['id'] == 'id' assert access_token_info['email'] == 'test@email' assert access_token_info['user'] == 'test' iat = access_token_info['iat'] assert access_token_info['exp'] == iat + cfg.access_token_expiration assert access_token_info['renew'] > 0 assert access_token_info['renew'] < 1000.0 * ( cfg.access_token_expiration - cfg.access_token_renew_window) refresh_token_info = auth.decode_token(auth_info.refresh_token) assert refresh_token_info['app'] == 'test_app.test' assert refresh_token_info['id'] == 'id' assert refresh_token_info['email'] == 'test@email' assert refresh_token_info['user'] == 'test' iat = refresh_token_info['iat'] assert refresh_token_info['exp'] == iat + cfg.refresh_token_expiration assert auth_info.user_info == ContextUserInfo(id='id', user='******', email='test@email') assert auth_info.access_token_expiration == cfg.access_token_expiration assert auth_info.refresh_token_expiration == cfg.refresh_token_expiration assert auth_info.renew == access_token_info['renew'] return auth_info
async def mock_handle_postprocess(app_config, *, payload, expected, expected_response): handler = EventHandler(app_config=app_config, plugins=[], effective_events=app_config.events) context = EventContext(app_config=app_config, plugin_config=app_config, event_name='plugin_event', track_ids={}, auth_info={ 'auth_type': AuthType.UNSECURED, 'allowed': 'true' }) intermediate_result = None async for res in handler.handle_async_event(context=context, query_args={}, payload=None): intermediate_result = res hook = PostprocessHook() response = await handler.postprocess(context=context, payload=intermediate_result, response=hook) assert hook.headers == expected_response['headers'] assert hook.cookies == expected_response['cookies'] assert hook.status == expected_response['status'] assert response == expected
def _validate_authorization(app_config: AppConfig, context: EventContext, auth_types: List[AuthType], request: web.Request): """ Validates Authorization header from request to provide valid credentials for the methods supported in event configuration. :raise `Unauthorized` if authorization is not valid """ auth_methods = context.event_info.auth if (len(auth_methods) == 0) and (app_config.server is not None): auth_methods = app_config.server.auth.default_auth_methods auth_header = _extract_authorization(auth_methods, request, context) try: method, data = auth_header.split(" ") except ValueError as e: raise BadRequest("Malformed Authorization") from e context.auth_info['allowed'] = False for auth_type in auth_types: if method.upper() == auth_type.name.upper(): auth.validate_auth_method(auth_type, data, context) if context.auth_info.get('allowed'): return None raise Unauthorized(method)
async def invoke_execute(engine: AppEngine, from_app: AppConfig, event_name: str, query_args: dict, payload: DataObject, expected: DataObject, track_ids: Dict[str, str], postprocess_expected: Optional[dict] = None): event_settings = get_event_settings(from_app.effective_settings, event_name) # type: ignore context = EventContext(app_config=from_app, plugin_config=engine.app_config, event_name=event_name, settings=event_settings, track_ids=track_ids, auth_info={ 'auth_type': AuthType.UNSECURED, 'allowed': 'true' }) res = await engine.execute(context=context, query_args=query_args, payload=payload) assert res == expected if postprocess_expected: hook = PostprocessHook() await engine.postprocess(context=context, payload=res, response=hook) assert hook.headers == postprocess_expected['headers'] assert hook.cookies == postprocess_expected['cookies'] assert hook.status == postprocess_expected['status']
def create_test_context(app_config: AppConfig, event_name: str, track_ids: Optional[dict] = None, auth_info: Optional[dict] = None) -> EventContext: """ Creates an EventContext object to be used in tests :param app_config: AppConfig, app confioguration :param event_name: str, event_name to be called :param track_ids: dict, optional: addictional key/values to add to track_ids section. By default required track_ids will be generated with default test or empty values. :param auth_info: dict, optional: additional auth_info to inject. By default Unsecured auth_info will be generated. """ return EventContext( app_config=app_config, plugin_config=app_config, event_name=event_name, settings=get_event_settings(app_config.effective_settings, event_name), # type: ignore track_ids={ **{k: '' for k in app_config.engine.track_headers}, 'track.operation_id': 'test_operation_id', 'track.request_id': 'test_request_id', 'track.request_ts': datetime.now(tz=timezone.utc).isoformat(), **({} if track_ids is None else track_ids) }, auth_info={ 'auth_type': AuthType.UNSECURED, 'allowed': 'true', **({} if auth_info is None else auth_info) })
def _setup_client_context(app_config: AppConfig, server_app_config: AppConfig, register_client_key: bool = True) -> EventContext: _init_engine_logger(app_config) assert app_config.server assert server_app_config.server app_config.server.auth = AuthConfig( secrets_location=f"/tmp/{uuid.uuid4()}", auth_passphrase='test_passphrase', enabled=True, create_keys=True ) auth.init(app_config.app_key(), app_config.server.auth) if register_client_key: os.rename( pathlib.Path(app_config.server.auth.secrets_location) / 'public' / f'{app_config.app_key()}_pub.pem', pathlib.Path(server_app_config.server.auth.secrets_location) / 'public' / f'{app_config.app_key()}_pub.pem' ) return EventContext( app_config=app_config, plugin_config=app_config, event_name='mock_event', settings=get_event_settings(app_config.effective_settings, 'mock_event'), # type: ignore track_ids={}, auth_info={} )
def test_stream_metrics(mock_app_config): # noqa: F811 context = EventContext(app_config=mock_app_config, plugin_config=mock_app_config, event_name='mock_event', track_ids={'track.request_ts': ZERO_TS.isoformat()}, auth_info={ 'auth_type': AuthType.UNSECURED, 'allowed': 'true' }) context.track_ids['stream.submit_ts'] = ONE_TS.isoformat() context.track_ids['stream.read_ts'] = TWO_TS.isoformat() context.creation_ts = ZERO_TS metrics.datetime = MockDatetime MockDatetime.ts = 3.0 result = metrics.stream_metrics(context) assert result[ 'extra'] == 'metrics.stream_age=1000.000 | metrics.request_elapsed=3000.000'
async def test_async_collector(mock_app_config): # noqa: F811 context = EventContext(app_config=mock_app_config, plugin_config=mock_app_config, event_name='mock_event', track_ids={}, auth_info={}) collector = await AsyncCollector()\ .input("0")\ .steps(('step1', step1), ('step2', step2), ('step3', step3))\ .run(context) result = await collector['step3'] assert result == "((0+mock_event+step1)&(0+mock_event+step2)+mock_event+step3)"
def _event_context(mock_app_config, plugin_config): # noqa: F811 return EventContext(app_config=mock_app_config, plugin_config=plugin_config, event_name='login', track_ids={}, auth_info={ 'allowed': True, 'auth_type': AuthType.BASIC, 'payload': 'dGVzdDpwYXNz' })
async def _save_partition(partition_key: str, items: List[DataObject], context: EventContext): settings = context.settings(datatype=FileStorageSettings) path = Path(settings.path) / partition_key file = path / f"{uuid.uuid4()}{SUFFIX}" logger.info(context, f"Saving {file}...") os.makedirs(path.resolve(), exist_ok=True) async with aiofiles.open(file, 'w') as f: for item in items: await f.write(Payload.to_json(item) + "\n")
def _request_start(app_engine: AppEngine, plugin: AppEngine, event_name: str, request: web.Request) -> EventContext: """ Extracts context and track information from a request and logs start of event """ context = EventContext(app_config=app_engine.app_config, plugin_config=plugin.app_config, event_name=event_name, track_ids=_track_ids(request), auth_info=auth_info_default) logger.start(context) return context
async def live_stats(collector: Collector, context: EventContext) -> Collector: settings = context.settings(key='apps_visualizer', datatype=AppsVisualizerSettings) apps = await collector['runtime_apps'] options = await collector['payload'] host_pids = set((server.host_name, server.pid) for _, runtime_info in apps.apps.items() for server in runtime_info.servers if options.host_filter in server.url) event_stats = get_stats( host_pids=host_pids, time_window_secs=settings.live_active_treshold_seconds, recent_secs=settings.live_recent_treshold_seconds) graph = await collector['cytoscape_data'] if len(event_stats) == 0: return graph for item_id, item in graph.data.items(): label = item['data'].get('label', '') source = item['data'].get('source', '') target = item['data'].get('target', '') if label and source and source[ 0] == '>' and source[-len(label):] != label: source += '.' + label if target and target[0] == '>': target = '' keys = filter(bool, [item_id, source, target]) classes = [] for key in keys: s = event_stats.get(key) if s: if s.started: classes.append('STARTED') if s.pending: classes.append('PENDING') if s.recent: classes.append('RECENT') if s.failed: classes.append('FAILED') if s.ignored: classes.append('IGNORED') item['classes'] = _classes(item, classes) target = item['data'].get('target', ' ') if target[0] == '>': target_item = graph.data[target] target_item['classes'] = _classes(target_item, classes) if len(source) > 5 and (source[-5:] == ".POST" or source[-4:] == ".GET" or source[-8:] == "MULTIPART"): source_item = graph.data[source] source_item['classes'] = _classes(source_item, classes) return graph
def test_context() -> EventContext: app_config = AppConfig( app=AppDescriptor(name='test_steps', version='test_version'), events={'test_steps': EventDescriptor(type=EventType.POST)} ) return EventContext( app_config=app_config, plugin_config=app_config, event_name='test_steps', track_ids={}, auth_info={} )
def test_context() -> EventContext: app_config = AppConfig( app=AppDescriptor(name='test_steps', version='test_version'), events={'test_steps': EventDescriptor(type=EventType.POST)} ).setup() return EventContext( app_config=app_config, plugin_config=app_config, event_name='test_steps', settings=get_event_settings(app_config.effective_settings, 'test_steps'), track_ids={}, auth_info={} )
def _event_context(mock_app_config): # noqa: F811 return EventContext(app_config=mock_app_config, plugin_config=mock_app_config, event_name='mock_event_logging', track_ids={ 'track.operation_id': 'test_operation_id', 'track.request_id': 'test_request_id', 'track.request_ts': '2020-01-01T00:00:00Z', 'track.session_id': 'test_session_id' }, auth_info={ 'auth_type': AuthType.UNSECURED, 'allowed': 'true' })
async def __service__(context: EventContext) -> Spawn[FlushSignal]: settings: FileStorageSettings = context.settings( datatype=FileStorageSettings) if settings.flush_seconds: while True: await asyncio.sleep(settings.flush_seconds) for partition_key in list(buffer.keys()): yield FlushSignal(partition_key=partition_key) else: if settings.flush_max_size == 0: logger.warning( context, "Flushing partitions by size and time are disabled." "Specify either `flush_seconds` or `flush_max_size`" "to enable flushing the buffer periodically") else: logger.info(context, "Flushing partitions by time disabled.")
def create_test_context(app_config: AppConfig, event_name: str) -> EventContext: return EventContext(app_config=app_config, plugin_config=app_config, event_name=event_name, track_ids={ 'track.operation_id': 'test_operation_id', 'track.request_id': 'test_request_id', 'track.request_ts': datetime.now(tz=timezone.utc).isoformat() }, auth_info={ 'auth_type': AuthType.UNSECURED, 'allowed': 'true' })
def _setup_event_context(mock_app_config: AppConfig) -> EventContext: # noqa: F811 _init_engine_logger(mock_app_config) assert mock_app_config.server mock_app_config.server.auth = AuthConfig( secrets_location=f"/tmp/{uuid.uuid4()}", auth_passphrase='test_passphrase', enabled=True, create_keys=True ) auth.init(mock_app_config.server.auth) return EventContext( app_config=mock_app_config, plugin_config=mock_app_config, event_name='mock_event', track_ids={}, auth_info={} )
async def mock_handle_request_response_event(app_config, *, payload, expected): handler = EventHandler(app_config=app_config, plugins=[], effective_events=app_config.events) context = EventContext(app_config=app_config, plugin_config=app_config, event_name='mock_post_event', track_ids=MockStreamManager.test_track_ids, auth_info={ 'auth_type': AuthType.UNSECURED, 'allowed': 'true' }) async for response in handler.handle_async_event( context=context, query_args={"query_arg1": payload.value}, payload=payload): assert response == expected
async def buffer_item(payload: DataObject, context: EventContext) -> Optional[FlushSignal]: """ Consumes any Dataobject type from stream and put it local memory buffer to be flushed later """ settings: FileStorageSettings = context.settings( datatype=FileStorageSettings) partition_key = get_partition_key(payload, settings.partition_dateformat or "") async with buffer_lock: partition = buffer.get(partition_key, Partition()) buffer[partition_key] = partition async with partition.lock: partition.items.append(payload) if settings.flush_max_size and len( partition.items) >= settings.flush_max_size: return FlushSignal(partition_key=partition_key) return None
def _setup_server_context(app_config: AppConfig) -> EventContext: _init_engine_logger(app_config) assert app_config.server app_config.server.auth = AuthConfig( secrets_location=f"/tmp/{uuid.uuid4()}", auth_passphrase='test_passphrase', enabled=True, create_keys=True ) auth.init(app_config.app_key(), app_config.server.auth) return EventContext( app_config=app_config, plugin_config=app_config, event_name='mock_event', settings=get_event_settings(app_config.effective_settings, 'mock_event'), # type: ignore track_ids={}, auth_info={} )
def _event_context(mock_app_config, plugin_config): # noqa: F811 iat = datetime.now() timeout = plugin_config.env['auth']['access_token_expiration'] return EventContext(app_config=mock_app_config, plugin_config=plugin_config, event_name='login', track_ids={}, auth_info={ 'allowed': True, 'auth_type': AuthType.REFRESH, 'payload': { 'id': 'id', 'user': '******', 'email': 'test@email', 'iat': iat, 'exp': iat + timedelta(seconds=timeout) } })
async def get_runtime_apps(context: EventContext, *, refresh: bool = False, expand_events: bool = False) -> RuntimeApps: """ Extract current runtime app_config objects """ global _apps, _expire if not refresh and _lock.locked(): raise RuntimeError("Events graph request in process. Ignoring") settings = context.settings(key="apps_visualizer", datatype=AppsVisualizerSettings) now_ts = datetime.now(tz=timezone.utc).timestamp() async with _lock: if _apps is None or refresh or now_ts > _expire: logger.info(context, "Contacting hosts config-manager...") _apps = await get_apps_config(settings.hosts, context, expand_events=expand_events) _expire = now_ts + settings.refresh_hosts_seconds else: logger.info(context, "Using cached runtime apps information.") return _apps
async def mock_handle_spawn_event(app_config, *, payload, expected, stream_name): handler = EventHandler(app_config=app_config, plugins=[], effective_events=app_config.events) context = EventContext(app_config=app_config, plugin_config=app_config, event_name='mock_spawn_event', track_ids=MockStreamManager.test_track_ids, auth_info={ 'auth_type': AuthType.UNSECURED, 'allowed': 'true' }) event_count = 0 async for result in handler.handle_async_event(context=context, query_args={}, payload=payload.value): assert result.value.startswith(expected.value) event_count += 1 assert event_count == 3
def _event_context(mock_app_config, plugin_config): # noqa: F811 settings = get_event_settings(plugin_config.effective_settings, "logout") cfg = settings(key='auth', datatype=AuthSettings) iat = datetime.now(tz=timezone.utc) timeout = cfg.access_token_expiration return EventContext(app_config=mock_app_config, plugin_config=plugin_config, event_name='logout', settings=settings, track_ids={}, auth_info={ 'allowed': True, 'auth_type': AuthType.REFRESH, 'payload': { 'id': 'id', 'user': '******', 'email': 'test@email', 'iat': iat, 'exp': iat + timedelta(seconds=timeout) } })
def _service_event_context( self, event_name: str, previous_context: Optional[EventContext] = None): if previous_context is None: track_ids = { 'track.request_id': str(uuid.uuid4()), 'track.request_ts': datetime.now().astimezone(timezone.utc).isoformat() } else: track_ids = previous_context.track_ids return EventContext(app_config=self.app_config, plugin_config=self.app_config, event_name=event_name, track_ids={ **track_ids, 'track.operation_id': str(uuid.uuid4()), }, auth_info={})