def test_metrics(mock_app_config): # noqa: F811
context = EventContext(app_config=mock_app_config,
plugin_config=mock_app_config,
event_name='mock_event',
track_ids={},
auth_info={
'auth_type': AuthType.UNSECURED,
'allowed': 'true'
})
context.creation_ts = ZERO_TS
metrics.datetime = MockDatetime
MockDatetime.ts = 3.0
result = metrics.metrics(context)
assert result['extra'] == 'metrics.duration=3000.000'
async def process_log_data(payload: LogRawBatch,
context: EventContext) -> Optional[LogBatch]:
"""
Receives emitted batch of raw log lines from service handler, process and filter in order
to emit processed batch to a stream.
"""
config = context.settings(datatype=LogReaderConfig)
logger.info(context,
"Processing batch of log entries...",
extra=extra(batch_size=len(payload.data)))
try:
entries: List[LogEntry] = []
for entry in payload.data:
processed_entry = await _process_log_entry(entry, context)
if processed_entry is not None:
entries.append(processed_entry) # type: ignore
if len(entries) > 0:
return LogBatch(entries=entries)
logger.info("Filtered out all entries in batch.")
return None
except Exception as e: # pylint: disable=broad-except # pragma: no cover
logger.error(context, e)
return None
finally:
await asyncio.sleep(config.batch_wait_interval_secs)
async def __service__(context: EventContext) -> Spawn[LogRawBatch]:
config = context.settings(datatype=LogReaderConfig)
event_handler = LogFileHandler(config, context)
logger.info(context,
"Starting LogFileHandler...",
extra=extra(logs_path=config.logs_path,
checkpoint_path=config.checkpoint_path))
observer = start_observer(event_handler, config.logs_path)
logger.info(context, "LogFileHandler started.")
try:
while True:
batch = await event_handler.get_and_reset_batch()
if len(batch) == 0:
logger.info(
context,
"LogFileHandler returned empty batch. Sleeping...")
await asyncio.sleep(config.batch_wait_interval_secs)
else:
for i in range(0, len(batch), config.batch_size):
yield LogRawBatch(data=batch[i:i + config.batch_size + 1])
await asyncio.sleep(config.batch_wait_interval_secs)
event_handler.close_inactive_files()
except KeyboardInterrupt: # pragma: no cover
pass
except Exception as e: # pylint: disable=broad-except # pragma: no cover
logger.error(context, e)
finally:
observer.stop()
observer.join()
def authorize(context: EventContext, user_info: ContextUserInfo,
now: datetime) -> AuthInfoExtended:
"""
Authorize user and returns auth info containing tokens for api access and authorization renewal
:param context: event context from app requesting authorization or login happened
:param user_info: already validated user info to be encoded in tokens:
Notice this method wont check if user is valid, invoking app should ensure this.
:param now: current datetime, fixed as start of authorization process
:return: AuthInfoExtended, containing new access and refresh tokens
"""
cfg: AuthSettings = context.settings(key="auth", datatype=AuthSettings)
renew_in = int(1000.0 * max(
1.0 * cfg.access_token_expiration -
1.0 * cfg.access_token_renew_window *
(1.0 + 0.5 * random.random()), 0.5 * cfg.access_token_expiration *
(0.5 * random.random() + 0.5)))
token = _new_access_token(asdict(user_info), context, now,
cfg.access_token_expiration, renew_in)
refresh_token = _new_refresh_token(asdict(user_info), context, now,
cfg.refresh_token_expiration)
result = AuthInfoExtended(
app=context.app_key,
access_token=token,
refresh_token=refresh_token,
token_type=AuthType.BEARER.name,
access_token_expiration=cfg.access_token_expiration,
refresh_token_expiration=cfg.refresh_token_expiration,
renew=renew_in,
user_info=user_info)
return result
async def invoke_login(context: EventContext):
auth_info = await login.login(None, context)
cfg = context.settings(key='auth', datatype=AuthSettings)
assert auth_info.token_type == 'BEARER'
access_token_info = auth.decode_token(auth_info.access_token)
assert access_token_info['app'] == 'test_app.test'
assert access_token_info['id'] == 'id'
assert access_token_info['email'] == 'test@email'
assert access_token_info['user'] == 'test'
iat = access_token_info['iat']
assert access_token_info['exp'] == iat + cfg.access_token_expiration
assert access_token_info['renew'] > 0
assert access_token_info['renew'] < 1000.0 * (
cfg.access_token_expiration - cfg.access_token_renew_window)
refresh_token_info = auth.decode_token(auth_info.refresh_token)
assert refresh_token_info['app'] == 'test_app.test'
assert refresh_token_info['id'] == 'id'
assert refresh_token_info['email'] == 'test@email'
assert refresh_token_info['user'] == 'test'
iat = refresh_token_info['iat']
assert refresh_token_info['exp'] == iat + cfg.refresh_token_expiration
assert auth_info.user_info == ContextUserInfo(id='id',
user='******',
email='test@email')
assert auth_info.access_token_expiration == cfg.access_token_expiration
assert auth_info.refresh_token_expiration == cfg.refresh_token_expiration
assert auth_info.renew == access_token_info['renew']
return auth_info
async def mock_handle_postprocess(app_config, *, payload, expected,
expected_response):
handler = EventHandler(app_config=app_config,
plugins=[],
effective_events=app_config.events)
context = EventContext(app_config=app_config,
plugin_config=app_config,
event_name='plugin_event',
track_ids={},
auth_info={
'auth_type': AuthType.UNSECURED,
'allowed': 'true'
})
intermediate_result = None
async for res in handler.handle_async_event(context=context,
query_args={},
payload=None):
intermediate_result = res
hook = PostprocessHook()
response = await handler.postprocess(context=context,
payload=intermediate_result,
response=hook)
assert hook.headers == expected_response['headers']
assert hook.cookies == expected_response['cookies']
assert hook.status == expected_response['status']
assert response == expected
def _validate_authorization(app_config: AppConfig, context: EventContext,
auth_types: List[AuthType], request: web.Request):
"""
Validates Authorization header from request to provide valid credentials
for the methods supported in event configuration.
:raise `Unauthorized` if authorization is not valid
"""
auth_methods = context.event_info.auth
if (len(auth_methods) == 0) and (app_config.server is not None):
auth_methods = app_config.server.auth.default_auth_methods
auth_header = _extract_authorization(auth_methods, request, context)
try:
method, data = auth_header.split(" ")
except ValueError as e:
raise BadRequest("Malformed Authorization") from e
context.auth_info['allowed'] = False
for auth_type in auth_types:
if method.upper() == auth_type.name.upper():
auth.validate_auth_method(auth_type, data, context)
if context.auth_info.get('allowed'):
return None
raise Unauthorized(method)
async def invoke_execute(engine: AppEngine,
from_app: AppConfig,
event_name: str,
query_args: dict,
payload: DataObject,
expected: DataObject,
track_ids: Dict[str, str],
postprocess_expected: Optional[dict] = None):
event_settings = get_event_settings(from_app.effective_settings,
event_name) # type: ignore
context = EventContext(app_config=from_app,
plugin_config=engine.app_config,
event_name=event_name,
settings=event_settings,
track_ids=track_ids,
auth_info={
'auth_type': AuthType.UNSECURED,
'allowed': 'true'
})
res = await engine.execute(context=context,
query_args=query_args,
payload=payload)
assert res == expected
if postprocess_expected:
hook = PostprocessHook()
await engine.postprocess(context=context, payload=res, response=hook)
assert hook.headers == postprocess_expected['headers']
assert hook.cookies == postprocess_expected['cookies']
assert hook.status == postprocess_expected['status']
def create_test_context(app_config: AppConfig,
event_name: str,
track_ids: Optional[dict] = None,
auth_info: Optional[dict] = None) -> EventContext:
"""
Creates an EventContext object to be used in tests
:param app_config: AppConfig, app confioguration
:param event_name: str, event_name to be called
:param track_ids: dict, optional: addictional key/values to add to track_ids section. By default
required track_ids will be generated with default test or empty values.
:param auth_info: dict, optional: additional auth_info to inject. By default Unsecured auth_info
will be generated.
"""
return EventContext(
app_config=app_config,
plugin_config=app_config,
event_name=event_name,
settings=get_event_settings(app_config.effective_settings,
event_name), # type: ignore
track_ids={
**{k: ''
for k in app_config.engine.track_headers}, 'track.operation_id':
'test_operation_id',
'track.request_id': 'test_request_id',
'track.request_ts': datetime.now(tz=timezone.utc).isoformat(),
**({} if track_ids is None else track_ids)
},
auth_info={
'auth_type': AuthType.UNSECURED,
'allowed': 'true',
**({} if auth_info is None else auth_info)
})
def _setup_client_context(app_config: AppConfig, server_app_config: AppConfig,
register_client_key: bool = True) -> EventContext:
_init_engine_logger(app_config)
assert app_config.server
assert server_app_config.server
app_config.server.auth = AuthConfig(
secrets_location=f"/tmp/{uuid.uuid4()}",
auth_passphrase='test_passphrase',
enabled=True,
create_keys=True
)
auth.init(app_config.app_key(), app_config.server.auth)
if register_client_key:
os.rename(
pathlib.Path(app_config.server.auth.secrets_location) / 'public' / f'{app_config.app_key()}_pub.pem',
pathlib.Path(server_app_config.server.auth.secrets_location) / 'public' / f'{app_config.app_key()}_pub.pem'
)
return EventContext(
app_config=app_config,
plugin_config=app_config,
event_name='mock_event',
settings=get_event_settings(app_config.effective_settings, 'mock_event'), # type: ignore
track_ids={},
auth_info={}
)
def test_stream_metrics(mock_app_config): # noqa: F811
context = EventContext(app_config=mock_app_config,
plugin_config=mock_app_config,
event_name='mock_event',
track_ids={'track.request_ts': ZERO_TS.isoformat()},
auth_info={
'auth_type': AuthType.UNSECURED,
'allowed': 'true'
})
context.track_ids['stream.submit_ts'] = ONE_TS.isoformat()
context.track_ids['stream.read_ts'] = TWO_TS.isoformat()
context.creation_ts = ZERO_TS
metrics.datetime = MockDatetime
MockDatetime.ts = 3.0
result = metrics.stream_metrics(context)
assert result[
'extra'] == 'metrics.stream_age=1000.000 | metrics.request_elapsed=3000.000'
async def test_async_collector(mock_app_config): # noqa: F811
context = EventContext(app_config=mock_app_config, plugin_config=mock_app_config, event_name='mock_event',
track_ids={}, auth_info={})
collector = await AsyncCollector()\
.input("0")\
.steps(('step1', step1), ('step2', step2), ('step3', step3))\
.run(context)
result = await collector['step3']
assert result == "((0+mock_event+step1)&(0+mock_event+step2)+mock_event+step3)"
def _event_context(mock_app_config, plugin_config): # noqa: F811
return EventContext(app_config=mock_app_config,
plugin_config=plugin_config,
event_name='login',
track_ids={},
auth_info={
'allowed': True,
'auth_type': AuthType.BASIC,
'payload': 'dGVzdDpwYXNz'
})
async def _save_partition(partition_key: str, items: List[DataObject],
context: EventContext):
settings = context.settings(datatype=FileStorageSettings)
path = Path(settings.path) / partition_key
file = path / f"{uuid.uuid4()}{SUFFIX}"
logger.info(context, f"Saving {file}...")
os.makedirs(path.resolve(), exist_ok=True)
async with aiofiles.open(file, 'w') as f:
for item in items:
await f.write(Payload.to_json(item) + "\n")
def _request_start(app_engine: AppEngine, plugin: AppEngine, event_name: str,
request: web.Request) -> EventContext:
"""
Extracts context and track information from a request and logs start of event
"""
context = EventContext(app_config=app_engine.app_config,
plugin_config=plugin.app_config,
event_name=event_name,
track_ids=_track_ids(request),
auth_info=auth_info_default)
logger.start(context)
return context
async def live_stats(collector: Collector, context: EventContext) -> Collector:
settings = context.settings(key='apps_visualizer',
datatype=AppsVisualizerSettings)
apps = await collector['runtime_apps']
options = await collector['payload']
host_pids = set((server.host_name, server.pid)
for _, runtime_info in apps.apps.items()
for server in runtime_info.servers
if options.host_filter in server.url)
event_stats = get_stats(
host_pids=host_pids,
time_window_secs=settings.live_active_treshold_seconds,
recent_secs=settings.live_recent_treshold_seconds)
graph = await collector['cytoscape_data']
if len(event_stats) == 0:
return graph
for item_id, item in graph.data.items():
label = item['data'].get('label', '')
source = item['data'].get('source', '')
target = item['data'].get('target', '')
if label and source and source[
0] == '>' and source[-len(label):] != label:
source += '.' + label
if target and target[0] == '>':
target = ''
keys = filter(bool, [item_id, source, target])
classes = []
for key in keys:
s = event_stats.get(key)
if s:
if s.started:
classes.append('STARTED')
if s.pending:
classes.append('PENDING')
if s.recent:
classes.append('RECENT')
if s.failed:
classes.append('FAILED')
if s.ignored:
classes.append('IGNORED')
item['classes'] = _classes(item, classes)
target = item['data'].get('target', ' ')
if target[0] == '>':
target_item = graph.data[target]
target_item['classes'] = _classes(target_item, classes)
if len(source) > 5 and (source[-5:] == ".POST"
or source[-4:] == ".GET"
or source[-8:] == "MULTIPART"):
source_item = graph.data[source]
source_item['classes'] = _classes(source_item, classes)
return graph
def test_context() -> EventContext:
app_config = AppConfig(
app=AppDescriptor(name='test_steps', version='test_version'),
events={'test_steps': EventDescriptor(type=EventType.POST)}
)
return EventContext(
app_config=app_config,
plugin_config=app_config,
event_name='test_steps',
track_ids={},
auth_info={}
)
def test_context() -> EventContext:
app_config = AppConfig(
app=AppDescriptor(name='test_steps', version='test_version'),
events={'test_steps': EventDescriptor(type=EventType.POST)}
).setup()
return EventContext(
app_config=app_config,
plugin_config=app_config,
event_name='test_steps',
settings=get_event_settings(app_config.effective_settings, 'test_steps'),
track_ids={},
auth_info={}
)
def _event_context(mock_app_config): # noqa: F811
return EventContext(app_config=mock_app_config,
plugin_config=mock_app_config,
event_name='mock_event_logging',
track_ids={
'track.operation_id': 'test_operation_id',
'track.request_id': 'test_request_id',
'track.request_ts': '2020-01-01T00:00:00Z',
'track.session_id': 'test_session_id'
},
auth_info={
'auth_type': AuthType.UNSECURED,
'allowed': 'true'
})
async def __service__(context: EventContext) -> Spawn[FlushSignal]:
settings: FileStorageSettings = context.settings(
datatype=FileStorageSettings)
if settings.flush_seconds:
while True:
await asyncio.sleep(settings.flush_seconds)
for partition_key in list(buffer.keys()):
yield FlushSignal(partition_key=partition_key)
else:
if settings.flush_max_size == 0:
logger.warning(
context, "Flushing partitions by size and time are disabled."
"Specify either `flush_seconds` or `flush_max_size`"
"to enable flushing the buffer periodically")
else:
logger.info(context, "Flushing partitions by time disabled.")
def create_test_context(app_config: AppConfig,
event_name: str) -> EventContext:
return EventContext(app_config=app_config,
plugin_config=app_config,
event_name=event_name,
track_ids={
'track.operation_id':
'test_operation_id',
'track.request_id':
'test_request_id',
'track.request_ts':
datetime.now(tz=timezone.utc).isoformat()
},
auth_info={
'auth_type': AuthType.UNSECURED,
'allowed': 'true'
})
def _setup_event_context(mock_app_config: AppConfig) -> EventContext: # noqa: F811
_init_engine_logger(mock_app_config)
assert mock_app_config.server
mock_app_config.server.auth = AuthConfig(
secrets_location=f"/tmp/{uuid.uuid4()}",
auth_passphrase='test_passphrase',
enabled=True,
create_keys=True
)
auth.init(mock_app_config.server.auth)
return EventContext(
app_config=mock_app_config,
plugin_config=mock_app_config,
event_name='mock_event',
track_ids={},
auth_info={}
)
async def mock_handle_request_response_event(app_config, *, payload, expected):
handler = EventHandler(app_config=app_config,
plugins=[],
effective_events=app_config.events)
context = EventContext(app_config=app_config,
plugin_config=app_config,
event_name='mock_post_event',
track_ids=MockStreamManager.test_track_ids,
auth_info={
'auth_type': AuthType.UNSECURED,
'allowed': 'true'
})
async for response in handler.handle_async_event(
context=context,
query_args={"query_arg1": payload.value},
payload=payload):
assert response == expected
async def buffer_item(payload: DataObject,
context: EventContext) -> Optional[FlushSignal]:
"""
Consumes any Dataobject type from stream and put it local memory buffer to be flushed later
"""
settings: FileStorageSettings = context.settings(
datatype=FileStorageSettings)
partition_key = get_partition_key(payload, settings.partition_dateformat
or "")
async with buffer_lock:
partition = buffer.get(partition_key, Partition())
buffer[partition_key] = partition
async with partition.lock:
partition.items.append(payload)
if settings.flush_max_size and len(
partition.items) >= settings.flush_max_size:
return FlushSignal(partition_key=partition_key)
return None
def _setup_server_context(app_config: AppConfig) -> EventContext:
_init_engine_logger(app_config)
assert app_config.server
app_config.server.auth = AuthConfig(
secrets_location=f"/tmp/{uuid.uuid4()}",
auth_passphrase='test_passphrase',
enabled=True,
create_keys=True
)
auth.init(app_config.app_key(), app_config.server.auth)
return EventContext(
app_config=app_config,
plugin_config=app_config,
event_name='mock_event',
settings=get_event_settings(app_config.effective_settings, 'mock_event'), # type: ignore
track_ids={},
auth_info={}
)
def _event_context(mock_app_config, plugin_config): # noqa: F811
iat = datetime.now()
timeout = plugin_config.env['auth']['access_token_expiration']
return EventContext(app_config=mock_app_config,
plugin_config=plugin_config,
event_name='login',
track_ids={},
auth_info={
'allowed': True,
'auth_type': AuthType.REFRESH,
'payload': {
'id': 'id',
'user': '******',
'email': 'test@email',
'iat': iat,
'exp': iat + timedelta(seconds=timeout)
}
})
async def get_runtime_apps(context: EventContext,
*, refresh: bool = False,
expand_events: bool = False) -> RuntimeApps:
"""
Extract current runtime app_config objects
"""
global _apps, _expire
if not refresh and _lock.locked():
raise RuntimeError("Events graph request in process. Ignoring")
settings = context.settings(key="apps_visualizer", datatype=AppsVisualizerSettings)
now_ts = datetime.now(tz=timezone.utc).timestamp()
async with _lock:
if _apps is None or refresh or now_ts > _expire:
logger.info(context, "Contacting hosts config-manager...")
_apps = await get_apps_config(settings.hosts, context, expand_events=expand_events)
_expire = now_ts + settings.refresh_hosts_seconds
else:
logger.info(context, "Using cached runtime apps information.")
return _apps
async def mock_handle_spawn_event(app_config, *, payload, expected,
stream_name):
handler = EventHandler(app_config=app_config,
plugins=[],
effective_events=app_config.events)
context = EventContext(app_config=app_config,
plugin_config=app_config,
event_name='mock_spawn_event',
track_ids=MockStreamManager.test_track_ids,
auth_info={
'auth_type': AuthType.UNSECURED,
'allowed': 'true'
})
event_count = 0
async for result in handler.handle_async_event(context=context,
query_args={},
payload=payload.value):
assert result.value.startswith(expected.value)
event_count += 1
assert event_count == 3
def _event_context(mock_app_config, plugin_config): # noqa: F811
settings = get_event_settings(plugin_config.effective_settings, "logout")
cfg = settings(key='auth', datatype=AuthSettings)
iat = datetime.now(tz=timezone.utc)
timeout = cfg.access_token_expiration
return EventContext(app_config=mock_app_config,
plugin_config=plugin_config,
event_name='logout',
settings=settings,
track_ids={},
auth_info={
'allowed': True,
'auth_type': AuthType.REFRESH,
'payload': {
'id': 'id',
'user': '******',
'email': 'test@email',
'iat': iat,
'exp': iat + timedelta(seconds=timeout)
}
})
def _service_event_context(
self,
event_name: str,
previous_context: Optional[EventContext] = None):
if previous_context is None:
track_ids = {
'track.request_id':
str(uuid.uuid4()),
'track.request_ts':
datetime.now().astimezone(timezone.utc).isoformat()
}
else:
track_ids = previous_context.track_ids
return EventContext(app_config=self.app_config,
plugin_config=self.app_config,
event_name=event_name,
track_ids={
**track_ids,
'track.operation_id':
str(uuid.uuid4()),
},
auth_info={})