async def test_execute_shuffle_event(mock_app_config): # noqa: F811
result = await execute_event(mock_app_config, 'mock_shuffle_event', 'ok')
assert result == [
MockResult(value='ok: ok.0', processed=True),
MockResult(value='ok: ok.1', processed=True),
MockResult(value='ok: ok.2', processed=True)
]
async def test_execute_multiple_spawn_steps():
steps = [
(0, 'step_spawn', (step_spawn, None, Spawn[MockData], True)),
(1, 'step1', (step1, MockData, MockData, False)),
(2, 'step2', (step2, MockData, MockData, False)),
(3, 'step3', (step3, MockData, MockData, False)),
(4, 'step_respawn', (step_respawn, MockData, Spawn[MockData], True)),
(5, 'step4', (step4, MockData, Union[MockData, str], False)),
(6, 'step5a', (step5a, MockData, MockResult, False)),
(7, 'step5b', (step5b, str, MockResult, False)),
(8, 'step6', (step6, MockResult, MockResult, False))
]
i, j, count = 0, 0, 0
async for result in execute_steps(steps=steps, payload=None, context=test_context(), query_arg1='a'):
assert result == MockResult(f"a {i} step1 step2 step3 respawn:{j} step4 step5a step6")
i, j = i + 1 if j == 2 else i, j + 1 if j < 2 else 0
count += 1
assert count <= 9
assert count == 9
i, j, count = 0, 0, 0
async for result in execute_steps(steps=steps, payload=None, context=test_context(), query_arg1='b'):
assert result == MockResult(f"b {i} step1 step2 step3 respawn:{j} step4 step5b step6")
i, j = i + 1 if j == 2 else i, j + 1 if j < 2 else 0
count += 1
assert count <= 9
assert count == 9
async def test_execute_shuffle_event_default_step(
mock_app_config): # noqa: F811
result = await execute_event(mock_app_config, 'mock_shuffle_event', 'none')
assert result == [
MockResult(value='default', processed=True),
MockResult(value='default', processed=True),
MockResult(value='default', processed=True)
]
async def test_execute_decision_steps():
steps = {'step1': (step1, MockData, MockData),
'step2': (step2, MockData, MockData),
'step3': (step3, MockData, MockData),
'step4': (step4, MockData, Union[MockData, str]),
'step5a': (step5a, MockData, MockResult),
'step5b': (step5b, str, MockResult),
'step6': (step6, MockResult, MockResult)}
async for result in execute_steps(steps=steps, payload=MockData('a'), context=test_context()):
assert result == MockResult("a step1 step2 step3 step4 step5a step6")
async for result in execute_steps(steps=steps, payload=MockData('b'), context=test_context()):
assert result == MockResult("b step1 step2 step3 step4 step5b step6")
async def test_execute_decision_steps():
steps = [
(0, 'step1', (step1, MockData, MockData, False)),
(1, 'step2', (step2, MockData, MockData, False)),
(2, 'step3', (step3, MockData, MockData, False)),
(3, 'step4', (step4, MockData, Union[MockData, str], False)),
(4, 'step5a', (step5a, MockData, MockResult, False)),
(5, 'step5b', (step5b, str, MockResult, False)),
(6, 'step6', (step6, MockResult, MockResult, False))
]
async for result in execute_steps(steps=steps, payload=MockData('a'), context=test_context()):
assert result == MockResult("a step1 step2 step3 step4 step5a step6")
async for result in execute_steps(steps=steps, payload=MockData('b'), context=test_context()):
assert result == MockResult("b step1 step2 step3 step4 step5b step6")
async def wait(payload: MockData, context: EventContext) -> MockResult:
logger.info(context, "mock_stream_timeout.wait")
if payload.value == "timeout":
logger.info(context, "simulating timeout...")
await asyncio.sleep(30.0)
logger.info(context, "done simulating timeout.")
return MockResult("ok: ok")
async def test_write_stream(monkeypatch, mock_app_config,
mock_plugin_config): # noqa: F811
payload = MockData("ok")
expected = MockResult("ok: ok")
setup_mocks(monkeypatch)
monkeypatch.setattr(MockEventHandler, 'input_payload', payload)
monkeypatch.setattr(MockEventHandler, 'expected_result', expected)
monkeypatch.setattr(MockStreamManager, 'test_payload', payload)
monkeypatch.setattr(MockEventHandler, 'test_track_ids', None)
engine = await create_engine(app_config=mock_app_config,
plugin=mock_plugin_config)
stream_manager = MockStreamManager(address='test')
monkeypatch.setattr(engine, 'stream_manager', stream_manager)
event_info = mock_app_config.events['mock_write_stream_event']
await invoke_execute(engine=engine,
from_app=engine.app_config,
event_name='mock_write_stream_event',
query_args={},
payload=payload,
expected=expected,
track_ids={
"track.request_id": "test_request_id",
"track.request_ts":
"2020-02-05T17:07:37.771396+00:00",
"track.session_id": "test_session_id"
})
assert stream_manager.write_stream_name == event_info.write_stream.name
assert stream_manager.write_stream_payload == expected
assert stream_manager.write_target_max_len == 10
await engine.stop()
async def test_execute_collector(monkeypatch, mock_app_config,
mock_plugin_config): # noqa: F811
payload = MockData(value="ok")
expected = MockResult(value="step3: ok")
setup_mocks(monkeypatch)
monkeypatch.setattr(MockEventHandler, 'input_payload', payload)
monkeypatch.setattr(MockEventHandler, 'expected_result', expected)
monkeypatch.setattr(MockStreamManager, 'test_payload', expected)
monkeypatch.setattr(MockEventHandler, 'test_track_ids', None)
engine = await create_engine(app_config=mock_app_config,
plugin=mock_plugin_config)
await invoke_execute(engine=engine,
from_app=engine.app_config,
event_name='mock_collector',
query_args={},
payload=payload,
expected=expected,
track_ids={
"track.request_id": "test_request_id",
"track.request_ts":
"2020-02-05T17:07:37.771396+00:00",
"track.session_id": "test_session_id"
})
await engine.stop()
async def test_read_write_stream_new_queue(
monkeypatch,
mock_app_config,
mock_plugin_config # noqa: F811
):
payload = MockData("ok")
expected = MockResult("ok: ok")
setup_mocks(monkeypatch)
monkeypatch.setattr(MockEventHandler, 'input_payload', payload)
monkeypatch.setattr(MockEventHandler, 'expected_result', expected)
monkeypatch.setattr(
MockEventHandler, 'test_track_ids', {
'track.operation_id': 'test_operation_id',
'track.request_id': 'test_request_id',
'track.request_ts': '2020-02-05T17:07:37.771396+00:00',
'track.session_id': 'test_session_id',
'stream.name': 'test_stream',
'stream.msg_id': '0000000000-0',
'stream.consumer_group': 'test_group',
'event.app': 'mock_app.test'
})
monkeypatch.setattr(MockStreamManager, 'test_payload', payload)
mock_app_config.events['mock_read_write_stream'].write_stream.queues = \
['custom']
engine = await create_engine(app_config=mock_app_config,
plugin=mock_plugin_config)
monkeypatch.setattr(engine, 'stream_manager',
MockStreamManager(address='test'))
res = await engine.read_stream(event_name='mock_read_write_stream',
test_mode=True)
assert res == expected
assert engine.stream_manager.write_stream_name == 'mock_read_write_stream.write.custom'
assert engine.stream_manager.write_stream_queue == 'custom'
assert engine.stream_manager.write_stream_payload == expected
await engine.stop()
async def test_execute_spawn_initial_steps():
steps = {'step_spawn': (step_spawn, None, Spawn[MockData]),
'step1': (step1, MockData, MockData),
'step2': (step2, MockData, MockData),
'step3': (step3, MockData, MockData),
'step4': (step4, MockData, Union[MockData, str]),
'step5a': (step5a, MockData, MockResult),
'step5b': (step5b, str, MockResult),
'step6': (step6, MockResult, MockResult)}
i = 0
async for result in execute_steps(steps=steps, payload=None, context=test_context(), query_arg1='a'):
assert result == MockResult(f"a {i} step1 step2 step3 step4 step5a step6")
i += 1
assert i == 3
i = 0
async for result in execute_steps(steps=steps, payload=None, context=test_context(), query_arg1='b'):
assert result == MockResult(f"b {i} step1 step2 step3 step4 step5b step6")
i += 1
assert i == 3
async def test_read_stream_event_fail_and_process_next(
monkeypatch, mock_app_config, mock_plugin_config): # noqa: F811
setup_mocks(monkeypatch)
payload = MockData("cancel")
expected = MockResult("none")
monkeypatch.setattr(MockEventHandler, 'input_payload', payload)
monkeypatch.setattr(MockEventHandler, 'expected_result', expected)
monkeypatch.setattr(MockStreamManager, 'test_payload', payload)
engine = await create_engine(app_config=mock_app_config,
plugin=mock_plugin_config)
monkeypatch.setattr(engine, 'stream_manager',
MockStreamManager(address='test'))
res = await engine.read_stream(event_name='mock_stream_timeout',
test_mode=True)
assert isinstance(res, asyncio.CancelledError)
payload = MockData("fail")
expected = MockResult("none")
monkeypatch.setattr(MockEventHandler, 'input_payload', payload)
monkeypatch.setattr(MockEventHandler, 'expected_result', expected)
monkeypatch.setattr(MockStreamManager, 'test_payload', payload)
engine = await create_engine(app_config=mock_app_config,
plugin=mock_plugin_config)
monkeypatch.setattr(engine, 'stream_manager',
MockStreamManager(address='test'))
res = await engine.read_stream(event_name='mock_stream_timeout',
test_mode=True)
assert isinstance(res, ValueError)
payload = MockData("ok")
expected = MockResult("ok: ok")
monkeypatch.setattr(MockEventHandler, 'input_payload', payload)
monkeypatch.setattr(MockEventHandler, 'expected_result', expected)
monkeypatch.setattr(MockStreamManager, 'test_payload', payload)
res = await engine.read_stream(event_name='mock_stream_timeout',
test_mode=True)
assert res == expected
await engine.stop()
async def test_execute_spawn_initial_steps():
steps = [
(0, 'step_spawn', (step_spawn, None, Spawn[MockData], True)),
(1, 'step1', (step1, MockData, MockData, False)),
(2, 'step2', (step2, MockData, MockData, False)),
(3, 'step3', (step3, MockData, MockData, False)),
(4, 'step4', (step4, MockData, Union[MockData, str], False)),
(5, 'step5a', (step5a, MockData, MockResult, False)),
(6, 'step5b', (step5b, str, MockResult, False)),
(7, 'step6', (step6, MockResult, MockResult, False))
]
i = 0
async for result in execute_steps(steps=steps, payload=None, context=test_context(), query_arg1='a'):
assert result == MockResult(f"a {i} step1 step2 step3 step4 step5a step6")
i += 1
assert i == 3
i = 0
async for result in execute_steps(steps=steps, payload=None, context=test_context(), query_arg1='b'):
assert result == MockResult(f"b {i} step1 step2 step3 step4 step5b step6")
i += 1
assert i == 3
async def test_read_write_stream_multiple_queues_propagate(
monkeypatch,
mock_app_config,
mock_plugin_config # noqa: F811
):
payload = MockData("ok")
expected = MockResult("ok: ok")
setup_mocks(monkeypatch)
monkeypatch.setattr(MockEventHandler, 'input_payload', payload)
monkeypatch.setattr(MockEventHandler, 'expected_result', expected)
monkeypatch.setattr(
MockEventHandler, 'test_track_ids', {
'track.operation_id': 'test_operation_id',
'track.request_id': 'test_request_id',
'track.request_ts': '2020-02-05T17:07:37.771396+00:00',
'track.session_id': 'test_session_id',
'stream.name': 'test_stream',
'stream.msg_id': '0000000000-0',
'stream.consumer_group': 'test_group',
'event.app': 'mock_app.test'
})
monkeypatch.setattr(MockStreamManager, 'test_payload', payload)
monkeypatch.setattr(MockStreamManager, 'test_queue',
None) # Will use last part of stream name
mock_app_config.events['mock_read_write_stream'].read_stream.queues = \
['q1', 'q2']
mock_app_config.events['mock_read_write_stream'].write_stream.queues = \
['q3', 'q4']
mock_app_config.events['mock_read_write_stream'].write_stream.queue_strategy = \
StreamQueueStrategy.PROPAGATE
engine = await create_engine(app_config=mock_app_config,
plugin=mock_plugin_config)
monkeypatch.setattr(engine, 'stream_manager',
MockStreamManager(address='test'))
res = await engine.read_stream(event_name='mock_read_write_stream',
test_mode=True)
assert res == expected
assert engine.stream_manager.write_stream_name == 'mock_read_write_stream.write.q4'
assert engine.stream_manager.write_stream_queue == 'q2'
assert engine.stream_manager.write_stream_payload == expected
assert MockStreamManager.last_read_stream_names[-2:] == \
['mock_read_write_stream.read.q1', 'mock_read_write_stream.read.q2']
assert MockStreamManager.last_read_queue_names[-2:] == \
['q1', 'q2']
assert MockStreamManager.last_write_stream_names[-4:] == [
'mock_read_write_stream.write.q3', 'mock_read_write_stream.write.q4',
'mock_read_write_stream.write.q3', 'mock_read_write_stream.write.q4'
]
assert MockStreamManager.last_write_queue_names[-4:] == \
['q1', 'q1', 'q2', 'q2']
await engine.stop()
class MockEventHandler(EventHandler):
input_query_args = {"query_arg1": "ok"}
input_payload = MockData("ok")
expected_result = MockResult("ok: ok", processed=True)
test_track_ids = {
'track.operation_id': 'test_operation_id',
'track.request_id': 'test_request_id',
'track.request_ts': '2020-02-05T17:07:37.771396+00:00',
'track.session_id': 'test_session_id'
}
expected_track_ids = test_track_ids
call_function = None
def __init__(self, *, app_config: AppConfig, plugins: List[AppConfig],
effective_events: Dict[str, EventDescriptor],
settings: Dict[str, Any]):
self.app_config = app_config
self.plugins = plugins
self.effective_events = effective_events
self.settings = settings
async def handle_async_event(
self, *, context: EventContext, query_args: Optional[dict],
payload: Optional[EventPayload]
) -> AsyncGenerator[Optional[EventPayload], None]:
assert payload == MockEventHandler.input_payload
if MockEventHandler.call_function is not None:
yield MockEventHandler.call_function(payload, context)
elif isinstance(payload,
MockData) and payload.value == 'fail': # type: ignore
raise ValueError("Test for error")
elif isinstance(
payload,
MockData) and payload.value == 'cancel': # type: ignore
raise asyncio.CancelledError("Test for cancellation")
elif isinstance(
payload,
MockData) and payload.value == 'timeout': # type: ignore
await asyncio.sleep(5.0)
yield MockEventHandler.expected_result
async def postprocess(self, *, context: EventContext,
payload: Optional[EventPayload],
response: PostprocessHook) -> Optional[EventPayload]:
response.set_header("PluginHeader", "PluginHeaderValue")
response.set_cookie("PluginCookie", "PluginCookieValue")
response.set_status(999)
return payload
async def test_read_stream_timeout_ok(monkeypatch, mock_app_config,
mock_plugin_config): # noqa: F811
payload = MockData("ok")
expected = MockResult("ok: ok")
setup_mocks(monkeypatch)
monkeypatch.setattr(MockEventHandler, 'input_payload', payload)
monkeypatch.setattr(MockEventHandler, 'expected_result', expected)
monkeypatch.setattr(MockStreamManager, 'test_payload', payload)
engine = await create_engine(app_config=mock_app_config,
plugin=mock_plugin_config)
monkeypatch.setattr(engine, 'stream_manager',
MockStreamManager(address='test'))
res = await engine.read_stream(event_name='mock_stream_timeout',
test_mode=True)
assert res == expected
await engine.stop()
async def _setup(
monkeypatch,
mock_app_config, # noqa: F811
mock_plugin_config, # noqa: F811
aiohttp_client, # noqa: F811
streams=None,
enabled_groups: Optional[List[str]] = None):
stream_event = MockResult("ok: ok")
monkeypatch.setattr(MockStreamManager, 'test_payload', stream_event)
monkeypatch.setattr(MockEventHandler, 'test_track_ids', None)
api.clear()
if enabled_groups is None:
enabled_groups = []
await start_test_server(mock_app_config, mock_plugin_config, streams,
enabled_groups)
return await aiohttp_client(hopeit.server.web.web_server)
def _setup(
monkeypatch,
loop,
mock_app_config, # noqa: F811
mock_plugin_config, # noqa: F811
aiohttp_server, # noqa: F811
aiohttp_client, # noqa: F811
streams=None):
stream_event = MockResult("ok: ok")
monkeypatch.setattr(MockStreamManager, 'test_payload', stream_event)
monkeypatch.setattr(MockEventHandler, 'test_track_ids', None)
api.clear()
loop.run_until_complete(
start_test_server(mock_app_config, mock_plugin_config, aiohttp_server,
streams))
return loop.run_until_complete(aiohttp_client(
hopeit.server.web.web_server))
async def test_read_stream(monkeypatch, mock_app_config,
mock_plugin_config): # noqa: F811
payload = MockData("ok")
expected = MockResult("ok: ok")
setup_mocks(monkeypatch)
from mock_app import mock_event_dataobject_payload
monkeypatch.setattr(MockEventHandler, 'input_payload', payload)
monkeypatch.setattr(MockEventHandler, 'call_function',
mock_event_dataobject_payload.entry_point)
monkeypatch.setattr(MockStreamManager, 'test_payload', payload)
engine = await create_engine(app_config=mock_app_config,
plugin=mock_plugin_config)
monkeypatch.setattr(engine, 'stream_manager',
MockStreamManager(address='test'))
res = await engine.read_stream(event_name='mock_stream_event',
test_mode=True)
assert res == expected
await engine.stop()
async def test_execute(monkeypatch, mock_app_config,
mock_plugin_config): # noqa: F811
payload = MockData("ok")
expected = MockResult("ok: ok")
setup_mocks(monkeypatch)
monkeypatch.setattr(MockEventHandler, 'input_payload', payload)
monkeypatch.setattr(MockEventHandler, 'expected_result', expected)
monkeypatch.setattr(MockStreamManager, 'test_payload', expected)
monkeypatch.setattr(MockEventHandler, 'test_track_ids', None)
engine = await create_engine(app_config=mock_app_config,
plugin=mock_plugin_config)
await invoke_execute(engine=engine,
from_app=engine.app_config,
event_name='mock_post_event',
query_args={"query_arg1": "ok"},
payload=payload,
expected=expected,
track_ids={"track.session_id": "test_session_id"})
await engine.stop()
def step5b(payload: str, context: EventContext) -> MockResult:
return MockResult(payload + ' step5b')
async def wait(payload: MockData, context: EventContext) -> MockResult:
logger.info(context, "mock_stream_timeout.wait")
if payload.value == "timeout":
await asyncio.sleep(5.0)
return MockResult("ok: ok")
def step5a(payload: MockData, context: EventContext) -> MockResult:
return MockResult(payload.value + ' step5a')
def step6(payload: MockResult, context: EventContext) -> MockResult:
return MockResult(payload.value + ' step6')
class MockStreamManager(StreamManager):
test_queue = StreamQueue.AUTO
test_payload = MockResult("ok: ok", processed=True)
test_track_ids = {
'track.request_id': 'test_request_id',
'track.request_ts': '2020-02-05T17:07:37.771396+00:00',
'track.session_id': 'test_session_id',
'stream.consumer_group': 'test_group',
'stream.msg_id': '0000000000-0',
'stream.name': 'test_stream',
'stream.event_id': 'test_id',
'stream.event_ts': '',
'stream.read_ts': '2020-02-05T17:07:39.771396+00:00',
'stream.submit_ts': '2020-02-05T17:07:38.771396+00:00'
}
test_auth_info = {'allowed': True, 'auth_type': 'Unsecured'}
closed = True
last_read_message = None
error_pattern = [None]
last_write_stream_names: List[str] = []
last_write_queue_names: List[str] = []
last_read_stream_names: List[str] = []
last_read_queue_names: List[str] = []
def __init__(self, address: str):
self.address = address
self.write_stream_name: Optional[str] = None
self.write_stream_queue: Optional[str] = None
self.write_stream_payload: Optional[DataObject] = None # type: ignore
self.write_track_ids: Optional[Dict[str, str]] = None
self.write_auth_info: Optional[Dict[str, Any]] = None
self.write_target_max_len: Optional[int] = None
self.write_count = 0
async def connect(self):
MockStreamManager.closed = False
return self
async def close(self):
MockStreamManager.closed = True
async def write_stream(self,
*,
stream_name: str,
queue: str,
payload: EventPayload,
track_ids: Dict[str, str],
auth_info: Dict[str, Any],
target_max_len: int = 0,
compression: Compression,
serialization: Serialization) -> int:
if MockEventHandler.test_track_ids:
track_ids['track.operation_id'] = MockEventHandler.test_track_ids[
'track.operation_id']
track_ids['track.request_ts'] = MockEventHandler.test_track_ids[
'track.request_ts']
assert track_ids == MockEventHandler.test_track_ids
self.write_stream_name = stream_name
self.write_stream_queue = queue
self.write_stream_payload = payload
self.write_track_ids = track_ids
self.write_auth_info = auth_info
self.write_target_max_len = target_max_len
self.write_count += 1
self.last_write_stream_names.append(stream_name)
self.last_write_queue_names.append(queue)
return 1
async def ensure_consumer_group(self, *, stream_name: str,
consumer_group: str):
pass
async def ack_read_stream(self, *, stream_name: str, consumer_group: str,
stream_event: StreamEvent):
return 1
async def read_stream(
self, *, stream_name: str, consumer_group: str,
datatypes: Dict[str, type], track_headers: List[str], offset: str,
batch_size: int, timeout: int,
batch_interval: int) -> List[Union[StreamEvent, Exception]]:
if not MockStreamManager.closed:
MockStreamManager.last_read_message = StreamEvent(
msg_internal_id=b'0000000000-0',
queue=MockStreamManager.test_queue
or stream_name.split('.')[-1],
payload=MockStreamManager.test_payload,
track_ids=MockStreamManager.test_track_ids,
auth_info=MockStreamManager.test_auth_info)
results: List[Union[StreamEvent, Exception]] = []
for i, err_mode in enumerate(copy(
MockStreamManager.error_pattern)):
if err_mode is None:
results.append(MockStreamManager.last_read_message)
self.last_read_stream_names.append(stream_name)
self.last_read_queue_names.append(
MockStreamManager.last_read_message.queue)
await asyncio.sleep(timeout)
elif isinstance(err_mode, StreamOSError):
MockStreamManager.error_pattern = MockStreamManager.error_pattern[
i + 1:]
raise err_mode
else:
results.append(err_mode)
return results
return []
async def test_handle_async_event_special_case(mock_app_config): # noqa: F811
await mock_handle_request_response_event(mock_app_config,
payload=MockData("no-ok"),
expected=MockResult("None"))