async def test_stop_stops_apps_in_top_down_order(self,
three_level_app_chain):
with connection_manager() as conn:
pipeline = BasePipeline(connection_name=conn.name,
installed_apps=three_level_app_chain)
mock_parent = MagicMock()
# Mock all of the apps that have been configured for this pipeline.
for key, app in pipeline._installed_apps.items():
app_mock = MagicMock(app.__class__)
app_mock.name = app.name
setattr(mock_parent, app.name, app_mock)
pipeline._installed_apps[key] = app_mock
await pipeline.stop()
for app in pipeline.apps.values():
assert app.stop.called
assert app.stop.call_count == 1
call_order = [
call[0].rstrip("stop").rstrip(".")
for call in mock_parent.method_calls[-len(pipeline.apps):]
]
assert call_order == list(pipeline.apps.keys())
async def test_send_stop(self, three_level_stop_app_chain):
with connection_manager() as conn:
pipeline = BasePipeline(connection_name=conn.name,
installed_apps=three_level_stop_app_chain)
message = await pipeline.send(admin.TestRequestMessage("Test"))
assert message.TestReqID == "Test s3"
def test_load_apps_falls_back_to_settings(self):
with connection_manager() as conn:
pipeline = BasePipeline(connection_name=conn.name)
assert len(pipeline.apps) == len(pipeline.settings.PIPELINE_APPS)
assert all(f"{app.__class__.__module__}.{app.__class__.__name__}"
in pipeline.settings.PIPELINE_APPS
for app in pipeline.apps.values())
def test_prep_processing_pipeline_direction_unknown_raises_exception(
self, three_level_app_chain):
with pytest.raises(ValidationError):
with connection_manager() as conn:
pipeline = BasePipeline(connection_name=conn.name,
installed_apps=three_level_app_chain)
pipeline._setup_message_handling("inbound")
async def main():
logging.basicConfig(
level=settings.LOGGING_LEVEL,
format="%(asctime)s - %(threadName)s - %(module)s - %(levelname)s - %(message)s",
)
args = parser.parse_args()
exit_code = os.EX_OK
with connection_manager(args.connection) as conn:
fix_pipeline = BasePipeline(
connection_name=conn.name, new_session=args.new_session
)
try:
# Graceful shutdown on termination signals.
# See: https://docs.python.org/3.7/library/asyncio-eventloop.html#set-signal-handlers-for-sigint-and-sigterm
loop = asyncio.get_running_loop()
for sig_name in {"SIGINT", "SIGTERM"}:
loop.add_signal_handler(
getattr(signal, sig_name),
lambda: asyncio.create_task(
graceful_shutdown(fix_pipeline, sig_name=sig_name)
),
)
await fix_pipeline.start()
except ImproperlyConfigured as e:
# User needs to fix config issue before restart is attempted. Set os.EX_OK so that system process
# monitors like Supervisor do not attempt a restart immediately.
await graceful_shutdown(fix_pipeline, error=e)
except KeyboardInterrupt:
logger.info("Received keyboard interrupt! Initiating shutdown...")
await graceful_shutdown(fix_pipeline)
except Exception as e:
await graceful_shutdown(fix_pipeline, error=e)
exit_code = os.EX_UNAVAILABLE # Abnormal termination
finally:
# Report tasks that are still running after shutdown.
tasks = [
task
for task in asyncio.all_tasks()
if task is not asyncio.current_task() and not task.cancelled()
]
if tasks:
task_output = "\n".join(str(task) for task in tasks)
logger.warning(
f"There are still {len(tasks)} tasks running that have not been cancelled! Cancelling them now...\n"
f"{task_output}."
)
for task in tasks:
task.cancel()
sys.exit(exit_code)
async def test_start_starts_apps_in_reverse_order(self,
three_level_app_chain):
with connection_manager() as conn:
pipeline = BasePipeline(connection_name=conn.name,
installed_apps=three_level_app_chain)
mock_parent = MagicMock()
# Mock all of the apps that have been configured for this pipeline.
for key, app in pipeline._installed_apps.items():
app_mock = MagicMock(app.__class__)
app_mock.name = app.name
setattr(mock_parent, app.name, app_mock)
pipeline._installed_apps[key] = app_mock
pipeline.stopped_event.set(
) # Stop pipeline immediately after start has been executed
await pipeline.start()
for app in pipeline.apps.values():
assert app.start.called
assert app.start.call_count == 1
call_order = [
call[0].rstrip(".start")
for call in mock_parent.method_calls[-len(pipeline.apps):]
]
assert call_order == list(reversed(pipeline.apps.keys()))
await pipeline.stop()
async def test_stop_allows_only_one_stop_process_to_run_concurrently(
self, three_level_app_chain):
with connection_manager() as conn:
pipeline = BasePipeline(connection_name=conn.name,
installed_apps=three_level_app_chain)
# Mock all of the apps that have been configured for this pipeline.
for key, app in pipeline._installed_apps.items():
app_mock = MagicMock(app.__class__)
app_mock.stop = get_mock_async()
pipeline._installed_apps[key] = app_mock
asyncio.create_task(pipeline.stop())
asyncio.create_task(pipeline.stop())
asyncio.create_task(pipeline.stop())
await pipeline.stop()
for app in pipeline.apps.values():
assert app.stop.called
assert app.stop.call_count == 1
# Wait for separate tasks to complete
tasks = asyncio.all_tasks()
await asyncio.wait(tasks, timeout=0.1)
def base_pipeline():
"""
Basic mock pipeline that can be used to instantiate new apps in tests.
:return: A pipeline mock with a client session initialized.
"""
with connection_manager() as conn:
pipeline = MagicMock(BasePipeline)
pipeline.settings = ConnectionSettings(conn.name)
client_session = ClientSessionApp(pipeline, new_session=True)
client_session.sender = pipeline.settings.SENDER
client_session.target = pipeline.settings.TARGET
pipeline.apps = {ClientSessionApp.name: client_session}
# Mock a future message that will allow us to await pipeline.send and pipeline.receive.
# Only useful in situations where we are not interested in the actual message result :(
mock_future_message = MagicMock(return_value=Future())
mock_future_message.return_value.set_result({})
pipeline.send = mock_future_message
pipeline.receive = MagicMock(return_value=mock_coroutine())
# Simulate the pipeline shutting down
pipeline.stop = MagicMock(return_value=mock_coroutine())
yield pipeline
try:
os.remove(client_session._sid_path)
except FileNotFoundError:
# File does not exist - skip deletion
pass
def test_group_templates_getter_initializes_with_empty_template_group(
self):
settings.CONNECTIONS["another_session"] = {}
with connection_manager(name="another_session"):
assert GroupTemplateMixin().group_templates == {}
del settings.CONNECTIONS["another_session"]
def test_load_apps_raises_exception_if_no_apps_installed(self):
with pytest.raises(ImproperlyConfigured):
with connection_manager() as conn:
orig_apps = settings.CONNECTIONS[conn.name]["PIPELINE_APPS"]
settings.CONNECTIONS[conn.name]["PIPELINE_APPS"] = []
_ = BasePipeline(connection_name=conn.name)
settings.CONNECTIONS[conn.name]["PIPELINE_APPS"] = orig_apps
async def test_send_stop(self, three_level_stop_app_chain):
with connection_manager() as conn:
pipeline = BasePipeline(connection_name=conn.name,
installed_apps=three_level_stop_app_chain)
# Simulate all apps active
pipeline._active_apps = OrderedDict(pipeline.apps.items())
message = await pipeline.send(admin.TestRequestMessage("Test"))
assert message.TestReqID == "Test s3"
async def main():
logging.basicConfig(
level=settings.LOGGING_LEVEL,
format=
"%(asctime)s - %(threadName)s - %(module)s - %(levelname)s - %(message)s",
)
args = parser.parse_args()
with connection_manager(args.connection) as conn:
fix_pipeline = BasePipeline(connection_name=conn.name,
new_session=args.new_session)
try:
# Graceful shutdown on termination signals.
# See: https://docs.python.org/3.7/library/asyncio-eventloop.html#set-signal-handlers-for-sigint-and-sigterm
loop = asyncio.get_running_loop()
for sig_name in {"SIGINT", "SIGTERM"}:
loop.add_signal_handler(
getattr(signal, sig_name),
lambda: asyncio.ensure_future(
graceful_shutdown(sig_name, fix_pipeline)),
)
await fix_pipeline.start()
except asyncio.TimeoutError as e:
logger.error(e)
sys.exit(os.EX_UNAVAILABLE)
except KeyboardInterrupt:
logger.info("Received keyboard interrupt! Initiating shutdown...")
sys.exit(os.EX_OK)
except futures.CancelledError as e:
logger.error(f"Cancelled: connection terminated abnormally! ({e})")
sys.exit(os.EX_UNAVAILABLE)
except ImproperlyConfigured as e:
logger.error(e)
sys.exit(
os.EX_OK
) # User needs to fix config issue before restart is attempted
except Exception as e:
logger.exception(e)
sys.exit(os.EX_UNAVAILABLE)
finally:
try:
await fix_pipeline.stop()
except futures.CancelledError as e:
logger.error(
f"Cancelled: connection terminated abnormally! ({e})")
sys.exit(os.EX_UNAVAILABLE)
def test_prep_processing_pipeline_inbound_order(self,
three_level_app_chain):
with connection_manager() as conn:
pipeline = BasePipeline(connection_name=conn.name,
installed_apps=three_level_app_chain)
func, app_chain = pipeline._setup_message_handling(
pipeline.INBOUND_PROCESSING)
assert func == "on_receive"
assert next(app_chain).name == "below"
assert next(app_chain).name == "middle"
assert next(app_chain).name == "top"
def test_pre_processing_pipeline_outbound_order(self,
three_level_app_chain):
with connection_manager() as conn:
pipeline = BasePipeline(connection_name=conn.name,
installed_apps=three_level_app_chain)
# Simulate all apps active
pipeline._active_apps = OrderedDict(pipeline.apps.items())
func, app_chain = pipeline._setup_message_handling(
pipeline.OUTBOUND_PROCESSING)
assert func == "on_send"
assert next(app_chain).name == "top"
assert next(app_chain).name == "middle"
assert next(app_chain).name == "below"
async def test_stop_no_op_if_already_stopped(self, three_level_app_chain):
with connection_manager() as conn:
pipeline = BasePipeline(connection_name=conn.name,
installed_apps=three_level_app_chain)
# Mock all of the apps that have been configured for this pipeline.
for key, app in pipeline._installed_apps.items():
app_mock = MagicMock(app.__class__)
pipeline._installed_apps[key] = app_mock
await pipeline.stop()
await pipeline.stop()
for app in pipeline.apps.values():
assert app.stop.called
assert app.stop.call_count == 1
async def test_stop_cancels_all_tasks_on_stop_timeout(
self, three_level_app_chain):
with pytest.raises(futures.CancelledError):
with connection_manager() as conn:
pipeline = BasePipeline(connection_name=conn.name,
installed_apps=three_level_app_chain)
settings.STOP_TIMEOUT = 0.1
# Mock all of the apps that have been configured for this pipeline.
for key, app in pipeline._installed_apps.items():
app_mock = MagicMock(app.__class__)
app_mock.stop = get_slow_mock_async(settings.STOP_TIMEOUT +
0.1)
pipeline._installed_apps[key] = app_mock
await pipeline.stop()
async def test_start_raises_exception_on_initialize_timeout(
self, three_level_app_chain, create_mock_coro):
mock_, _ = create_mock_coro(
runtime=settings.INIT_TIMEOUT + 0.1,
to_patch="wtfix.apps.base.BaseApp.initialize",
)
with pytest.raises(asyncio.exceptions.TimeoutError):
with connection_manager() as conn:
pipeline = BasePipeline(connection_name=conn.name,
installed_apps=three_level_app_chain)
settings.INIT_TIMEOUT = 0.1
await pipeline.start()
await pipeline.stop()
assert mock_.call_count == 1
async def test_initialize_initializes_each_app_exactly_once(
self, three_level_app_chain):
with connection_manager() as conn:
pipeline = BasePipeline(connection_name=conn.name,
installed_apps=three_level_app_chain)
# Mock all of the apps that have been configured for this pipeline.
for key, app in pipeline._installed_apps.items():
app_mock = MagicMock(app.__class__)
app_mock.name = app.name
pipeline._installed_apps[key] = app_mock
await pipeline.initialize()
for app in pipeline.apps.values():
assert app.initialize.called
assert app.initialize.call_count == 1
await pipeline.stop()
async def test_start_raises_exception_on_initialize_timeout(
self, three_level_app_chain):
with pytest.raises(asyncio.TimeoutError):
with connection_manager() as conn:
pipeline = BasePipeline(connection_name=conn.name,
installed_apps=three_level_app_chain)
settings.INIT_TIMEOUT = 0.1
# Mock all of the apps that have been configured for this pipeline.
for key, app in pipeline._installed_apps.items():
app_mock = MagicMock(app.__class__)
app_mock.initialize = get_slow_mock_async(
settings.INIT_TIMEOUT + 0.1)
app_mock.stop = get_mock_async()
pipeline._installed_apps[key] = app_mock
await pipeline.start()
await pipeline.stop()
def base_pipeline():
"""
Basic mock pipeline that can be used to instantiate new apps in tests.
:return: A pipeline mock with a client session initialized.
"""
with connection_manager() as conn:
pipeline = MagicMock(BasePipeline)
pipeline.settings = ConnectionSettings(conn.name)
client_session = ClientSessionApp(pipeline, new_session=True)
client_session.sender = pipeline.settings.SENDER
client_session.target = pipeline.settings.TARGET
pipeline.apps = {ClientSessionApp.name: client_session}
yield pipeline
try:
os.remove(client_session._sid_path)
except FileNotFoundError:
# File does not exist - skip deletion
pass
def test_load_apps_installs_apps_in_pipeline(self, three_level_app_chain):
with connection_manager() as conn:
pipeline = BasePipeline(connection_name=conn.name,
installed_apps=three_level_app_chain)
assert len(pipeline.apps) == 3
