async def test_inventory_build():
"""Test inventory build
"""
inv = _init_inventory()
with patch.multiple(Node, _init_ssh=get_async_task_mock(),
init_node=get_async_task_mock()):
nodes = await inv.build_inventory()
# Check if nodes are registered in the inventory
assert set(nodes) == set(inv.nodes)
# Check if all the nodes have been registered
assert len(inv.nodes) == 2
# Check if all the parameters have been correctly set in the nodes
for node in sample_inventory:
key = f"{node['namespace']}.{node['address']}"
invnode = nodes[key]
assert node['namespace'] == invnode.nsname
assert inv.connect_timeout == invnode.connect_timeout
assert node['jump_host'].split("@")[1] == invnode.jump_host
assert invnode.jump_host_key
for arg, arg_value in node.items():
if arg in ['namespace', 'ssh_keyfile', 'jump_host',
'jump_host_key_file', 'passphrase']:
continue
assert arg_value == getattr(invnode, arg)
async def test_coalescer_keep_on_failing(coalescer_cfg):
"""Try to start a coalescer which keep on failing
"""
cfg_file = coalescer_cfg
cfg = load_sq_config(config_file=cfg_file)
cl = CoalescerLauncher(cfg_file, cfg, MOCK_COALESCER)
monitor_process_fn = get_async_task_mock()
dummy_process = MagicMock()
dummy_process.returncode = 1
start_coalescer_fn = get_async_task_mock(dummy_process)
try:
with patch.multiple(CoalescerLauncher,
_start_coalescer=start_coalescer_fn), \
patch.multiple(coalescer_launcher_module,
monitor_process=monitor_process_fn):
await asyncio.wait_for(cl.start_and_monitor_coalescer(), 5)
# Check if multiple attempts have been performed
attempts_done = start_coalescer_fn.call_count
assert attempts_done == cl.max_attempts, \
f'Expected {cl.max_attempts} attempts, {attempts_done} done'
except asyncio.TimeoutError:
pytest.fail(
'The coalescer launcher task expected to fail but it does not')
async def ready_inventory():
"""Fixture returning an already built inventory
"""
inv = _init_inventory()
with patch.multiple(Node, _init_ssh=get_async_task_mock(),
init_node=get_async_task_mock()):
await inv.build_inventory()
return inv
async def test_launch_worker(monkeypatch, manager_cfg):
"""Test the creation of the worker process
"""
fake_environ = {}
monkeypatch.setattr(os, 'environ', fake_environ)
manager = init_static_manager(manager_cfg)
fake_process = MagicMock()
create_subprocess_fn = get_async_task_mock(fake_process)
with patch.multiple(asyncio, create_subprocess_exec=create_subprocess_fn):
expected_worker_id = 1
await manager._launch_poller(expected_worker_id)
# Check if the process have been correctly created
create_subprocess_fn.assert_called()
assert manager._waiting_workers.get(expected_worker_id), \
'Worker not placed in the waiting queue'
assert manager._waiting_workers[expected_worker_id] == fake_process, \
'The worker process is not the expected one'
# Check the arguments passed to the worker
call_args = list(list(create_subprocess_fn.call_args_list[0])[0])
try:
worker_id_index = call_args.index('--worker-id') + 1
except ValueError:
pytest.fail('Worker id flag not provided')
assert len(call_args) > worker_id_index, \
'Worker id value not provided to worker'
assert call_args[worker_id_index] == str(expected_worker_id)
async def test_run_output_workers(data_to_write):
"""Check if the OutputWorkers are correctly launched
"""
parquet_write = get_async_task_mock()
gather_write = get_async_task_mock()
with patch.object(ParquetOutputWorker, 'write_data', parquet_write), \
patch.object(GatherOutputWorker, 'write_data', gather_write):
mgr = OutputWorkerManager(OUTPUT_TYPES, OUTPUT_ARGS)
run_task = asyncio.create_task(mgr.run_output_workers())
await mgr.output_queue.put(data_to_write)
# Wait to make the OutputWorkerManager consume the messages
await asyncio.sleep(0.5)
# Check if the writers have been called
parquet_write.assert_called()
gather_write.assert_called()
# Stop the OuputWorkerManager
run_task.cancel()
async def apply_with_mocks(manager: StaticManager,
chunks: List[Dict]) -> Tuple[MagicMock, MagicMock]:
"""Apply the chunks with mocks preventing the application to actually
start the workers
Args:
manager (StaticManager): the static manager to use
chunks (List[Dict]): the chunks to apply to each of the worker
Returns:
Tuple[MagicMock, MagicMock]: the MagicMocks representing the function
to start and stop the workers
"""
launch_fn = get_async_task_mock()
stop_fn = get_async_task_mock()
write_chunk_fn = get_async_task_mock()
with patch.multiple(StaticManager,
_launch_poller=launch_fn,
_stop_poller=stop_fn,
_write_chunk=write_chunk_fn):
await manager.apply(chunks)
return launch_fn, stop_fn, write_chunk_fn
async def test_launch_worker_process_creation_fails(monkeypatch, manager_cfg):
"""Test if an exception is raised if it is not possible to create the
poller worker process.
"""
fake_environ = {}
monkeypatch.setattr(os, 'environ', fake_environ)
manager = init_static_manager(manager_cfg)
create_subprocess_fn = get_async_task_mock()
with patch.multiple(asyncio, create_subprocess_exec=create_subprocess_fn):
expected_worker_id = 1
with pytest.raises(PollingError):
await manager._launch_poller(expected_worker_id)
async def run_poller_with_mocks(poller: Poller) -> Dict[str, MagicMock]:
"""Run the poller replacing component 'run' functions with mocks
"""
# Create a mock for each method
mks = {
'inventory': get_async_task_mock(),
'service_manager': get_async_task_mock(),
'output_worker_manager': get_async_task_mock()
}
with patch.multiple(Inventory,
schedule_nodes_run=mks['inventory']), \
patch.multiple(ServiceManager,
schedule_services_run=mks['service_manager']), \
patch.multiple(OutputWorkerManager,
run_output_workers=mks['output_worker_manager']), \
patch.multiple(asyncio,
get_event_loop=MagicMock()):
poller_run = asyncio.create_task(poller.run())
await asyncio.wait([poller_run])
return mks
def test_inventory_init():
"""Test init of Inventory class
"""
inventory_args = {
'add_task_fn': get_async_task_mock(),
'ssh_config_file': 'config/file',
'connect_timeout': 30
}
inv = _init_inventory(**inventory_args)
for arg, arg_value in inventory_args.items():
assert getattr(inv, arg) == arg_value
def test_init_service_manager():
"""Test correct ServiceManager initialization
"""
run_mode = 'forever'
interval = 30
add_tasks = get_async_task_mock()
service_manager = ServiceManager(add_tasks, SERVICE_DIR, SCHEMA_DIR,
asyncio.Queue(), run_mode, interval)
# Check if the service manager has been correctly initialized
assert service_manager.add_task_fn == add_tasks
assert service_manager.service_directory == SERVICE_DIR
assert service_manager.schema_dir == SCHEMA_DIR
assert service_manager.output_queue
assert service_manager.default_interval == 30
assert service_manager.run_mode == run_mode
# Check the list of services
assert set(service_manager.svcs_list) == set(_get_services())
async def test_exception_when_launching_worker(monkeypatch, manager_cfg):
"""Test if exception when launching the workers is properly handled
"""
fake_environ = {}
monkeypatch.setattr(os, 'environ', fake_environ)
manager = init_static_manager(manager_cfg)
write_chunk_fn = get_async_task_mock()
launch_res = asyncio.Future()
launch_res.set_exception(Exception('test'))
launch_fn = MagicMock(return_value=launch_res)
_, _, chunk1 = get_random_node_list(1)
with pytest.raises(Exception, match=r'test'):
with patch.multiple(StaticManager,
_launch_poller=launch_fn,
_write_chunk=write_chunk_fn):
await manager.apply([chunk1])
def _init_service_manager(add_tasks: Callable = None,
svc_dir: str = SERVICE_DIR,
schema_dir: str = SCHEMA_DIR,
queue: asyncio.Queue = asyncio.Queue,
run_mode: str = 'forever',
interval: int = 30,
service_only: str = None,
exclude_svcs: str = None):
# Create add tasks method mock
if not add_tasks:
add_tasks = get_async_task_mock()
# Construct additional parameters
other_params = {}
if service_only:
other_params['service_only'] = service_only
if exclude_svcs:
other_params['exclude_services'] = exclude_svcs
# Init ServiceManager
return ServiceManager(add_tasks, svc_dir, schema_dir, queue, run_mode,
interval, **other_params)
def _init_inventory(add_task_fn: Callable = None, **kwargs):
if not add_task_fn:
add_task_fn = get_async_task_mock()
return SampleInventory(add_task_fn,
**kwargs)
