async def test_start_execution_success():
""" Test _start_execution in normal conditions """
run_provider_instance = Mock(spec = RunProvider)
worker_storage_instance = Mock(spec = WorkerStorage)
worker_remote_instance = FakeRemoteWorker(worker_storage_instance)
worker_messenger = InProcessMessenger(worker_remote_instance._handle_request)
worker_local_instance = LocalWorker("my_worker", worker_messenger, lambda: None, run_provider_instance, None)
job = { "project": "my_project", "identifier": "my_job", "definition": {} }
run = { "project": "my_project", "identifier": "my_run", "job": "my_job", "status": "pending", "parameters": {} }
await worker_local_instance._start_execution(run, job)
async def test_finish_execution_success():
""" Test _finish_execution in normal conditions """
run_provider_instance = Mock(spec = RunProvider)
worker_storage_instance = Mock(spec = WorkerStorage)
worker_remote_instance = FakeRemoteWorker(worker_storage_instance)
worker_messenger = InProcessMessenger(worker_remote_instance._handle_request)
worker_local_instance = LocalWorker("my_worker", worker_messenger, lambda: None, run_provider_instance, None)
run = { "project": "my_project", "identifier": "my_run", "job": "my_job", "status": "succeeded" }
worker_remote_instance._active_executors.append(FakeExecutorWatcher(run["identifier"]))
await worker_local_instance._finish_execution(run)
async def test_process_success():
""" Test executing a run which succeeds """
database_client_instance = MemoryDatabaseClient()
data_storage_instance = MemoryDataStorage()
date_time_provider_instance = FakeDateTimeProvider()
worker_storage_instance = Mock(spec = WorkerStorage)
run_provider_instance = RunProvider(data_storage_instance, date_time_provider_instance)
worker_remote_instance = FakeRemoteWorker(worker_storage_instance)
worker_messenger = InProcessMessenger(worker_remote_instance._handle_request)
worker_local_instance = LocalWorker("my_worker", worker_messenger, lambda: database_client_instance, run_provider_instance, None)
job = { "project": "my_project", "identifier": "my_job", "definition": {} }
run = run_provider_instance.create(database_client_instance, job["project"], job["identifier"], {}, None)
assert run["status"] == "pending"
assert len(worker_local_instance.executors) == 0
worker_local_instance.assign_run(job, run)
local_executor = worker_local_instance.executors[0]
assert local_executor["local_status"] == "pending"
assert run["status"] == "pending"
assert len(worker_local_instance.executors) == 1
# pending => running (_start_execution)
await worker_local_instance._process_executor(database_client_instance, local_executor)
assert local_executor["local_status"] == "running"
assert run["status"] == "pending"
remote_executor = worker_remote_instance._find_executor(run["identifier"])
await worker_local_instance.receive_update({ "run": run["identifier"], "status": remote_executor.status })
await worker_local_instance._process_executor(database_client_instance, local_executor)
assert local_executor["local_status"] == "running"
assert run["status"] == "running"
remote_executor.succeed()
await worker_local_instance.receive_update({ "run": run["identifier"], "status": remote_executor.status })
await worker_local_instance._process_executor(database_client_instance, local_executor)
assert local_executor["local_status"] == "running"
assert run["status"] == "succeeded"
# running => verifying
await worker_local_instance._process_executor(database_client_instance, local_executor)
assert local_executor["local_status"] == "verifying"
assert run["status"] == "succeeded"
await worker_local_instance.receive_update({ "run": run["identifier"], "event": "synchronization_completed" })
await worker_local_instance._process_executor(database_client_instance, local_executor)
# verifying => finishing
await worker_local_instance._process_executor(database_client_instance, local_executor)
assert local_executor["local_status"] == "finishing"
assert run["status"] == "succeeded"
# finishing => done (_finish_execution)
await worker_local_instance._process_executor(database_client_instance, local_executor)
assert local_executor["local_status"] == "done"
assert run["status"] == "succeeded"
assert len(worker_local_instance.executors) == 1
async def test_process_recovery_after_execution(): # pylint: disable = too-many-statements
""" Test executing a run which gets recovered after a disconnection and after it completed """
database_client_instance = MemoryDatabaseClient()
data_storage_instance = MemoryDataStorage()
date_time_provider_instance = FakeDateTimeProvider()
worker_storage_instance = Mock(spec = WorkerStorage)
run_provider_instance = RunProvider(data_storage_instance, date_time_provider_instance)
worker_remote_instance = FakeRemoteWorker(worker_storage_instance)
worker_messenger = InProcessMessenger(worker_remote_instance._handle_request)
worker_local_instance = LocalWorker("my_worker", worker_messenger, lambda: database_client_instance, run_provider_instance, None)
job = { "project": "my_project", "identifier": "my_job", "definition": {} }
run = run_provider_instance.create(database_client_instance, job["project"], job["identifier"], {}, None)
request = { "project_identifier": run["project"], "run_identifier": run["identifier"], "job_definition": job["definition"], "parameters": {} }
worker_storage_instance.load_request.return_value = request
assert run["status"] == "pending"
assert len(worker_local_instance.executors) == 0
worker_local_instance.assign_run(job, run)
local_executor = worker_local_instance.executors[0]
assert local_executor["local_status"] == "pending"
assert run["status"] == "pending"
assert len(worker_local_instance.executors) == 1
# pending => running (_start_execution)
await worker_local_instance._process_executor(database_client_instance, local_executor)
assert local_executor["local_status"] == "running"
assert run["status"] == "pending"
assert len(worker_local_instance.executors) == 1
remote_executor = worker_remote_instance._find_executor(run["identifier"])
await worker_local_instance.receive_update({ "run": run["identifier"], "status": remote_executor.status })
await worker_local_instance._process_executor(database_client_instance, local_executor)
assert local_executor["local_status"] == "running"
assert run["status"] == "running"
assert len(worker_local_instance.executors) == 1
# New worker to simulate disconnection
worker_local_instance = LocalWorker("my_worker", worker_messenger, lambda: database_client_instance, run_provider_instance, None)
assert run["status"] == "running"
assert len(worker_local_instance.executors) == 0
remote_executor.succeed()
# none => running (_recover_execution)
worker_local_instance.executors = await worker_local_instance._recover_executors(database_client_instance)
local_executor = worker_local_instance.executors[0]
assert local_executor["local_status"] == "running"
await worker_local_instance.receive_update({ "run": run["identifier"], "status": remote_executor.status })
await worker_local_instance._process_executor(database_client_instance, local_executor)
assert local_executor["local_status"] == "running"
assert run["status"] == "succeeded"
# running => verifying
await worker_local_instance._process_executor(database_client_instance, local_executor)
assert local_executor["local_status"] == "verifying"
assert run["status"] == "succeeded"
await worker_local_instance.receive_update({ "run": run["identifier"], "event": "synchronization_completed" })
await worker_local_instance._process_executor(database_client_instance, local_executor)
# verifying => finishing
await worker_local_instance._process_executor(database_client_instance, local_executor)
assert local_executor["local_status"] == "finishing"
assert run["status"] == "succeeded"
# finishing => done (_finish_execution)
await worker_local_instance._process_executor(database_client_instance, local_executor)
assert local_executor["local_status"] == "done"
assert run["status"] == "succeeded"
assert len(worker_local_instance.executors) == 1