def test_abort_run_running_disconnected():
""" Test aborting an in progress run on a disconnected worker """
database_client_instance = MemoryDatabaseClient()
date_time_provider_instance = FakeDateTimeProvider()
run_provider_instance = RunProvider(None, date_time_provider_instance)
worker_instance = Worker("worker_test", None,
lambda: database_client_instance,
run_provider_instance, None)
supervisor_instance = Supervisor(None, None, None, None, None, None)
job_scheduler_instance = JobScheduler(lambda: database_client_instance,
None, run_provider_instance, None,
supervisor_instance, None,
date_time_provider_instance)
job = {"project": "examples", "identifier": "empty"}
run = run_provider_instance.create(database_client_instance,
job["project"], job["identifier"], {},
None)
worker_instance.assign_run(job, run)
run_provider_instance.update_status(database_client_instance,
run,
status="running")
assert run["status"] == "running"
assert run["worker"] == worker_instance.identifier
assert len(supervisor_instance._active_workers) == 0
assert len(worker_instance.executors) == 1
operation_result = job_scheduler_instance.abort_run(run)
assert operation_result is False
assert run["status"] == "running"
assert worker_instance.executors[0]["should_abort"] is False
def test_abort_run_completed():
""" Test aborting a completed run """
database_client_instance = MemoryDatabaseClient()
date_time_provider_instance = FakeDateTimeProvider()
run_provider_instance = RunProvider(None, date_time_provider_instance)
supervisor_instance = Supervisor(None, None, None, None, None, None)
job_scheduler_instance = JobScheduler(lambda: database_client_instance,
None, run_provider_instance, None,
supervisor_instance, None,
date_time_provider_instance)
job = {"project": "examples", "identifier": "empty"}
run = run_provider_instance.create(database_client_instance,
job["project"], job["identifier"], {},
None)
run_provider_instance.update_status(database_client_instance,
run,
status="succeeded")
assert run["status"] == "succeeded"
assert len(supervisor_instance._active_workers) == 0
with pytest.raises(ValueError):
job_scheduler_instance.abort_run(run)
assert run["status"] == "succeeded"
def create_application(arguments):
database_metadata = None
if arguments.database.startswith("postgresql://"):
database_metadata = importlib.import_module(
"bhamon_orchestra_model.database.sql_database_model").metadata
database_administration_factory = factory.create_database_administration_factory(
arguments.database, database_metadata)
database_client_factory = factory.create_database_client_factory(
arguments.database, database_metadata)
data_storage_instance = FileDataStorage(".")
date_time_provider_instance = DateTimeProvider()
application = types.SimpleNamespace()
application.database_administration_factory = database_administration_factory
application.database_client_factory = database_client_factory
application.authentication_provider = AuthenticationProvider(
date_time_provider_instance)
application.authorization_provider = AuthorizationProvider()
application.job_provider = JobProvider(date_time_provider_instance)
application.project_provider = ProjectProvider(date_time_provider_instance)
application.run_provider = RunProvider(data_storage_instance,
date_time_provider_instance)
application.schedule_provider = ScheduleProvider(
date_time_provider_instance)
application.user_provider = UserProvider(date_time_provider_instance)
application.worker_provider = WorkerProvider(date_time_provider_instance)
return application
def delete(self, database_client: DatabaseClient, worker_identifier: str,
run_provider: RunProvider) -> None:
worker_record = self.get(database_client, worker_identifier)
if worker_record is None:
raise ValueError("Worker '%s' does not exist" % worker_identifier)
if worker_record["is_enabled"]:
raise ValueError("Worker '%s' is enabled" % worker_identifier)
if worker_record["is_active"]:
raise ValueError("Worker '%s' is active" % worker_identifier)
if run_provider.count(worker=worker_identifier, status="pending") > 0:
raise ValueError("Worker '%s' has pending runs" %
worker_identifier)
if run_provider.count(worker=worker_identifier, status="running") > 0:
raise ValueError("Worker '%s' has running runs" %
worker_identifier)
database_client.delete_one(self.table,
{"identifier": worker_identifier})
def create_application(configuration):
database_metadata = None
if configuration["orchestra_database_uri"].startswith("postgresql://"):
database_metadata = importlib.import_module(
"bhamon_orchestra_model.database.sql_database_model").metadata
database_client_factory = factory.create_database_client_factory(
database_uri=configuration["orchestra_database_uri"],
database_authentication=configuration[
"orchestra_database_authentication"],
database_metadata=database_metadata,
)
data_storage_instance = FileDataStorage(
configuration["orchestra_file_storage_path"])
date_time_provider_instance = DateTimeProvider()
application = flask.Flask(__name__)
application.database_client_factory = database_client_factory
application.authentication_provider = AuthenticationProvider(
date_time_provider_instance)
application.authorization_provider = AuthorizationProvider()
application.job_provider = JobProvider(date_time_provider_instance)
application.project_provider = ProjectProvider(date_time_provider_instance)
application.run_provider = RunProvider(data_storage_instance,
date_time_provider_instance)
application.schedule_provider = ScheduleProvider(
date_time_provider_instance)
application.user_provider = UserProvider(date_time_provider_instance)
application.worker_provider = WorkerProvider(date_time_provider_instance)
application.run_result_transformer = transform_run_results
service.configure(application)
service.register_handlers(application)
service.register_routes(application)
application.external_services = {
"artifacts":
FileServerClient("Artifact Server",
configuration["artifact_server_web_url"]),
"github":
GitHubClient(configuration.get("github_access_token", None)),
"python_packages":
FileServerClient("Python Package Repository",
configuration["python_package_repository_web_url"]),
}
application.config["GITHUB_ACCESS_TOKEN"] = configuration.get(
"github_access_token", None)
return application
def __init__(self,
temporary_directory,
database_type,
database_suffix=None):
environment_instance = environment.load_test_context_environment(
str(temporary_directory), database_type)
self.temporary_directory = str(temporary_directory)
self.master_address = environment_instance["master_address"]
self.master_port = environment_instance["master_port"]
self.service_address = environment_instance["service_address"]
self.service_port = environment_instance["service_port"]
self.website_address = environment_instance["website_address"]
self.website_port = environment_instance["website_port"]
self.database_uri = environment_instance["database_uri"]
self.process_collection = []
if self.database_uri is not None:
self.database_uri += ("_" +
database_suffix) if database_suffix else ""
date_time_provider_instance = DateTimeProvider()
self.database_administration_factory = factory.create_database_administration_factory(
self.database_uri, sql_database_model.metadata)
self.database_client_factory = factory.create_database_client_factory(
self.database_uri, sql_database_model.metadata)
self.data_storage = FileDataStorage(
os.path.join(self.temporary_directory, "master"))
self.authentication_provider = AuthenticationProvider(
date_time_provider_instance)
self.authorization_provider = AuthorizationProvider()
self.job_provider = JobProvider(date_time_provider_instance)
self.project_provider = ProjectProvider(
date_time_provider_instance)
self.run_provider = RunProvider(self.data_storage,
date_time_provider_instance)
self.schedule_provider = ScheduleProvider(
date_time_provider_instance)
self.user_provider = UserProvider(date_time_provider_instance)
self.worker_provider = WorkerProvider(date_time_provider_instance)
def create_application(arguments):
database_metadata = None
if arguments.database.startswith("postgresql://"):
database_metadata = importlib.import_module(
"bhamon_orchestra_model.database.sql_database_model").metadata
database_client_factory = factory.create_database_client_factory(
arguments.database, database_metadata)
data_storage_instance = FileDataStorage(".")
date_time_provider_instance = DateTimeProvider()
application = flask.Flask(__name__)
application.database_client_factory = database_client_factory
application.authentication_provider = AuthenticationProvider(
date_time_provider_instance)
application.authorization_provider = AuthorizationProvider()
application.job_provider = JobProvider(date_time_provider_instance)
application.project_provider = ProjectProvider(date_time_provider_instance)
application.run_provider = RunProvider(data_storage_instance,
date_time_provider_instance)
application.schedule_provider = ScheduleProvider(
date_time_provider_instance)
application.user_provider = UserProvider(date_time_provider_instance)
application.worker_provider = WorkerProvider(date_time_provider_instance)
application.run_result_transformer = transform_run_results
application.external_services = {}
service.configure(application)
service.register_handlers(application)
service.register_routes(application)
application.add_url_rule("/me/routes",
methods=["GET"],
view_func=list_routes)
return application
def create_application(configuration): # pylint: disable = too-many-locals
database_metadata = None
if configuration["orchestra_database_uri"].startswith("postgresql://"):
database_metadata = importlib.import_module("bhamon_orchestra_model.database.sql_database_model").metadata
database_client_factory = factory.create_database_client_factory(
database_uri = configuration["orchestra_database_uri"],
database_authentication = configuration["orchestra_database_authentication"],
database_metadata = database_metadata,
)
data_storage_instance = FileDataStorage(configuration["orchestra_file_storage_path"])
date_time_provider_instance = DateTimeProvider()
authentication_provider_instance = AuthenticationProvider(date_time_provider_instance)
authorization_provider_instance = AuthorizationProvider()
job_provider_instance = JobProvider(date_time_provider_instance)
project_provider_instance = ProjectProvider(date_time_provider_instance)
run_provider_instance = RunProvider(data_storage_instance, date_time_provider_instance)
schedule_provider_instance = ScheduleProvider(date_time_provider_instance)
user_provider_instance = UserProvider(date_time_provider_instance)
worker_provider_instance = WorkerProvider(date_time_provider_instance)
protocol_factory = functools.partial(
WebSocketServerProtocol,
database_client_factory = database_client_factory,
user_provider = user_provider_instance,
authentication_provider = authentication_provider_instance,
authorization_provider = authorization_provider_instance,
)
supervisor_instance = Supervisor(
host = configuration["orchestra_master_listen_address"],
port = configuration["orchestra_master_listen_port"],
protocol_factory = protocol_factory,
database_client_factory = database_client_factory,
worker_provider = worker_provider_instance,
run_provider = run_provider_instance,
)
worker_selector_instance = WorkerSelector(
database_client_factory = database_client_factory,
worker_provider = worker_provider_instance,
supervisor = supervisor_instance,
)
job_scheduler_instance = JobScheduler(
database_client_factory = database_client_factory,
job_provider = job_provider_instance,
run_provider = run_provider_instance,
schedule_provider = schedule_provider_instance,
supervisor = supervisor_instance,
worker_selector = worker_selector_instance,
date_time_provider = date_time_provider_instance,
)
master_instance = Master(
database_client_factory = database_client_factory,
project_provider = project_provider_instance,
job_provider = job_provider_instance,
schedule_provider = schedule_provider_instance,
worker_provider = worker_provider_instance,
job_scheduler = job_scheduler_instance,
supervisor = supervisor_instance,
)
return master_instance
async def test_process_success():
""" Test executing a run which succeeds """
database_client_instance = MemoryDatabaseClient()
data_storage_instance = MemoryDataStorage()
date_time_provider_instance = FakeDateTimeProvider()
worker_storage_instance = Mock(spec = WorkerStorage)
run_provider_instance = RunProvider(data_storage_instance, date_time_provider_instance)
worker_remote_instance = FakeRemoteWorker(worker_storage_instance)
worker_messenger = InProcessMessenger(worker_remote_instance._handle_request)
worker_local_instance = LocalWorker("my_worker", worker_messenger, lambda: database_client_instance, run_provider_instance, None)
job = { "project": "my_project", "identifier": "my_job", "definition": {} }
run = run_provider_instance.create(database_client_instance, job["project"], job["identifier"], {}, None)
assert run["status"] == "pending"
assert len(worker_local_instance.executors) == 0
worker_local_instance.assign_run(job, run)
local_executor = worker_local_instance.executors[0]
assert local_executor["local_status"] == "pending"
assert run["status"] == "pending"
assert len(worker_local_instance.executors) == 1
# pending => running (_start_execution)
await worker_local_instance._process_executor(database_client_instance, local_executor)
assert local_executor["local_status"] == "running"
assert run["status"] == "pending"
remote_executor = worker_remote_instance._find_executor(run["identifier"])
await worker_local_instance.receive_update({ "run": run["identifier"], "status": remote_executor.status })
await worker_local_instance._process_executor(database_client_instance, local_executor)
assert local_executor["local_status"] == "running"
assert run["status"] == "running"
remote_executor.succeed()
await worker_local_instance.receive_update({ "run": run["identifier"], "status": remote_executor.status })
await worker_local_instance._process_executor(database_client_instance, local_executor)
assert local_executor["local_status"] == "running"
assert run["status"] == "succeeded"
# running => verifying
await worker_local_instance._process_executor(database_client_instance, local_executor)
assert local_executor["local_status"] == "verifying"
assert run["status"] == "succeeded"
await worker_local_instance.receive_update({ "run": run["identifier"], "event": "synchronization_completed" })
await worker_local_instance._process_executor(database_client_instance, local_executor)
# verifying => finishing
await worker_local_instance._process_executor(database_client_instance, local_executor)
assert local_executor["local_status"] == "finishing"
assert run["status"] == "succeeded"
# finishing => done (_finish_execution)
await worker_local_instance._process_executor(database_client_instance, local_executor)
assert local_executor["local_status"] == "done"
assert run["status"] == "succeeded"
assert len(worker_local_instance.executors) == 1
async def test_process_recovery_after_execution(): # pylint: disable = too-many-statements
""" Test executing a run which gets recovered after a disconnection and after it completed """
database_client_instance = MemoryDatabaseClient()
data_storage_instance = MemoryDataStorage()
date_time_provider_instance = FakeDateTimeProvider()
worker_storage_instance = Mock(spec = WorkerStorage)
run_provider_instance = RunProvider(data_storage_instance, date_time_provider_instance)
worker_remote_instance = FakeRemoteWorker(worker_storage_instance)
worker_messenger = InProcessMessenger(worker_remote_instance._handle_request)
worker_local_instance = LocalWorker("my_worker", worker_messenger, lambda: database_client_instance, run_provider_instance, None)
job = { "project": "my_project", "identifier": "my_job", "definition": {} }
run = run_provider_instance.create(database_client_instance, job["project"], job["identifier"], {}, None)
request = { "project_identifier": run["project"], "run_identifier": run["identifier"], "job_definition": job["definition"], "parameters": {} }
worker_storage_instance.load_request.return_value = request
assert run["status"] == "pending"
assert len(worker_local_instance.executors) == 0
worker_local_instance.assign_run(job, run)
local_executor = worker_local_instance.executors[0]
assert local_executor["local_status"] == "pending"
assert run["status"] == "pending"
assert len(worker_local_instance.executors) == 1
# pending => running (_start_execution)
await worker_local_instance._process_executor(database_client_instance, local_executor)
assert local_executor["local_status"] == "running"
assert run["status"] == "pending"
assert len(worker_local_instance.executors) == 1
remote_executor = worker_remote_instance._find_executor(run["identifier"])
await worker_local_instance.receive_update({ "run": run["identifier"], "status": remote_executor.status })
await worker_local_instance._process_executor(database_client_instance, local_executor)
assert local_executor["local_status"] == "running"
assert run["status"] == "running"
assert len(worker_local_instance.executors) == 1
# New worker to simulate disconnection
worker_local_instance = LocalWorker("my_worker", worker_messenger, lambda: database_client_instance, run_provider_instance, None)
assert run["status"] == "running"
assert len(worker_local_instance.executors) == 0
remote_executor.succeed()
# none => running (_recover_execution)
worker_local_instance.executors = await worker_local_instance._recover_executors(database_client_instance)
local_executor = worker_local_instance.executors[0]
assert local_executor["local_status"] == "running"
await worker_local_instance.receive_update({ "run": run["identifier"], "status": remote_executor.status })
await worker_local_instance._process_executor(database_client_instance, local_executor)
assert local_executor["local_status"] == "running"
assert run["status"] == "succeeded"
# running => verifying
await worker_local_instance._process_executor(database_client_instance, local_executor)
assert local_executor["local_status"] == "verifying"
assert run["status"] == "succeeded"
await worker_local_instance.receive_update({ "run": run["identifier"], "event": "synchronization_completed" })
await worker_local_instance._process_executor(database_client_instance, local_executor)
# verifying => finishing
await worker_local_instance._process_executor(database_client_instance, local_executor)
assert local_executor["local_status"] == "finishing"
assert run["status"] == "succeeded"
# finishing => done (_finish_execution)
await worker_local_instance._process_executor(database_client_instance, local_executor)
assert local_executor["local_status"] == "done"
assert run["status"] == "succeeded"
assert len(worker_local_instance.executors) == 1
def create_application(arguments): # pylint: disable = too-many-locals
database_metadata = None
if arguments.database.startswith("postgresql://"):
database_metadata = importlib.import_module(
"bhamon_orchestra_model.database.sql_database_model").metadata
database_client_factory = factory.create_database_client_factory(
arguments.database, database_metadata)
data_storage_instance = FileDataStorage(".")
date_time_provider_instance = DateTimeProvider()
authentication_provider_instance = AuthenticationProvider(
date_time_provider_instance)
authorization_provider_instance = AuthorizationProvider()
job_provider_instance = JobProvider(date_time_provider_instance)
project_provider_instance = ProjectProvider(date_time_provider_instance)
run_provider_instance = RunProvider(data_storage_instance,
date_time_provider_instance)
schedule_provider_instance = ScheduleProvider(date_time_provider_instance)
user_provider_instance = UserProvider(date_time_provider_instance)
worker_provider_instance = WorkerProvider(date_time_provider_instance)
protocol_factory = functools.partial(
WebSocketServerProtocol,
database_client_factory=database_client_factory,
user_provider=user_provider_instance,
authentication_provider=authentication_provider_instance,
authorization_provider=authorization_provider_instance,
)
supervisor_instance = Supervisor(
host=arguments.address,
port=arguments.port,
protocol_factory=protocol_factory,
database_client_factory=database_client_factory,
worker_provider=worker_provider_instance,
run_provider=run_provider_instance,
)
worker_selector_instance = WorkerSelector(
database_client_factory=database_client_factory,
worker_provider=worker_provider_instance,
supervisor=supervisor_instance,
)
job_scheduler_instance = JobScheduler(
database_client_factory=database_client_factory,
job_provider=job_provider_instance,
run_provider=run_provider_instance,
schedule_provider=schedule_provider_instance,
supervisor=supervisor_instance,
worker_selector=worker_selector_instance,
date_time_provider=date_time_provider_instance,
)
master_instance = Master(
database_client_factory=database_client_factory,
project_provider=project_provider_instance,
job_provider=job_provider_instance,
schedule_provider=schedule_provider_instance,
worker_provider=worker_provider_instance,
job_scheduler=job_scheduler_instance,
supervisor=supervisor_instance,
)
# Rapid updates to reduce delays in tests
job_scheduler_instance.update_interval_seconds = 1
supervisor_instance.update_interval_seconds = 1
return master_instance