def create_application(arguments, authentication):
properties = {
"project": ["examples"],
"is_controller": arguments.identifier == "controller",
"executor_limit": 100 if arguments.identifier == "controller" else 1,
}
data_storage_instance = FileDataStorage(".")
worker_storage_instance = WorkerStorage(data_storage_instance)
master_client_instance = MasterClient(
master_uri=arguments.master_uri,
worker_identifier=arguments.identifier,
worker_version=bhamon_orchestra_worker.__version__,
user=authentication["user"],
secret=authentication["secret"],
)
executor_command_factory = lambda run_request: [
sys.executable, "-m", "test.integration.executor_main", run_request[
"run_identifier"]
]
worker_instance = Worker(
storage=worker_storage_instance,
master_client=master_client_instance,
display_name=arguments.identifier,
properties=properties,
executor_command_factory=executor_command_factory,
)
return worker_instance
def create_application(arguments):
database_metadata = None
if arguments.database.startswith("postgresql://"):
database_metadata = importlib.import_module(
"bhamon_orchestra_model.database.sql_database_model").metadata
database_administration_factory = factory.create_database_administration_factory(
arguments.database, database_metadata)
database_client_factory = factory.create_database_client_factory(
arguments.database, database_metadata)
data_storage_instance = FileDataStorage(".")
date_time_provider_instance = DateTimeProvider()
application = types.SimpleNamespace()
application.database_administration_factory = database_administration_factory
application.database_client_factory = database_client_factory
application.authentication_provider = AuthenticationProvider(
date_time_provider_instance)
application.authorization_provider = AuthorizationProvider()
application.job_provider = JobProvider(date_time_provider_instance)
application.project_provider = ProjectProvider(date_time_provider_instance)
application.run_provider = RunProvider(data_storage_instance,
date_time_provider_instance)
application.schedule_provider = ScheduleProvider(
date_time_provider_instance)
application.user_provider = UserProvider(date_time_provider_instance)
application.worker_provider = WorkerProvider(date_time_provider_instance)
return application
def create_application(run_identifier, configuration, authentication):
data_storage_instance = FileDataStorage(".")
date_time_provider_instance = DateTimeProvider()
worker_storage_instance = WorkerStorage(data_storage_instance)
service_client_instance = WebServiceClient(
configuration["orchestra_service_url"],
authorization=(authentication["user"], authentication["secret"]))
request = worker_storage_instance.load_request(run_identifier)
if request["job_definition"]["type"] == "job":
executor_instance = JobExecutor(
storage=worker_storage_instance,
date_time_provider=date_time_provider_instance,
)
elif request["job_definition"]["type"] == "pipeline":
executor_instance = PipelineExecutor(
storage=worker_storage_instance,
date_time_provider=date_time_provider_instance,
service_client=service_client_instance,
)
else:
raise RuntimeError("Unsupported job definition")
return executor_instance
def instantiate_implementation(temporary_directory,
storage_implementation: str) -> DataStorage:
if storage_implementation == "memory":
return MemoryDataStorage()
if storage_implementation == "file":
return FileDataStorage(str(temporary_directory))
raise ValueError("Unsupported implementation '%s'" %
storage_implementation)
def create_application(configuration):
database_metadata = None
if configuration["orchestra_database_uri"].startswith("postgresql://"):
database_metadata = importlib.import_module(
"bhamon_orchestra_model.database.sql_database_model").metadata
database_client_factory = factory.create_database_client_factory(
database_uri=configuration["orchestra_database_uri"],
database_authentication=configuration[
"orchestra_database_authentication"],
database_metadata=database_metadata,
)
data_storage_instance = FileDataStorage(
configuration["orchestra_file_storage_path"])
date_time_provider_instance = DateTimeProvider()
application = flask.Flask(__name__)
application.database_client_factory = database_client_factory
application.authentication_provider = AuthenticationProvider(
date_time_provider_instance)
application.authorization_provider = AuthorizationProvider()
application.job_provider = JobProvider(date_time_provider_instance)
application.project_provider = ProjectProvider(date_time_provider_instance)
application.run_provider = RunProvider(data_storage_instance,
date_time_provider_instance)
application.schedule_provider = ScheduleProvider(
date_time_provider_instance)
application.user_provider = UserProvider(date_time_provider_instance)
application.worker_provider = WorkerProvider(date_time_provider_instance)
application.run_result_transformer = transform_run_results
service.configure(application)
service.register_handlers(application)
service.register_routes(application)
application.external_services = {
"artifacts":
FileServerClient("Artifact Server",
configuration["artifact_server_web_url"]),
"github":
GitHubClient(configuration.get("github_access_token", None)),
"python_packages":
FileServerClient("Python Package Repository",
configuration["python_package_repository_web_url"]),
}
application.config["GITHUB_ACCESS_TOKEN"] = configuration.get(
"github_access_token", None)
return application
def invoke_executor(self, worker_identifier, run_request):
worker_directory = os.path.join(self.temporary_directory,
worker_identifier)
file_data_storage_instance = FileDataStorage(worker_directory)
worker_storage_instance = WorkerStorage(file_data_storage_instance)
worker_storage_instance.create_run(run_request["run_identifier"])
worker_storage_instance.save_request(run_request["run_identifier"],
run_request)
return self.invoke(
identifier=worker_identifier + "_" + "executor",
module="test.integration.executor_main",
arguments=[run_request["run_identifier"]],
workspace=worker_directory,
)
def __init__(self,
temporary_directory,
database_type,
database_suffix=None):
environment_instance = environment.load_test_context_environment(
str(temporary_directory), database_type)
self.temporary_directory = str(temporary_directory)
self.master_address = environment_instance["master_address"]
self.master_port = environment_instance["master_port"]
self.service_address = environment_instance["service_address"]
self.service_port = environment_instance["service_port"]
self.website_address = environment_instance["website_address"]
self.website_port = environment_instance["website_port"]
self.database_uri = environment_instance["database_uri"]
self.process_collection = []
if self.database_uri is not None:
self.database_uri += ("_" +
database_suffix) if database_suffix else ""
date_time_provider_instance = DateTimeProvider()
self.database_administration_factory = factory.create_database_administration_factory(
self.database_uri, sql_database_model.metadata)
self.database_client_factory = factory.create_database_client_factory(
self.database_uri, sql_database_model.metadata)
self.data_storage = FileDataStorage(
os.path.join(self.temporary_directory, "master"))
self.authentication_provider = AuthenticationProvider(
date_time_provider_instance)
self.authorization_provider = AuthorizationProvider()
self.job_provider = JobProvider(date_time_provider_instance)
self.project_provider = ProjectProvider(
date_time_provider_instance)
self.run_provider = RunProvider(self.data_storage,
date_time_provider_instance)
self.schedule_provider = ScheduleProvider(
date_time_provider_instance)
self.user_provider = UserProvider(date_time_provider_instance)
self.worker_provider = WorkerProvider(date_time_provider_instance)
def test_list_runs(tmpdir):
""" Test listing runs """
data_storage_instance = FileDataStorage(str(tmpdir))
worker_storage_instance = WorkerStorage(data_storage_instance)
uuid_run_identifier = "2152bd45-dd77-4cbb-998a-af21a52e4cea"
arbitrary_run_identifier = "my_run"
assert not worker_storage_instance.run_exists(uuid_run_identifier)
assert not worker_storage_instance.run_exists(arbitrary_run_identifier)
assert worker_storage_instance.list_runs() == []
worker_storage_instance.create_run(uuid_run_identifier)
worker_storage_instance.save_request(uuid_run_identifier, {})
worker_storage_instance.create_run(arbitrary_run_identifier)
worker_storage_instance.save_request(arbitrary_run_identifier, {})
assert worker_storage_instance.run_exists(uuid_run_identifier)
assert worker_storage_instance.run_exists(arbitrary_run_identifier)
assert list(sorted(worker_storage_instance.list_runs())) == [ uuid_run_identifier, arbitrary_run_identifier ]
def create_application(arguments):
database_metadata = None
if arguments.database.startswith("postgresql://"):
database_metadata = importlib.import_module(
"bhamon_orchestra_model.database.sql_database_model").metadata
database_client_factory = factory.create_database_client_factory(
arguments.database, database_metadata)
data_storage_instance = FileDataStorage(".")
date_time_provider_instance = DateTimeProvider()
application = flask.Flask(__name__)
application.database_client_factory = database_client_factory
application.authentication_provider = AuthenticationProvider(
date_time_provider_instance)
application.authorization_provider = AuthorizationProvider()
application.job_provider = JobProvider(date_time_provider_instance)
application.project_provider = ProjectProvider(date_time_provider_instance)
application.run_provider = RunProvider(data_storage_instance,
date_time_provider_instance)
application.schedule_provider = ScheduleProvider(
date_time_provider_instance)
application.user_provider = UserProvider(date_time_provider_instance)
application.worker_provider = WorkerProvider(date_time_provider_instance)
application.run_result_transformer = transform_run_results
application.external_services = {}
service.configure(application)
service.register_handlers(application)
service.register_routes(application)
application.add_url_rule("/me/routes",
methods=["GET"],
view_func=list_routes)
return application
def create_application(local_worker_identifier, configuration,
executor_script):
worker_definition = configuration["orchestra_workers"][
local_worker_identifier]
worker_identifier = worker_definition["identifier"].format(
host=socket.gethostname())
worker_display_name = worker_definition.get(
"display_name", worker_identifier).format(host=socket.gethostname())
write_local_configuration(configuration)
authentication = load_authentication()
properties = load_properties(worker_definition)
data_storage_instance = FileDataStorage(".")
worker_storage_instance = WorkerStorage(data_storage_instance)
master_client_instance = MasterClient(
master_uri=configuration["orchestra_master_url"],
worker_identifier=worker_identifier,
worker_version=bhamon_orchestra_worker.__version__,
user=authentication["user"],
secret=authentication["secret"],
)
executor_command_factory = lambda run_request: [
sys.executable, executor_script, run_request["run_identifier"]
]
worker_instance = Worker(
storage=worker_storage_instance,
master_client=master_client_instance,
display_name=worker_display_name,
properties=properties,
executor_command_factory=executor_command_factory,
)
return worker_instance
def create_application(configuration): # pylint: disable = too-many-locals
database_metadata = None
if configuration["orchestra_database_uri"].startswith("postgresql://"):
database_metadata = importlib.import_module("bhamon_orchestra_model.database.sql_database_model").metadata
database_client_factory = factory.create_database_client_factory(
database_uri = configuration["orchestra_database_uri"],
database_authentication = configuration["orchestra_database_authentication"],
database_metadata = database_metadata,
)
data_storage_instance = FileDataStorage(configuration["orchestra_file_storage_path"])
date_time_provider_instance = DateTimeProvider()
authentication_provider_instance = AuthenticationProvider(date_time_provider_instance)
authorization_provider_instance = AuthorizationProvider()
job_provider_instance = JobProvider(date_time_provider_instance)
project_provider_instance = ProjectProvider(date_time_provider_instance)
run_provider_instance = RunProvider(data_storage_instance, date_time_provider_instance)
schedule_provider_instance = ScheduleProvider(date_time_provider_instance)
user_provider_instance = UserProvider(date_time_provider_instance)
worker_provider_instance = WorkerProvider(date_time_provider_instance)
protocol_factory = functools.partial(
WebSocketServerProtocol,
database_client_factory = database_client_factory,
user_provider = user_provider_instance,
authentication_provider = authentication_provider_instance,
authorization_provider = authorization_provider_instance,
)
supervisor_instance = Supervisor(
host = configuration["orchestra_master_listen_address"],
port = configuration["orchestra_master_listen_port"],
protocol_factory = protocol_factory,
database_client_factory = database_client_factory,
worker_provider = worker_provider_instance,
run_provider = run_provider_instance,
)
worker_selector_instance = WorkerSelector(
database_client_factory = database_client_factory,
worker_provider = worker_provider_instance,
supervisor = supervisor_instance,
)
job_scheduler_instance = JobScheduler(
database_client_factory = database_client_factory,
job_provider = job_provider_instance,
run_provider = run_provider_instance,
schedule_provider = schedule_provider_instance,
supervisor = supervisor_instance,
worker_selector = worker_selector_instance,
date_time_provider = date_time_provider_instance,
)
master_instance = Master(
database_client_factory = database_client_factory,
project_provider = project_provider_instance,
job_provider = job_provider_instance,
schedule_provider = schedule_provider_instance,
worker_provider = worker_provider_instance,
job_scheduler = job_scheduler_instance,
supervisor = supervisor_instance,
)
return master_instance
def create_application(arguments): # pylint: disable = too-many-locals
database_metadata = None
if arguments.database.startswith("postgresql://"):
database_metadata = importlib.import_module(
"bhamon_orchestra_model.database.sql_database_model").metadata
database_client_factory = factory.create_database_client_factory(
arguments.database, database_metadata)
data_storage_instance = FileDataStorage(".")
date_time_provider_instance = DateTimeProvider()
authentication_provider_instance = AuthenticationProvider(
date_time_provider_instance)
authorization_provider_instance = AuthorizationProvider()
job_provider_instance = JobProvider(date_time_provider_instance)
project_provider_instance = ProjectProvider(date_time_provider_instance)
run_provider_instance = RunProvider(data_storage_instance,
date_time_provider_instance)
schedule_provider_instance = ScheduleProvider(date_time_provider_instance)
user_provider_instance = UserProvider(date_time_provider_instance)
worker_provider_instance = WorkerProvider(date_time_provider_instance)
protocol_factory = functools.partial(
WebSocketServerProtocol,
database_client_factory=database_client_factory,
user_provider=user_provider_instance,
authentication_provider=authentication_provider_instance,
authorization_provider=authorization_provider_instance,
)
supervisor_instance = Supervisor(
host=arguments.address,
port=arguments.port,
protocol_factory=protocol_factory,
database_client_factory=database_client_factory,
worker_provider=worker_provider_instance,
run_provider=run_provider_instance,
)
worker_selector_instance = WorkerSelector(
database_client_factory=database_client_factory,
worker_provider=worker_provider_instance,
supervisor=supervisor_instance,
)
job_scheduler_instance = JobScheduler(
database_client_factory=database_client_factory,
job_provider=job_provider_instance,
run_provider=run_provider_instance,
schedule_provider=schedule_provider_instance,
supervisor=supervisor_instance,
worker_selector=worker_selector_instance,
date_time_provider=date_time_provider_instance,
)
master_instance = Master(
database_client_factory=database_client_factory,
project_provider=project_provider_instance,
job_provider=job_provider_instance,
schedule_provider=schedule_provider_instance,
worker_provider=worker_provider_instance,
job_scheduler=job_scheduler_instance,
supervisor=supervisor_instance,
)
# Rapid updates to reduce delays in tests
job_scheduler_instance.update_interval_seconds = 1
supervisor_instance.update_interval_seconds = 1
return master_instance