def create_application(arguments, authentication): properties = { "project": ["examples"], "is_controller": arguments.identifier == "controller", "executor_limit": 100 if arguments.identifier == "controller" else 1, } data_storage_instance = FileDataStorage(".") worker_storage_instance = WorkerStorage(data_storage_instance) master_client_instance = MasterClient( master_uri=arguments.master_uri, worker_identifier=arguments.identifier, worker_version=bhamon_orchestra_worker.__version__, user=authentication["user"], secret=authentication["secret"], ) executor_command_factory = lambda run_request: [ sys.executable, "-m", "test.integration.executor_main", run_request[ "run_identifier"] ] worker_instance = Worker( storage=worker_storage_instance, master_client=master_client_instance, display_name=arguments.identifier, properties=properties, executor_command_factory=executor_command_factory, ) return worker_instance
def create_application(arguments): database_metadata = None if arguments.database.startswith("postgresql://"): database_metadata = importlib.import_module( "bhamon_orchestra_model.database.sql_database_model").metadata database_administration_factory = factory.create_database_administration_factory( arguments.database, database_metadata) database_client_factory = factory.create_database_client_factory( arguments.database, database_metadata) data_storage_instance = FileDataStorage(".") date_time_provider_instance = DateTimeProvider() application = types.SimpleNamespace() application.database_administration_factory = database_administration_factory application.database_client_factory = database_client_factory application.authentication_provider = AuthenticationProvider( date_time_provider_instance) application.authorization_provider = AuthorizationProvider() application.job_provider = JobProvider(date_time_provider_instance) application.project_provider = ProjectProvider(date_time_provider_instance) application.run_provider = RunProvider(data_storage_instance, date_time_provider_instance) application.schedule_provider = ScheduleProvider( date_time_provider_instance) application.user_provider = UserProvider(date_time_provider_instance) application.worker_provider = WorkerProvider(date_time_provider_instance) return application
def create_application(run_identifier, configuration, authentication): data_storage_instance = FileDataStorage(".") date_time_provider_instance = DateTimeProvider() worker_storage_instance = WorkerStorage(data_storage_instance) service_client_instance = WebServiceClient( configuration["orchestra_service_url"], authorization=(authentication["user"], authentication["secret"])) request = worker_storage_instance.load_request(run_identifier) if request["job_definition"]["type"] == "job": executor_instance = JobExecutor( storage=worker_storage_instance, date_time_provider=date_time_provider_instance, ) elif request["job_definition"]["type"] == "pipeline": executor_instance = PipelineExecutor( storage=worker_storage_instance, date_time_provider=date_time_provider_instance, service_client=service_client_instance, ) else: raise RuntimeError("Unsupported job definition") return executor_instance
def instantiate_implementation(temporary_directory, storage_implementation: str) -> DataStorage: if storage_implementation == "memory": return MemoryDataStorage() if storage_implementation == "file": return FileDataStorage(str(temporary_directory)) raise ValueError("Unsupported implementation '%s'" % storage_implementation)
def create_application(configuration): database_metadata = None if configuration["orchestra_database_uri"].startswith("postgresql://"): database_metadata = importlib.import_module( "bhamon_orchestra_model.database.sql_database_model").metadata database_client_factory = factory.create_database_client_factory( database_uri=configuration["orchestra_database_uri"], database_authentication=configuration[ "orchestra_database_authentication"], database_metadata=database_metadata, ) data_storage_instance = FileDataStorage( configuration["orchestra_file_storage_path"]) date_time_provider_instance = DateTimeProvider() application = flask.Flask(__name__) application.database_client_factory = database_client_factory application.authentication_provider = AuthenticationProvider( date_time_provider_instance) application.authorization_provider = AuthorizationProvider() application.job_provider = JobProvider(date_time_provider_instance) application.project_provider = ProjectProvider(date_time_provider_instance) application.run_provider = RunProvider(data_storage_instance, date_time_provider_instance) application.schedule_provider = ScheduleProvider( date_time_provider_instance) application.user_provider = UserProvider(date_time_provider_instance) application.worker_provider = WorkerProvider(date_time_provider_instance) application.run_result_transformer = transform_run_results service.configure(application) service.register_handlers(application) service.register_routes(application) application.external_services = { "artifacts": FileServerClient("Artifact Server", configuration["artifact_server_web_url"]), "github": GitHubClient(configuration.get("github_access_token", None)), "python_packages": FileServerClient("Python Package Repository", configuration["python_package_repository_web_url"]), } application.config["GITHUB_ACCESS_TOKEN"] = configuration.get( "github_access_token", None) return application
def invoke_executor(self, worker_identifier, run_request): worker_directory = os.path.join(self.temporary_directory, worker_identifier) file_data_storage_instance = FileDataStorage(worker_directory) worker_storage_instance = WorkerStorage(file_data_storage_instance) worker_storage_instance.create_run(run_request["run_identifier"]) worker_storage_instance.save_request(run_request["run_identifier"], run_request) return self.invoke( identifier=worker_identifier + "_" + "executor", module="test.integration.executor_main", arguments=[run_request["run_identifier"]], workspace=worker_directory, )
def __init__(self, temporary_directory, database_type, database_suffix=None): environment_instance = environment.load_test_context_environment( str(temporary_directory), database_type) self.temporary_directory = str(temporary_directory) self.master_address = environment_instance["master_address"] self.master_port = environment_instance["master_port"] self.service_address = environment_instance["service_address"] self.service_port = environment_instance["service_port"] self.website_address = environment_instance["website_address"] self.website_port = environment_instance["website_port"] self.database_uri = environment_instance["database_uri"] self.process_collection = [] if self.database_uri is not None: self.database_uri += ("_" + database_suffix) if database_suffix else "" date_time_provider_instance = DateTimeProvider() self.database_administration_factory = factory.create_database_administration_factory( self.database_uri, sql_database_model.metadata) self.database_client_factory = factory.create_database_client_factory( self.database_uri, sql_database_model.metadata) self.data_storage = FileDataStorage( os.path.join(self.temporary_directory, "master")) self.authentication_provider = AuthenticationProvider( date_time_provider_instance) self.authorization_provider = AuthorizationProvider() self.job_provider = JobProvider(date_time_provider_instance) self.project_provider = ProjectProvider( date_time_provider_instance) self.run_provider = RunProvider(self.data_storage, date_time_provider_instance) self.schedule_provider = ScheduleProvider( date_time_provider_instance) self.user_provider = UserProvider(date_time_provider_instance) self.worker_provider = WorkerProvider(date_time_provider_instance)
def test_list_runs(tmpdir): """ Test listing runs """ data_storage_instance = FileDataStorage(str(tmpdir)) worker_storage_instance = WorkerStorage(data_storage_instance) uuid_run_identifier = "2152bd45-dd77-4cbb-998a-af21a52e4cea" arbitrary_run_identifier = "my_run" assert not worker_storage_instance.run_exists(uuid_run_identifier) assert not worker_storage_instance.run_exists(arbitrary_run_identifier) assert worker_storage_instance.list_runs() == [] worker_storage_instance.create_run(uuid_run_identifier) worker_storage_instance.save_request(uuid_run_identifier, {}) worker_storage_instance.create_run(arbitrary_run_identifier) worker_storage_instance.save_request(arbitrary_run_identifier, {}) assert worker_storage_instance.run_exists(uuid_run_identifier) assert worker_storage_instance.run_exists(arbitrary_run_identifier) assert list(sorted(worker_storage_instance.list_runs())) == [ uuid_run_identifier, arbitrary_run_identifier ]
def create_application(arguments): database_metadata = None if arguments.database.startswith("postgresql://"): database_metadata = importlib.import_module( "bhamon_orchestra_model.database.sql_database_model").metadata database_client_factory = factory.create_database_client_factory( arguments.database, database_metadata) data_storage_instance = FileDataStorage(".") date_time_provider_instance = DateTimeProvider() application = flask.Flask(__name__) application.database_client_factory = database_client_factory application.authentication_provider = AuthenticationProvider( date_time_provider_instance) application.authorization_provider = AuthorizationProvider() application.job_provider = JobProvider(date_time_provider_instance) application.project_provider = ProjectProvider(date_time_provider_instance) application.run_provider = RunProvider(data_storage_instance, date_time_provider_instance) application.schedule_provider = ScheduleProvider( date_time_provider_instance) application.user_provider = UserProvider(date_time_provider_instance) application.worker_provider = WorkerProvider(date_time_provider_instance) application.run_result_transformer = transform_run_results application.external_services = {} service.configure(application) service.register_handlers(application) service.register_routes(application) application.add_url_rule("/me/routes", methods=["GET"], view_func=list_routes) return application
def create_application(local_worker_identifier, configuration, executor_script): worker_definition = configuration["orchestra_workers"][ local_worker_identifier] worker_identifier = worker_definition["identifier"].format( host=socket.gethostname()) worker_display_name = worker_definition.get( "display_name", worker_identifier).format(host=socket.gethostname()) write_local_configuration(configuration) authentication = load_authentication() properties = load_properties(worker_definition) data_storage_instance = FileDataStorage(".") worker_storage_instance = WorkerStorage(data_storage_instance) master_client_instance = MasterClient( master_uri=configuration["orchestra_master_url"], worker_identifier=worker_identifier, worker_version=bhamon_orchestra_worker.__version__, user=authentication["user"], secret=authentication["secret"], ) executor_command_factory = lambda run_request: [ sys.executable, executor_script, run_request["run_identifier"] ] worker_instance = Worker( storage=worker_storage_instance, master_client=master_client_instance, display_name=worker_display_name, properties=properties, executor_command_factory=executor_command_factory, ) return worker_instance
def create_application(configuration): # pylint: disable = too-many-locals database_metadata = None if configuration["orchestra_database_uri"].startswith("postgresql://"): database_metadata = importlib.import_module("bhamon_orchestra_model.database.sql_database_model").metadata database_client_factory = factory.create_database_client_factory( database_uri = configuration["orchestra_database_uri"], database_authentication = configuration["orchestra_database_authentication"], database_metadata = database_metadata, ) data_storage_instance = FileDataStorage(configuration["orchestra_file_storage_path"]) date_time_provider_instance = DateTimeProvider() authentication_provider_instance = AuthenticationProvider(date_time_provider_instance) authorization_provider_instance = AuthorizationProvider() job_provider_instance = JobProvider(date_time_provider_instance) project_provider_instance = ProjectProvider(date_time_provider_instance) run_provider_instance = RunProvider(data_storage_instance, date_time_provider_instance) schedule_provider_instance = ScheduleProvider(date_time_provider_instance) user_provider_instance = UserProvider(date_time_provider_instance) worker_provider_instance = WorkerProvider(date_time_provider_instance) protocol_factory = functools.partial( WebSocketServerProtocol, database_client_factory = database_client_factory, user_provider = user_provider_instance, authentication_provider = authentication_provider_instance, authorization_provider = authorization_provider_instance, ) supervisor_instance = Supervisor( host = configuration["orchestra_master_listen_address"], port = configuration["orchestra_master_listen_port"], protocol_factory = protocol_factory, database_client_factory = database_client_factory, worker_provider = worker_provider_instance, run_provider = run_provider_instance, ) worker_selector_instance = WorkerSelector( database_client_factory = database_client_factory, worker_provider = worker_provider_instance, supervisor = supervisor_instance, ) job_scheduler_instance = JobScheduler( database_client_factory = database_client_factory, job_provider = job_provider_instance, run_provider = run_provider_instance, schedule_provider = schedule_provider_instance, supervisor = supervisor_instance, worker_selector = worker_selector_instance, date_time_provider = date_time_provider_instance, ) master_instance = Master( database_client_factory = database_client_factory, project_provider = project_provider_instance, job_provider = job_provider_instance, schedule_provider = schedule_provider_instance, worker_provider = worker_provider_instance, job_scheduler = job_scheduler_instance, supervisor = supervisor_instance, ) return master_instance
def create_application(arguments): # pylint: disable = too-many-locals database_metadata = None if arguments.database.startswith("postgresql://"): database_metadata = importlib.import_module( "bhamon_orchestra_model.database.sql_database_model").metadata database_client_factory = factory.create_database_client_factory( arguments.database, database_metadata) data_storage_instance = FileDataStorage(".") date_time_provider_instance = DateTimeProvider() authentication_provider_instance = AuthenticationProvider( date_time_provider_instance) authorization_provider_instance = AuthorizationProvider() job_provider_instance = JobProvider(date_time_provider_instance) project_provider_instance = ProjectProvider(date_time_provider_instance) run_provider_instance = RunProvider(data_storage_instance, date_time_provider_instance) schedule_provider_instance = ScheduleProvider(date_time_provider_instance) user_provider_instance = UserProvider(date_time_provider_instance) worker_provider_instance = WorkerProvider(date_time_provider_instance) protocol_factory = functools.partial( WebSocketServerProtocol, database_client_factory=database_client_factory, user_provider=user_provider_instance, authentication_provider=authentication_provider_instance, authorization_provider=authorization_provider_instance, ) supervisor_instance = Supervisor( host=arguments.address, port=arguments.port, protocol_factory=protocol_factory, database_client_factory=database_client_factory, worker_provider=worker_provider_instance, run_provider=run_provider_instance, ) worker_selector_instance = WorkerSelector( database_client_factory=database_client_factory, worker_provider=worker_provider_instance, supervisor=supervisor_instance, ) job_scheduler_instance = JobScheduler( database_client_factory=database_client_factory, job_provider=job_provider_instance, run_provider=run_provider_instance, schedule_provider=schedule_provider_instance, supervisor=supervisor_instance, worker_selector=worker_selector_instance, date_time_provider=date_time_provider_instance, ) master_instance = Master( database_client_factory=database_client_factory, project_provider=project_provider_instance, job_provider=job_provider_instance, schedule_provider=schedule_provider_instance, worker_provider=worker_provider_instance, job_scheduler=job_scheduler_instance, supervisor=supervisor_instance, ) # Rapid updates to reduce delays in tests job_scheduler_instance.update_interval_seconds = 1 supervisor_instance.update_interval_seconds = 1 return master_instance