def create_application(arguments):
database_metadata = None
if arguments.database.startswith("postgresql://"):
database_metadata = importlib.import_module(
"bhamon_orchestra_model.database.sql_database_model").metadata
database_administration_factory = factory.create_database_administration_factory(
arguments.database, database_metadata)
database_client_factory = factory.create_database_client_factory(
arguments.database, database_metadata)
data_storage_instance = FileDataStorage(".")
date_time_provider_instance = DateTimeProvider()
application = types.SimpleNamespace()
application.database_administration_factory = database_administration_factory
application.database_client_factory = database_client_factory
application.authentication_provider = AuthenticationProvider(
date_time_provider_instance)
application.authorization_provider = AuthorizationProvider()
application.job_provider = JobProvider(date_time_provider_instance)
application.project_provider = ProjectProvider(date_time_provider_instance)
application.run_provider = RunProvider(data_storage_instance,
date_time_provider_instance)
application.schedule_provider = ScheduleProvider(
date_time_provider_instance)
application.user_provider = UserProvider(date_time_provider_instance)
application.worker_provider = WorkerProvider(date_time_provider_instance)
return application
def __init__(self,
temporary_directory,
database_type,
database_suffix=None):
environment_instance = environment.load_test_context_environment(
str(temporary_directory), database_type)
self.temporary_directory = str(temporary_directory)
self.master_address = environment_instance["master_address"]
self.master_port = environment_instance["master_port"]
self.service_address = environment_instance["service_address"]
self.service_port = environment_instance["service_port"]
self.website_address = environment_instance["website_address"]
self.website_port = environment_instance["website_port"]
self.database_uri = environment_instance["database_uri"]
self.process_collection = []
if self.database_uri is not None:
self.database_uri += ("_" +
database_suffix) if database_suffix else ""
date_time_provider_instance = DateTimeProvider()
self.database_administration_factory = factory.create_database_administration_factory(
self.database_uri, sql_database_model.metadata)
self.database_client_factory = factory.create_database_client_factory(
self.database_uri, sql_database_model.metadata)
self.data_storage = FileDataStorage(
os.path.join(self.temporary_directory, "master"))
self.authentication_provider = AuthenticationProvider(
date_time_provider_instance)
self.authorization_provider = AuthorizationProvider()
self.job_provider = JobProvider(date_time_provider_instance)
self.project_provider = ProjectProvider(
date_time_provider_instance)
self.run_provider = RunProvider(self.data_storage,
date_time_provider_instance)
self.schedule_provider = ScheduleProvider(
date_time_provider_instance)
self.user_provider = UserProvider(date_time_provider_instance)
self.worker_provider = WorkerProvider(date_time_provider_instance)
def create_application(configuration):
database_metadata = None
if configuration["orchestra_database_uri"].startswith("postgresql://"):
database_metadata = importlib.import_module(
"bhamon_orchestra_model.database.sql_database_model").metadata
database_client_factory = factory.create_database_client_factory(
database_uri=configuration["orchestra_database_uri"],
database_authentication=configuration[
"orchestra_database_authentication"],
database_metadata=database_metadata,
)
data_storage_instance = FileDataStorage(
configuration["orchestra_file_storage_path"])
date_time_provider_instance = DateTimeProvider()
application = flask.Flask(__name__)
application.database_client_factory = database_client_factory
application.authentication_provider = AuthenticationProvider(
date_time_provider_instance)
application.authorization_provider = AuthorizationProvider()
application.job_provider = JobProvider(date_time_provider_instance)
application.project_provider = ProjectProvider(date_time_provider_instance)
application.run_provider = RunProvider(data_storage_instance,
date_time_provider_instance)
application.schedule_provider = ScheduleProvider(
date_time_provider_instance)
application.user_provider = UserProvider(date_time_provider_instance)
application.worker_provider = WorkerProvider(date_time_provider_instance)
application.run_result_transformer = transform_run_results
service.configure(application)
service.register_handlers(application)
service.register_routes(application)
application.external_services = {
"artifacts":
FileServerClient("Artifact Server",
configuration["artifact_server_web_url"]),
"github":
GitHubClient(configuration.get("github_access_token", None)),
"python_packages":
FileServerClient("Python Package Repository",
configuration["python_package_repository_web_url"]),
}
application.config["GITHUB_ACCESS_TOKEN"] = configuration.get(
"github_access_token", None)
return application
def create_application(arguments):
database_metadata = None
if arguments.database.startswith("postgresql://"):
database_metadata = importlib.import_module(
"bhamon_orchestra_model.database.sql_database_model").metadata
database_client_factory = factory.create_database_client_factory(
arguments.database, database_metadata)
data_storage_instance = FileDataStorage(".")
date_time_provider_instance = DateTimeProvider()
application = flask.Flask(__name__)
application.database_client_factory = database_client_factory
application.authentication_provider = AuthenticationProvider(
date_time_provider_instance)
application.authorization_provider = AuthorizationProvider()
application.job_provider = JobProvider(date_time_provider_instance)
application.project_provider = ProjectProvider(date_time_provider_instance)
application.run_provider = RunProvider(data_storage_instance,
date_time_provider_instance)
application.schedule_provider = ScheduleProvider(
date_time_provider_instance)
application.user_provider = UserProvider(date_time_provider_instance)
application.worker_provider = WorkerProvider(date_time_provider_instance)
application.run_result_transformer = transform_run_results
application.external_services = {}
service.configure(application)
service.register_handlers(application)
service.register_routes(application)
application.add_url_rule("/me/routes",
methods=["GET"],
view_func=list_routes)
return application
def create_application(configuration): # pylint: disable = too-many-locals
database_metadata = None
if configuration["orchestra_database_uri"].startswith("postgresql://"):
database_metadata = importlib.import_module("bhamon_orchestra_model.database.sql_database_model").metadata
database_client_factory = factory.create_database_client_factory(
database_uri = configuration["orchestra_database_uri"],
database_authentication = configuration["orchestra_database_authentication"],
database_metadata = database_metadata,
)
data_storage_instance = FileDataStorage(configuration["orchestra_file_storage_path"])
date_time_provider_instance = DateTimeProvider()
authentication_provider_instance = AuthenticationProvider(date_time_provider_instance)
authorization_provider_instance = AuthorizationProvider()
job_provider_instance = JobProvider(date_time_provider_instance)
project_provider_instance = ProjectProvider(date_time_provider_instance)
run_provider_instance = RunProvider(data_storage_instance, date_time_provider_instance)
schedule_provider_instance = ScheduleProvider(date_time_provider_instance)
user_provider_instance = UserProvider(date_time_provider_instance)
worker_provider_instance = WorkerProvider(date_time_provider_instance)
protocol_factory = functools.partial(
WebSocketServerProtocol,
database_client_factory = database_client_factory,
user_provider = user_provider_instance,
authentication_provider = authentication_provider_instance,
authorization_provider = authorization_provider_instance,
)
supervisor_instance = Supervisor(
host = configuration["orchestra_master_listen_address"],
port = configuration["orchestra_master_listen_port"],
protocol_factory = protocol_factory,
database_client_factory = database_client_factory,
worker_provider = worker_provider_instance,
run_provider = run_provider_instance,
)
worker_selector_instance = WorkerSelector(
database_client_factory = database_client_factory,
worker_provider = worker_provider_instance,
supervisor = supervisor_instance,
)
job_scheduler_instance = JobScheduler(
database_client_factory = database_client_factory,
job_provider = job_provider_instance,
run_provider = run_provider_instance,
schedule_provider = schedule_provider_instance,
supervisor = supervisor_instance,
worker_selector = worker_selector_instance,
date_time_provider = date_time_provider_instance,
)
master_instance = Master(
database_client_factory = database_client_factory,
project_provider = project_provider_instance,
job_provider = job_provider_instance,
schedule_provider = schedule_provider_instance,
worker_provider = worker_provider_instance,
job_scheduler = job_scheduler_instance,
supervisor = supervisor_instance,
)
return master_instance
def create_application(arguments): # pylint: disable = too-many-locals
database_metadata = None
if arguments.database.startswith("postgresql://"):
database_metadata = importlib.import_module(
"bhamon_orchestra_model.database.sql_database_model").metadata
database_client_factory = factory.create_database_client_factory(
arguments.database, database_metadata)
data_storage_instance = FileDataStorage(".")
date_time_provider_instance = DateTimeProvider()
authentication_provider_instance = AuthenticationProvider(
date_time_provider_instance)
authorization_provider_instance = AuthorizationProvider()
job_provider_instance = JobProvider(date_time_provider_instance)
project_provider_instance = ProjectProvider(date_time_provider_instance)
run_provider_instance = RunProvider(data_storage_instance,
date_time_provider_instance)
schedule_provider_instance = ScheduleProvider(date_time_provider_instance)
user_provider_instance = UserProvider(date_time_provider_instance)
worker_provider_instance = WorkerProvider(date_time_provider_instance)
protocol_factory = functools.partial(
WebSocketServerProtocol,
database_client_factory=database_client_factory,
user_provider=user_provider_instance,
authentication_provider=authentication_provider_instance,
authorization_provider=authorization_provider_instance,
)
supervisor_instance = Supervisor(
host=arguments.address,
port=arguments.port,
protocol_factory=protocol_factory,
database_client_factory=database_client_factory,
worker_provider=worker_provider_instance,
run_provider=run_provider_instance,
)
worker_selector_instance = WorkerSelector(
database_client_factory=database_client_factory,
worker_provider=worker_provider_instance,
supervisor=supervisor_instance,
)
job_scheduler_instance = JobScheduler(
database_client_factory=database_client_factory,
job_provider=job_provider_instance,
run_provider=run_provider_instance,
schedule_provider=schedule_provider_instance,
supervisor=supervisor_instance,
worker_selector=worker_selector_instance,
date_time_provider=date_time_provider_instance,
)
master_instance = Master(
database_client_factory=database_client_factory,
project_provider=project_provider_instance,
job_provider=job_provider_instance,
schedule_provider=schedule_provider_instance,
worker_provider=worker_provider_instance,
job_scheduler=job_scheduler_instance,
supervisor=supervisor_instance,
)
# Rapid updates to reduce delays in tests
job_scheduler_instance.update_interval_seconds = 1
supervisor_instance.update_interval_seconds = 1
return master_instance
class OrchestraContext: # pylint: disable = too-many-instance-attributes
def __init__(self,
temporary_directory,
database_type,
database_suffix=None):
environment_instance = environment.load_test_context_environment(
str(temporary_directory), database_type)
self.temporary_directory = str(temporary_directory)
self.master_address = environment_instance["master_address"]
self.master_port = environment_instance["master_port"]
self.service_address = environment_instance["service_address"]
self.service_port = environment_instance["service_port"]
self.website_address = environment_instance["website_address"]
self.website_port = environment_instance["website_port"]
self.database_uri = environment_instance["database_uri"]
self.process_collection = []
if self.database_uri is not None:
self.database_uri += ("_" +
database_suffix) if database_suffix else ""
date_time_provider_instance = DateTimeProvider()
self.database_administration_factory = factory.create_database_administration_factory(
self.database_uri, sql_database_model.metadata)
self.database_client_factory = factory.create_database_client_factory(
self.database_uri, sql_database_model.metadata)
self.data_storage = FileDataStorage(
os.path.join(self.temporary_directory, "master"))
self.authentication_provider = AuthenticationProvider(
date_time_provider_instance)
self.authorization_provider = AuthorizationProvider()
self.job_provider = JobProvider(date_time_provider_instance)
self.project_provider = ProjectProvider(
date_time_provider_instance)
self.run_provider = RunProvider(self.data_storage,
date_time_provider_instance)
self.schedule_provider = ScheduleProvider(
date_time_provider_instance)
self.user_provider = UserProvider(date_time_provider_instance)
self.worker_provider = WorkerProvider(date_time_provider_instance)
def __enter__(self):
if self.database_uri is not None and self.database_uri.startswith(
"mongodb://"):
with pymongo.MongoClient(
self.database_uri,
serverSelectionTimeoutMS=5000) as mongo_client:
mongo_client.drop_database(mongo_client.get_database())
if self.database_uri is not None and self.database_uri.startswith(
"postgresql://"):
if sqlalchemy_utils.database_exists(self.database_uri):
sqlalchemy_utils.drop_database(self.database_uri)
sqlalchemy_utils.create_database(self.database_uri)
if self.database_uri is not None:
with self.database_administration_factory(
) as database_administration:
database_administration.initialize(simulate=False)
return self
def __exit__(self, exception_type, exception_value, traceback):
for process in self.process_collection:
self.terminate(str(process.pid), process, "ContextExit")
self.process_collection.clear()
if self.database_uri is not None and self.database_uri.startswith(
"mongodb://"):
with pymongo.MongoClient(
self.database_uri,
serverSelectionTimeoutMS=5000) as mongo_client:
mongo_client.drop_database(mongo_client.get_database())
if self.database_uri is not None and self.database_uri.startswith(
"postgresql://"):
if sqlalchemy_utils.database_exists(self.database_uri):
sqlalchemy_utils.drop_database(self.database_uri)
def get_service_uri(self):
return "http://%s:%s" % (self.service_address, self.service_port)
def get_website_uri(self):
return "http://%s:%s" % (self.website_address, self.website_port)
def invoke_master(self):
return self.invoke(
identifier="master",
module="test.integration.master_main",
arguments=[
"--address", self.master_address, "--port",
str(self.master_port), "--database", self.database_uri
],
workspace=os.path.join(self.temporary_directory, "master"),
)
def invoke_worker(self, worker_identifier):
return self.invoke(
identifier=worker_identifier,
module="test.integration.worker_main",
arguments=[
"--identifier", worker_identifier, "--master-uri",
"ws://%s:%s" % (self.master_address, self.master_port)
],
workspace=os.path.join(self.temporary_directory,
worker_identifier),
)
def invoke_executor(self, worker_identifier, run_request):
worker_directory = os.path.join(self.temporary_directory,
worker_identifier)
file_data_storage_instance = FileDataStorage(worker_directory)
worker_storage_instance = WorkerStorage(file_data_storage_instance)
worker_storage_instance.create_run(run_request["run_identifier"])
worker_storage_instance.save_request(run_request["run_identifier"],
run_request)
return self.invoke(
identifier=worker_identifier + "_" + "executor",
module="test.integration.executor_main",
arguments=[run_request["run_identifier"]],
workspace=worker_directory,
)
def invoke_service(self):
return self.invoke(
identifier="service",
module="test.integration.service_main",
arguments=[
"--address", self.service_address, "--port",
str(self.service_port), "--database", self.database_uri
],
workspace=os.path.join(self.temporary_directory, "master"),
)
def invoke_website(self):
return self.invoke(
identifier="website",
module="test.integration.website_main",
arguments=[
"--address", self.website_address, "--port",
str(self.website_port)
],
workspace=os.path.join(self.temporary_directory, "website"),
)
def invoke(self, identifier, module, arguments, workspace):
logger.info("Invoking subprocess '%s'", identifier)
command = [sys.executable, "-m", module] + arguments
process_environment = os.environ.copy()
process_environment["PYTHONPATH"] = os.getcwd()
os.makedirs(workspace, exist_ok=True)
with open(os.path.join(self.temporary_directory,
identifier + "_" + "stdout.log"),
mode="a",
encoding="utf-8") as stdout_file:
with open(os.path.join(self.temporary_directory,
identifier + "_" + "stderr.log"),
mode="a",
encoding="utf-8") as stderr_file:
process = subprocess.Popen(command,
cwd=workspace,
env=process_environment,
stdout=stdout_file,
stderr=stderr_file,
creationflags=subprocess_flags)
logger.info("New subprocess '%s' (PID: %s)", identifier, process.pid)
self.process_collection.append(process)
time.sleep(1) # Wait for initialization
return {
"identifier":
identifier,
"process":
process,
"stdout_file_path":
os.path.join(self.temporary_directory,
identifier + "_" + "stdout.log"),
"stderr_file_path":
os.path.join(self.temporary_directory,
identifier + "_" + "stderr.log"),
}
def terminate(self, identifier, process, reason):
if process not in self.process_collection:
raise ValueError("Unknown process '%s' (PID: %s)" %
(identifier, process.pid))
logger.info("Terminating subprocess '%s' (PID: %s, Reason: '%s')",
identifier, process.pid, reason)
if process.poll() is None:
logger.info("Requesting subprocess '%s' for termination (PID: %s)",
identifier, process.pid)
os.kill(process.pid, shutdown_signal)
try:
process.wait(termination_timeout_seconds)
except subprocess.TimeoutExpired:
pass
if process.poll() is None:
logger.error("Forcing subprocess '%s' termination (PID: %s)",
identifier, process.pid)
process.kill()
try:
process.wait(termination_timeout_seconds)
except subprocess.TimeoutExpired:
pass
if process.poll() is None:
logger.error("Terminating subprocess '%s' failed (PID: %s)",
identifier, process.pid)
if process.poll() is not None:
logger.info("Terminating subprocess '%s' succeeded (PID: %s)",
identifier, process.pid)
def configure_worker_authentication(self, worker_collection):
with self.database_client_factory() as database_client:
user = self.user_provider.create(database_client, "worker",
"Worker")
self.user_provider.update_roles(database_client, user, "Worker")
token = self.authentication_provider.create_token(
database_client, "worker", None, None)
for worker in worker_collection:
worker_directory = os.path.join(self.temporary_directory, worker)
os.makedirs(worker_directory, exist_ok=True)
with open(os.path.join(worker_directory, "authentication.json"),
mode="w",
encoding="utf-8") as authentication_file:
json.dump(token, authentication_file, indent=4)
def configure_service_authentication(self, user_identifier, user_roles):
with self.database_client_factory() as database_client:
user = self.user_provider.create(database_client, user_identifier,
user_identifier)
self.user_provider.update_roles(database_client, user, user_roles)
token = self.authentication_provider.create_token(
database_client, user_identifier, None, None)
return (user_identifier, token["secret"])
def configure_website_authentication(self, user_identifier, user_roles):
with self.database_client_factory() as database_client:
user = self.user_provider.create(database_client, user_identifier,
user_identifier)
self.user_provider.update_roles(database_client, user, user_roles)
self.authentication_provider.set_password(database_client,
user_identifier,
"password")
return (user_identifier, "password")