def __update_plan_type(self, plan_type: PlanType): self.__plan_type = plan_type with DatabaseManager.connect() as database_connection: database_connection.execute( "UPDATE magic_castles SET plan_type = ? WHERE hostname = ?", (self.__plan_type.value, self.__hostname), ) database_connection.commit()
def get_status(self) -> ClusterStatusCode: if self.__status is None: with DatabaseManager.connect() as database_connection: result = database_connection.execute( "SELECT status FROM magic_castles WHERE hostname = ?", (self.get_hostname(), ), ).fetchone() if result: self.__status = ClusterStatusCode(result[0]) else: self.__status = ClusterStatusCode.NOT_FOUND return self.__status
def get_owner(self): if self.__owner is None: with DatabaseManager.connect() as database_connection: result = database_connection.execute( "SELECT owner FROM magic_castles WHERE hostname = ?", (self.get_hostname(), ), ).fetchone() if result: self.__owner = result[0] else: self.__owner = None return self.__owner
def get_plan_type(self) -> PlanType: if self.__plan_type is None: with DatabaseManager.connect() as database_connection: result = database_connection.execute( "SELECT plan_type FROM magic_castles WHERE hostname = ?", (self.get_hostname(), ), ).fetchone() if result: self.__plan_type = PlanType(result[0]) else: self.__plan_type = PlanType.NONE return self.__plan_type
def __update_status(self, status: ClusterStatusCode): self.__status = status with DatabaseManager.connect() as database_connection: database_connection.execute( "UPDATE magic_castles SET status = ? WHERE hostname = ?", (self.__status.value, self.__hostname), ) database_connection.commit() # Log cluster status updates for log analytics print( json.dumps({ "hostname": self.get_hostname(), "status": self.__status.value, "owner": self.get_owner(), }), flush=True, )
def decorator(*args, **kwargs): auth_type = AuthType(config.get("auth_type") or "NONE") with DatabaseManager.connect() as database_connection: if auth_type == AuthType.SAML: try: # Note: Request headers are interpreted as ISO Latin 1 encoded strings. # Therefore, special characters and accents in givenName and surname are not correctly decoded. user = AuthenticatedUser( database_connection, edu_person_principal_name=request. headers["eduPersonPrincipalName"], given_name=request.headers["givenName"], surname=request.headers["surname"], mail=request.headers["mail"], ) except KeyError: # Missing an authentication header raise UnauthenticatedException else: user = AnonymousUser(database_connection) return route_handler(user, *args, **kwargs)
from flask import Flask, send_file, send_from_directory from resources.magic_castle_api import MagicCastleAPI from resources.progress_api import ProgressAPI from resources.available_resources_api import AvailableResourcesApi from resources.user_api import UserAPI from flask_cors import CORS from models.cloud.openstack_manager import OpenStackManager from database.schema_manager import SchemaManager from database.database_manager import DatabaseManager from models.configuration import config # Exit with an error if the clouds.yaml is not found or the OpenStack API can't be reached OpenStackManager.test_connection() # Update the database schema to the latest version with DatabaseManager.connect() as database_connection: SchemaManager(database_connection).update_schema() app = Flask(__name__) # Allows all origins on all routes (not safe for production) CORS( app, origins=config["cors_allowed_origins"], ) magic_castle_view = MagicCastleAPI.as_view("magic_castle") app.add_url_rule( "/api/magic-castles", view_func=magic_castle_view, defaults={"hostname": None},
def terraform_apply(destroy: bool): try: self.__rotate_terraform_logs(apply=True) with open( self.__get_cluster_path(TERRAFORM_APPLY_LOG_FILENAME), "w") as output_file: environment_variables = environ.copy() dns_manager = DnsManager(self.get_domain()) environment_variables.update( dns_manager.get_environment_variables()) environment_variables["OS_CLOUD"] = DEFAULT_CLOUD if destroy: environment_variables["TF_WARN_OUTPUT_ERRORS"] = "1" run( [ "terraform", "apply", "-input=false", "-no-color", "-auto-approve", self.__get_cluster_path( TERRAFORM_PLAN_BINARY_FILENAME), ], cwd=self.__get_cluster_path(), stdout=output_file, stderr=output_file, check=True, env=environment_variables, ) if destroy: # Removes the content of the cluster's folder, even if not empty rmtree(self.__get_cluster_path(), ignore_errors=True) with DatabaseManager.connect() as database_connection: database_connection.execute( "DELETE FROM magic_castles WHERE hostname = ?", (self.get_hostname(), ), ) database_connection.commit() else: self.__update_status( ClusterStatusCode.PROVISIONING_RUNNING) if not destroy: provisioning_manager = ProvisioningManager( self.get_hostname()) # Avoid multiple threads polling the same cluster if not provisioning_manager.is_busy(): try: provisioning_manager.poll_until_success() status_code = ClusterStatusCode.PROVISIONING_SUCCESS except PuppetTimeoutException: status_code = ClusterStatusCode.PROVISIONING_ERROR self.__update_status(status_code) except CalledProcessError: logging.info( "An error occurred while running terraform apply.") self.__update_status(ClusterStatusCode.DESTROY_ERROR if destroy else ClusterStatusCode.BUILD_ERROR) finally: self.__remove_existing_plan()
def __plan(self, *, destroy, existing_cluster): plan_type = PlanType.DESTROY if destroy else PlanType.BUILD if existing_cluster: self.__remove_existing_plan() previous_status = self.get_status() else: with DatabaseManager.connect() as database_connection: database_connection.execute( "INSERT INTO magic_castles (hostname, cluster_name, domain, status, plan_type, owner) VALUES (?, ?, ?, ?, ?, ?)", ( self.get_hostname(), self.get_cluster_name(), self.get_domain(), ClusterStatusCode.CREATED.value, plan_type.value, self.get_owner(), ), ) database_connection.commit() mkdir(self.__get_cluster_path()) previous_status = ClusterStatusCode.CREATED self.__update_status(ClusterStatusCode.PLAN_RUNNING) self.__update_plan_type(plan_type) if not destroy: self.__configuration.update_main_tf_json_file() try: run( ["terraform", "init", "-no-color", "-input=false"], cwd=self.__get_cluster_path(), capture_output=True, check=True, ) except CalledProcessError: self.__update_status(previous_status) raise PlanException( "An error occurred while initializing Terraform.", additional_details=f"hostname: {self.get_hostname()}", ) self.__rotate_terraform_logs(apply=False) with open(self.__get_cluster_path(TERRAFORM_PLAN_LOG_FILENAME), "w") as output_file: environment_variables = environ.copy() dns_manager = DnsManager(self.get_domain()) environment_variables.update( dns_manager.get_environment_variables()) environment_variables["OS_CLOUD"] = DEFAULT_CLOUD try: run( [ "terraform", "plan", "-input=false", "-no-color", "-destroy=" + ("true" if destroy else "false"), "-out=" + self.__get_cluster_path( TERRAFORM_PLAN_BINARY_FILENAME), ], cwd=self.__get_cluster_path(), env=environment_variables, stdout=output_file, stderr=output_file, check=True, ) except CalledProcessError: if destroy: # Terraform returns an error if we try to destroy a cluster when the image # it was created with does not exist anymore (e.g. CentOS-7-x64-2019-07). In these cases, # not refreshing the terraform state (refresh=false) solves the issue. try: run( [ "terraform", "plan", "-refresh=false", "-input=false", "-no-color", "-destroy=" + ("true" if destroy else "false"), "-out=" + self.__get_cluster_path( TERRAFORM_PLAN_BINARY_FILENAME), ], cwd=self.__get_cluster_path(), env=environment_variables, stdout=output_file, stderr=output_file, check=True, ) except CalledProcessError: # terraform plan fails even without refreshing the state self.__update_status(previous_status) raise PlanException( "An error occurred while planning changes.", additional_details= f"hostname: {self.get_hostname()}", ) else: self.__update_status(previous_status) raise PlanException( "An error occurred while planning changes.", additional_details=f"hostname: {self.get_hostname()}", ) with open(self.__get_cluster_path(TERRAFORM_PLAN_JSON_FILENAME), "w") as output_file: try: run( [ "terraform", "show", "-no-color", "-json", TERRAFORM_PLAN_BINARY_FILENAME, ], cwd=self.__get_cluster_path(), stdout=output_file, check=True, ) except CalledProcessError: self.__update_status(previous_status) raise PlanException( "An error occurred while exporting planned changes.", additional_details=f"hostname: {self.get_hostname()}", ) self.__update_status(previous_status)
def database_connection(mocker): # Using an in-memory database for faster unit tests with less disk IO mocker.patch("database.database_manager.DATABASE_PATH", new=":memory:") with DatabaseManager.connect() as database_connection: # The database :memory: only exist within a single connection. # Therefore, the DatabaseConnection object is mocked to always return the same connection. class MockDatabaseConnection: def __init__(self): self.__connection = None def __enter__(self) -> sqlite3.Connection: return database_connection def __exit__(self, type, value, traceback): pass mocker.patch( "database.database_manager.DatabaseManager.connect", return_value=MockDatabaseConnection(), ) # Creating the DB schema SchemaManager(database_connection).update_schema() # Seeding test data test_magic_castle_rows_with_owner = [ ( "buildplanning.calculquebec.cloud", "buildplanning", "calculquebec.cloud", "plan_running", "build", "*****@*****.**", ), ( "created.calculquebec.cloud", "created", "calculquebec.cloud", "created", "build", "*****@*****.**", ), ( "empty.calculquebec.cloud", "empty", "calculquebec.cloud", "build_error", "none", "*****@*****.**", ), ( "missingfloatingips.c3.ca", "missingfloatingips", "c3.ca", "build_running", "none", "*****@*****.**", ), ( "missingnodes.sub.example.com", "missingnodes", "sub.example.com", "build_error", "none", "*****@*****.**", ), ( "valid1.calculquebec.cloud", "valid1", "calculquebec.cloud", "provisioning_success", "destroy", "*****@*****.**", ), ] test_magic_castle_rows_without_owner = [ ( "noowner.calculquebec.cloud", "noowner", "calculquebec.cloud", "provisioning_success", "destroy", ), ] database_connection.executemany( "INSERT INTO magic_castles (hostname, cluster_name, domain, status, plan_type, owner) values (?, ?, ?, ?, ?, ?)", test_magic_castle_rows_with_owner, ) database_connection.executemany( "INSERT INTO magic_castles (hostname, cluster_name, domain, status, plan_type) values (?, ?, ?, ?, ?)", test_magic_castle_rows_without_owner, ) database_connection.commit() yield database_connection