def _copy_supervisor_files(parent_folder: str, tmp_zip_path: str, layer_code_path: str) -> None: supervisor_path = FileUtils.join_paths(tmp_zip_path, parent_folder, 'faassupervisor') shutil.move( supervisor_path, FileUtils.join_paths(layer_code_path, 'python', 'faassupervisor'))
def _create_layer(self) -> str: # Create tmp folders tmp_path = FileUtils.create_tmp_dir() layer_code_path = FileUtils.create_tmp_dir() # Extract 'extra' and 'faassupervisor' from supervisor_zip_path with zipfile.ZipFile(self.supervisor_zip_path) as thezip: for file in thezip.namelist(): # Remove the parent folder path parent_folder, file_name = file.split('/', 1) if file_name.startswith('extra') or file_name.startswith( 'faassupervisor'): thezip.extract(file, tmp_path.name) # Extract content of 'extra' files in layer_code_path extra_folder_path = FileUtils.join_paths(tmp_path.name, parent_folder, 'extra') files = FileUtils.get_all_files_in_directory(extra_folder_path) for file_path in files: FileUtils.unzip_folder(file_path, layer_code_path.name) # Copy 'faassupervisor' to layer_code_path supervisor_folder_path = FileUtils.join_paths(tmp_path.name, parent_folder, 'faassupervisor') shutil.move( supervisor_folder_path, FileUtils.join_paths(layer_code_path.name, 'python', 'faassupervisor')) # Create layer zip with content of layer_code_path layer_zip_path = FileUtils.join_paths(tmp_path.name, f'{self.layer_name}.zip') FileUtils.zip_folder(layer_zip_path, layer_code_path.name) # Register the layer props = self._get_supervisor_layer_props(layer_zip_path) response = self.layer.create(**props) return response['LayerVersionArn']
def __init__(self, resources_info: str, tmp_payload_folder_path: str, supervisor_zip_path: str): self.resources_info = resources_info self._tmp_payload_folder_path = tmp_payload_folder_path self._udocker_dir = FileUtils.join_paths(self._tmp_payload_folder_path, "udocker") self._udocker_dir_orig = "" self._udocker_code = FileUtils.join_paths(self._udocker_dir, "udocker.py") self._udocker_exec = ['python3', self._udocker_code] self._install_udocker(supervisor_zip_path)
def _add_init_script(self): if hasattr(self.aws.lambdaf, "init_script"): if hasattr(self.aws, "config_path"): self.aws.lambdaf.init_script = FileUtils.join_paths(self.aws.config_path, self.aws.lambdaf.init_script) FileUtils.copy_file(self.aws.lambdaf.init_script, FileUtils.join_paths(self.scar_tmp_function_folder_path, _INIT_SCRIPT_NAME)) self.aws.lambdaf.environment['Variables']['INIT_SCRIPT_PATH'] = \ f"/var/task/{_INIT_SCRIPT_NAME}"
def _extract_handler_code(self) -> None: function_handler_dest = FileUtils.join_paths(self.scar_tmp_function_folder_path, f"{self.aws.lambdaf.name}.py") file_path = "" with ZipFile(self._supervisor_zip_path) as thezip: for file in thezip.namelist(): if file.endswith("function_handler.py"): file_path = FileUtils.join_paths(self.aws.lambdaf.tmp_folder_path, file) thezip.extract(file, self.aws.lambdaf.tmp_folder_path) break FileUtils.copy_file(file_path, function_handler_dest)
class ConfigFileParser(): """Class to manage the SCAR configuration file creation, update and load.""" _CONFIG_FOLDER_PATH = ".scar" _CONFIG_FILE_PATH = "scar.cfg" _CONFIG_FILE_NAME_BCK = "scar.cfg_old" config_file_folder = FileUtils.join_paths(SysUtils.get_user_home_path(), _CONFIG_FOLDER_PATH) config_file_path = FileUtils.join_paths(config_file_folder, _CONFIG_FILE_PATH) backup_file_path = FileUtils.join_paths(config_file_folder, _CONFIG_FILE_NAME_BCK) @exception(logger) def __init__(self): # Check if the config file exists if FileUtils.is_file(self.config_file_path): with open(self.config_file_path) as cfg_file: self.cfg_data = json.load(cfg_file) if not self._is_config_file_updated(): self._update_config_file() else: self._create_scar_config_folder_and_file() def _is_config_file_updated(self): if 'config_version' not in self.cfg_data['scar']: return False return StrUtils.compare_versions(self.cfg_data.get('scar', {}).get("config_version", ""), _DEFAULT_CFG['scar']["config_version"]) >= 0 def get_properties(self): """Returns the configuration data of the configuration file.""" return self.cfg_data def get_udocker_zip_url(self): """Returns the url where the udocker zip is stored.""" return self.cfg_data['scar']['udocker_info']['zip_url'] def _create_scar_config_folder_and_file(self): FileUtils.create_folder(self.config_file_folder) self._create_new_config_file() raise ScarConfigFileError(file_path=self.config_file_path) def _create_new_config_file(self): FileUtils.create_file_with_content(self.config_file_path, json.dumps(_DEFAULT_CFG, indent=2)) def _update_config_file(self): logger.info(("SCAR configuration file deprecated.\n" "Updating your SCAR configuration file.")) FileUtils.copy_file(self.config_file_path, self.backup_file_path) logger.info(f"Old configuration file saved in '{self.backup_file_path}'.") self._create_new_config_file() logger.info((f"New configuration file saved in '{self.config_file_path}'.\n" "Please fill your new configuration file with your account information.")) SysUtils.finish_scar_execution()
def create_ecr_image(resources_info: Dict, supervisor_version: str) -> str: """Creates an ECR image using the user provided image adding the supervisor tools.""" # If the user set an already prepared image return the image name image_name = ContainerImage._ecr_image_name_prepared( resources_info.get('lambda').get('container')) if image_name: return image_name tmp_folder = FileUtils.create_tmp_dir() # Create function config file FileUtils.write_yaml( FileUtils.join_paths(tmp_folder.name, "function_config.yaml"), create_function_config(resources_info)) init_script_path = resources_info.get('lambda').get('init_script') # Copy the init script defined by the user to the payload folder if init_script_path: FileUtils.copy_file( init_script_path, FileUtils.join_paths( tmp_folder.name, FileUtils.get_file_name(init_script_path))) # Get supervisor zip supervisor_zip_path = ContainerImage.get_supervisor_zip( resources_info, supervisor_version) # Unzip the supervisor file to the temp file FileUtils.unzip_folder(supervisor_zip_path, tmp_folder.name) # Create dockerfile to generate the new ECR image FileUtils.create_file_with_content( "%s/Dockerfile" % tmp_folder.name, ContainerImage._create_dockerfile_ecr_image( resources_info.get('lambda'))) # Create the ECR Repo and get the image uri ecr_cli = ECR(resources_info) repo_name = resources_info.get('lambda').get('name') ecr_image = ecr_cli.get_repository_uri(repo_name) if not ecr_image: logger.info('Creating ECR repository: %s' % repo_name) ecr_image = ecr_cli.create_repository(repo_name) # Build and push the image to the ECR repo platform = None arch = resources_info.get('lambda').get('architectures', ['x86_64'])[0] if arch == 'arm64': platform = 'linux/arm64' return ContainerImage._build_push_ecr_image( tmp_folder.name, ecr_image, platform, ecr_cli.get_authorization_token())
def _add_config_file_path(scar_info: Dict, resources_info: Dict): if scar_info.get("conf_file", False): resources_info['lambda']['config_path'] = os.path.dirname(scar_info.get("conf_file")) # Update the path of the files based on the path of the yaml (if any) if resources_info['lambda'].get('init_script', False): resources_info['lambda']['init_script'] = FileUtils.join_paths(resources_info['lambda']['config_path'], resources_info['lambda']['init_script']) if resources_info['lambda'].get('image_file', False): resources_info['lambda']['image_file'] = FileUtils.join_paths(resources_info['lambda']['config_path'], resources_info['lambda']['image_file']) if resources_info['lambda'].get('run_script', False): resources_info['lambda']['run_script'] = FileUtils.join_paths(resources_info['lambda']['config_path'], resources_info['lambda']['run_script'])
def _extract_handler_code(self) -> None: function_handler_dest = FileUtils.join_paths(self.tmp_payload_folder.name, f"{self.resources_info.get('lambda').get('name')}.py") file_path = "" with ZipFile(self.supervisor_zip_path) as thezip: for file in thezip.namelist(): if file.endswith("function_handler.py"): file_path = FileUtils.join_paths(FileUtils.get_tmp_dir(), file) # Extracts the complete folder structure and the file (cannot avoid) thezip.extract(file, FileUtils.get_tmp_dir()) break if file_path: # Copy only the handler to the payload folder FileUtils.copy_file(file_path, function_handler_dest)
def _add_init_script(self) -> None: """Copy the init script defined by the user to the payload folder.""" if self.resources_info.get('lambda').get('init_script', False): init_script_path = self.resources_info.get('lambda').get('init_script') FileUtils.copy_file(init_script_path, FileUtils.join_paths(self.tmp_payload_folder.name, FileUtils.get_file_name(init_script_path)))
def _copy_extra_files(parent_folder: str, tmp_zip_path: str, layer_code_path: str) -> None: extra_folder_path = FileUtils.join_paths(tmp_zip_path, parent_folder, 'extra') files = FileUtils.get_all_files_in_directory(extra_folder_path) for file_path in files: FileUtils.unzip_folder(file_path, layer_code_path)
def create_function(self): # Create tmp folders zip_payload_path = None supervisor_zip_path = None if self.function.get('runtime') == "image": # Create docker image in ECR self.function['container']['image'] = ContainerImage.create_ecr_image(self.resources_info, self.supervisor_version) else: # Check if supervisor's source is already cached cached, supervisor_zip_path = SupervisorUtils.is_supervisor_cached(self.supervisor_version) if not cached: # Download supervisor supervisor_zip_path = SupervisorUtils.download_supervisor(self.supervisor_version) # Manage supervisor layer self._manage_supervisor_layer(supervisor_zip_path) # Create function tmp_folder = FileUtils.create_tmp_dir() zip_payload_path = FileUtils.join_paths(tmp_folder.name, 'function.zip') self._set_image_id() self._set_fdl() creation_args = self._get_creations_args(zip_payload_path, supervisor_zip_path) response = self.client.create_function(**creation_args) if response and "FunctionArn" in response: self.function['arn'] = response.get('FunctionArn', "") return response
def __init__(self, aws_properties, supervisor_version): self.aws = aws_properties self.supervisor_version = supervisor_version self.scar_tmp_function_folder = FileUtils.create_tmp_dir() self.scar_tmp_function_folder_path = self.scar_tmp_function_folder.name self._supervisor_zip_path = FileUtils.join_paths(self.aws.lambdaf.tmp_folder_path, 'faas.zip') self.package_args = {}
def __init__(self, aws_properties, function_tmp_folder, supervisor_zip_path): self.aws = aws_properties self.function_tmp_folder = function_tmp_folder self.udocker_dir = FileUtils.join_paths(self.function_tmp_folder, "udocker") self.udocker_dir_orig = "" self._initialize_udocker(supervisor_zip_path)
def _extract_udocker_zip(supervisor_zip_path) -> None: file_path = "" with ZipFile(supervisor_zip_path) as thezip: for file in thezip.namelist(): if file.endswith("udocker.zip"): file_path = FileUtils.join_paths(FileUtils.get_tmp_dir(), file) thezip.extract(file, FileUtils.get_tmp_dir()) break return file_path
def _manage_udocker_images(self): if hasattr(self.aws.lambdaf, "image") and \ hasattr(self.aws, "s3") and \ hasattr(self.aws.s3, "deployment_bucket"): self.udocker.download_udocker_image() if hasattr(self.aws.lambdaf, "image_file"): if hasattr(self.aws, "config_path"): self.aws.lambdaf.image_file = FileUtils.join_paths(self.aws.config_path, self.aws.lambdaf.image_file) self.udocker.prepare_udocker_image()
def _create_layer(self) -> None: tmp_zip_path, layer_code_path = _create_tmp_folders() layer_zip_path = FileUtils.join_paths( FileUtils.get_tmp_dir(), f"{self._SUPERVISOR_LAYER_NAME}.zip") parent_folder = _download_supervisor(self.supervisor_version, tmp_zip_path) _copy_supervisor_files(parent_folder, tmp_zip_path, layer_code_path) _copy_extra_files(parent_folder, tmp_zip_path, layer_code_path) _create_layer_zip(layer_zip_path, layer_code_path) self.layer.create(**self._get_supervisor_layer_props(layer_zip_path)) FileUtils.delete_file(layer_zip_path)
def get_file_key(self, folder_name=None, file_path=None, file_key=None): if file_key: return file_key file_key = '' if file_path: file_key = os.path.basename(file_path) if folder_name: file_key = FileUtils.join_paths(folder_name, file_key) elif folder_name: file_key = folder_name if folder_name.endswith('/') else '{0}/'.format(folder_name) return file_key
def _initialize_properties(self, aws_properties): self.aws.lambdaf.environment = {'Variables': {}} self.aws.lambdaf.invocation_type = "RequestResponse" self.aws.lambdaf.log_type = "Tail" self.aws.lambdaf.layers = [] self.aws.lambdaf.tmp_folder = FileUtils.create_tmp_dir() self.aws.lambdaf.tmp_folder_path = self.aws.lambdaf.tmp_folder.name self.aws.lambdaf.zip_file_path = FileUtils.join_paths(self.aws.lambdaf.tmp_folder_path, 'function.zip') if hasattr(self.aws.lambdaf, "name"): self.aws.lambdaf.handler = "{0}.lambda_handler".format(self.aws.lambdaf.name) if not hasattr(self.aws.lambdaf, "asynchronous"): self.aws.lambdaf.asynchronous = False self._set_default_call_parameters()
def prepare_udocker_image(self): self.save_tmp_udocker_env() image_path = FileUtils.join_paths(FileUtils.get_tmp_dir(), "udocker_image.tar.gz") FileUtils.copy_file(self.aws.lambdaf.image_file, image_path) cmd_out = SysUtils.execute_command_with_msg( self.udocker_exec + ["load", "-i", image_path], cli_msg="Loading image file") # Get the image name from the command output self.aws.lambdaf.image = cmd_out.split('\n')[1] self._create_udocker_container() self.aws.lambdaf.environment['Variables'][ 'IMAGE_ID'] = self.aws.lambdaf.image self._set_udocker_local_registry() self.restore_udocker_env()
def _validate_container_size(self, max_payload_size): if FileUtils.get_tree_size(self.udocker_dir) < (max_payload_size / 2): ucmd = self.udocker_exec + [ "create", "--name=lambda_cont", self.aws.lambdaf.image ] SysUtils.execute_command_with_msg( ucmd, cli_msg="Creating container structure") elif FileUtils.get_tree_size(self.udocker_dir) > max_payload_size: FileUtils.delete_folder( FileUtils.join_paths(self.udocker_dir, "containers")) else: self.aws.lambdaf.environment['Variables']['UDOCKER_LAYERS'] = \ '/var/task/udocker/containers/'
def _get_invocation_payload(self): # Default payload payload = self.aws.lambdaf.payload if hasattr(self.aws.lambdaf, 'payload') else {} if not payload: # Check for defined run script if hasattr(self.aws.lambdaf, "run_script"): script_path = self.aws.lambdaf.run_script if hasattr(self.aws, "config_path"): script_path = FileUtils.join_paths(self.aws.config_path, script_path) # We first code to base64 in bytes and then decode those bytes to allow the json lib to parse the data # https://stackoverflow.com/questions/37225035/serialize-in-json-a-base64-encoded-data#37239382 payload = { "script" : StrUtils.bytes_to_base64str(FileUtils.read_file(script_path, 'rb')) } # Check for defined commands # This overrides any other function payload if hasattr(self.aws.lambdaf, "c_args"): payload = {"cmd_args" : json.dumps(self.aws.lambdaf.c_args)} return json.dumps(payload)
def _copy_function_configuration(self): cfg_file_path = FileUtils.join_paths(self.tmp_payload_folder.name, "function_config.yaml") function_cfg = create_function_config(self.resources_info) FileUtils.write_yaml(cfg_file_path, function_cfg)
def _initialize_udocker(self, supervisor_zip_path): self.udocker_code = FileUtils.join_paths(self.udocker_dir, "udocker.py") self.udocker_exec = ['python3', self.udocker_code] self._install_udocker(supervisor_zip_path)
def _get_download_file_path(self, file_key=None): file_path = file_key if self.scar_info.get('path', False): file_path = FileUtils.join_paths(self.scar_info.get('path'), file_path) return file_path
def _get_download_file_path(self, file_key=None): file_path = file_key if hasattr(self.scar_properties, "path") and self.scar_properties.path: file_path = FileUtils.join_paths(self.scar_properties.path, file_path) return file_path