class ConfigFileParser(object): config_file_name = "scar.cfg" backup_config_file_name = "scar.cfg_old" config_folder_name = ".scar" config_file_folder = utils.join_paths(os.path.expanduser("~"), config_folder_name) config_file_path = utils.join_paths(config_file_folder, config_file_name) backup_file_path = utils.join_paths(config_file_folder, backup_config_file_name) @excp.exception(logger) def __init__(self): # Check if the config file exists if os.path.isfile(self.config_file_path): with open(self.config_file_path) as cfg_file: self.__setattr__("cfg_data", json.load(cfg_file)) if 'region' not in self.cfg_data[ 'aws'] or 'boto_profile' not in self.cfg_data[ 'aws'] or 'execution_mode' not in self.cfg_data['aws']: self.add_missing_attributes() else: # Create scar config dir os.makedirs(self.config_file_folder, exist_ok=True) self.create_default_config_file() raise excp.ScarConfigFileError(file_path=self.config_file_path) def create_default_config_file(self): with open(self.config_file_path, mode='w') as cfg_file: cfg_file.write(json.dumps(default_cfg, indent=2)) def get_properties(self): return self.cfg_data def add_missing_attributes(self): logger.info("Updating old scar config file '{0}'.\n".format( self.config_file_path)) shutil.copy(self.config_file_path, self.backup_file_path) logger.info("Old scar config file saved in '{0}'.\n".format( self.backup_file_path)) self.merge_files(self.cfg_data, default_cfg) self.delete_unused_data() with open(self.config_file_path, mode='w') as cfg_file: cfg_file.write(json.dumps(self.cfg_data, indent=2)) def merge_files(self, cfg_data, default_data): for k, v in default_data.items(): if k not in cfg_data: cfg_data[k] = v elif type(cfg_data[k]) is dict: self.merge_files(cfg_data[k], default_data[k]) def delete_unused_data(self): if 'region' in self.cfg_data['aws']['lambda']: region = self.cfg_data['aws']['lambda'].pop('region', None) if region: self.cfg_data['aws']['region'] = region
def save_tmp_udocker_env(cls): #Avoid override global variables if utils.is_value_in_dict(os.environ, 'UDOCKER_TARBALL'): cls.udocker_tarball = os.environ['UDOCKER_TARBALL'] if utils.is_value_in_dict(os.environ, 'UDOCKER_DIR'): cls.udocker_dir = os.environ['UDOCKER_DIR'] # Set temporal global vars udocker_tarball = utils.resource_path( utils.join_paths(cls.lambda_code_files_path, "udocker", "udocker-1.1.3.tar.gz")) utils.set_environment_variable('UDOCKER_TARBALL', udocker_tarball) utils.set_environment_variable( 'UDOCKER_DIR', utils.join_paths(cls.scar_temporal_folder, "udocker"))
def get_function_payload_props(self): package_args = { 'FunctionName': self.properties['name'], 'EnvironmentVariables': self.properties['environment']['Variables'], 'ZipFilePath': self.properties['zip_file_path'], } if 'init_script' in self.properties: if 'config_path' in self.aws_properties: package_args['Script'] = utils.join_paths( self.aws_properties['config_path'], self.properties['init_script']) else: package_args['Script'] = self.properties['init_script'] if 'extra_payload' in self.properties: package_args['ExtraPayload'] = self.properties['extra_payload'] if 'image_id' in self.properties: package_args['ImageId'] = self.properties['image_id'] if 'image_file' in self.properties: package_args['ImageFile'] = self.properties['image_file'] if 's3' in self.aws_properties: if 'deployment_bucket' in self.aws_properties['s3']: package_args['DeploymentBucket'] = self.aws_properties['s3'][ 'deployment_bucket'] if 'DeploymentBucket' in package_args: package_args['FileKey'] = 'lambda/{0}.zip'.format( self.properties['name']) return package_args
def __init__(self, function_args): self.registry_name = utils.get_environment_variable("DOCKER_REGISTRY") self.function_args = function_args self.function_image_folder = utils.join_paths( '/pv/kaniko-builds', utils.get_random_uuid4_str()) self.root_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))) self.job_name = '{0}-build-job'.format(function_args['name'])
def add_init_script(self): if 'Script' in self.properties: shutil.copy( self.properties['Script'], utils.join_paths(self.scar_temporal_folder, self.init_script_name)) self.properties['EnvironmentVariables'][ 'INIT_SCRIPT_PATH'] = self.init_script_path
def add_mandatory_files(self): os.makedirs(self.scar_temporal_folder, exist_ok=True) shutil.copy(utils.resource_path(self.supervisor_source), self.supervisor_dest) shutil.copy(utils.resource_path(self.udocker_source), self.udocker_dest) os.makedirs(utils.join_paths(self.scar_temporal_folder, "src"), exist_ok=True) initpy_source = utils.resource_path( utils.join_paths(self.lambda_code_files_path, "__init__.py")) self.initpy_dest = utils.join_paths(self.scar_temporal_folder, "src/__init__.py") shutil.copy(initpy_source, self.initpy_dest) utils_source = utils.resource_path( utils.join_paths(self.src_path, "utils.py")) self.utils_dest = utils.join_paths(self.scar_temporal_folder, "src/utils.py") shutil.copy(utils_source, self.utils_dest) exceptions_source = utils.resource_path( utils.join_paths(self.src_path, "exceptions.py")) self.exceptions_dest = utils.join_paths(self.scar_temporal_folder, "src/exceptions.py") shutil.copy(exceptions_source, self.exceptions_dest) self.set_environment_variable('UDOCKER_DIR', "/tmp/home/udocker") self.set_environment_variable('UDOCKER_LIB', "/var/task/udocker/lib/") self.set_environment_variable('UDOCKER_BIN', "/var/task/udocker/bin/") self.create_udocker_files()
def _copy_dockerfile(self): # Get function Dockerfile paths func_dockerfile_path = utils.join_paths(self.root_path, 'src', 'providers', 'onpremises', 'function_template', 'Dockerfile') func_dockerfile_dest_path = utils.join_paths( self.function_image_folder, 'Dockerfile') # Modify Dockerfile with open(func_dockerfile_path, 'r') as f_in: with open(func_dockerfile_dest_path, 'w') as f_out: for line in f_in: f_out.write(line.replace( 'FROM ubuntu', 'FROM {0}'.format(self.function_args['image'])))
def prepare_udocker_image(self): image_path = utils.join_paths(self.os_tmp_folder, "udocker_image.tar.gz") shutil.copy(self.properties['ImageFile'], image_path) cmd_out = self.execute_command(self.udocker_exec + ["load", "-i", image_path], cli_msg="Loading image file") self.create_udocker_container(cmd_out) self.set_environment_variable('IMAGE_ID', cmd_out) self.set_udocker_local_registry()
def create_udocker_container(self, image_id): if (utils.get_tree_size(self.scar_temporal_folder) < MAX_S3_PAYLOAD_SIZE / 2): self.execute_command(self.udocker_exec + ["create", "--name=lambda_cont", image_id], cli_msg="Creating container structure") if (utils.get_tree_size(self.scar_temporal_folder) > MAX_S3_PAYLOAD_SIZE): shutil.rmtree( utils.join_paths(self.scar_temporal_folder, "udocker/containers/"))
def get_download_file_path(self, s3_file_key, file_prefix): file_path = s3_file_key # Parse file path if file_prefix: # Get folder name dir_name_to_add = os.path.basename(os.path.dirname(file_prefix)) # Don't replace last '/' file_path = s3_file_key.replace(file_prefix[:-1], dir_name_to_add) if 'path' in self.scar_properties and self.scar_properties['path']: path_to_download = self.scar_properties['path'] file_path = utils.join_paths(path_to_download, file_path) return file_path
def get_file_key(self, folder_name=None, file_path=None, file_key=None): if file_key: return file_key file_key = '' if file_path: file_key = os.path.basename(file_path) if folder_name: file_key = utils.join_paths(folder_name, file_key) elif folder_name: file_key = folder_name if folder_name.endswith( '/') else '{0}/'.format(folder_name) return file_key
def __init__(self, aws_properties): GenericClient.__init__(self, aws_properties) self.properties = aws_properties['lambda'] self.properties['environment'] = {'Variables': {}} self.properties['zip_file_path'] = utils.join_paths( utils.get_temp_dir(), 'function.zip') self.properties['invocation_type'] = 'RequestResponse' self.properties['log_type'] = 'Tail' if 'name' in self.properties: self.properties['handler'] = "{0}.lambda_handler".format( self.properties['name']) if 'asynchronous' not in self.properties: self.properties['asynchronous'] = False
def get_payload(self): # Default payload payload = {} if 'run_script' in self.properties: if 'config_path' in self.aws_properties: script_path = utils.join_paths( self.aws_properties['config_path'], self.properties['run_script']) else: script_path = self.properties['run_script'] file_content = utils.read_file(script_path, 'rb') # We first code to base64 in bytes and then decode those bytes to allow the json lib to parse the data # https://stackoverflow.com/questions/37225035/serialize-in-json-a-base64-encoded-data#37239382 payload = {"script": utils.utf8_to_base64_string(file_content)} if 'c_args' in self.properties: payload = {"cmd_args": json.dumps(self.properties['c_args'])} return json.dumps(payload)
def add_mandatory_files(self): os.makedirs(self.scar_temporal_folder, exist_ok=True) shutil.copy(utils.resource_path(self.supervisor_source), self.supervisor_dest) self.execute_command(['chmod', '0664', self.supervisor_dest]) shutil.copy(utils.resource_path(self.udocker_source), self.udocker_dest) self.execute_command(['chmod', '0775', self.udocker_dest]) os.makedirs(utils.join_paths(self.scar_temporal_folder, "src"), exist_ok=True) os.makedirs(utils.join_paths(self.scar_temporal_folder, "src", "clients"), exist_ok=True) files = ["utils.py", "exceptions.py"] for file in files: file_source = utils.resource_path( utils.join_paths(self.src_path, file)) self.file_dest = utils.join_paths(self.scar_temporal_folder, "src/{0}".format(file)) shutil.copy(file_source, self.file_dest) self.execute_command(['chmod', '0664', self.file_dest]) files = [ "apigateway.py", "batch.py", "lambdafunction.py", "s3.py", "udocker.py" ] for file in files: file_source = utils.resource_path( utils.join_paths(self.lambda_code_files_path, 'clients', file)) self.file_dest = utils.join_paths(self.scar_temporal_folder, "src/clients/{0}".format(file)) shutil.copy(file_source, self.file_dest) self.execute_command(['chmod', '0664', self.file_dest]) self.set_environment_variable('UDOCKER_DIR', "/tmp/home/udocker") self.set_environment_variable('UDOCKER_LIB', "/var/task/udocker/lib/") self.set_environment_variable('UDOCKER_BIN', "/var/task/udocker/bin/") self.create_udocker_files()
def __init__(self, package_props): self.properties = package_props self.lambda_code_name = "{0}.py".format( self.properties['FunctionName']) self.supervisor_dest = utils.join_paths(self.scar_temporal_folder, self.lambda_code_name)
class FunctionPackageCreator(): src_path = os.path.dirname( os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) aws_src_path = os.path.dirname(os.path.abspath(__file__)) lambda_code_files_path = utils.join_paths(aws_src_path, "cloud/lambda/") os_tmp_folder = tempfile.gettempdir() scar_temporal_folder = utils.join_paths(os_tmp_folder, "scar/") supervisor_source = utils.join_paths(lambda_code_files_path, "scarsupervisor.py") udocker_file = "udockerb" if utils.is_binary_execution() else "udocker.py" udocker_source = utils.join_paths(lambda_code_files_path, "udocker", udocker_file) udocker_dest = utils.join_paths(scar_temporal_folder, "udockerb") udocker_exec = [udocker_dest] udocker_tarball = "" udocker_dir = "" init_script_name = "init_script.sh" init_script_path = "/var/task/{0}".format(init_script_name) extra_payload_path = "/var/task" def __init__(self, package_props): self.properties = package_props self.lambda_code_name = "{0}.py".format( self.properties['FunctionName']) self.supervisor_dest = utils.join_paths(self.scar_temporal_folder, self.lambda_code_name) @excp.exception(logger) def prepare_lambda_code(self): self.clean_tmp_folders() self.add_mandatory_files() if 'DeploymentBucket' in self.properties and 'ImageId' in self.properties: self.download_udocker_image() if 'ImageFile' in self.properties: self.prepare_udocker_image() self.add_init_script() self.add_extra_payload() self.zip_scar_folder() self.check_code_size() def add_mandatory_files(self): os.makedirs(self.scar_temporal_folder, exist_ok=True) shutil.copy(utils.resource_path(self.supervisor_source), self.supervisor_dest) self.execute_command(['chmod', '0664', self.supervisor_dest]) shutil.copy(utils.resource_path(self.udocker_source), self.udocker_dest) self.execute_command(['chmod', '0775', self.udocker_dest]) os.makedirs(utils.join_paths(self.scar_temporal_folder, "src"), exist_ok=True) os.makedirs(utils.join_paths(self.scar_temporal_folder, "src", "clients"), exist_ok=True) files = ["utils.py", "exceptions.py"] for file in files: file_source = utils.resource_path( utils.join_paths(self.src_path, file)) self.file_dest = utils.join_paths(self.scar_temporal_folder, "src/{0}".format(file)) shutil.copy(file_source, self.file_dest) self.execute_command(['chmod', '0664', self.file_dest]) files = [ "apigateway.py", "batch.py", "lambdafunction.py", "s3.py", "udocker.py" ] for file in files: file_source = utils.resource_path( utils.join_paths(self.lambda_code_files_path, 'clients', file)) self.file_dest = utils.join_paths(self.scar_temporal_folder, "src/clients/{0}".format(file)) shutil.copy(file_source, self.file_dest) self.execute_command(['chmod', '0664', self.file_dest]) self.set_environment_variable('UDOCKER_DIR', "/tmp/home/udocker") self.set_environment_variable('UDOCKER_LIB', "/var/task/udocker/lib/") self.set_environment_variable('UDOCKER_BIN', "/var/task/udocker/bin/") self.create_udocker_files() @udocker_env def create_udocker_files(self): self.execute_command(self.udocker_exec + ["help"], cli_msg="Packing udocker files") def add_init_script(self): if 'Script' in self.properties: shutil.copy( self.properties['Script'], utils.join_paths(self.scar_temporal_folder, self.init_script_name)) self.properties['EnvironmentVariables'][ 'INIT_SCRIPT_PATH'] = self.init_script_path def add_extra_payload(self): if 'ExtraPayload' in self.properties: logger.info("Adding extra payload from {0}".format( self.properties['ExtraPayload'])) dir_util.copy_tree(self.properties['ExtraPayload'], self.scar_temporal_folder) self.set_environment_variable('EXTRA_PAYLOAD', self.extra_payload_path) def check_code_size(self): # Check if the code size fits within the aws limits if 'DeploymentBucket' in self.properties: AWSValidator.validate_s3_code_size(self.scar_temporal_folder, MAX_S3_PAYLOAD_SIZE) else: AWSValidator.validate_function_code_size(self.scar_temporal_folder, MAX_PAYLOAD_SIZE) def clean_tmp_folders(self): if os.path.isfile(self.properties['ZipFilePath']): utils.delete_file(self.properties['ZipFilePath']) # Delete created temporal files if os.path.isdir(self.scar_temporal_folder): shutil.rmtree(self.scar_temporal_folder, ignore_errors=True) def zip_scar_folder(self): zip_exe = utils.resource_path("src/bin/zip", bin_path='/usr/bin/zip') self.execute_command( [zip_exe, "-r9y", self.properties['ZipFilePath'], "."], cmd_wd=self.scar_temporal_folder, cli_msg="Creating function package") @classmethod def save_tmp_udocker_env(cls): #Avoid override global variables if utils.is_value_in_dict(os.environ, 'UDOCKER_TARBALL'): cls.udocker_tarball = os.environ['UDOCKER_TARBALL'] if utils.is_value_in_dict(os.environ, 'UDOCKER_DIR'): cls.udocker_dir = os.environ['UDOCKER_DIR'] # Set temporal global vars udocker_tarball = utils.resource_path( utils.join_paths(cls.lambda_code_files_path, "udocker", "udocker-1.1.3.tar.gz")) utils.set_environment_variable('UDOCKER_TARBALL', udocker_tarball) utils.set_environment_variable( 'UDOCKER_DIR', utils.join_paths(cls.scar_temporal_folder, "udocker")) @classmethod def restore_udocker_env(cls): cls.restore_environ_var('UDOCKER_TARBALL', cls.udocker_tarball) cls.restore_environ_var('UDOCKER_DIR', cls.udocker_dir) @classmethod def restore_environ_var(cls, key, var): if var: utils.set_environment_variable(key, var) else: del os.environ[key] def execute_command(self, command, cmd_wd=None, cli_msg=None): cmd_out = subprocess.check_output(command, cwd=cmd_wd).decode("utf-8") logger.info(cli_msg, cmd_out) return cmd_out[:-1] @udocker_env def prepare_udocker_image(self): image_path = utils.join_paths(self.os_tmp_folder, "udocker_image.tar.gz") shutil.copy(self.properties['ImageFile'], image_path) cmd_out = self.execute_command(self.udocker_exec + ["load", "-i", image_path], cli_msg="Loading image file") self.create_udocker_container(cmd_out) self.set_environment_variable('IMAGE_ID', cmd_out) self.set_udocker_local_registry() @udocker_env def download_udocker_image(self): self.execute_command(self.udocker_exec + ["pull", self.properties['ImageId']], cli_msg="Downloading container image") self.create_udocker_container(self.properties['ImageId']) self.set_udocker_local_registry() def create_udocker_container(self, image_id): if (utils.get_tree_size(self.scar_temporal_folder) < MAX_S3_PAYLOAD_SIZE / 2): self.execute_command(self.udocker_exec + ["create", "--name=lambda_cont", image_id], cli_msg="Creating container structure") if (utils.get_tree_size(self.scar_temporal_folder) > MAX_S3_PAYLOAD_SIZE): shutil.rmtree( utils.join_paths(self.scar_temporal_folder, "udocker/containers/")) def set_udocker_local_registry(self): self.set_environment_variable('UDOCKER_REPOS', '/var/task/udocker/repos/') self.set_environment_variable('UDOCKER_LAYERS', '/var/task/udocker/layers/') def set_environment_variable(self, key, val): if key and val: self.properties['EnvironmentVariables'][key] = val
def _copy_user_script(self): utils.create_file_with_content( utils.join_paths(self.function_image_folder, 'user_script.sh'), utils.base64_to_utf8_string(self.function_args['script']))