def _copy_supervisor_files(parent_folder: str, tmp_zip_path: str,
layer_code_path: str) -> None:
supervisor_path = FileUtils.join_paths(tmp_zip_path, parent_folder,
'faassupervisor')
shutil.move(
supervisor_path,
FileUtils.join_paths(layer_code_path, 'python', 'faassupervisor'))
def _create_layer(self) -> str:
# Create tmp folders
tmp_path = FileUtils.create_tmp_dir()
layer_code_path = FileUtils.create_tmp_dir()
# Extract 'extra' and 'faassupervisor' from supervisor_zip_path
with zipfile.ZipFile(self.supervisor_zip_path) as thezip:
for file in thezip.namelist():
# Remove the parent folder path
parent_folder, file_name = file.split('/', 1)
if file_name.startswith('extra') or file_name.startswith(
'faassupervisor'):
thezip.extract(file, tmp_path.name)
# Extract content of 'extra' files in layer_code_path
extra_folder_path = FileUtils.join_paths(tmp_path.name, parent_folder,
'extra')
files = FileUtils.get_all_files_in_directory(extra_folder_path)
for file_path in files:
FileUtils.unzip_folder(file_path, layer_code_path.name)
# Copy 'faassupervisor' to layer_code_path
supervisor_folder_path = FileUtils.join_paths(tmp_path.name,
parent_folder,
'faassupervisor')
shutil.move(
supervisor_folder_path,
FileUtils.join_paths(layer_code_path.name, 'python',
'faassupervisor'))
# Create layer zip with content of layer_code_path
layer_zip_path = FileUtils.join_paths(tmp_path.name,
f'{self.layer_name}.zip')
FileUtils.zip_folder(layer_zip_path, layer_code_path.name)
# Register the layer
props = self._get_supervisor_layer_props(layer_zip_path)
response = self.layer.create(**props)
return response['LayerVersionArn']
def __init__(self, resources_info: str, tmp_payload_folder_path: str, supervisor_zip_path: str):
self.resources_info = resources_info
self._tmp_payload_folder_path = tmp_payload_folder_path
self._udocker_dir = FileUtils.join_paths(self._tmp_payload_folder_path, "udocker")
self._udocker_dir_orig = ""
self._udocker_code = FileUtils.join_paths(self._udocker_dir, "udocker.py")
self._udocker_exec = ['python3', self._udocker_code]
self._install_udocker(supervisor_zip_path)
def _add_init_script(self):
if hasattr(self.aws.lambdaf, "init_script"):
if hasattr(self.aws, "config_path"):
self.aws.lambdaf.init_script = FileUtils.join_paths(self.aws.config_path,
self.aws.lambdaf.init_script)
FileUtils.copy_file(self.aws.lambdaf.init_script,
FileUtils.join_paths(self.scar_tmp_function_folder_path, _INIT_SCRIPT_NAME))
self.aws.lambdaf.environment['Variables']['INIT_SCRIPT_PATH'] = \
f"/var/task/{_INIT_SCRIPT_NAME}"
def _extract_handler_code(self) -> None:
function_handler_dest = FileUtils.join_paths(self.scar_tmp_function_folder_path, f"{self.aws.lambdaf.name}.py")
file_path = ""
with ZipFile(self._supervisor_zip_path) as thezip:
for file in thezip.namelist():
if file.endswith("function_handler.py"):
file_path = FileUtils.join_paths(self.aws.lambdaf.tmp_folder_path, file)
thezip.extract(file, self.aws.lambdaf.tmp_folder_path)
break
FileUtils.copy_file(file_path, function_handler_dest)
class ConfigFileParser():
"""Class to manage the SCAR configuration file creation, update and load."""
_CONFIG_FOLDER_PATH = ".scar"
_CONFIG_FILE_PATH = "scar.cfg"
_CONFIG_FILE_NAME_BCK = "scar.cfg_old"
config_file_folder = FileUtils.join_paths(SysUtils.get_user_home_path(), _CONFIG_FOLDER_PATH)
config_file_path = FileUtils.join_paths(config_file_folder, _CONFIG_FILE_PATH)
backup_file_path = FileUtils.join_paths(config_file_folder, _CONFIG_FILE_NAME_BCK)
@exception(logger)
def __init__(self):
# Check if the config file exists
if FileUtils.is_file(self.config_file_path):
with open(self.config_file_path) as cfg_file:
self.cfg_data = json.load(cfg_file)
if not self._is_config_file_updated():
self._update_config_file()
else:
self._create_scar_config_folder_and_file()
def _is_config_file_updated(self):
if 'config_version' not in self.cfg_data['scar']:
return False
return StrUtils.compare_versions(self.cfg_data.get('scar', {}).get("config_version", ""),
_DEFAULT_CFG['scar']["config_version"]) >= 0
def get_properties(self):
"""Returns the configuration data of the configuration file."""
return self.cfg_data
def get_udocker_zip_url(self):
"""Returns the url where the udocker zip is stored."""
return self.cfg_data['scar']['udocker_info']['zip_url']
def _create_scar_config_folder_and_file(self):
FileUtils.create_folder(self.config_file_folder)
self._create_new_config_file()
raise ScarConfigFileError(file_path=self.config_file_path)
def _create_new_config_file(self):
FileUtils.create_file_with_content(self.config_file_path,
json.dumps(_DEFAULT_CFG, indent=2))
def _update_config_file(self):
logger.info(("SCAR configuration file deprecated.\n"
"Updating your SCAR configuration file."))
FileUtils.copy_file(self.config_file_path, self.backup_file_path)
logger.info(f"Old configuration file saved in '{self.backup_file_path}'.")
self._create_new_config_file()
logger.info((f"New configuration file saved in '{self.config_file_path}'.\n"
"Please fill your new configuration file with your account information."))
SysUtils.finish_scar_execution()
def create_ecr_image(resources_info: Dict, supervisor_version: str) -> str:
"""Creates an ECR image using the user provided image adding the supervisor tools."""
# If the user set an already prepared image return the image name
image_name = ContainerImage._ecr_image_name_prepared(
resources_info.get('lambda').get('container'))
if image_name:
return image_name
tmp_folder = FileUtils.create_tmp_dir()
# Create function config file
FileUtils.write_yaml(
FileUtils.join_paths(tmp_folder.name, "function_config.yaml"),
create_function_config(resources_info))
init_script_path = resources_info.get('lambda').get('init_script')
# Copy the init script defined by the user to the payload folder
if init_script_path:
FileUtils.copy_file(
init_script_path,
FileUtils.join_paths(
tmp_folder.name,
FileUtils.get_file_name(init_script_path)))
# Get supervisor zip
supervisor_zip_path = ContainerImage.get_supervisor_zip(
resources_info, supervisor_version)
# Unzip the supervisor file to the temp file
FileUtils.unzip_folder(supervisor_zip_path, tmp_folder.name)
# Create dockerfile to generate the new ECR image
FileUtils.create_file_with_content(
"%s/Dockerfile" % tmp_folder.name,
ContainerImage._create_dockerfile_ecr_image(
resources_info.get('lambda')))
# Create the ECR Repo and get the image uri
ecr_cli = ECR(resources_info)
repo_name = resources_info.get('lambda').get('name')
ecr_image = ecr_cli.get_repository_uri(repo_name)
if not ecr_image:
logger.info('Creating ECR repository: %s' % repo_name)
ecr_image = ecr_cli.create_repository(repo_name)
# Build and push the image to the ECR repo
platform = None
arch = resources_info.get('lambda').get('architectures', ['x86_64'])[0]
if arch == 'arm64':
platform = 'linux/arm64'
return ContainerImage._build_push_ecr_image(
tmp_folder.name, ecr_image, platform,
ecr_cli.get_authorization_token())
def _add_config_file_path(scar_info: Dict, resources_info: Dict):
if scar_info.get("conf_file", False):
resources_info['lambda']['config_path'] = os.path.dirname(scar_info.get("conf_file"))
# Update the path of the files based on the path of the yaml (if any)
if resources_info['lambda'].get('init_script', False):
resources_info['lambda']['init_script'] = FileUtils.join_paths(resources_info['lambda']['config_path'],
resources_info['lambda']['init_script'])
if resources_info['lambda'].get('image_file', False):
resources_info['lambda']['image_file'] = FileUtils.join_paths(resources_info['lambda']['config_path'],
resources_info['lambda']['image_file'])
if resources_info['lambda'].get('run_script', False):
resources_info['lambda']['run_script'] = FileUtils.join_paths(resources_info['lambda']['config_path'],
resources_info['lambda']['run_script'])
def _extract_handler_code(self) -> None:
function_handler_dest = FileUtils.join_paths(self.tmp_payload_folder.name, f"{self.resources_info.get('lambda').get('name')}.py")
file_path = ""
with ZipFile(self.supervisor_zip_path) as thezip:
for file in thezip.namelist():
if file.endswith("function_handler.py"):
file_path = FileUtils.join_paths(FileUtils.get_tmp_dir(), file)
# Extracts the complete folder structure and the file (cannot avoid)
thezip.extract(file, FileUtils.get_tmp_dir())
break
if file_path:
# Copy only the handler to the payload folder
FileUtils.copy_file(file_path, function_handler_dest)
def _add_init_script(self) -> None:
"""Copy the init script defined by the user to the payload folder."""
if self.resources_info.get('lambda').get('init_script', False):
init_script_path = self.resources_info.get('lambda').get('init_script')
FileUtils.copy_file(init_script_path,
FileUtils.join_paths(self.tmp_payload_folder.name,
FileUtils.get_file_name(init_script_path)))
def _copy_extra_files(parent_folder: str, tmp_zip_path: str,
layer_code_path: str) -> None:
extra_folder_path = FileUtils.join_paths(tmp_zip_path, parent_folder,
'extra')
files = FileUtils.get_all_files_in_directory(extra_folder_path)
for file_path in files:
FileUtils.unzip_folder(file_path, layer_code_path)
def create_function(self):
# Create tmp folders
zip_payload_path = None
supervisor_zip_path = None
if self.function.get('runtime') == "image":
# Create docker image in ECR
self.function['container']['image'] = ContainerImage.create_ecr_image(self.resources_info,
self.supervisor_version)
else:
# Check if supervisor's source is already cached
cached, supervisor_zip_path = SupervisorUtils.is_supervisor_cached(self.supervisor_version)
if not cached:
# Download supervisor
supervisor_zip_path = SupervisorUtils.download_supervisor(self.supervisor_version)
# Manage supervisor layer
self._manage_supervisor_layer(supervisor_zip_path)
# Create function
tmp_folder = FileUtils.create_tmp_dir()
zip_payload_path = FileUtils.join_paths(tmp_folder.name, 'function.zip')
self._set_image_id()
self._set_fdl()
creation_args = self._get_creations_args(zip_payload_path, supervisor_zip_path)
response = self.client.create_function(**creation_args)
if response and "FunctionArn" in response:
self.function['arn'] = response.get('FunctionArn', "")
return response
def __init__(self, aws_properties, supervisor_version):
self.aws = aws_properties
self.supervisor_version = supervisor_version
self.scar_tmp_function_folder = FileUtils.create_tmp_dir()
self.scar_tmp_function_folder_path = self.scar_tmp_function_folder.name
self._supervisor_zip_path = FileUtils.join_paths(self.aws.lambdaf.tmp_folder_path, 'faas.zip')
self.package_args = {}
def __init__(self, aws_properties, function_tmp_folder,
supervisor_zip_path):
self.aws = aws_properties
self.function_tmp_folder = function_tmp_folder
self.udocker_dir = FileUtils.join_paths(self.function_tmp_folder,
"udocker")
self.udocker_dir_orig = ""
self._initialize_udocker(supervisor_zip_path)
def _extract_udocker_zip(supervisor_zip_path) -> None:
file_path = ""
with ZipFile(supervisor_zip_path) as thezip:
for file in thezip.namelist():
if file.endswith("udocker.zip"):
file_path = FileUtils.join_paths(FileUtils.get_tmp_dir(), file)
thezip.extract(file, FileUtils.get_tmp_dir())
break
return file_path
def _manage_udocker_images(self):
if hasattr(self.aws.lambdaf, "image") and \
hasattr(self.aws, "s3") and \
hasattr(self.aws.s3, "deployment_bucket"):
self.udocker.download_udocker_image()
if hasattr(self.aws.lambdaf, "image_file"):
if hasattr(self.aws, "config_path"):
self.aws.lambdaf.image_file = FileUtils.join_paths(self.aws.config_path,
self.aws.lambdaf.image_file)
self.udocker.prepare_udocker_image()
def _create_layer(self) -> None:
tmp_zip_path, layer_code_path = _create_tmp_folders()
layer_zip_path = FileUtils.join_paths(
FileUtils.get_tmp_dir(), f"{self._SUPERVISOR_LAYER_NAME}.zip")
parent_folder = _download_supervisor(self.supervisor_version,
tmp_zip_path)
_copy_supervisor_files(parent_folder, tmp_zip_path, layer_code_path)
_copy_extra_files(parent_folder, tmp_zip_path, layer_code_path)
_create_layer_zip(layer_zip_path, layer_code_path)
self.layer.create(**self._get_supervisor_layer_props(layer_zip_path))
FileUtils.delete_file(layer_zip_path)
def get_file_key(self, folder_name=None, file_path=None, file_key=None):
if file_key:
return file_key
file_key = ''
if file_path:
file_key = os.path.basename(file_path)
if folder_name:
file_key = FileUtils.join_paths(folder_name, file_key)
elif folder_name:
file_key = folder_name if folder_name.endswith('/') else '{0}/'.format(folder_name)
return file_key
def _initialize_properties(self, aws_properties):
self.aws.lambdaf.environment = {'Variables': {}}
self.aws.lambdaf.invocation_type = "RequestResponse"
self.aws.lambdaf.log_type = "Tail"
self.aws.lambdaf.layers = []
self.aws.lambdaf.tmp_folder = FileUtils.create_tmp_dir()
self.aws.lambdaf.tmp_folder_path = self.aws.lambdaf.tmp_folder.name
self.aws.lambdaf.zip_file_path = FileUtils.join_paths(self.aws.lambdaf.tmp_folder_path, 'function.zip')
if hasattr(self.aws.lambdaf, "name"):
self.aws.lambdaf.handler = "{0}.lambda_handler".format(self.aws.lambdaf.name)
if not hasattr(self.aws.lambdaf, "asynchronous"):
self.aws.lambdaf.asynchronous = False
self._set_default_call_parameters()
def prepare_udocker_image(self):
self.save_tmp_udocker_env()
image_path = FileUtils.join_paths(FileUtils.get_tmp_dir(),
"udocker_image.tar.gz")
FileUtils.copy_file(self.aws.lambdaf.image_file, image_path)
cmd_out = SysUtils.execute_command_with_msg(
self.udocker_exec + ["load", "-i", image_path],
cli_msg="Loading image file")
# Get the image name from the command output
self.aws.lambdaf.image = cmd_out.split('\n')[1]
self._create_udocker_container()
self.aws.lambdaf.environment['Variables'][
'IMAGE_ID'] = self.aws.lambdaf.image
self._set_udocker_local_registry()
self.restore_udocker_env()
def _validate_container_size(self, max_payload_size):
if FileUtils.get_tree_size(self.udocker_dir) < (max_payload_size / 2):
ucmd = self.udocker_exec + [
"create", "--name=lambda_cont", self.aws.lambdaf.image
]
SysUtils.execute_command_with_msg(
ucmd, cli_msg="Creating container structure")
elif FileUtils.get_tree_size(self.udocker_dir) > max_payload_size:
FileUtils.delete_folder(
FileUtils.join_paths(self.udocker_dir, "containers"))
else:
self.aws.lambdaf.environment['Variables']['UDOCKER_LAYERS'] = \
'/var/task/udocker/containers/'
def _get_invocation_payload(self):
# Default payload
payload = self.aws.lambdaf.payload if hasattr(self.aws.lambdaf, 'payload') else {}
if not payload:
# Check for defined run script
if hasattr(self.aws.lambdaf, "run_script"):
script_path = self.aws.lambdaf.run_script
if hasattr(self.aws, "config_path"):
script_path = FileUtils.join_paths(self.aws.config_path, script_path)
# We first code to base64 in bytes and then decode those bytes to allow the json lib to parse the data
# https://stackoverflow.com/questions/37225035/serialize-in-json-a-base64-encoded-data#37239382
payload = { "script" : StrUtils.bytes_to_base64str(FileUtils.read_file(script_path, 'rb')) }
# Check for defined commands
# This overrides any other function payload
if hasattr(self.aws.lambdaf, "c_args"):
payload = {"cmd_args" : json.dumps(self.aws.lambdaf.c_args)}
return json.dumps(payload)
def _copy_function_configuration(self):
cfg_file_path = FileUtils.join_paths(self.tmp_payload_folder.name,
"function_config.yaml")
function_cfg = create_function_config(self.resources_info)
FileUtils.write_yaml(cfg_file_path, function_cfg)
def _initialize_udocker(self, supervisor_zip_path):
self.udocker_code = FileUtils.join_paths(self.udocker_dir,
"udocker.py")
self.udocker_exec = ['python3', self.udocker_code]
self._install_udocker(supervisor_zip_path)
def _get_download_file_path(self, file_key=None):
file_path = file_key
if self.scar_info.get('path', False):
file_path = FileUtils.join_paths(self.scar_info.get('path'),
file_path)
return file_path
def _get_download_file_path(self, file_key=None):
file_path = file_key
if hasattr(self.scar_properties, "path") and self.scar_properties.path:
file_path = FileUtils.join_paths(self.scar_properties.path, file_path)
return file_path