class ConfigFileParser(object):
config_file_name = "scar.cfg"
backup_config_file_name = "scar.cfg_old"
config_folder_name = ".scar"
config_file_folder = utils.join_paths(os.path.expanduser("~"),
config_folder_name)
config_file_path = utils.join_paths(config_file_folder, config_file_name)
backup_file_path = utils.join_paths(config_file_folder,
backup_config_file_name)
@excp.exception(logger)
def __init__(self):
# Check if the config file exists
if os.path.isfile(self.config_file_path):
with open(self.config_file_path) as cfg_file:
self.__setattr__("cfg_data", json.load(cfg_file))
if 'region' not in self.cfg_data[
'aws'] or 'boto_profile' not in self.cfg_data[
'aws'] or 'execution_mode' not in self.cfg_data['aws']:
self.add_missing_attributes()
else:
# Create scar config dir
os.makedirs(self.config_file_folder, exist_ok=True)
self.create_default_config_file()
raise excp.ScarConfigFileError(file_path=self.config_file_path)
def create_default_config_file(self):
with open(self.config_file_path, mode='w') as cfg_file:
cfg_file.write(json.dumps(default_cfg, indent=2))
def get_properties(self):
return self.cfg_data
def add_missing_attributes(self):
logger.info("Updating old scar config file '{0}'.\n".format(
self.config_file_path))
shutil.copy(self.config_file_path, self.backup_file_path)
logger.info("Old scar config file saved in '{0}'.\n".format(
self.backup_file_path))
self.merge_files(self.cfg_data, default_cfg)
self.delete_unused_data()
with open(self.config_file_path, mode='w') as cfg_file:
cfg_file.write(json.dumps(self.cfg_data, indent=2))
def merge_files(self, cfg_data, default_data):
for k, v in default_data.items():
if k not in cfg_data:
cfg_data[k] = v
elif type(cfg_data[k]) is dict:
self.merge_files(cfg_data[k], default_data[k])
def delete_unused_data(self):
if 'region' in self.cfg_data['aws']['lambda']:
region = self.cfg_data['aws']['lambda'].pop('region', None)
if region:
self.cfg_data['aws']['region'] = region
def save_tmp_udocker_env(cls):
#Avoid override global variables
if utils.is_value_in_dict(os.environ, 'UDOCKER_TARBALL'):
cls.udocker_tarball = os.environ['UDOCKER_TARBALL']
if utils.is_value_in_dict(os.environ, 'UDOCKER_DIR'):
cls.udocker_dir = os.environ['UDOCKER_DIR']
# Set temporal global vars
udocker_tarball = utils.resource_path(
utils.join_paths(cls.lambda_code_files_path, "udocker",
"udocker-1.1.3.tar.gz"))
utils.set_environment_variable('UDOCKER_TARBALL', udocker_tarball)
utils.set_environment_variable(
'UDOCKER_DIR', utils.join_paths(cls.scar_temporal_folder,
"udocker"))
def get_function_payload_props(self):
package_args = {
'FunctionName': self.properties['name'],
'EnvironmentVariables':
self.properties['environment']['Variables'],
'ZipFilePath': self.properties['zip_file_path'],
}
if 'init_script' in self.properties:
if 'config_path' in self.aws_properties:
package_args['Script'] = utils.join_paths(
self.aws_properties['config_path'],
self.properties['init_script'])
else:
package_args['Script'] = self.properties['init_script']
if 'extra_payload' in self.properties:
package_args['ExtraPayload'] = self.properties['extra_payload']
if 'image_id' in self.properties:
package_args['ImageId'] = self.properties['image_id']
if 'image_file' in self.properties:
package_args['ImageFile'] = self.properties['image_file']
if 's3' in self.aws_properties:
if 'deployment_bucket' in self.aws_properties['s3']:
package_args['DeploymentBucket'] = self.aws_properties['s3'][
'deployment_bucket']
if 'DeploymentBucket' in package_args:
package_args['FileKey'] = 'lambda/{0}.zip'.format(
self.properties['name'])
return package_args
def __init__(self, function_args):
self.registry_name = utils.get_environment_variable("DOCKER_REGISTRY")
self.function_args = function_args
self.function_image_folder = utils.join_paths(
'/pv/kaniko-builds',
utils.get_random_uuid4_str())
self.root_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
self.job_name = '{0}-build-job'.format(function_args['name'])
def add_init_script(self):
if 'Script' in self.properties:
shutil.copy(
self.properties['Script'],
utils.join_paths(self.scar_temporal_folder,
self.init_script_name))
self.properties['EnvironmentVariables'][
'INIT_SCRIPT_PATH'] = self.init_script_path
def add_mandatory_files(self):
os.makedirs(self.scar_temporal_folder, exist_ok=True)
shutil.copy(utils.resource_path(self.supervisor_source),
self.supervisor_dest)
shutil.copy(utils.resource_path(self.udocker_source),
self.udocker_dest)
os.makedirs(utils.join_paths(self.scar_temporal_folder, "src"),
exist_ok=True)
initpy_source = utils.resource_path(
utils.join_paths(self.lambda_code_files_path, "__init__.py"))
self.initpy_dest = utils.join_paths(self.scar_temporal_folder,
"src/__init__.py")
shutil.copy(initpy_source, self.initpy_dest)
utils_source = utils.resource_path(
utils.join_paths(self.src_path, "utils.py"))
self.utils_dest = utils.join_paths(self.scar_temporal_folder,
"src/utils.py")
shutil.copy(utils_source, self.utils_dest)
exceptions_source = utils.resource_path(
utils.join_paths(self.src_path, "exceptions.py"))
self.exceptions_dest = utils.join_paths(self.scar_temporal_folder,
"src/exceptions.py")
shutil.copy(exceptions_source, self.exceptions_dest)
self.set_environment_variable('UDOCKER_DIR', "/tmp/home/udocker")
self.set_environment_variable('UDOCKER_LIB', "/var/task/udocker/lib/")
self.set_environment_variable('UDOCKER_BIN', "/var/task/udocker/bin/")
self.create_udocker_files()
def _copy_dockerfile(self):
# Get function Dockerfile paths
func_dockerfile_path = utils.join_paths(self.root_path,
'src',
'providers',
'onpremises',
'function_template',
'Dockerfile')
func_dockerfile_dest_path = utils.join_paths(
self.function_image_folder,
'Dockerfile')
# Modify Dockerfile
with open(func_dockerfile_path, 'r') as f_in:
with open(func_dockerfile_dest_path, 'w') as f_out:
for line in f_in:
f_out.write(line.replace(
'FROM ubuntu',
'FROM {0}'.format(self.function_args['image'])))
def prepare_udocker_image(self):
image_path = utils.join_paths(self.os_tmp_folder,
"udocker_image.tar.gz")
shutil.copy(self.properties['ImageFile'], image_path)
cmd_out = self.execute_command(self.udocker_exec +
["load", "-i", image_path],
cli_msg="Loading image file")
self.create_udocker_container(cmd_out)
self.set_environment_variable('IMAGE_ID', cmd_out)
self.set_udocker_local_registry()
def create_udocker_container(self, image_id):
if (utils.get_tree_size(self.scar_temporal_folder) <
MAX_S3_PAYLOAD_SIZE / 2):
self.execute_command(self.udocker_exec +
["create", "--name=lambda_cont", image_id],
cli_msg="Creating container structure")
if (utils.get_tree_size(self.scar_temporal_folder) >
MAX_S3_PAYLOAD_SIZE):
shutil.rmtree(
utils.join_paths(self.scar_temporal_folder,
"udocker/containers/"))
def get_download_file_path(self, s3_file_key, file_prefix):
file_path = s3_file_key
# Parse file path
if file_prefix:
# Get folder name
dir_name_to_add = os.path.basename(os.path.dirname(file_prefix))
# Don't replace last '/'
file_path = s3_file_key.replace(file_prefix[:-1], dir_name_to_add)
if 'path' in self.scar_properties and self.scar_properties['path']:
path_to_download = self.scar_properties['path']
file_path = utils.join_paths(path_to_download, file_path)
return file_path
def get_file_key(self, folder_name=None, file_path=None, file_key=None):
if file_key:
return file_key
file_key = ''
if file_path:
file_key = os.path.basename(file_path)
if folder_name:
file_key = utils.join_paths(folder_name, file_key)
elif folder_name:
file_key = folder_name if folder_name.endswith(
'/') else '{0}/'.format(folder_name)
return file_key
def __init__(self, aws_properties):
GenericClient.__init__(self, aws_properties)
self.properties = aws_properties['lambda']
self.properties['environment'] = {'Variables': {}}
self.properties['zip_file_path'] = utils.join_paths(
utils.get_temp_dir(), 'function.zip')
self.properties['invocation_type'] = 'RequestResponse'
self.properties['log_type'] = 'Tail'
if 'name' in self.properties:
self.properties['handler'] = "{0}.lambda_handler".format(
self.properties['name'])
if 'asynchronous' not in self.properties:
self.properties['asynchronous'] = False
def get_payload(self):
# Default payload
payload = {}
if 'run_script' in self.properties:
if 'config_path' in self.aws_properties:
script_path = utils.join_paths(
self.aws_properties['config_path'],
self.properties['run_script'])
else:
script_path = self.properties['run_script']
file_content = utils.read_file(script_path, 'rb')
# We first code to base64 in bytes and then decode those bytes to allow the json lib to parse the data
# https://stackoverflow.com/questions/37225035/serialize-in-json-a-base64-encoded-data#37239382
payload = {"script": utils.utf8_to_base64_string(file_content)}
if 'c_args' in self.properties:
payload = {"cmd_args": json.dumps(self.properties['c_args'])}
return json.dumps(payload)
def add_mandatory_files(self):
os.makedirs(self.scar_temporal_folder, exist_ok=True)
shutil.copy(utils.resource_path(self.supervisor_source),
self.supervisor_dest)
self.execute_command(['chmod', '0664', self.supervisor_dest])
shutil.copy(utils.resource_path(self.udocker_source),
self.udocker_dest)
self.execute_command(['chmod', '0775', self.udocker_dest])
os.makedirs(utils.join_paths(self.scar_temporal_folder, "src"),
exist_ok=True)
os.makedirs(utils.join_paths(self.scar_temporal_folder, "src",
"clients"),
exist_ok=True)
files = ["utils.py", "exceptions.py"]
for file in files:
file_source = utils.resource_path(
utils.join_paths(self.src_path, file))
self.file_dest = utils.join_paths(self.scar_temporal_folder,
"src/{0}".format(file))
shutil.copy(file_source, self.file_dest)
self.execute_command(['chmod', '0664', self.file_dest])
files = [
"apigateway.py", "batch.py", "lambdafunction.py", "s3.py",
"udocker.py"
]
for file in files:
file_source = utils.resource_path(
utils.join_paths(self.lambda_code_files_path, 'clients', file))
self.file_dest = utils.join_paths(self.scar_temporal_folder,
"src/clients/{0}".format(file))
shutil.copy(file_source, self.file_dest)
self.execute_command(['chmod', '0664', self.file_dest])
self.set_environment_variable('UDOCKER_DIR', "/tmp/home/udocker")
self.set_environment_variable('UDOCKER_LIB', "/var/task/udocker/lib/")
self.set_environment_variable('UDOCKER_BIN', "/var/task/udocker/bin/")
self.create_udocker_files()
def __init__(self, package_props):
self.properties = package_props
self.lambda_code_name = "{0}.py".format(
self.properties['FunctionName'])
self.supervisor_dest = utils.join_paths(self.scar_temporal_folder,
self.lambda_code_name)
class FunctionPackageCreator():
src_path = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
aws_src_path = os.path.dirname(os.path.abspath(__file__))
lambda_code_files_path = utils.join_paths(aws_src_path, "cloud/lambda/")
os_tmp_folder = tempfile.gettempdir()
scar_temporal_folder = utils.join_paths(os_tmp_folder, "scar/")
supervisor_source = utils.join_paths(lambda_code_files_path,
"scarsupervisor.py")
udocker_file = "udockerb" if utils.is_binary_execution() else "udocker.py"
udocker_source = utils.join_paths(lambda_code_files_path, "udocker",
udocker_file)
udocker_dest = utils.join_paths(scar_temporal_folder, "udockerb")
udocker_exec = [udocker_dest]
udocker_tarball = ""
udocker_dir = ""
init_script_name = "init_script.sh"
init_script_path = "/var/task/{0}".format(init_script_name)
extra_payload_path = "/var/task"
def __init__(self, package_props):
self.properties = package_props
self.lambda_code_name = "{0}.py".format(
self.properties['FunctionName'])
self.supervisor_dest = utils.join_paths(self.scar_temporal_folder,
self.lambda_code_name)
@excp.exception(logger)
def prepare_lambda_code(self):
self.clean_tmp_folders()
self.add_mandatory_files()
if 'DeploymentBucket' in self.properties and 'ImageId' in self.properties:
self.download_udocker_image()
if 'ImageFile' in self.properties:
self.prepare_udocker_image()
self.add_init_script()
self.add_extra_payload()
self.zip_scar_folder()
self.check_code_size()
def add_mandatory_files(self):
os.makedirs(self.scar_temporal_folder, exist_ok=True)
shutil.copy(utils.resource_path(self.supervisor_source),
self.supervisor_dest)
self.execute_command(['chmod', '0664', self.supervisor_dest])
shutil.copy(utils.resource_path(self.udocker_source),
self.udocker_dest)
self.execute_command(['chmod', '0775', self.udocker_dest])
os.makedirs(utils.join_paths(self.scar_temporal_folder, "src"),
exist_ok=True)
os.makedirs(utils.join_paths(self.scar_temporal_folder, "src",
"clients"),
exist_ok=True)
files = ["utils.py", "exceptions.py"]
for file in files:
file_source = utils.resource_path(
utils.join_paths(self.src_path, file))
self.file_dest = utils.join_paths(self.scar_temporal_folder,
"src/{0}".format(file))
shutil.copy(file_source, self.file_dest)
self.execute_command(['chmod', '0664', self.file_dest])
files = [
"apigateway.py", "batch.py", "lambdafunction.py", "s3.py",
"udocker.py"
]
for file in files:
file_source = utils.resource_path(
utils.join_paths(self.lambda_code_files_path, 'clients', file))
self.file_dest = utils.join_paths(self.scar_temporal_folder,
"src/clients/{0}".format(file))
shutil.copy(file_source, self.file_dest)
self.execute_command(['chmod', '0664', self.file_dest])
self.set_environment_variable('UDOCKER_DIR', "/tmp/home/udocker")
self.set_environment_variable('UDOCKER_LIB', "/var/task/udocker/lib/")
self.set_environment_variable('UDOCKER_BIN', "/var/task/udocker/bin/")
self.create_udocker_files()
@udocker_env
def create_udocker_files(self):
self.execute_command(self.udocker_exec + ["help"],
cli_msg="Packing udocker files")
def add_init_script(self):
if 'Script' in self.properties:
shutil.copy(
self.properties['Script'],
utils.join_paths(self.scar_temporal_folder,
self.init_script_name))
self.properties['EnvironmentVariables'][
'INIT_SCRIPT_PATH'] = self.init_script_path
def add_extra_payload(self):
if 'ExtraPayload' in self.properties:
logger.info("Adding extra payload from {0}".format(
self.properties['ExtraPayload']))
dir_util.copy_tree(self.properties['ExtraPayload'],
self.scar_temporal_folder)
self.set_environment_variable('EXTRA_PAYLOAD',
self.extra_payload_path)
def check_code_size(self):
# Check if the code size fits within the aws limits
if 'DeploymentBucket' in self.properties:
AWSValidator.validate_s3_code_size(self.scar_temporal_folder,
MAX_S3_PAYLOAD_SIZE)
else:
AWSValidator.validate_function_code_size(self.scar_temporal_folder,
MAX_PAYLOAD_SIZE)
def clean_tmp_folders(self):
if os.path.isfile(self.properties['ZipFilePath']):
utils.delete_file(self.properties['ZipFilePath'])
# Delete created temporal files
if os.path.isdir(self.scar_temporal_folder):
shutil.rmtree(self.scar_temporal_folder, ignore_errors=True)
def zip_scar_folder(self):
zip_exe = utils.resource_path("src/bin/zip", bin_path='/usr/bin/zip')
self.execute_command(
[zip_exe, "-r9y", self.properties['ZipFilePath'], "."],
cmd_wd=self.scar_temporal_folder,
cli_msg="Creating function package")
@classmethod
def save_tmp_udocker_env(cls):
#Avoid override global variables
if utils.is_value_in_dict(os.environ, 'UDOCKER_TARBALL'):
cls.udocker_tarball = os.environ['UDOCKER_TARBALL']
if utils.is_value_in_dict(os.environ, 'UDOCKER_DIR'):
cls.udocker_dir = os.environ['UDOCKER_DIR']
# Set temporal global vars
udocker_tarball = utils.resource_path(
utils.join_paths(cls.lambda_code_files_path, "udocker",
"udocker-1.1.3.tar.gz"))
utils.set_environment_variable('UDOCKER_TARBALL', udocker_tarball)
utils.set_environment_variable(
'UDOCKER_DIR', utils.join_paths(cls.scar_temporal_folder,
"udocker"))
@classmethod
def restore_udocker_env(cls):
cls.restore_environ_var('UDOCKER_TARBALL', cls.udocker_tarball)
cls.restore_environ_var('UDOCKER_DIR', cls.udocker_dir)
@classmethod
def restore_environ_var(cls, key, var):
if var:
utils.set_environment_variable(key, var)
else:
del os.environ[key]
def execute_command(self, command, cmd_wd=None, cli_msg=None):
cmd_out = subprocess.check_output(command, cwd=cmd_wd).decode("utf-8")
logger.info(cli_msg, cmd_out)
return cmd_out[:-1]
@udocker_env
def prepare_udocker_image(self):
image_path = utils.join_paths(self.os_tmp_folder,
"udocker_image.tar.gz")
shutil.copy(self.properties['ImageFile'], image_path)
cmd_out = self.execute_command(self.udocker_exec +
["load", "-i", image_path],
cli_msg="Loading image file")
self.create_udocker_container(cmd_out)
self.set_environment_variable('IMAGE_ID', cmd_out)
self.set_udocker_local_registry()
@udocker_env
def download_udocker_image(self):
self.execute_command(self.udocker_exec +
["pull", self.properties['ImageId']],
cli_msg="Downloading container image")
self.create_udocker_container(self.properties['ImageId'])
self.set_udocker_local_registry()
def create_udocker_container(self, image_id):
if (utils.get_tree_size(self.scar_temporal_folder) <
MAX_S3_PAYLOAD_SIZE / 2):
self.execute_command(self.udocker_exec +
["create", "--name=lambda_cont", image_id],
cli_msg="Creating container structure")
if (utils.get_tree_size(self.scar_temporal_folder) >
MAX_S3_PAYLOAD_SIZE):
shutil.rmtree(
utils.join_paths(self.scar_temporal_folder,
"udocker/containers/"))
def set_udocker_local_registry(self):
self.set_environment_variable('UDOCKER_REPOS',
'/var/task/udocker/repos/')
self.set_environment_variable('UDOCKER_LAYERS',
'/var/task/udocker/layers/')
def set_environment_variable(self, key, val):
if key and val:
self.properties['EnvironmentVariables'][key] = val
def _copy_user_script(self):
utils.create_file_with_content(
utils.join_paths(self.function_image_folder, 'user_script.sh'),
utils.base64_to_utf8_string(self.function_args['script']))
