def _add_container_volumes(self):
self.cont_cmd.extend(["-v", SysUtils.get_env_var("TMP_INPUT_DIR")])
self.cont_cmd.extend(["-v", SysUtils.get_env_var("TMP_OUTPUT_DIR")])
self.cont_cmd.extend(
["-v", "/dev", "-v", "/proc", "-v", "/etc/hosts", "--nosysdirs"])
if SysUtils.is_var_in_env('EXTRA_PAYLOAD'):
self.cont_cmd.extend(["-v", self.lambda_instance.PERMANENT_FOLDER])
def execute_function(self):
script_path = self._get_script_path()
if script_path:
try:
pyinstaller_library_path = SysUtils.get_env_var(
'LD_LIBRARY_PATH')
orig_library_path = SysUtils.get_env_var(
'LD_LIBRARY_PATH_ORIG')
if orig_library_path:
SysUtils.set_env_var('LD_LIBRARY_PATH', orig_library_path)
else:
SysUtils.delete_env_var('LD_LIBRARY_PATH')
self.output = subprocess.check_output(
['/bin/sh', script_path],
stderr=subprocess.STDOUT).decode("latin-1")
SysUtils.set_env_var('LD_LIBRARY_PATH',
pyinstaller_library_path)
get_logger().debug("CONTAINER OUTPUT:\n %s", self.output)
except subprocess.CalledProcessError as cpe:
# Exit with user script return code if an
# error occurs (Kubernetes handles the error)
get_logger().error(cpe.output.decode('latin-1'))
sys.exit(cpe.returncode)
else:
get_logger().error('No user script found!')
def execute_function(self):
script_path = self._get_script_path()
if script_path:
try:
pyinstaller_library_path = SysUtils.get_env_var(
'LD_LIBRARY_PATH')
orig_library_path = SysUtils.get_env_var(
'LD_LIBRARY_PATH_ORIG')
if orig_library_path:
SysUtils.set_env_var('LD_LIBRARY_PATH', orig_library_path)
else:
SysUtils.delete_env_var('LD_LIBRARY_PATH')
proc = subprocess.Popen(['/bin/bash', script_path],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
encoding='utf-8',
errors='ignore')
SysUtils.set_env_var('LD_LIBRARY_PATH',
pyinstaller_library_path)
get_logger().debug("CONTAINER OUTPUT:\n %s", self.output)
for line in proc.stdout:
get_logger().debug(line.strip())
self.output = self.output + line
except subprocess.CalledProcessError as cpe:
# Exit with user script return code if an
# error occurs (Kubernetes handles the error)
get_logger().error(
cpe.output.decode(encoding='utf-8', errors='ignore'))
sys.exit(cpe.returncode)
else:
get_logger().error('No user script found!')
def execute_function(self):
if SysUtils.is_var_in_env('SCRIPT'):
script_path = SysUtils.join_paths(
SysUtils.get_env_var("TMP_INPUT_DIR"), self._SCRIPT_FILE_NAME)
script_content = StrUtils.base64_to_str(
SysUtils.get_env_var('SCRIPT'))
FileUtils.create_file_with_content(script_path, script_content)
get_logger().info("Script file created in '%s'", script_path)
FileUtils.set_file_execution_rights(script_path)
get_logger().info("Executing user defined script: '%s'",
script_path)
try:
pyinstaller_library_path = SysUtils.get_env_var(
'LD_LIBRARY_PATH')
orig_library_path = SysUtils.get_env_var(
'LD_LIBRARY_PATH_ORIG')
if orig_library_path:
SysUtils.set_env_var('LD_LIBRARY_PATH', orig_library_path)
self.output = subprocess.check_output(
['/bin/sh', script_path],
stderr=subprocess.STDOUT).decode("latin-1")
SysUtils.set_env_var('LD_LIBRARY_PATH',
pyinstaller_library_path)
get_logger().debug("CONTAINER OUTPUT:\n %s", self.output)
except subprocess.CalledProcessError as cpe:
# Exit with user script return code if an
# error occurs (Kubernetes handles the error)
get_logger().error(cpe.output.decode('latin-1'))
sys.exit(cpe.returncode)
else:
get_logger().error('No user script found!')
def test_get_env_var(self):
with mock.patch.dict('os.environ', {
"K1": "V1",
"K2": "V2"
},
clear=True):
self.assertEqual(SysUtils.get_env_var("K2"), "V2")
self.assertEqual(SysUtils.get_env_var("K3"), "")
def _add_aws_access_keys(self):
self.cont_cmd.extend(
_parse_cont_env_var("AWS_ACCESS_KEY_ID",
SysUtils.get_env_var("AWS_ACCESS_KEY_ID")))
self.cont_cmd.extend(
_parse_cont_env_var("AWS_SECRET_ACCESS_KEY",
SysUtils.get_env_var("AWS_SECRET_ACCESS_KEY")))
self.cont_cmd.extend(
_parse_cont_env_var("AWS_SESSION_TOKEN",
SysUtils.get_env_var("AWS_SESSION_TOKEN")))
def __init__(self, lambda_instance):
self.lambda_instance = lambda_instance
# Create required udocker folder
FileUtils.create_folder(SysUtils.get_env_var("UDOCKER_DIR"))
# Init the udocker command that will be executed
self.udocker_exec = [SysUtils.get_env_var("UDOCKER_EXEC")]
self.cont_cmd = self.udocker_exec + ["--quiet", "run"]
self.cont_img_id = ConfigUtils.read_cfg_var('container').get('image')
if not self.cont_img_id:
raise ContainerImageNotFoundError()
def _get_script_path(self):
script_path = None
if SysUtils.is_var_in_env('SCRIPT'):
script_path = SysUtils.join_paths(
SysUtils.get_env_var("TMP_INPUT_DIR"), self._SCRIPT_FILE_NAME)
script_content = StrUtils.base64_to_str(
SysUtils.get_env_var('SCRIPT'))
FileUtils.create_file_with_content(script_path, script_content)
get_logger().info("Script file created in '%s'", script_path)
elif FileUtils.is_file(self._OSCAR_SCRIPT_PATH):
script_path = self._OSCAR_SCRIPT_PATH
return script_path
def get_remaining_time_in_seconds(self):
"""Returns the amount of time remaining for the invocation in seconds."""
remaining_time = int(self.context.get_remaining_time_in_millis() / 1000)
timeout_threshold = SysUtils.get_env_var('TIMEOUT_THRESHOLD')
if timeout_threshold is '':
timeout_threshold = ConfigUtils.read_cfg_var('container')['timeout_threshold']
return remaining_time - int(timeout_threshold)
def _get_client(self):
"""Return Minio client with user configuration."""
endpoint = SysUtils.get_env_var('MINIO_ENDPOINT')
if not endpoint:
endpoint = self._DEFAULT_MINIO_ENDPOINT
return boto3.client(
's3',
endpoint_url=endpoint,
aws_access_key_id=self.stg_auth.get_credential('USER'),
aws_secret_access_key=self.stg_auth.get_credential('PASS'))
def _add_iam_credentials(self):
iam_creds = {
'CONT_VAR_AWS_ACCESS_KEY_ID': 'AWS_ACCESS_KEY_ID',
'CONT_VAR_AWS_SECRET_ACCESS_KEY': 'AWS_SECRET_ACCESS_KEY',
'CONT_VAR_AWS_SESSION_TOKEN': 'AWS_SESSION_TOKEN'
}
# Add IAM credentials
for key, value in iam_creds.items():
if SysUtils.is_var_in_env(key):
self.cont_cmd.extend(
_parse_cont_env_var(value, SysUtils.get_env_var(key)))
def _parse_exec_script_and_commands(self):
# Check for script in function event
if 'script' in self.raw_event:
self.script_path = f"{self.input_folder}/script.sh"
script_content = StrUtils.base64_to_str(self.raw_event['script'])
FileUtils.create_file_with_content(self.script_path,
script_content)
# Container invoked with arguments
elif 'cmd_args' in self.raw_event:
# Add args
self.cmd_args = json.loads(self.raw_event['cmd_args'])
# Script to be executed every time (if defined)
elif SysUtils.is_var_in_env('INIT_SCRIPT_PATH'):
# Add init script
self.init_script_path = f"{self.input_folder}/init_script.sh"
FileUtils.cp_file(SysUtils.get_env_var("INIT_SCRIPT_PATH"),
self.init_script_path)
def _parse_input(self):
"""Download input data from storage provider
or save data from POST request.
A function can have information from several storage providers
but one event always represents only one file (so far), so only
one provider is going to be used for each event received.
"""
stg_prov = self._get_input_provider()
get_logger().info("Found '%s' input provider", stg_prov.get_type())
if stg_prov:
get_logger().info("Downloading input file using '%s' event",
self.parsed_event.get_type())
input_file_path = storage.download_input(
stg_prov, self.parsed_event,
SysUtils.get_env_var("TMP_INPUT_DIR"))
if input_file_path and FileUtils.is_file(input_file_path):
SysUtils.set_env_var("INPUT_FILE_PATH", input_file_path)
get_logger().info("INPUT_FILE_PATH variable set to '%s'",
input_file_path)
def _is_lambda_batch_execution():
return SysUtils.get_env_var("EXECUTION_MODE") == "lambda-batch"
def _get_log_level():
loglevel = logging.INFO
if SysUtils.is_var_in_env("LOG_LEVEL"):
loglevel = logging.getLevelName(SysUtils.get_env_var("LOG_LEVEL"))
return loglevel
def _is_lambda_environment():
return (
SysUtils.is_var_in_env('AWS_EXECUTION_ENV') and
SysUtils.get_env_var('AWS_EXECUTION_ENV').startswith('AWS_Lambda_'))
def _add_extra_payload_path(self):
self.cont_cmd.extend(
_parse_cont_env_var("EXTRA_PAYLOAD",
SysUtils.get_env_var("EXTRA_PAYLOAD")))
def _add_storage_object_key(self):
self.cont_cmd.extend(
_parse_cont_env_var("STORAGE_OBJECT_KEY",
SysUtils.get_env_var("STORAGE_OBJECT_KEY")))
def _add_output_dir(self):
self.cont_cmd.extend(
_parse_cont_env_var("TMP_OUTPUT_DIR",
SysUtils.get_env_var("TMP_OUTPUT_DIR")))
def _add_input_file(self):
self.cont_cmd.extend(
_parse_cont_env_var("INPUT_FILE_PATH",
SysUtils.get_env_var("INPUT_FILE_PATH")))
def _set_tmp_folders(self):
self.input_folder = SysUtils.get_env_var("TMP_INPUT_DIR")
self.output_folder = SysUtils.get_env_var("TMP_OUTPUT_DIR")
def _parse_output(self):
for stg_prov in self._get_output_providers():
get_logger().info("Found '%s' output provider",
stg_prov.get_type())
storage.upload_output(stg_prov,
SysUtils.get_env_var("TMP_OUTPUT_DIR"))
def get_remaining_time_in_seconds(self):
"""Returns the amount of time remaining for the invocation in seconds."""
remaining_time = int(self.context.get_remaining_time_in_millis() /
1000)
timeout_threshold = int(SysUtils.get_env_var('TIMEOUT_THRESHOLD'))
return remaining_time - timeout_threshold
