def execute_function(self):
if SysUtils.is_var_in_env('SCRIPT'):
script_path = SysUtils.join_paths(
SysUtils.get_env_var("TMP_INPUT_DIR"), self._SCRIPT_FILE_NAME)
script_content = StrUtils.base64_to_str(
SysUtils.get_env_var('SCRIPT'))
FileUtils.create_file_with_content(script_path, script_content)
get_logger().info("Script file created in '%s'", script_path)
FileUtils.set_file_execution_rights(script_path)
get_logger().info("Executing user defined script: '%s'",
script_path)
try:
pyinstaller_library_path = SysUtils.get_env_var(
'LD_LIBRARY_PATH')
orig_library_path = SysUtils.get_env_var(
'LD_LIBRARY_PATH_ORIG')
if orig_library_path:
SysUtils.set_env_var('LD_LIBRARY_PATH', orig_library_path)
self.output = subprocess.check_output(
['/bin/sh', script_path],
stderr=subprocess.STDOUT).decode("latin-1")
SysUtils.set_env_var('LD_LIBRARY_PATH',
pyinstaller_library_path)
get_logger().debug("CONTAINER OUTPUT:\n %s", self.output)
except subprocess.CalledProcessError as cpe:
# Exit with user script return code if an
# error occurs (Kubernetes handles the error)
get_logger().error(cpe.output.decode('latin-1'))
sys.exit(cpe.returncode)
else:
get_logger().error('No user script found!')
def save_event(self, input_dir_path):
file_path = SysUtils.join_paths(input_dir_path, self._FILE_NAME)
if self.has_json_body():
FileUtils.create_file_with_content(file_path, self.body)
else:
FileUtils.create_file_with_content(file_path,
base64.b64decode(self.body),
mode='wb')
return file_path
def _get_script_path(self):
script_path = None
if SysUtils.is_var_in_env('SCRIPT'):
script_path = SysUtils.join_paths(
SysUtils.get_env_var("TMP_INPUT_DIR"), self._SCRIPT_FILE_NAME)
script_content = StrUtils.base64_to_str(
SysUtils.get_env_var('SCRIPT'))
FileUtils.create_file_with_content(script_path, script_content)
get_logger().info("Script file created in '%s'", script_path)
elif FileUtils.is_file(self._OSCAR_SCRIPT_PATH):
script_path = self._OSCAR_SCRIPT_PATH
return script_path
def download_file(self, parsed_event, input_dir_path):
"""Downloads a file from a minio bucket."""
file_download_path = SysUtils.join_paths(input_dir_path,
parsed_event.file_name)
get_logger().info("Downloading item from bucket '%s' with key '%s'",
parsed_event.bucket_name, parsed_event.file_name)
with open(file_download_path, 'wb') as data:
self._get_client().download_fileobj(parsed_event.bucket_name,
parsed_event.file_name, data)
get_logger().info(
"Successful download of file '%s' from bucket '%s' in path '%s'",
parsed_event.file_name, parsed_event.bucket_name,
file_download_path)
return file_download_path
def download_file(self, parsed_event, input_dir_path):
""" Downloads the file from the S3 bucket and
returns the path were the download is placed. """
file_download_path = SysUtils.join_paths(input_dir_path,
parsed_event.file_name)
get_logger().info("Downloading item from bucket '%s' with key '%s'",
parsed_event.bucket_name, parsed_event.object_key)
with open(file_download_path, 'wb') as data:
_get_client().download_fileobj(parsed_event.bucket_name,
parsed_event.object_key, data)
get_logger().info(
"Successful download of file '%s' from bucket '%s' in path '%s'",
parsed_event.object_key, parsed_event.bucket_name,
file_download_path)
return file_download_path
def save_event(self, input_dir_path):
"""Stores the unknown event and returns
the file path where the file is stored."""
file_path = SysUtils.join_paths(input_dir_path, self._file_name)
try:
json.loads(self.event)
except ValueError:
FileUtils.create_file_with_content(file_path,
base64.b64decode(self.event),
mode='wb')
except TypeError:
FileUtils.create_file_with_content(file_path, self.event)
else:
FileUtils.create_file_with_content(file_path, self.event)
return file_path
def download_file(self, parsed_event, input_dir_path):
"""Downloads the file from the space of Onedata and
returns the path were the download is placed. """
file_download_path = ""
url = f'https://{self.oneprovider_host}{self._CDMI_PATH}{parsed_event.object_key}'
get_logger().info('Downloading item from host \'%s\' with key \'%s\'',
self.oneprovider_host, parsed_event.object_key)
response = requests.get(url, headers=self.headers)
if response.status_code == 200:
file_download_path = SysUtils.join_paths(input_dir_path,
parsed_event.file_name)
FileUtils.create_file_with_content(file_download_path,
response.content,
mode='wb')
get_logger().info(
'Successful download of file \'%s\' with key \'%s\' in path \'%s\'',
parsed_event.file_name, parsed_event.object_key,
file_download_path)
else:
raise OnedataDownloadError(file_name=parsed_event.object_key,
status_code=response.status_code)
return file_download_path
def download_file(self, parsed_event, input_dir_path):
""" Downloads the file from the space of Onedata and
returns the path were the download is placed. """
file_download_path = ""
url = f"https://{self.oneprovider_host}/{self._CDMI_PATH}{parsed_event.object_key}"
get_logger().info("Downloading item from host '%s' with key '%s'",
self.oneprovider_host, parsed_event.object_key)
response = requests.get(url, headers=self.headers)
if response.status_code == 200:
file_download_path = SysUtils.join_paths(input_dir_path,
parsed_event.file_name)
FileUtils.create_file_with_content(file_download_path,
response.content,
mode='wb')
get_logger().info(
"Successful download of file '%s' with key '%s' in path '%s'",
parsed_event.file_name, parsed_event.object_key,
file_download_path)
else:
get_logger().error(
"File '%s' download from Onedata host '%s' failed!",
parsed_event.file_name, self.oneprovider_host)
return file_download_path
class Udocker():
"""Class in charge of managing the udocker binary."""
_CONTAINER_OUTPUT_FILE = SysUtils.join_paths(FileUtils.get_tmp_dir(),
"container-stdout")
_CONTAINER_NAME = "udocker_container"
_SCRIPT_EXEC = "/bin/bash"
def __init__(self, lambda_instance):
self.lambda_instance = lambda_instance
# Create required udocker folder
FileUtils.create_folder(SysUtils.get_env_var("UDOCKER_DIR"))
# Init the udocker command that will be executed
self.udocker_exec = [SysUtils.get_env_var("UDOCKER_EXEC")]
self.cont_cmd = self.udocker_exec + ["--quiet", "run"]
self.cont_img_id = ConfigUtils.read_cfg_var('container').get('image')
if not self.cont_img_id:
raise ContainerImageNotFoundError()
def _list_udocker_images_cmd(self):
return self.udocker_exec + ["images"]
def _load_udocker_image_cmd(self):
return self.udocker_exec + ["load", "-i", self.cont_img_id]
def _download_udocker_image_cmd(self):
return self.udocker_exec + ["pull", self.cont_img_id]
def _list_udocker_containers_cmd(self):
return self.udocker_exec + ["ps"]
def _create_udocker_container_cmd(self):
return self.udocker_exec + [
"create", f"--name={self._CONTAINER_NAME}", self.cont_img_id
]
def _set_udocker_container_execution_mode_cmd(self):
return self.udocker_exec + [
"setup", "--execmode=F1", self._CONTAINER_NAME
]
def _is_container_image_downloaded(self):
cmd_out = SysUtils.execute_cmd_and_return_output(
self._list_udocker_images_cmd())
return self.cont_img_id in cmd_out
def _load_local_container_image(self):
get_logger().info("Loading container image '%s'", self.cont_img_id)
SysUtils.execute_cmd(self._load_udocker_image_cmd())
def _download_container_image(self):
get_logger().info("Pulling container '%s' from Docker Hub",
self.cont_img_id)
SysUtils.execute_cmd(self._download_udocker_image_cmd())
def _is_container_available(self):
cmd_out = SysUtils.execute_cmd_and_return_output(
self._list_udocker_containers_cmd())
return self._CONTAINER_NAME in cmd_out
def _create_image(self):
if self._is_container_image_downloaded():
get_logger().info("Container image '%s' already available",
self.cont_img_id)
else:
if SysUtils.is_var_in_env("IMAGE_FILE"):
self._load_local_container_image()
else:
self._download_container_image()
def _create_container(self):
if self._is_container_available():
get_logger().info("Container already available")
else:
get_logger().info("Creating container based on image '%s'.",
self.cont_img_id)
SysUtils.execute_cmd(self._create_udocker_container_cmd())
SysUtils.execute_cmd(self._set_udocker_container_execution_mode_cmd())
def _create_command(self):
self._add_container_volumes()
self._add_container_environment_variables()
# Container running script
if hasattr(self.lambda_instance, 'script_path'):
# Add script in memory as entrypoint
self.cont_cmd += [(f"--entrypoint={self._SCRIPT_EXEC} "
f"{self.lambda_instance.script_path}"),
self._CONTAINER_NAME]
# Container with args
elif hasattr(self.lambda_instance, 'cmd_args'):
# Add args
self.cont_cmd += [self._CONTAINER_NAME]
self.cont_cmd += self.lambda_instance.cmd_args
# Script to be executed every time (if defined)
elif hasattr(self.lambda_instance, 'init_script_path'):
# Add init script
self.cont_cmd += [(f"--entrypoint={self._SCRIPT_EXEC} "
f"{self.lambda_instance.init_script_path}"),
self._CONTAINER_NAME]
# Only container
else:
self.cont_cmd += [self._CONTAINER_NAME]
def _add_container_volumes(self):
self.cont_cmd.extend(["-v", SysUtils.get_env_var("TMP_INPUT_DIR")])
self.cont_cmd.extend(["-v", SysUtils.get_env_var("TMP_OUTPUT_DIR")])
self.cont_cmd.extend(
["-v", "/dev", "-v", "/proc", "-v", "/etc/hosts", "--nosysdirs"])
if SysUtils.is_var_in_env('EXTRA_PAYLOAD'):
self.cont_cmd.extend(["-v", self.lambda_instance.PERMANENT_FOLDER])
def _add_cont_env_vars(self):
for key, value in SysUtils.get_cont_env_vars().items():
self.cont_cmd.extend(_parse_cont_env_var(key, value))
def _add_input_file(self):
self.cont_cmd.extend(
_parse_cont_env_var("INPUT_FILE_PATH",
SysUtils.get_env_var("INPUT_FILE_PATH")))
def _add_output_dir(self):
self.cont_cmd.extend(
_parse_cont_env_var("TMP_OUTPUT_DIR",
SysUtils.get_env_var("TMP_OUTPUT_DIR")))
def _add_storage_object_key(self):
self.cont_cmd.extend(
_parse_cont_env_var("STORAGE_OBJECT_KEY",
SysUtils.get_env_var("STORAGE_OBJECT_KEY")))
def _add_extra_payload_path(self):
self.cont_cmd.extend(
_parse_cont_env_var("EXTRA_PAYLOAD",
SysUtils.get_env_var("EXTRA_PAYLOAD")))
def _add_function_request_id(self):
self.cont_cmd.extend(
_parse_cont_env_var("REQUEST_ID",
self.lambda_instance.get_request_id()))
def _add_aws_access_keys(self):
self.cont_cmd.extend(
_parse_cont_env_var("AWS_ACCESS_KEY_ID",
SysUtils.get_env_var("AWS_ACCESS_KEY_ID")))
self.cont_cmd.extend(
_parse_cont_env_var("AWS_SECRET_ACCESS_KEY",
SysUtils.get_env_var("AWS_SECRET_ACCESS_KEY")))
self.cont_cmd.extend(
_parse_cont_env_var("AWS_SESSION_TOKEN",
SysUtils.get_env_var("AWS_SESSION_TOKEN")))
def _add_function_ip(self):
self.cont_cmd.extend(
_parse_cont_env_var("INSTANCE_IP", get_function_ip()))
def _add_container_environment_variables(self):
self._add_function_request_id()
self._add_function_ip()
self._add_aws_access_keys()
self._add_cont_env_vars()
self._add_input_file()
self._add_output_dir()
self._add_storage_object_key()
self._add_extra_payload_path()
def prepare_container(self):
"""Prepares the environment to execute the udocker container."""
self._create_image()
self._create_container()
self._create_command()
def launch_udocker_container(self):
"""Launches the udocker container.
If the execution time of the container exceeds the defined execution time,
the container is killed and a warning is raised."""
remaining_seconds = self.lambda_instance.get_remaining_time_in_seconds(
)
get_logger().info(
"Executing udocker container. Timeout set to '%d' seconds",
remaining_seconds)
get_logger().debug("Udocker command: '%s'", self.cont_cmd)
with open(self._CONTAINER_OUTPUT_FILE, "wb") as out:
with subprocess.Popen(self.cont_cmd,
stderr=subprocess.STDOUT,
stdout=out,
start_new_session=True) as process:
try:
process.wait(timeout=remaining_seconds)
except subprocess.TimeoutExpired:
get_logger().info("Stopping process '%s'", process)
process.kill()
raise ContainerTimeoutExpiredWarning()
udocker_output = b''
if FileUtils.is_file(self._CONTAINER_OUTPUT_FILE):
udocker_output = FileUtils.read_file(self._CONTAINER_OUTPUT_FILE,
file_mode="rb")
return udocker_output
def test_join_paths(self):
paths = ['It', 'works', '!']
# Works in Linux systems
self.assertEqual(SysUtils.join_paths(*paths), "/".join(paths))
def save_event(self, input_dir_path):
"""Stores the unknown event and returns
the file path where the file is stored."""
file_path = SysUtils.join_paths(input_dir_path, self._FILE_NAME)
FileUtils.create_file_with_content(file_path, self.event)
return file_path
