def lambda_handler(event, context):
logger.debug("Received event: " + json.dumps(event))
set_instance_properties(event, context)
supervisor = Supervisor()
try:
supervisor.parse_input()
if utils.is_variable_in_environment("EXECUTION_MODE"):
if(utils.get_environment_variable("EXECUTION_MODE")=="lambda-batch"):
logger.info("Mode lambda - batch")
supervisor.prepare_udocker()
supervisor.execute_udocker_batch()
elif (utils.get_environment_variable("EXECUTION_MODE")=="batch"):
logger.info("Mode batch")
supervisor.execute_batch()
else:
supervisor.prepare_udocker()
supervisor.execute_udocker()
supervisor.parse_output()
else:
supervisor.prepare_udocker()
supervisor.execute_udocker()
supervisor.parse_output()
except Exception:
logger.error("Exception launched:\n {0}".format(traceback.format_exc()))
supervisor.body["exception"] = traceback.format_exc()
supervisor.status_code = 500
return supervisor.create_response()
def get_create_job_definition_args(self,name,image):
return {
'type':'container',
'containerProperties':{
'command': [],
'mountPoints': [
{"sourceVolume": "SCAR_INPUT_DIR",
"readOnly": False,
"containerPath": lambda_instance.input_folder},
{"sourceVolume": "SCAR_OUTPUT_DIR",
"readOnly": False,
"containerPath": lambda_instance.output_folder},
],
'jobRoleArn': utils.get_environment_variable("ROLE"),
"volumes": [
{"host": {"sourcePath": lambda_instance.input_folder},
"name": "SCAR_INPUT_DIR"},
{"host":{"sourcePath": lambda_instance.output_folder},
"name": "SCAR_OUTPUT_DIR"},
],
'readonlyRootFilesystem':False,
"privileged": True,
'image': image,
'memory':int(utils.get_environment_variable("MEMORY")),
'vcpus': 1,
},
'jobDefinitionName':name,
}
def __init__(self):
# Get k8s api host and port
self.kubernetes_service_host = utils.get_environment_variable('KUBERNETES_SERVICE_HOST')
if not self.kubernetes_service_host:
self.kubernetes_service_host = 'kubernetes.default'
self.kubernetes_service_port = utils.get_environment_variable('KUBERNETES_SERVICE_PORT')
if not self.kubernetes_service_port:
self.kubernetes_service_port = '443'
# Get k8s api token
self.kube_token = utils.read_file('/var/run/secrets/kubernetes.io/serviceaccount/token')
# Get k8s api certs
if os.path.isfile('/var/run/secrets/kubernetes.io/serviceaccount/ca.crt'):
self.cert_verify = '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt'
else:
self.cert_verify = False
def get_register_job_definition_args(self, job_name, step):
self.set_container_variables(step)
job_def_args = {
'jobDefinitionName': job_name,
"type": "container",
"containerProperties": {
"image": self.scar_batch_io_image_id,
"vcpus": 1,
"memory": self.lambda_instance.memory,
"command": ["scar-batch-io"],
"volumes": [
{"host": {
"sourcePath": self.lambda_instance.input_folder},
"name": "SCAR_INPUT_DIR"},
{"host":{
"sourcePath": self.lambda_instance.output_folder},
"name": "SCAR_OUTPUT_DIR"},
],
"environment" : self.container_environment_variables,
'mountPoints': [
{"sourceVolume": "SCAR_INPUT_DIR",
"containerPath": self.lambda_instance.input_folder},
{"sourceVolume": "SCAR_OUTPUT_DIR",
"containerPath": self.lambda_instance.output_folder},
],
},
}
if step == "MED":
job_def_args["containerProperties"]["command"] = []
if self.script != "":
job_def_args["containerProperties"]["command"] = ["{0}/script.sh".format(self.lambda_instance.input_folder)]
job_def_args["containerProperties"]["image"] = utils.get_environment_variable("IMAGE_ID")
return job_def_args
def get_extra_payload_path(self):
ppath = []
if utils.is_variable_in_environment('EXTRA_PAYLOAD'):
ppath += self.parse_container_environment_variable(
"EXTRA_PAYLOAD",
utils.get_environment_variable("EXTRA_PAYLOAD"))
return ppath
def __init__(self, function_args, minio_id):
self.function_name = function_args['name']
if minio_id and 'envVars' in function_args:
if 'STORAGE_PATH_OUTPUT_{}'.format(
minio_id) in function_args['envVars']:
self.output_bucket = function_args['envVars'][
f'STORAGE_PATH_OUTPUT_{minio_id}']
elif 'OUTPUT_BUCKET' in function_args['envVars']:
self.output_bucket = function_args['envVars']['OUTPUT_BUCKET']
self.access_key = utils.get_environment_variable('MINIO_USER')
self.secret_key = utils.get_environment_variable('MINIO_PASS')
self.client = minio.Minio(
utils.get_environment_variable('MINIO_ENDPOINT'),
access_key=self.access_key,
secret_key=self.secret_key,
secure=False)
def invoke_function_batch(self,scar_input_file,variables):
self.register_job_definition(lambda_instance.function_name, utils.get_environment_variable("IMAGE_ID"))
data=""
if utils.is_variable_in_environment('INIT_SCRIPT_PATH') :
with open(str(os.environ['INIT_SCRIPT_PATH'])) as myfile:
data = myfile.read()
if utils.is_value_in_dict(lambda_instance.event, 'script'):
data=utils.base64_to_utf8_string(lambda_instance.event['script'])
if lambda_instance.has_input_bucket():
self.bucket_name_input = lambda_instance.input_bucket
if lambda_instance.has_output_bucket():
self.bucket_name_output = lambda_instance.output_bucket
if (data!=""):
self.register_job_definition("scarfiles",self.scarfile_id_image)
response = self.submit_job("scarfiles",1,"INIT",data,self.bucket_name_input,self.bucket_name_output,"script.sh",scar_input_file,variables)
idJob = self.submit_job(lambda_instance.function_name,1,"MED",data,self.bucket_name_input,self.bucket_name_output,lambda_instance.input_folder+"/script.sh",scar_input_file,variables,idJob=response["jobId"])["jobId"]
response = self.submit_job("scarfiles",1,"FINISH",data,self.bucket_name_input,self.bucket_name_output,"script.sh",scar_input_file,variables,idJob=idJob)
elif(self.bucket_name_output!="NO" or self.bucket_name_input!="NO"):
self.register_job_definition("scarfiles",self.scarfile_id_image)
response = self.submit_job("scarfiles",1,"INIT",data,self.bucket_name_input,self.bucket_name_output,"",scar_input_file,variables)
idJob = self.submit_job(lambda_instance.function_name,1,"MED",data,self.bucket_name_input,self.bucket_name_output,"",scar_input_file,variables,idJob=response["jobId"])["jobId"]
response = self.submit_job("scarfiles",1,"FINISH",data,self.bucket_name_input,self.bucket_name_output,"",scar_input_file,variables,idJob=idJob)
else:
idJob = self.submit_job(lambda_instance.function_name,1,"MED",data,self.bucket_name_input,self.bucket_name_output,"",scar_input_file,variables)["jobId"]
return idJob
def get_lambda_output_variable(self):
out_lambda = []
if utils.is_variable_in_environment('OUTPUT_LAMBDA'):
utils.set_environment_variable("OUTPUT_LAMBDA_FILE", "/tmp/{0}/lambda_output".format(lambda_instance.request_id))
out_lambda += self.parse_container_environment_variable("OUTPUT_LAMBDA_FILE",
utils.get_environment_variable("EXTRA_PAYLOAD"))
return out_lambda
def get_user_defined_variables(self):
user_vars = []
for key in os.environ.keys():
# Find global variables with the specified prefix
if re.match("CONT_VAR_.*", key):
user_vars += self.parse_container_environment_variable(key.replace("CONT_VAR_", ""),
utils.get_environment_variable(key))
return user_vars
def __init__(self, function_args):
self.registry_name = utils.get_environment_variable("DOCKER_REGISTRY")
self.function_args = function_args
self.function_image_folder = utils.join_paths(
'/pv/kaniko-builds',
utils.get_random_uuid4_str())
self.root_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
self.job_name = '{0}-build-job'.format(function_args['name'])
def __init__(self, function_args, onedata_id):
self.function_name = function_args['name']
self.onedata_id = onedata_id
self.endpoint = utils.get_environment_variable('OPENFAAS_ENDPOINT')
self.onetrigger_version = utils.get_environment_variable(
'ONETRIGGER_VERSION')
if not self.onetrigger_version:
self.onetrigger_version = 'latest'
if ('envVars' in function_args
and 'OUTPUT_BUCKET' in function_args['envVars']):
self.output_bucket = function_args['envVars'][
'OUTPUT_BUCKET'].strip('/ ')
self.oneprovider_host = function_args['envVars']['ONEPROVIDER_HOST']
self.onedata_access_token = function_args['envVars'][
'ONEDATA_ACCESS_TOKEN']
self.onedata_space = function_args['envVars']['ONEDATA_SPACE'].strip(
'/ ')
def get_user_script(self):
script = ""
if utils.is_variable_in_environment('INIT_SCRIPT_PATH'):
file_content = utils.read_file(utils.get_environment_variable('INIT_SCRIPT_PATH'), 'rb')
script = utils.utf8_to_base64_string(file_content)
if utils.is_value_in_dict(self.lambda_instance.event, 'script'):
script = self.lambda_instance.event['script']
return script
def __init__(self, scar_input_file):
self.container_output_file = "{0}/container-stdout.txt".format(lambda_instance.temporal_folder)
self.scar_input_file = scar_input_file
if utils.is_variable_in_environment("IMAGE_ID"):
self.container_image_id = utils.get_environment_variable("IMAGE_ID")
self.set_udocker_commands()
else:
raise Exception("Container image id not specified.")
def get_iam_credentials(self):
credentials = []
iam_creds = {'CONT_VAR_AWS_ACCESS_KEY_ID':'AWS_ACCESS_KEY_ID',
'CONT_VAR_AWS_SECRET_ACCESS_KEY':'AWS_SECRET_ACCESS_KEY',
'CONT_VAR_AWS_SESSION_TOKEN':'AWS_SESSION_TOKEN'}
# Add IAM credentials
for key,value in iam_creds.items():
if not utils.is_variable_in_environment(key):
credentials += self.parse_container_environment_variable(value, utils.get_environment_variable(value))
return credentials
def execute_udocker(self):
try:
udocker_output = self.udocker.launch_udocker_container()
logger.info("CONTAINER OUTPUT:\n " + udocker_output)
self.body["udocker_output"] = udocker_output
except subprocess.TimeoutExpired:
logger.warning("Container execution timed out")
if (utils.get_environment_variable("EXECUTION_MODE") ==
"lambda-batch"):
self.execute_batch()
def get_decrypted_variables(self):
decrypted_vars = []
session = boto3.session.Session()
client = session.client('kms')
for key in os.environ.keys():
if re.match("CONT_VAR_KMS_ENC_.*", key):
secret = utils.get_environment_variable(key)
decrypt = client.decrypt(CiphertextBlob=bytes(base64.b64decode(secret)))["Plaintext"]
decrypted_vars += self.parse_container_environment_variable(key.replace("CONT_VAR_KMS_ENC", "KMS_DEC"),
decrypt.decode("utf-8"))
return decrypted_vars
def __init__(self, function_args):
self.endpoint = utils.get_environment_variable('OPENFAAS_ENDPOINT')
self.openfaas_envvars = {'read_timeout': '90',
#{'sprocess': '/tmp/user_script.sh',
'write_timeout': '90'}
self.openfaas_labels = {'com.openfaas.scale.zero': 'true'}
self.set_function_args(function_args)
self.basic_auth = None
if (os.path.isfile('/var/secrets/basic-auth-user') and
os.path.isfile('/var/secrets/basic-auth-password')):
self.basic_auth = (utils.read_file('/var/secrets/basic-auth-user'),
utils.read_file('/var/secrets/basic-auth-password'))
def _download_binaries(self):
# Download latest fwatchdog binary and set exec permissions
utils.download_github_asset('openfaas',
'faas',
'fwatchdog',
self.function_image_folder)
fwatchdog_path = os.path.join(self.function_image_folder, 'fwatchdog')
fwatchdog_st = os.stat(fwatchdog_path)
os.chmod(fwatchdog_path, fwatchdog_st.st_mode | stat.S_IEXEC)
# Download faas-supervisor binary and set exec permissions
release = utils.get_environment_variable('SUPERVISOR_VERSION')
utils.download_github_asset('grycap',
'faas-supervisor',
'supervisor',
self.function_image_folder,
release=release)
supervisor_path = os.path.join(self.function_image_folder, 'supervisor')
supervisor_st = os.stat(supervisor_path)
os.chmod(supervisor_path, supervisor_st.st_mode | stat.S_IEXEC)
def create_command(self):
self.add_container_volumes()
self.add_container_environment_variables()
# Container running script
if utils.is_value_in_dict(self.lambda_instance.event, 'script'):
# Add script in memory as entrypoint
script_path = "{0}/script.sh".format(self.lambda_instance.temporal_folder)
script_content = utils.base64_to_utf8_string(self.lambda_instance.event['script'])
utils.create_file_with_content(script_path, script_content)
self.cmd_container_execution += ["--entrypoint={0} {1}".format(self.script_exec, script_path), self.container_name]
# Container with args
elif utils.is_value_in_dict(self.lambda_instance.event,'cmd_args'):
# Add args
self.cmd_container_execution += [self.container_name]
self.cmd_container_execution += json.loads(self.lambda_instance.event['cmd_args'])
# Script to be executed every time (if defined)
elif utils.is_variable_in_environment('INIT_SCRIPT_PATH'):
# Add init script
init_script_path = "{0}/init_script.sh".format(self.lambda_instance.temporal_folder)
shutil.copyfile(utils.get_environment_variable("INIT_SCRIPT_PATH"), init_script_path)
self.cmd_container_execution += ["--entrypoint={0} {1}".format(self.script_exec, init_script_path), self.container_name]
# Only container
else:
self.cmd_container_execution += [self.container_name]
def get_invocation_remaining_seconds(self):
return int(self.context.get_remaining_time_in_millis() / 1000) - int(
utils.get_environment_variable('TIMEOUT_THRESHOLD'))
def input_bucket(self):
input_bucket = utils.get_environment_variable('INPUT_BUCKET')
return input_bucket
def output_bucket_folder(self):
output_folder = utils.get_environment_variable('OUTPUT_FOLDER')
return output_folder
def output_bucket(self):
output_bucket = utils.get_environment_variable('OUTPUT_BUCKET')
return output_bucket
def is_batch_execution(self):
return utils.get_environment_variable("EXECUTION_MODE") == "batch"
import subprocess
import traceback
import socket
import uuid
import base64
import sys
from urllib.parse import unquote_plus
sys.path.append("..")
sys.path.append(".")
# Works in lambda environment
import src.utils as utils
logger = logging.getLogger()
if utils.is_variable_in_environment('LOG_LEVEL'):
logger.setLevel(utils.get_environment_variable('LOG_LEVEL'))
else:
logger.setLevel('INFO')
logger.info('SCAR: Loading lambda function')
lambda_instance = None
#######################################
# S3 RELATED FUNCTIONS #
#######################################
class S3():
@utils.lazy_property
def client(self):
client = boto3.client('s3')
return client
