def lambda_handler(event, context): logger.debug("Received event: " + json.dumps(event)) set_instance_properties(event, context) supervisor = Supervisor() try: supervisor.parse_input() if utils.is_variable_in_environment("EXECUTION_MODE"): if(utils.get_environment_variable("EXECUTION_MODE")=="lambda-batch"): logger.info("Mode lambda - batch") supervisor.prepare_udocker() supervisor.execute_udocker_batch() elif (utils.get_environment_variable("EXECUTION_MODE")=="batch"): logger.info("Mode batch") supervisor.execute_batch() else: supervisor.prepare_udocker() supervisor.execute_udocker() supervisor.parse_output() else: supervisor.prepare_udocker() supervisor.execute_udocker() supervisor.parse_output() except Exception: logger.error("Exception launched:\n {0}".format(traceback.format_exc())) supervisor.body["exception"] = traceback.format_exc() supervisor.status_code = 500 return supervisor.create_response()
def get_create_job_definition_args(self,name,image): return { 'type':'container', 'containerProperties':{ 'command': [], 'mountPoints': [ {"sourceVolume": "SCAR_INPUT_DIR", "readOnly": False, "containerPath": lambda_instance.input_folder}, {"sourceVolume": "SCAR_OUTPUT_DIR", "readOnly": False, "containerPath": lambda_instance.output_folder}, ], 'jobRoleArn': utils.get_environment_variable("ROLE"), "volumes": [ {"host": {"sourcePath": lambda_instance.input_folder}, "name": "SCAR_INPUT_DIR"}, {"host":{"sourcePath": lambda_instance.output_folder}, "name": "SCAR_OUTPUT_DIR"}, ], 'readonlyRootFilesystem':False, "privileged": True, 'image': image, 'memory':int(utils.get_environment_variable("MEMORY")), 'vcpus': 1, }, 'jobDefinitionName':name, }
def __init__(self): # Get k8s api host and port self.kubernetes_service_host = utils.get_environment_variable('KUBERNETES_SERVICE_HOST') if not self.kubernetes_service_host: self.kubernetes_service_host = 'kubernetes.default' self.kubernetes_service_port = utils.get_environment_variable('KUBERNETES_SERVICE_PORT') if not self.kubernetes_service_port: self.kubernetes_service_port = '443' # Get k8s api token self.kube_token = utils.read_file('/var/run/secrets/kubernetes.io/serviceaccount/token') # Get k8s api certs if os.path.isfile('/var/run/secrets/kubernetes.io/serviceaccount/ca.crt'): self.cert_verify = '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt' else: self.cert_verify = False
def get_register_job_definition_args(self, job_name, step): self.set_container_variables(step) job_def_args = { 'jobDefinitionName': job_name, "type": "container", "containerProperties": { "image": self.scar_batch_io_image_id, "vcpus": 1, "memory": self.lambda_instance.memory, "command": ["scar-batch-io"], "volumes": [ {"host": { "sourcePath": self.lambda_instance.input_folder}, "name": "SCAR_INPUT_DIR"}, {"host":{ "sourcePath": self.lambda_instance.output_folder}, "name": "SCAR_OUTPUT_DIR"}, ], "environment" : self.container_environment_variables, 'mountPoints': [ {"sourceVolume": "SCAR_INPUT_DIR", "containerPath": self.lambda_instance.input_folder}, {"sourceVolume": "SCAR_OUTPUT_DIR", "containerPath": self.lambda_instance.output_folder}, ], }, } if step == "MED": job_def_args["containerProperties"]["command"] = [] if self.script != "": job_def_args["containerProperties"]["command"] = ["{0}/script.sh".format(self.lambda_instance.input_folder)] job_def_args["containerProperties"]["image"] = utils.get_environment_variable("IMAGE_ID") return job_def_args
def get_extra_payload_path(self): ppath = [] if utils.is_variable_in_environment('EXTRA_PAYLOAD'): ppath += self.parse_container_environment_variable( "EXTRA_PAYLOAD", utils.get_environment_variable("EXTRA_PAYLOAD")) return ppath
def __init__(self, function_args, minio_id): self.function_name = function_args['name'] if minio_id and 'envVars' in function_args: if 'STORAGE_PATH_OUTPUT_{}'.format( minio_id) in function_args['envVars']: self.output_bucket = function_args['envVars'][ f'STORAGE_PATH_OUTPUT_{minio_id}'] elif 'OUTPUT_BUCKET' in function_args['envVars']: self.output_bucket = function_args['envVars']['OUTPUT_BUCKET'] self.access_key = utils.get_environment_variable('MINIO_USER') self.secret_key = utils.get_environment_variable('MINIO_PASS') self.client = minio.Minio( utils.get_environment_variable('MINIO_ENDPOINT'), access_key=self.access_key, secret_key=self.secret_key, secure=False)
def invoke_function_batch(self,scar_input_file,variables): self.register_job_definition(lambda_instance.function_name, utils.get_environment_variable("IMAGE_ID")) data="" if utils.is_variable_in_environment('INIT_SCRIPT_PATH') : with open(str(os.environ['INIT_SCRIPT_PATH'])) as myfile: data = myfile.read() if utils.is_value_in_dict(lambda_instance.event, 'script'): data=utils.base64_to_utf8_string(lambda_instance.event['script']) if lambda_instance.has_input_bucket(): self.bucket_name_input = lambda_instance.input_bucket if lambda_instance.has_output_bucket(): self.bucket_name_output = lambda_instance.output_bucket if (data!=""): self.register_job_definition("scarfiles",self.scarfile_id_image) response = self.submit_job("scarfiles",1,"INIT",data,self.bucket_name_input,self.bucket_name_output,"script.sh",scar_input_file,variables) idJob = self.submit_job(lambda_instance.function_name,1,"MED",data,self.bucket_name_input,self.bucket_name_output,lambda_instance.input_folder+"/script.sh",scar_input_file,variables,idJob=response["jobId"])["jobId"] response = self.submit_job("scarfiles",1,"FINISH",data,self.bucket_name_input,self.bucket_name_output,"script.sh",scar_input_file,variables,idJob=idJob) elif(self.bucket_name_output!="NO" or self.bucket_name_input!="NO"): self.register_job_definition("scarfiles",self.scarfile_id_image) response = self.submit_job("scarfiles",1,"INIT",data,self.bucket_name_input,self.bucket_name_output,"",scar_input_file,variables) idJob = self.submit_job(lambda_instance.function_name,1,"MED",data,self.bucket_name_input,self.bucket_name_output,"",scar_input_file,variables,idJob=response["jobId"])["jobId"] response = self.submit_job("scarfiles",1,"FINISH",data,self.bucket_name_input,self.bucket_name_output,"",scar_input_file,variables,idJob=idJob) else: idJob = self.submit_job(lambda_instance.function_name,1,"MED",data,self.bucket_name_input,self.bucket_name_output,"",scar_input_file,variables)["jobId"] return idJob
def get_lambda_output_variable(self): out_lambda = [] if utils.is_variable_in_environment('OUTPUT_LAMBDA'): utils.set_environment_variable("OUTPUT_LAMBDA_FILE", "/tmp/{0}/lambda_output".format(lambda_instance.request_id)) out_lambda += self.parse_container_environment_variable("OUTPUT_LAMBDA_FILE", utils.get_environment_variable("EXTRA_PAYLOAD")) return out_lambda
def get_user_defined_variables(self): user_vars = [] for key in os.environ.keys(): # Find global variables with the specified prefix if re.match("CONT_VAR_.*", key): user_vars += self.parse_container_environment_variable(key.replace("CONT_VAR_", ""), utils.get_environment_variable(key)) return user_vars
def __init__(self, function_args): self.registry_name = utils.get_environment_variable("DOCKER_REGISTRY") self.function_args = function_args self.function_image_folder = utils.join_paths( '/pv/kaniko-builds', utils.get_random_uuid4_str()) self.root_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))) self.job_name = '{0}-build-job'.format(function_args['name'])
def __init__(self, function_args, onedata_id): self.function_name = function_args['name'] self.onedata_id = onedata_id self.endpoint = utils.get_environment_variable('OPENFAAS_ENDPOINT') self.onetrigger_version = utils.get_environment_variable( 'ONETRIGGER_VERSION') if not self.onetrigger_version: self.onetrigger_version = 'latest' if ('envVars' in function_args and 'OUTPUT_BUCKET' in function_args['envVars']): self.output_bucket = function_args['envVars'][ 'OUTPUT_BUCKET'].strip('/ ') self.oneprovider_host = function_args['envVars']['ONEPROVIDER_HOST'] self.onedata_access_token = function_args['envVars'][ 'ONEDATA_ACCESS_TOKEN'] self.onedata_space = function_args['envVars']['ONEDATA_SPACE'].strip( '/ ')
def get_user_script(self): script = "" if utils.is_variable_in_environment('INIT_SCRIPT_PATH'): file_content = utils.read_file(utils.get_environment_variable('INIT_SCRIPT_PATH'), 'rb') script = utils.utf8_to_base64_string(file_content) if utils.is_value_in_dict(self.lambda_instance.event, 'script'): script = self.lambda_instance.event['script'] return script
def __init__(self, scar_input_file): self.container_output_file = "{0}/container-stdout.txt".format(lambda_instance.temporal_folder) self.scar_input_file = scar_input_file if utils.is_variable_in_environment("IMAGE_ID"): self.container_image_id = utils.get_environment_variable("IMAGE_ID") self.set_udocker_commands() else: raise Exception("Container image id not specified.")
def get_iam_credentials(self): credentials = [] iam_creds = {'CONT_VAR_AWS_ACCESS_KEY_ID':'AWS_ACCESS_KEY_ID', 'CONT_VAR_AWS_SECRET_ACCESS_KEY':'AWS_SECRET_ACCESS_KEY', 'CONT_VAR_AWS_SESSION_TOKEN':'AWS_SESSION_TOKEN'} # Add IAM credentials for key,value in iam_creds.items(): if not utils.is_variable_in_environment(key): credentials += self.parse_container_environment_variable(value, utils.get_environment_variable(value)) return credentials
def execute_udocker(self): try: udocker_output = self.udocker.launch_udocker_container() logger.info("CONTAINER OUTPUT:\n " + udocker_output) self.body["udocker_output"] = udocker_output except subprocess.TimeoutExpired: logger.warning("Container execution timed out") if (utils.get_environment_variable("EXECUTION_MODE") == "lambda-batch"): self.execute_batch()
def get_decrypted_variables(self): decrypted_vars = [] session = boto3.session.Session() client = session.client('kms') for key in os.environ.keys(): if re.match("CONT_VAR_KMS_ENC_.*", key): secret = utils.get_environment_variable(key) decrypt = client.decrypt(CiphertextBlob=bytes(base64.b64decode(secret)))["Plaintext"] decrypted_vars += self.parse_container_environment_variable(key.replace("CONT_VAR_KMS_ENC", "KMS_DEC"), decrypt.decode("utf-8")) return decrypted_vars
def __init__(self, function_args): self.endpoint = utils.get_environment_variable('OPENFAAS_ENDPOINT') self.openfaas_envvars = {'read_timeout': '90', #{'sprocess': '/tmp/user_script.sh', 'write_timeout': '90'} self.openfaas_labels = {'com.openfaas.scale.zero': 'true'} self.set_function_args(function_args) self.basic_auth = None if (os.path.isfile('/var/secrets/basic-auth-user') and os.path.isfile('/var/secrets/basic-auth-password')): self.basic_auth = (utils.read_file('/var/secrets/basic-auth-user'), utils.read_file('/var/secrets/basic-auth-password'))
def _download_binaries(self): # Download latest fwatchdog binary and set exec permissions utils.download_github_asset('openfaas', 'faas', 'fwatchdog', self.function_image_folder) fwatchdog_path = os.path.join(self.function_image_folder, 'fwatchdog') fwatchdog_st = os.stat(fwatchdog_path) os.chmod(fwatchdog_path, fwatchdog_st.st_mode | stat.S_IEXEC) # Download faas-supervisor binary and set exec permissions release = utils.get_environment_variable('SUPERVISOR_VERSION') utils.download_github_asset('grycap', 'faas-supervisor', 'supervisor', self.function_image_folder, release=release) supervisor_path = os.path.join(self.function_image_folder, 'supervisor') supervisor_st = os.stat(supervisor_path) os.chmod(supervisor_path, supervisor_st.st_mode | stat.S_IEXEC)
def create_command(self): self.add_container_volumes() self.add_container_environment_variables() # Container running script if utils.is_value_in_dict(self.lambda_instance.event, 'script'): # Add script in memory as entrypoint script_path = "{0}/script.sh".format(self.lambda_instance.temporal_folder) script_content = utils.base64_to_utf8_string(self.lambda_instance.event['script']) utils.create_file_with_content(script_path, script_content) self.cmd_container_execution += ["--entrypoint={0} {1}".format(self.script_exec, script_path), self.container_name] # Container with args elif utils.is_value_in_dict(self.lambda_instance.event,'cmd_args'): # Add args self.cmd_container_execution += [self.container_name] self.cmd_container_execution += json.loads(self.lambda_instance.event['cmd_args']) # Script to be executed every time (if defined) elif utils.is_variable_in_environment('INIT_SCRIPT_PATH'): # Add init script init_script_path = "{0}/init_script.sh".format(self.lambda_instance.temporal_folder) shutil.copyfile(utils.get_environment_variable("INIT_SCRIPT_PATH"), init_script_path) self.cmd_container_execution += ["--entrypoint={0} {1}".format(self.script_exec, init_script_path), self.container_name] # Only container else: self.cmd_container_execution += [self.container_name]
def get_invocation_remaining_seconds(self): return int(self.context.get_remaining_time_in_millis() / 1000) - int( utils.get_environment_variable('TIMEOUT_THRESHOLD'))
def input_bucket(self): input_bucket = utils.get_environment_variable('INPUT_BUCKET') return input_bucket
def output_bucket_folder(self): output_folder = utils.get_environment_variable('OUTPUT_FOLDER') return output_folder
def output_bucket(self): output_bucket = utils.get_environment_variable('OUTPUT_BUCKET') return output_bucket
def is_batch_execution(self): return utils.get_environment_variable("EXECUTION_MODE") == "batch"
import subprocess import traceback import socket import uuid import base64 import sys from urllib.parse import unquote_plus sys.path.append("..") sys.path.append(".") # Works in lambda environment import src.utils as utils logger = logging.getLogger() if utils.is_variable_in_environment('LOG_LEVEL'): logger.setLevel(utils.get_environment_variable('LOG_LEVEL')) else: logger.setLevel('INFO') logger.info('SCAR: Loading lambda function') lambda_instance = None ####################################### # S3 RELATED FUNCTIONS # ####################################### class S3(): @utils.lazy_property def client(self): client = boto3.client('s3') return client