def test_cp_r(self, tmp_path): pytest.skip( "this test does not work on python3.7 due to an issue shutil used by cp_r" ) source = tmp_path / "source" target = tmp_path / "target" f1 = source / "f1.txt" f2 = source / "d1" / "f2.txt" f3 = source / "d1" / "d2" / "f3.txt" source.mkdir() target.mkdir() f3.parent.mkdir(parents=True) f1.write_text("f1") f2.write_text("f2") f3.write_text("f3") common.cp_r(source, target) assert (target / "f1.txt").is_file() assert (target / "d1" / "f2.txt").is_file() assert (target / "d1" / "f2.txt").is_file() assert (target / "d1" / "d2" / "f3.txt").is_file() assert (target / "d1" / "d2" / "f3.txt").read_text() == "f3"
def get_lambda_code_param(params, _include_arch=False, **kwargs): code = params.get("Code", {}) zip_file = code.get("ZipFile") if zip_file and not is_base64(zip_file) and not is_zip_file( to_bytes(zip_file)): tmp_dir = new_tmp_dir() handler_file = get_handler_file_from_name( params["Handler"], runtime=params["Runtime"]) tmp_file = os.path.join(tmp_dir, handler_file) save_file(tmp_file, zip_file) # add 'cfn-response' module to archive - see: # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cfn-lambda-function-code-cfnresponsemodule.html cfn_response_tmp_file = get_cfn_response_mod_file() cfn_response_mod_dir = os.path.join(tmp_dir, "node_modules", "cfn-response") mkdir(cfn_response_mod_dir) cp_r( cfn_response_tmp_file, os.path.join(cfn_response_mod_dir, "index.js"), ) # create zip file zip_file = create_zip_file(tmp_dir, get_content=True) code["ZipFile"] = zip_file rm_rf(tmp_dir) if _include_arch and "Architectures" in params: code["Architectures"] = params.get("Architectures") return code
def _execute(self, func_arn, func_details, event, context=None, version=None): lambda_cwd = func_details.cwd runtime = func_details.runtime handler = func_details.handler environment = func_details.envvars.copy() # configure USE_SSL in environment if config.USE_SSL: environment['USE_SSL'] = '1' # prepare event body if not event: LOG.warning('Empty event body specified for invocation of Lambda "%s"' % func_arn) event = {} event_body = json.dumps(json_safe(event)) stdin = self.prepare_event(environment, event_body) docker_host = config.DOCKER_HOST_FROM_CONTAINER environment['HOSTNAME'] = docker_host environment['LOCALSTACK_HOSTNAME'] = docker_host environment['_HANDLER'] = handler if func_details.timeout: environment['AWS_LAMBDA_FUNCTION_TIMEOUT'] = str(func_details.timeout) if context: environment['AWS_LAMBDA_FUNCTION_NAME'] = context.function_name environment['AWS_LAMBDA_FUNCTION_VERSION'] = context.function_version environment['AWS_LAMBDA_FUNCTION_INVOKED_ARN'] = context.invoked_function_arn # custom command to execute in the container command = '' # if running a Java Lambda, set up classpath arguments if is_java_lambda(runtime): java_opts = Util.get_java_opts() stdin = None # copy executor jar into temp directory target_file = os.path.join(lambda_cwd, os.path.basename(LAMBDA_EXECUTOR_JAR)) if not os.path.exists(target_file): cp_r(LAMBDA_EXECUTOR_JAR, target_file) # TODO cleanup once we have custom Java Docker image taskdir = '/var/task' save_file(os.path.join(lambda_cwd, LAMBDA_EVENT_FILE), event_body) classpath = Util.get_java_classpath(target_file) command = ("bash -c 'cd %s; java %s -cp \"%s\" \"%s\" \"%s\" \"%s\"'" % (taskdir, java_opts, classpath, LAMBDA_EXECUTOR_CLASS, handler, LAMBDA_EVENT_FILE)) # accept any self-signed certificates for outgoing calls from the Lambda if is_nodejs_runtime(runtime): environment['NODE_TLS_REJECT_UNAUTHORIZED'] = '0' # determine the command to be executed (implemented by subclasses) cmd = self.prepare_execution(func_arn, environment, runtime, command, handler, lambda_cwd) # lambci writes the Lambda result to stdout and logs to stderr, fetch it from there! LOG.info('Running lambda cmd: %s' % cmd) result = self.run_lambda_executor(cmd, stdin, env_vars=environment, func_details=func_details) return result
def _execute(self, func_arn, func_details, event, context=None, version=None): lambda_cwd = func_details.cwd runtime = func_details.runtime handler = func_details.handler environment = func_details.envvars.copy() # configure USE_SSL in environment if config.USE_SSL: environment['USE_SSL'] = '1' # prepare event body if not event: LOG.warning('Empty event body specified for invocation of Lambda "%s"' % func_arn) event = {} event_body = json.dumps(json_safe(event)) stdin = self.prepare_event(environment, event_body) docker_host = config.DOCKER_HOST_FROM_CONTAINER environment['HOSTNAME'] = docker_host environment['LOCALSTACK_HOSTNAME'] = docker_host if context: environment['AWS_LAMBDA_FUNCTION_NAME'] = context.function_name environment['AWS_LAMBDA_FUNCTION_VERSION'] = context.function_version environment['AWS_LAMBDA_FUNCTION_INVOKED_ARN'] = context.invoked_function_arn # custom command to execute in the container command = '' # if running a Java Lambda, set up classpath arguments if runtime == LAMBDA_RUNTIME_JAVA8: java_opts = Util.get_java_opts() stdin = None # copy executor jar into temp directory target_file = os.path.join(lambda_cwd, os.path.basename(LAMBDA_EXECUTOR_JAR)) if not os.path.exists(target_file): cp_r(LAMBDA_EXECUTOR_JAR, target_file) # TODO cleanup once we have custom Java Docker image taskdir = '/var/task' save_file(os.path.join(lambda_cwd, LAMBDA_EVENT_FILE), event_body) classpath = Util.get_java_classpath(target_file) command = ("bash -c 'cd %s; java %s -cp \"%s\" \"%s\" \"%s\" \"%s\"'" % (taskdir, java_opts, classpath, LAMBDA_EXECUTOR_CLASS, handler, LAMBDA_EVENT_FILE)) # determine the command to be executed (implemented by subclasses) cmd = self.prepare_execution(func_arn, environment, runtime, command, handler, lambda_cwd) # lambci writes the Lambda result to stdout and logs to stderr, fetch it from there! LOG.debug('Running lambda cmd: %s' % cmd) result, log_output = self.run_lambda_executor(cmd, stdin, environment) log_formatted = log_output.strip().replace('\n', '\n> ') LOG.debug('Lambda %s result / log output:\n%s\n>%s' % (func_arn, result.strip(), log_formatted)) return result, log_output
def execute(self, func_arn, func_details, event, context=None, version=None, asynchronous=False): lambda_cwd = func_details.cwd runtime = func_details.runtime handler = func_details.handler environment = func_details.envvars.copy() # configure USE_SSL in environment if config.USE_SSL: environment['USE_SSL'] = '1' # prepare event body if not event: LOG.warning('Empty event body specified for invocation of Lambda "%s"' % func_arn) event = {} event_body = json.dumps(event) event_body_escaped = event_body.replace("'", "\\'") docker_host = config.DOCKER_HOST_FROM_CONTAINER # amend the environment variables for execution environment['AWS_LAMBDA_EVENT_BODY'] = event_body_escaped environment['HOSTNAME'] = docker_host environment['LOCALSTACK_HOSTNAME'] = docker_host if context: environment['AWS_LAMBDA_FUNCTION_NAME'] = context.function_name environment['AWS_LAMBDA_FUNCTION_VERSION'] = context.function_version environment['AWS_LAMBDA_FUNCTION_INVOKED_ARN'] = context.invoked_function_arn # custom command to execute in the container command = '' # if running a Java Lambda, set up classpath arguments if runtime == LAMBDA_RUNTIME_JAVA8: # copy executor jar into temp directory cp_r(LAMBDA_EXECUTOR_JAR, lambda_cwd) # TODO cleanup once we have custom Java Docker image taskdir = '/var/task' save_file(os.path.join(lambda_cwd, LAMBDA_EVENT_FILE), event_body) command = ("bash -c 'cd %s; java -cp .:`ls *.jar | tr \"\\n\" \":\"` \"%s\" \"%s\" \"%s\"'" % (taskdir, LAMBDA_EXECUTOR_CLASS, handler, LAMBDA_EVENT_FILE)) # determine the command to be executed (implemented by subclasses) cmd = self.prepare_execution(func_arn, environment, runtime, command, handler, lambda_cwd) # lambci writes the Lambda result to stdout and logs to stderr, fetch it from there! LOG.debug('Running lambda cmd: %s' % cmd) result, log_output = self.run_lambda_executor(cmd, environment, asynchronous) LOG.debug('Lambda result / log output:\n%s\n>%s' % (result.strip(), log_output.strip().replace('\n', '\n> '))) return result, log_output
def _execute(self, func_arn, func_details, event, context=None, version=None): lambda_cwd = func_details.cwd runtime = func_details.runtime handler = func_details.handler environment = self._prepare_environment(func_details) # configure USE_SSL in environment if config.USE_SSL: environment['USE_SSL'] = '1' # prepare event body if not event: LOG.warning('Empty event body specified for invocation of Lambda "%s"' % func_arn) event = {} event_body = json.dumps(json_safe(event)) stdin = self.prepare_event(environment, event_body) main_endpoint = get_main_endpoint_from_container() environment['LOCALSTACK_HOSTNAME'] = main_endpoint environment['EDGE_PORT'] = str(config.EDGE_PORT) environment['_HANDLER'] = handler if os.environ.get('HTTP_PROXY'): environment['HTTP_PROXY'] = os.environ['HTTP_PROXY'] if func_details.timeout: environment['AWS_LAMBDA_FUNCTION_TIMEOUT'] = str(func_details.timeout) if context: environment['AWS_LAMBDA_FUNCTION_NAME'] = context.function_name environment['AWS_LAMBDA_FUNCTION_VERSION'] = context.function_version environment['AWS_LAMBDA_FUNCTION_INVOKED_ARN'] = context.invoked_function_arn environment['AWS_LAMBDA_COGNITO_IDENTITY'] = json.dumps(context.cognito_identity or {}) if context.client_context is not None: environment['AWS_LAMBDA_CLIENT_CONTEXT'] = json.dumps(to_str( base64.b64decode(to_bytes(context.client_context)))) # custom command to execute in the container command = '' events_file = '' if config.LAMBDA_JAVA_OPTS and is_java_lambda(runtime): # if running a Java Lambda with our custom executor, set up classpath arguments java_opts = Util.get_java_opts() stdin = None # copy executor jar into temp directory target_file = os.path.join(lambda_cwd, os.path.basename(LAMBDA_EXECUTOR_JAR)) if not os.path.exists(target_file): cp_r(LAMBDA_EXECUTOR_JAR, target_file) # TODO cleanup once we have custom Java Docker image taskdir = '/var/task' events_file = '_lambda.events.%s.json' % short_uid() save_file(os.path.join(lambda_cwd, events_file), event_body) classpath = Util.get_java_classpath(target_file) command = ("bash -c 'cd %s; java %s -cp \"%s\" \"%s\" \"%s\" \"%s\"'" % (taskdir, java_opts, classpath, LAMBDA_EXECUTOR_CLASS, handler, events_file)) # accept any self-signed certificates for outgoing calls from the Lambda if is_nodejs_runtime(runtime): environment['NODE_TLS_REJECT_UNAUTHORIZED'] = '0' # determine the command to be executed (implemented by subclasses) cmd = self.prepare_execution(func_details, environment, command) # run Lambda executor and fetch invocation result LOG.info('Running lambda cmd: %s' % cmd) result = self.run_lambda_executor(cmd, stdin, env_vars=environment, func_details=func_details) # clean up events file events_file and os.path.exists(events_file) and rm_rf(events_file) return result
def run_lambda(func, event, context, func_arn, suppress_output=False): if suppress_output: stdout_ = sys.stdout stderr_ = sys.stderr stream = StringIO() sys.stdout = stream sys.stderr = stream lambda_cwd = arn_to_lambda.get(func_arn).cwd result = None try: runtime = arn_to_lambda.get(func_arn).runtime handler = arn_to_lambda.get(func_arn).handler environment = arn_to_lambda.get(func_arn).envvars if use_docker(): handler_args = '"%s"' % handler entrypoint = '' # prepare event body if not event: LOG.warning('Empty event body specified for invocation of Lambda "%s"' % func_arn) event = {} event_body = json.dumps(event) # if running a Java Lambda, set up classpath arguments if runtime == LAMBDA_RUNTIME_JAVA8: # copy executor jar into temp directory cp_r(LAMBDA_EXECUTOR_JAR, lambda_cwd) # TODO cleanup once we have custom Java Docker image taskdir = '/var/task' event_file = 'event_file.json' save_file(os.path.join(lambda_cwd, event_file), event_body) handler_args = ("bash -c 'cd %s; java -cp .:`ls *.jar | tr \"\\n\" \":\"` \"%s\" \"%s\" \"%s\"'" % (taskdir, LAMBDA_EXECUTOR_CLASS, handler, event_file)) entrypoint = ' --entrypoint ""' env_vars = ' '.join(['-e {}={}'.format(k, cmd_quote(v)) for (k, v) in environment.items()]) if config.LAMBDA_REMOTE_DOCKER: cmd = ( 'CONTAINER_ID="$(docker create' '%s -e AWS_LAMBDA_EVENT_BODY="$AWS_LAMBDA_EVENT_BODY"' ' -e HOSTNAME="$HOSTNAME"' ' -e LOCALSTACK_HOSTNAME="$LOCALSTACK_HOSTNAME"' ' %s' ' "lambci/lambda:%s" %s' ')";' 'docker cp "%s/." "$CONTAINER_ID:/var/task";' 'docker start -a "$CONTAINER_ID";' ) % (entrypoint, runtime, env_vars, handler_args, lambda_cwd) else: lambda_cwd_on_host = get_host_path_for_path_in_docker(lambda_cwd) cmd = ( 'docker run' '%s -v "%s":/var/task' ' -e AWS_LAMBDA_EVENT_BODY="$AWS_LAMBDA_EVENT_BODY"' ' -e HOSTNAME="$HOSTNAME"' ' -e LOCALSTACK_HOSTNAME="$LOCALSTACK_HOSTNAME"' ' %s' ' --rm' ' "lambci/lambda:%s" %s' ) % (entrypoint, lambda_cwd_on_host, env_vars, runtime, handler_args) print(cmd) event_body_escaped = event_body.replace("'", "\\'") docker_host = config.DOCKER_HOST_FROM_CONTAINER env_vars = { 'AWS_LAMBDA_EVENT_BODY': event_body_escaped, 'HOSTNAME': docker_host, 'LOCALSTACK_HOSTNAME': docker_host } # lambci writes the Lambda result to stdout and logs to stderr, fetch it from there! result, log_output = run_lambda_executor(cmd, env_vars) LOG.debug('Lambda log output:\n%s' % log_output) else: # execute the Lambda function in a forked sub-process, sync result via queue queue = Queue() def do_execute(): # now we're executing in the child process, safe to change CWD and ENV if lambda_cwd: os.chdir(lambda_cwd) if environment: os.environ.update(environment) result = func(event, context) queue.put(result) process = Process(target=do_execute) process.run() result = queue.get() except Exception as e: return error_response('Error executing Lambda function: %s %s' % (e, traceback.format_exc())) finally: if suppress_output: sys.stdout = stdout_ sys.stderr = stderr_ return result
if use_docker(): handler_args = '"%s"' % handler entrypoint = '' # prepare event body if not event: LOG.warning( 'Empty event body specified for invocation of Lambda "%s"' % func_arn) event = {} event_body = json.dumps(event) # if running a Java Lambda, set up classpath arguments if runtime == LAMBDA_RUNTIME_JAVA8: # copy executor jar into temp directory cp_r(LAMBDA_EXECUTOR_JAR, lambda_cwd) # TODO cleanup once we have custom Java Docker image taskdir = '/var/task' event_file = 'event_file.json' save_file(os.path.join(lambda_cwd, event_file), event_body) handler_args = ( "bash -c 'cd %s; java -cp .:`ls *.jar | tr \"\\n\" \":\"` \"%s\" \"%s\" \"%s\"'" % (taskdir, LAMBDA_EXECUTOR_CLASS, handler, event_file)) entrypoint = ' --entrypoint ""' env_vars = ' '.join([ '-e {}={}'.format(k, cmd_quote(v)) for (k, v) in environment.items() ]) if config.LAMBDA_REMOTE_DOCKER:
def _execute(self, func_arn, func_details, event, context=None, version=None): lambda_cwd = func_details.cwd runtime = func_details.runtime handler = func_details.handler environment = self._prepare_environment(func_details) # configure USE_SSL in environment if config.USE_SSL: environment["USE_SSL"] = "1" # prepare event body if not event: LOG.info( 'Empty event body specified for invocation of Lambda "%s"' % func_arn) event = {} event_body = json.dumps(json_safe(event)) stdin = self.prepare_event(environment, event_body) main_endpoint = get_main_endpoint_from_container() environment["LOCALSTACK_HOSTNAME"] = main_endpoint environment["EDGE_PORT"] = str(config.EDGE_PORT) environment["_HANDLER"] = handler if os.environ.get("HTTP_PROXY"): environment["HTTP_PROXY"] = os.environ["HTTP_PROXY"] if func_details.timeout: environment["AWS_LAMBDA_FUNCTION_TIMEOUT"] = str( func_details.timeout) if context: environment["AWS_LAMBDA_FUNCTION_NAME"] = context.function_name environment[ "AWS_LAMBDA_FUNCTION_VERSION"] = context.function_version environment[ "AWS_LAMBDA_FUNCTION_INVOKED_ARN"] = context.invoked_function_arn environment["AWS_LAMBDA_COGNITO_IDENTITY"] = json.dumps( context.cognito_identity or {}) if context.client_context is not None: environment["AWS_LAMBDA_CLIENT_CONTEXT"] = json.dumps( to_str(base64.b64decode(to_bytes(context.client_context)))) # custom command to execute in the container command = "" events_file_path = "" if config.LAMBDA_JAVA_OPTS and is_java_lambda(runtime): # if running a Java Lambda with our custom executor, set up classpath arguments java_opts = Util.get_java_opts() stdin = None # copy executor jar into temp directory target_file = os.path.join(lambda_cwd, os.path.basename(LAMBDA_EXECUTOR_JAR)) if not os.path.exists(target_file): cp_r(LAMBDA_EXECUTOR_JAR, target_file) # TODO cleanup once we have custom Java Docker image events_file = "_lambda.events.%s.json" % short_uid() events_file_path = os.path.join(lambda_cwd, events_file) save_file(events_file_path, event_body) # construct Java command classpath = Util.get_java_classpath(target_file) command = 'bash -c \'cd %s; java %s -cp "%s" "%s" "%s" "%s"\'' % ( DOCKER_TASK_FOLDER, java_opts, classpath, LAMBDA_EXECUTOR_CLASS, handler, events_file, ) # accept any self-signed certificates for outgoing calls from the Lambda if is_nodejs_runtime(runtime): environment["NODE_TLS_REJECT_UNAUTHORIZED"] = "0" # determine the command to be executed (implemented by subclasses) cmd = self.prepare_execution(func_details, environment, command) # copy events file into container, if necessary if events_file_path: container_name = self.get_container_name(func_details.arn()) self.copy_into_container(events_file_path, container_name, DOCKER_TASK_FOLDER) # run Lambda executor and fetch invocation result LOG.info("Running lambda cmd: %s" % cmd) result = self.run_lambda_executor(cmd, event=stdin, env_vars=environment, func_details=func_details) # clean up events file events_file_path and os.path.exists(events_file_path) and rm_rf( events_file_path) # TODO: delete events file from container! return result