def test_cp_r(self, tmp_path):
pytest.skip(
"this test does not work on python3.7 due to an issue shutil used by cp_r"
)
source = tmp_path / "source"
target = tmp_path / "target"
f1 = source / "f1.txt"
f2 = source / "d1" / "f2.txt"
f3 = source / "d1" / "d2" / "f3.txt"
source.mkdir()
target.mkdir()
f3.parent.mkdir(parents=True)
f1.write_text("f1")
f2.write_text("f2")
f3.write_text("f3")
common.cp_r(source, target)
assert (target / "f1.txt").is_file()
assert (target / "d1" / "f2.txt").is_file()
assert (target / "d1" / "f2.txt").is_file()
assert (target / "d1" / "d2" / "f3.txt").is_file()
assert (target / "d1" / "d2" / "f3.txt").read_text() == "f3"
def get_lambda_code_param(params, _include_arch=False, **kwargs):
code = params.get("Code", {})
zip_file = code.get("ZipFile")
if zip_file and not is_base64(zip_file) and not is_zip_file(
to_bytes(zip_file)):
tmp_dir = new_tmp_dir()
handler_file = get_handler_file_from_name(
params["Handler"], runtime=params["Runtime"])
tmp_file = os.path.join(tmp_dir, handler_file)
save_file(tmp_file, zip_file)
# add 'cfn-response' module to archive - see:
# https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cfn-lambda-function-code-cfnresponsemodule.html
cfn_response_tmp_file = get_cfn_response_mod_file()
cfn_response_mod_dir = os.path.join(tmp_dir, "node_modules",
"cfn-response")
mkdir(cfn_response_mod_dir)
cp_r(
cfn_response_tmp_file,
os.path.join(cfn_response_mod_dir, "index.js"),
)
# create zip file
zip_file = create_zip_file(tmp_dir, get_content=True)
code["ZipFile"] = zip_file
rm_rf(tmp_dir)
if _include_arch and "Architectures" in params:
code["Architectures"] = params.get("Architectures")
return code
def _execute(self, func_arn, func_details, event, context=None, version=None):
lambda_cwd = func_details.cwd
runtime = func_details.runtime
handler = func_details.handler
environment = func_details.envvars.copy()
# configure USE_SSL in environment
if config.USE_SSL:
environment['USE_SSL'] = '1'
# prepare event body
if not event:
LOG.warning('Empty event body specified for invocation of Lambda "%s"' % func_arn)
event = {}
event_body = json.dumps(json_safe(event))
stdin = self.prepare_event(environment, event_body)
docker_host = config.DOCKER_HOST_FROM_CONTAINER
environment['HOSTNAME'] = docker_host
environment['LOCALSTACK_HOSTNAME'] = docker_host
environment['_HANDLER'] = handler
if func_details.timeout:
environment['AWS_LAMBDA_FUNCTION_TIMEOUT'] = str(func_details.timeout)
if context:
environment['AWS_LAMBDA_FUNCTION_NAME'] = context.function_name
environment['AWS_LAMBDA_FUNCTION_VERSION'] = context.function_version
environment['AWS_LAMBDA_FUNCTION_INVOKED_ARN'] = context.invoked_function_arn
# custom command to execute in the container
command = ''
# if running a Java Lambda, set up classpath arguments
if is_java_lambda(runtime):
java_opts = Util.get_java_opts()
stdin = None
# copy executor jar into temp directory
target_file = os.path.join(lambda_cwd, os.path.basename(LAMBDA_EXECUTOR_JAR))
if not os.path.exists(target_file):
cp_r(LAMBDA_EXECUTOR_JAR, target_file)
# TODO cleanup once we have custom Java Docker image
taskdir = '/var/task'
save_file(os.path.join(lambda_cwd, LAMBDA_EVENT_FILE), event_body)
classpath = Util.get_java_classpath(target_file)
command = ("bash -c 'cd %s; java %s -cp \"%s\" \"%s\" \"%s\" \"%s\"'" %
(taskdir, java_opts, classpath, LAMBDA_EXECUTOR_CLASS, handler, LAMBDA_EVENT_FILE))
# accept any self-signed certificates for outgoing calls from the Lambda
if is_nodejs_runtime(runtime):
environment['NODE_TLS_REJECT_UNAUTHORIZED'] = '0'
# determine the command to be executed (implemented by subclasses)
cmd = self.prepare_execution(func_arn, environment, runtime, command, handler, lambda_cwd)
# lambci writes the Lambda result to stdout and logs to stderr, fetch it from there!
LOG.info('Running lambda cmd: %s' % cmd)
result = self.run_lambda_executor(cmd, stdin, env_vars=environment, func_details=func_details)
return result
def _execute(self, func_arn, func_details, event, context=None, version=None):
lambda_cwd = func_details.cwd
runtime = func_details.runtime
handler = func_details.handler
environment = func_details.envvars.copy()
# configure USE_SSL in environment
if config.USE_SSL:
environment['USE_SSL'] = '1'
# prepare event body
if not event:
LOG.warning('Empty event body specified for invocation of Lambda "%s"' % func_arn)
event = {}
event_body = json.dumps(json_safe(event))
stdin = self.prepare_event(environment, event_body)
docker_host = config.DOCKER_HOST_FROM_CONTAINER
environment['HOSTNAME'] = docker_host
environment['LOCALSTACK_HOSTNAME'] = docker_host
if context:
environment['AWS_LAMBDA_FUNCTION_NAME'] = context.function_name
environment['AWS_LAMBDA_FUNCTION_VERSION'] = context.function_version
environment['AWS_LAMBDA_FUNCTION_INVOKED_ARN'] = context.invoked_function_arn
# custom command to execute in the container
command = ''
# if running a Java Lambda, set up classpath arguments
if runtime == LAMBDA_RUNTIME_JAVA8:
java_opts = Util.get_java_opts()
stdin = None
# copy executor jar into temp directory
target_file = os.path.join(lambda_cwd, os.path.basename(LAMBDA_EXECUTOR_JAR))
if not os.path.exists(target_file):
cp_r(LAMBDA_EXECUTOR_JAR, target_file)
# TODO cleanup once we have custom Java Docker image
taskdir = '/var/task'
save_file(os.path.join(lambda_cwd, LAMBDA_EVENT_FILE), event_body)
classpath = Util.get_java_classpath(target_file)
command = ("bash -c 'cd %s; java %s -cp \"%s\" \"%s\" \"%s\" \"%s\"'" %
(taskdir, java_opts, classpath, LAMBDA_EXECUTOR_CLASS, handler, LAMBDA_EVENT_FILE))
# determine the command to be executed (implemented by subclasses)
cmd = self.prepare_execution(func_arn, environment, runtime, command, handler, lambda_cwd)
# lambci writes the Lambda result to stdout and logs to stderr, fetch it from there!
LOG.debug('Running lambda cmd: %s' % cmd)
result, log_output = self.run_lambda_executor(cmd, stdin, environment)
log_formatted = log_output.strip().replace('\n', '\n> ')
LOG.debug('Lambda %s result / log output:\n%s\n>%s' % (func_arn, result.strip(), log_formatted))
return result, log_output
def execute(self, func_arn, func_details, event, context=None, version=None, asynchronous=False):
lambda_cwd = func_details.cwd
runtime = func_details.runtime
handler = func_details.handler
environment = func_details.envvars.copy()
# configure USE_SSL in environment
if config.USE_SSL:
environment['USE_SSL'] = '1'
# prepare event body
if not event:
LOG.warning('Empty event body specified for invocation of Lambda "%s"' % func_arn)
event = {}
event_body = json.dumps(event)
event_body_escaped = event_body.replace("'", "\\'")
docker_host = config.DOCKER_HOST_FROM_CONTAINER
# amend the environment variables for execution
environment['AWS_LAMBDA_EVENT_BODY'] = event_body_escaped
environment['HOSTNAME'] = docker_host
environment['LOCALSTACK_HOSTNAME'] = docker_host
if context:
environment['AWS_LAMBDA_FUNCTION_NAME'] = context.function_name
environment['AWS_LAMBDA_FUNCTION_VERSION'] = context.function_version
environment['AWS_LAMBDA_FUNCTION_INVOKED_ARN'] = context.invoked_function_arn
# custom command to execute in the container
command = ''
# if running a Java Lambda, set up classpath arguments
if runtime == LAMBDA_RUNTIME_JAVA8:
# copy executor jar into temp directory
cp_r(LAMBDA_EXECUTOR_JAR, lambda_cwd)
# TODO cleanup once we have custom Java Docker image
taskdir = '/var/task'
save_file(os.path.join(lambda_cwd, LAMBDA_EVENT_FILE), event_body)
command = ("bash -c 'cd %s; java -cp .:`ls *.jar | tr \"\\n\" \":\"` \"%s\" \"%s\" \"%s\"'" %
(taskdir, LAMBDA_EXECUTOR_CLASS, handler, LAMBDA_EVENT_FILE))
# determine the command to be executed (implemented by subclasses)
cmd = self.prepare_execution(func_arn, environment, runtime, command, handler, lambda_cwd)
# lambci writes the Lambda result to stdout and logs to stderr, fetch it from there!
LOG.debug('Running lambda cmd: %s' % cmd)
result, log_output = self.run_lambda_executor(cmd, environment, asynchronous)
LOG.debug('Lambda result / log output:\n%s\n>%s' % (result.strip(), log_output.strip().replace('\n', '\n> ')))
return result, log_output
def _execute(self, func_arn, func_details, event, context=None, version=None):
lambda_cwd = func_details.cwd
runtime = func_details.runtime
handler = func_details.handler
environment = self._prepare_environment(func_details)
# configure USE_SSL in environment
if config.USE_SSL:
environment['USE_SSL'] = '1'
# prepare event body
if not event:
LOG.warning('Empty event body specified for invocation of Lambda "%s"' % func_arn)
event = {}
event_body = json.dumps(json_safe(event))
stdin = self.prepare_event(environment, event_body)
main_endpoint = get_main_endpoint_from_container()
environment['LOCALSTACK_HOSTNAME'] = main_endpoint
environment['EDGE_PORT'] = str(config.EDGE_PORT)
environment['_HANDLER'] = handler
if os.environ.get('HTTP_PROXY'):
environment['HTTP_PROXY'] = os.environ['HTTP_PROXY']
if func_details.timeout:
environment['AWS_LAMBDA_FUNCTION_TIMEOUT'] = str(func_details.timeout)
if context:
environment['AWS_LAMBDA_FUNCTION_NAME'] = context.function_name
environment['AWS_LAMBDA_FUNCTION_VERSION'] = context.function_version
environment['AWS_LAMBDA_FUNCTION_INVOKED_ARN'] = context.invoked_function_arn
environment['AWS_LAMBDA_COGNITO_IDENTITY'] = json.dumps(context.cognito_identity or {})
if context.client_context is not None:
environment['AWS_LAMBDA_CLIENT_CONTEXT'] = json.dumps(to_str(
base64.b64decode(to_bytes(context.client_context))))
# custom command to execute in the container
command = ''
events_file = ''
if config.LAMBDA_JAVA_OPTS and is_java_lambda(runtime):
# if running a Java Lambda with our custom executor, set up classpath arguments
java_opts = Util.get_java_opts()
stdin = None
# copy executor jar into temp directory
target_file = os.path.join(lambda_cwd, os.path.basename(LAMBDA_EXECUTOR_JAR))
if not os.path.exists(target_file):
cp_r(LAMBDA_EXECUTOR_JAR, target_file)
# TODO cleanup once we have custom Java Docker image
taskdir = '/var/task'
events_file = '_lambda.events.%s.json' % short_uid()
save_file(os.path.join(lambda_cwd, events_file), event_body)
classpath = Util.get_java_classpath(target_file)
command = ("bash -c 'cd %s; java %s -cp \"%s\" \"%s\" \"%s\" \"%s\"'" %
(taskdir, java_opts, classpath, LAMBDA_EXECUTOR_CLASS, handler, events_file))
# accept any self-signed certificates for outgoing calls from the Lambda
if is_nodejs_runtime(runtime):
environment['NODE_TLS_REJECT_UNAUTHORIZED'] = '0'
# determine the command to be executed (implemented by subclasses)
cmd = self.prepare_execution(func_details, environment, command)
# run Lambda executor and fetch invocation result
LOG.info('Running lambda cmd: %s' % cmd)
result = self.run_lambda_executor(cmd, stdin, env_vars=environment, func_details=func_details)
# clean up events file
events_file and os.path.exists(events_file) and rm_rf(events_file)
return result
def run_lambda(func, event, context, func_arn, suppress_output=False):
if suppress_output:
stdout_ = sys.stdout
stderr_ = sys.stderr
stream = StringIO()
sys.stdout = stream
sys.stderr = stream
lambda_cwd = arn_to_lambda.get(func_arn).cwd
result = None
try:
runtime = arn_to_lambda.get(func_arn).runtime
handler = arn_to_lambda.get(func_arn).handler
environment = arn_to_lambda.get(func_arn).envvars
if use_docker():
handler_args = '"%s"' % handler
entrypoint = ''
# prepare event body
if not event:
LOG.warning('Empty event body specified for invocation of Lambda "%s"' % func_arn)
event = {}
event_body = json.dumps(event)
# if running a Java Lambda, set up classpath arguments
if runtime == LAMBDA_RUNTIME_JAVA8:
# copy executor jar into temp directory
cp_r(LAMBDA_EXECUTOR_JAR, lambda_cwd)
# TODO cleanup once we have custom Java Docker image
taskdir = '/var/task'
event_file = 'event_file.json'
save_file(os.path.join(lambda_cwd, event_file), event_body)
handler_args = ("bash -c 'cd %s; java -cp .:`ls *.jar | tr \"\\n\" \":\"` \"%s\" \"%s\" \"%s\"'" %
(taskdir, LAMBDA_EXECUTOR_CLASS, handler, event_file))
entrypoint = ' --entrypoint ""'
env_vars = ' '.join(['-e {}={}'.format(k, cmd_quote(v)) for (k, v) in environment.items()])
if config.LAMBDA_REMOTE_DOCKER:
cmd = (
'CONTAINER_ID="$(docker create'
'%s -e AWS_LAMBDA_EVENT_BODY="$AWS_LAMBDA_EVENT_BODY"'
' -e HOSTNAME="$HOSTNAME"'
' -e LOCALSTACK_HOSTNAME="$LOCALSTACK_HOSTNAME"'
' %s'
' "lambci/lambda:%s" %s'
')";'
'docker cp "%s/." "$CONTAINER_ID:/var/task";'
'docker start -a "$CONTAINER_ID";'
) % (entrypoint, runtime, env_vars, handler_args, lambda_cwd)
else:
lambda_cwd_on_host = get_host_path_for_path_in_docker(lambda_cwd)
cmd = (
'docker run'
'%s -v "%s":/var/task'
' -e AWS_LAMBDA_EVENT_BODY="$AWS_LAMBDA_EVENT_BODY"'
' -e HOSTNAME="$HOSTNAME"'
' -e LOCALSTACK_HOSTNAME="$LOCALSTACK_HOSTNAME"'
' %s'
' --rm'
' "lambci/lambda:%s" %s'
) % (entrypoint, lambda_cwd_on_host, env_vars, runtime, handler_args)
print(cmd)
event_body_escaped = event_body.replace("'", "\\'")
docker_host = config.DOCKER_HOST_FROM_CONTAINER
env_vars = {
'AWS_LAMBDA_EVENT_BODY': event_body_escaped,
'HOSTNAME': docker_host,
'LOCALSTACK_HOSTNAME': docker_host
}
# lambci writes the Lambda result to stdout and logs to stderr, fetch it from there!
result, log_output = run_lambda_executor(cmd, env_vars)
LOG.debug('Lambda log output:\n%s' % log_output)
else:
# execute the Lambda function in a forked sub-process, sync result via queue
queue = Queue()
def do_execute():
# now we're executing in the child process, safe to change CWD and ENV
if lambda_cwd:
os.chdir(lambda_cwd)
if environment:
os.environ.update(environment)
result = func(event, context)
queue.put(result)
process = Process(target=do_execute)
process.run()
result = queue.get()
except Exception as e:
return error_response('Error executing Lambda function: %s %s' % (e, traceback.format_exc()))
finally:
if suppress_output:
sys.stdout = stdout_
sys.stderr = stderr_
return result
if use_docker():
handler_args = '"%s"' % handler
entrypoint = ''
# prepare event body
if not event:
LOG.warning(
'Empty event body specified for invocation of Lambda "%s"'
% func_arn)
event = {}
event_body = json.dumps(event)
# if running a Java Lambda, set up classpath arguments
if runtime == LAMBDA_RUNTIME_JAVA8:
# copy executor jar into temp directory
cp_r(LAMBDA_EXECUTOR_JAR, lambda_cwd)
# TODO cleanup once we have custom Java Docker image
taskdir = '/var/task'
event_file = 'event_file.json'
save_file(os.path.join(lambda_cwd, event_file), event_body)
handler_args = (
"bash -c 'cd %s; java -cp .:`ls *.jar | tr \"\\n\" \":\"` \"%s\" \"%s\" \"%s\"'"
% (taskdir, LAMBDA_EXECUTOR_CLASS, handler, event_file))
entrypoint = ' --entrypoint ""'
env_vars = ' '.join([
'-e {}={}'.format(k, cmd_quote(v))
for (k, v) in environment.items()
])
if config.LAMBDA_REMOTE_DOCKER:
def _execute(self,
func_arn,
func_details,
event,
context=None,
version=None):
lambda_cwd = func_details.cwd
runtime = func_details.runtime
handler = func_details.handler
environment = self._prepare_environment(func_details)
# configure USE_SSL in environment
if config.USE_SSL:
environment["USE_SSL"] = "1"
# prepare event body
if not event:
LOG.info(
'Empty event body specified for invocation of Lambda "%s"' %
func_arn)
event = {}
event_body = json.dumps(json_safe(event))
stdin = self.prepare_event(environment, event_body)
main_endpoint = get_main_endpoint_from_container()
environment["LOCALSTACK_HOSTNAME"] = main_endpoint
environment["EDGE_PORT"] = str(config.EDGE_PORT)
environment["_HANDLER"] = handler
if os.environ.get("HTTP_PROXY"):
environment["HTTP_PROXY"] = os.environ["HTTP_PROXY"]
if func_details.timeout:
environment["AWS_LAMBDA_FUNCTION_TIMEOUT"] = str(
func_details.timeout)
if context:
environment["AWS_LAMBDA_FUNCTION_NAME"] = context.function_name
environment[
"AWS_LAMBDA_FUNCTION_VERSION"] = context.function_version
environment[
"AWS_LAMBDA_FUNCTION_INVOKED_ARN"] = context.invoked_function_arn
environment["AWS_LAMBDA_COGNITO_IDENTITY"] = json.dumps(
context.cognito_identity or {})
if context.client_context is not None:
environment["AWS_LAMBDA_CLIENT_CONTEXT"] = json.dumps(
to_str(base64.b64decode(to_bytes(context.client_context))))
# custom command to execute in the container
command = ""
events_file_path = ""
if config.LAMBDA_JAVA_OPTS and is_java_lambda(runtime):
# if running a Java Lambda with our custom executor, set up classpath arguments
java_opts = Util.get_java_opts()
stdin = None
# copy executor jar into temp directory
target_file = os.path.join(lambda_cwd,
os.path.basename(LAMBDA_EXECUTOR_JAR))
if not os.path.exists(target_file):
cp_r(LAMBDA_EXECUTOR_JAR, target_file)
# TODO cleanup once we have custom Java Docker image
events_file = "_lambda.events.%s.json" % short_uid()
events_file_path = os.path.join(lambda_cwd, events_file)
save_file(events_file_path, event_body)
# construct Java command
classpath = Util.get_java_classpath(target_file)
command = 'bash -c \'cd %s; java %s -cp "%s" "%s" "%s" "%s"\'' % (
DOCKER_TASK_FOLDER,
java_opts,
classpath,
LAMBDA_EXECUTOR_CLASS,
handler,
events_file,
)
# accept any self-signed certificates for outgoing calls from the Lambda
if is_nodejs_runtime(runtime):
environment["NODE_TLS_REJECT_UNAUTHORIZED"] = "0"
# determine the command to be executed (implemented by subclasses)
cmd = self.prepare_execution(func_details, environment, command)
# copy events file into container, if necessary
if events_file_path:
container_name = self.get_container_name(func_details.arn())
self.copy_into_container(events_file_path, container_name,
DOCKER_TASK_FOLDER)
# run Lambda executor and fetch invocation result
LOG.info("Running lambda cmd: %s" % cmd)
result = self.run_lambda_executor(cmd,
event=stdin,
env_vars=environment,
func_details=func_details)
# clean up events file
events_file_path and os.path.exists(events_file_path) and rm_rf(
events_file_path)
# TODO: delete events file from container!
return result
