def test_must_flush_underlying_stream(self):
stream_mock = Mock()
writer = StreamWriter(stream_mock)
writer.flush()
stream_mock.flush.assert_called_once_with()
def test_auto_flush_must_be_off_by_default(self):
stream_mock = Mock()
writer = StreamWriter(stream_mock)
writer.write("something")
stream_mock.flush.assert_not_called()
def test_auto_flush_must_be_off_by_default(self):
stream_mock = Mock()
writer = StreamWriter(stream_mock)
writer.write("something")
stream_mock.flush.assert_not_called()
def test_must_flush_underlying_stream(self):
stream_mock = Mock()
writer = StreamWriter(stream_mock)
writer.flush()
stream_mock.flush.assert_called_once_with()
def test_function_result_is_available_in_stdout_and_logs_in_stderr(self):
# This is the JSON result from Lambda function
# Convert to proper binary type to be compatible with Python 2 & 3
expected_output = b'{"a":"b"}'
expected_stderr = b"**This string is printed from Lambda function**"
layer_downloader = LayerDownloader("./", "./")
image_builder = LambdaImage(layer_downloader, False, False)
container = LambdaContainer(self.runtime, self.handler, self.code_dir,
self.layers, image_builder)
stdout_stream = io.BytesIO()
stderr_stream = io.BytesIO()
stdout_stream_writer = StreamWriter(stdout_stream)
stderr_stream_writer = StreamWriter(stderr_stream)
with self._create(container):
container.start()
container.wait_for_logs(stdout=stdout_stream_writer,
stderr=stderr_stream_writer)
function_output = stdout_stream.getvalue()
function_stderr = stderr_stream.getvalue()
self.assertEqual(function_output.strip(), expected_output)
self.assertIn(expected_stderr, function_stderr)
def test_must_write_to_stream(self):
buffer = "something"
stream_mock = Mock()
writer = StreamWriter(stream_mock)
writer.write(buffer)
stream_mock.write.assert_called_once_with(buffer)
def test_must_write_to_stream(self):
buffer = "something"
stream_mock = Mock()
writer = StreamWriter(stream_mock)
writer.write(buffer)
stream_mock.write.assert_called_once_with(buffer)
def __init__(self,
docker_client,
ecr_client,
ecr_repo,
tag="latest",
stream=stderr()):
self.docker_client = docker_client if docker_client else docker.from_env(
)
self.ecr_client = ecr_client
self.ecr_repo = ecr_repo
self.tag = tag
self.auth_config = {}
self.stream = StreamWriter(stream=stream, auto_flush=True)
def test_when_auto_flush_on_flush_after_each_write(self):
stream_mock = Mock()
flush_mock = Mock()
stream_mock.flush = flush_mock
lines = ["first", "second", "third"]
writer = StreamWriter(stream_mock, True)
for line in lines:
writer.write(line)
flush_mock.assert_called_once_with()
flush_mock.reset_mock()
def test_when_auto_flush_on_flush_after_each_write(self):
stream_mock = Mock()
flush_mock = Mock()
stream_mock.flush = flush_mock
lines = ["first", "second", "third"]
writer = StreamWriter(stream_mock, True)
for line in lines:
writer.write(line)
flush_mock.assert_called_once_with()
flush_mock.reset_mock()
def test_must_invoke(self):
input_event = '"some data"'
expected_env_vars = {
"var1": "override_value1",
"var2": "shell_env_value2"
}
manager = ContainerManager()
layer_downloader = LayerDownloader("./", "./")
lambda_image = LambdaImage(layer_downloader, False, False)
local_runtime = LambdaRuntime(manager, lambda_image)
runner = LocalLambdaRunner(local_runtime,
self.mock_function_provider,
self.cwd,
self.env_var_overrides,
debug_context=None)
# Append the real AWS credentials to the expected values.
creds = runner.get_aws_creds()
# default value for creds is not configured by the test. But coming from a downstream class
expected_env_vars["AWS_SECRET_ACCESS_KEY"] = creds.get(
"secret", "defaultsecret")
expected_env_vars["AWS_ACCESS_KEY_ID"] = creds.get("key", "defaultkey")
expected_env_vars["AWS_REGION"] = creds.get("region", "us-east-1")
stdout_stream = io.BytesIO()
stderr_stream = io.BytesIO()
stdout_stream_writer = StreamWriter(stdout_stream)
stderr_stream_writer = StreamWriter(stderr_stream)
runner.invoke(self.function_name,
input_event,
stdout=stdout_stream_writer,
stderr=stderr_stream_writer)
# stderr is where the Lambda container runtime logs are available. It usually contains requestId, start time
# etc. So it is non-zero in size
self.assertGreater(len(stderr_stream.getvalue().strip()), 0,
"stderr stream must contain data")
# This should contain all the environment variables passed to the function
actual_output = json.loads(
stdout_stream.getvalue().strip().decode("utf-8"))
for key, value in expected_env_vars.items():
self.assertTrue(key in actual_output,
"Key '{}' must be in function output".format(key))
self.assertEqual(actual_output.get(key), value)
def test_check_environment_variables(self):
variables = {"var1": "value1", "var2": "value2"}
aws_creds = {
"region": "ap-south-1",
"key": "mykey",
"secret": "mysecret"
}
timeout = 30
input_event = ""
stdout_stream = io.BytesIO()
stdout_stream_writer = StreamWriter(stdout_stream)
expected_output = {
"AWS_SAM_LOCAL": "true",
"AWS_LAMBDA_FUNCTION_MEMORY_SIZE": "1024",
"AWS_LAMBDA_FUNCTION_TIMEOUT": "30",
"AWS_LAMBDA_FUNCTION_HANDLER": "index.handler",
# Values coming from AWS Credentials
"AWS_REGION": "ap-south-1",
"AWS_DEFAULT_REGION": "ap-south-1",
"AWS_ACCESS_KEY_ID": "mykey",
"AWS_SECRET_ACCESS_KEY": "mysecret",
# Custom environment variables
"var1": "value1",
"var2": "value2",
}
config = FunctionConfig(
name="helloworld",
runtime=RUNTIME,
handler=HANDLER,
code_abs_path=self.code_dir["envvar"],
layers=[],
memory=MEMORY,
timeout=timeout,
)
# Set the appropriate environment variables
config.env_vars.variables = variables
config.env_vars.aws_creds = aws_creds
self.runtime.invoke(config, input_event, stdout=stdout_stream_writer)
actual_output = json.loads(stdout_stream.getvalue().strip().decode(
"utf-8")) # Output is a JSON String. Deserialize.
# Make sure all key/value from expected_output is present in actual_output
for key, value in expected_output.items():
# Do the key check first to print a nice error error message when it fails
self.assertTrue(
key in actual_output,
"'{}' should be in environment variable output".format(key))
self.assertEqual(
actual_output[key],
expected_output[key],
"Value of environment variable '{}' differs fromm expectation".
format(key),
)
def test_echo_function_with_zip_file(self, file_name_extension):
timeout = 3
input_event = '"this input should be echoed"'
expected_output = b'"this input should be echoed"'
code_dir = self.code_dir["echo"]
with make_zip(code_dir, file_name_extension) as code_zip_path:
config = FunctionConfig(
name="helloworld",
runtime=RUNTIME,
handler=HANDLER,
code_abs_path=code_zip_path,
layers=[],
timeout=timeout,
)
stdout_stream = io.BytesIO()
stdout_stream_writer = StreamWriter(stdout_stream)
self.runtime.invoke(config,
input_event,
stdout=stdout_stream_writer)
actual_output = stdout_stream.getvalue()
self.assertEqual(actual_output.strip(), expected_output)
def _invoke_sleep(self,
timeout,
sleep_duration,
check_stdout,
exceptions=None):
name = "sleepfunction_timeout_{}_sleep_{}".format(
timeout, sleep_duration)
print("Invoking function " + name)
try:
stdout_stream = io.BytesIO()
stdout_stream_writer = StreamWriter(stdout_stream)
config = FunctionConfig(name=name,
runtime=RUNTIME,
handler=HANDLER,
code_abs_path=self.code_dir,
layers=[],
memory=1024,
timeout=timeout)
self.runtime.invoke(config,
sleep_duration,
stdout=stdout_stream_writer)
actual_output = stdout_stream.getvalue().strip(
) # Must output the sleep duration
if check_stdout:
self.assertEquals(actual_output.decode('utf-8'),
str(sleep_duration))
except Exception as ex:
if exceptions is not None:
exceptions.append({"name": name, "error": ex})
else:
raise
def test_function_timeout(self):
"""
Setup a short timeout and verify that the container is stopped
"""
stdout_stream = io.BytesIO()
stdout_stream_writer = StreamWriter(stdout_stream)
timeout = 1 # 1 second timeout
sleep_seconds = 20 # Ask the function to sleep for 20 seconds
config = FunctionConfig(
name="sleep_timeout",
runtime=RUNTIME,
handler=HANDLER,
code_abs_path=self.code_dir["sleep"],
layers=[],
timeout=timeout,
)
# Measure the actual duration of execution
start = timer()
self.runtime.invoke(config, str(sleep_seconds), stdout=stdout_stream_writer)
end = timer()
# Make sure that the wall clock duration is around the ballpark of timeout value
wall_clock_func_duration = end - start
print ("Function completed in {} seconds".format(wall_clock_func_duration))
# The function should *not* preemptively stop
self.assertGreater(wall_clock_func_duration, timeout - 1)
# The function should not run for much longer than timeout.
self.assertLess(wall_clock_func_duration, timeout + self.CONTAINER_STARTUP_OVERHEAD_SECONDS)
# There should be no output from the function because timer was interrupted
actual_output = stdout_stream.getvalue()
self.assertEqual(actual_output.strip(), b"")
def build(self, runtime, layers, is_debug, stream=None):
"""
Build the image if one is not already on the system that matches the runtime and layers
Parameters
----------
runtime str
Name of the Lambda runtime
layers list(samcli.commands.local.lib.provider.Layer)
List of layers
Returns
-------
str
The image to be used (REPOSITORY:TAG)
"""
base_image = f"{self._INVOKE_REPO_PREFIX}-{runtime}:latest"
# Default image tag to be the base image with a tag of 'rapid' instead of latest
image_tag = f"{self._INVOKE_REPO_PREFIX}-{runtime}:rapid-{version}"
downloaded_layers = []
if layers:
downloaded_layers = self.layer_downloader.download_all(
layers, self.force_image_build)
docker_image_version = self._generate_docker_image_version(
downloaded_layers, runtime)
image_tag = f"{self._SAM_CLI_REPO_NAME}:{docker_image_version}"
image_not_found = False
is_debug_go = runtime == "go1.x" and is_debug
if is_debug_go:
image_tag = f"{self._INVOKE_REPO_PREFIX}-{runtime}:debug-{version}"
# If we are not using layers, build anyways to ensure any updates to rapid get added
try:
self.docker_client.images.get(image_tag)
except docker.errors.ImageNotFound:
LOG.info("Image was not found.")
image_not_found = True
if (self.force_image_build or image_not_found
or any(layer.is_defined_within_template
for layer in downloaded_layers)):
stream_writer = stream or StreamWriter(sys.stderr)
stream_writer.write("Building image...")
stream_writer.flush()
self._build_image(base_image,
image_tag,
downloaded_layers,
is_debug_go,
stream=stream_writer)
return image_tag
def _request_handler(self, **kwargs):
"""
We handle all requests to the host:port. The general flow of handling a request is as follows
* Fetch request from the Flask Global state. This is where Flask places the request and is per thread so
multiple requests are still handled correctly
* Find the Lambda function to invoke by doing a look up based on the request.endpoint and method
* If we don't find the function, we will throw a 502 (just like the 404 and 405 responses we get
from Flask.
* Since we found a Lambda function to invoke, we construct the Lambda Event from the request
* Then Invoke the Lambda function (docker container)
* We then transform the response or errors we get from the Invoke and return the data back to
the caller
Parameters
----------
kwargs dict
Keyword Args that are passed to the function from Flask. This happens when we have path parameters
Returns
-------
Response object
"""
route = self._get_current_route(request)
try:
event = self._construct_event(request, self.port, route.binary_types, route.stage_name,
route.stage_variables)
except UnicodeDecodeError:
return ServiceErrorResponses.lambda_failure_response()
stdout_stream = io.BytesIO()
stdout_stream_writer = StreamWriter(stdout_stream, self.is_debugging)
try:
self.lambda_runner.invoke(route.function_name, event, stdout=stdout_stream_writer, stderr=self.stderr)
except FunctionNotFound:
return ServiceErrorResponses.lambda_not_found_response()
lambda_response, lambda_logs, _ = LambdaOutputParser.get_lambda_output(stdout_stream)
if self.stderr and lambda_logs:
# Write the logs to stderr if available.
self.stderr.write(lambda_logs)
try:
(status_code, headers, body) = self._parse_lambda_output(lambda_response,
route.binary_types,
request)
except (KeyError, TypeError, ValueError):
LOG.error("Function returned an invalid response (must include one of: body, headers, multiValueHeaders or "
"statusCode in the response object). Response received: %s", lambda_response)
return ServiceErrorResponses.lambda_failure_response()
return self.service_response(body, headers, status_code)
def stderr(self) -> StreamWriter:
"""
Returns stream writer for stderr to output Lambda function errors to
Returns
-------
samcli.lib.utils.stream_writer.StreamWriter
Stream writer for stderr
"""
stream = self._log_file_handle if self._log_file_handle else osutils.stderr()
return StreamWriter(stream, self._is_debugging)
def pull_image(self, image_name, tag=None, stream=None):
"""
Ask Docker to pull the container image with given name.
Parameters
----------
image_name str
Name of the image
stream samcli.lib.utils.stream_writer.StreamWriter
Optional stream writer to output to. Defaults to stderr
Raises
------
DockerImagePullFailedException
If the Docker image was not available in the server
"""
if tag is None:
tag = image_name.split(":")[1] if ":" in image_name else "latest"
# use a global lock to get the image lock
with self._lock:
image_lock = self._lock_per_image.get(image_name)
if not image_lock:
image_lock = threading.Lock()
self._lock_per_image[image_name] = image_lock
# with specific image lock, pull this image only once
# since there are different locks for each image, different images can be pulled in parallel
with image_lock:
stream_writer = stream or StreamWriter(sys.stderr)
try:
result_itr = self.docker_client.api.pull(image_name,
tag=tag,
stream=True,
decode=True)
except docker.errors.APIError as ex:
LOG.debug("Failed to download image with name %s", image_name)
raise DockerImagePullFailedException(str(ex)) from ex
# io streams, especially StringIO, work only with unicode strings
stream_writer.write(
"\nFetching {} Docker container image...".format(image_name))
# Each line contains information on progress of the pull. Each line is a JSON string
for _ in result_itr:
# For every line, print a dot to show progress
stream_writer.write(".")
stream_writer.flush()
# We are done. Go to the next line
stream_writer.write("\n")
def _invoke_request_handler(self, function_name):
"""
Request Handler for the Local Lambda Invoke path. This method is responsible for understanding the incoming
request and invoking the Local Lambda Function
Parameters
----------
function_name str
Name of the function to invoke
Returns
-------
A Flask Response response object as if it was returned from Lambda
"""
flask_request = request
request_data = flask_request.get_data()
if not request_data:
request_data = b'{}'
request_data = request_data.decode('utf-8')
stdout_stream = io.BytesIO()
stdout_stream_writer = StreamWriter(stdout_stream, self.is_debugging)
try:
self.lambda_runner.invoke(function_name,
request_data,
stdout=stdout_stream_writer,
stderr=self.stderr)
except FunctionNotFound:
LOG.debug('%s was not found to invoke.', function_name)
return LambdaErrorResponses.resource_not_found(function_name)
lambda_response, lambda_logs, is_lambda_user_error_response = \
LambdaOutputParser.get_lambda_output(stdout_stream)
if self.stderr and lambda_logs:
# Write the logs to stderr if available.
self.stderr.write(lambda_logs)
if is_lambda_user_error_response:
return self.service_response(
lambda_response, {
'Content-Type': 'application/json',
'x-amz-function-error': 'Unhandled'
}, 200)
return self.service_response(lambda_response,
{'Content-Type': 'application/json'}, 200)
def test_echo_function(self):
timeout = 3
input_event = '{"a":"b"}'
expected_output = b'{"a":"b"}'
config = FunctionConfig(name="helloworld",
runtime=RUNTIME,
handler=HANDLER,
code_abs_path=self.code_dir["echo"],
layers=[],
timeout=timeout)
stdout_stream = io.BytesIO()
stdout_stream_writer = StreamWriter(stdout_stream)
self.runtime.invoke(config, input_event, stdout=stdout_stream_writer)
actual_output = stdout_stream.getvalue()
self.assertEquals(actual_output.strip(), expected_output)
def pull_image(self, image_name, stream=None):
"""
Ask Docker to pull the container image with given name.
Parameters
----------
image_name str
Name of the image
stream samcli.lib.utils.stream_writer.StreamWriter
Optional stream writer to output to. Defaults to stderr
Raises
------
DockerImagePullFailedException
If the Docker image was not available in the server
"""
stream_writer = stream or StreamWriter(sys.stderr)
try:
result_itr = self.docker_client.api.pull(image_name,
stream=True,
decode=True)
except docker.errors.APIError as ex:
LOG.debug("Failed to download image with name %s", image_name)
raise DockerImagePullFailedException(str(ex))
# io streams, especially StringIO, work only with unicode strings
stream_writer.write(
"\nFetching {} Docker container image...".format(image_name))
# Each line contains information on progress of the pull. Each line is a JSON string
for _ in result_itr:
# For every line, print a dot to show progress
stream_writer.write(".")
stream_writer.flush()
# We are done. Go to the next line
stream_writer.write("\n")
def _request_handler(self, **kwargs):
"""
We handle all requests to the host:port. The general flow of handling a request is as follows
* Fetch request from the Flask Global state. This is where Flask places the request and is per thread so
multiple requests are still handled correctly
* Find the Lambda function to invoke by doing a look up based on the request.endpoint and method
* If we don't find the function, we will throw a 502 (just like the 404 and 405 responses we get
from Flask.
* Since we found a Lambda function to invoke, we construct the Lambda Event from the request
* Then Invoke the Lambda function (docker container)
* We then transform the response or errors we get from the Invoke and return the data back to
the caller
Parameters
----------
kwargs dict
Keyword Args that are passed to the function from Flask. This happens when we have path parameters
Returns
-------
Response object
"""
route = self._get_current_route(request)
cors_headers = Cors.cors_to_headers(self.api.cors)
method, endpoint = self.get_request_methods_endpoints(request)
if method == "OPTIONS" and self.api.cors:
headers = Headers(cors_headers)
return self.service_response("", headers, 200)
try:
# the Lambda Event 2.0 is only used for the HTTP API gateway with defined payload format version equal 2.0
# or none, as the default value to be used is 2.0
# https://docs.aws.amazon.com/apigatewayv2/latest/api-reference/apis-apiid-integrations.html#apis-apiid-integrations-prop-createintegrationinput-payloadformatversion
if route.event_type == Route.HTTP and route.payload_format_version in [
None, "2.0"
]:
route_key = self._v2_route_key(method, endpoint,
route.is_default_route)
event = self._construct_v_2_0_event_http(
request,
self.port,
self.api.binary_media_types,
self.api.stage_name,
self.api.stage_variables,
route_key,
)
else:
event = self._construct_v_1_0_event(
request, self.port, self.api.binary_media_types,
self.api.stage_name, self.api.stage_variables)
except UnicodeDecodeError:
return ServiceErrorResponses.lambda_failure_response()
stdout_stream = io.BytesIO()
stdout_stream_writer = StreamWriter(stdout_stream, self.is_debugging)
try:
self.lambda_runner.invoke(route.function_name,
event,
stdout=stdout_stream_writer,
stderr=self.stderr)
except FunctionNotFound:
return ServiceErrorResponses.lambda_not_found_response()
lambda_response, lambda_logs, _ = LambdaOutputParser.get_lambda_output(
stdout_stream)
if self.stderr and lambda_logs:
# Write the logs to stderr if available.
self.stderr.write(lambda_logs)
try:
if route.event_type == Route.HTTP and (
not route.payload_format_version
or route.payload_format_version == "2.0"):
(status_code, headers,
body) = self._parse_v2_payload_format_lambda_output(
lambda_response, self.api.binary_media_types, request)
else:
(status_code, headers,
body) = self._parse_v1_payload_format_lambda_output(
lambda_response, self.api.binary_media_types, request)
except LambdaResponseParseException as ex:
LOG.error("Invalid lambda response received: %s", ex)
return ServiceErrorResponses.lambda_failure_response()
return self.service_response(body, headers, status_code)
class ECRUploader:
"""
Class to upload Images to ECR.
"""
def __init__(self,
docker_client,
ecr_client,
ecr_repo,
tag="latest",
stream=stderr()):
self.docker_client = docker_client if docker_client else docker.from_env(
)
self.ecr_client = ecr_client
self.ecr_repo = ecr_repo
self.tag = tag
self.auth_config = {}
self.stream = StreamWriter(stream=stream, auto_flush=True)
def login(self):
"""
Logs into the supplied ECR with credentials.
"""
try:
token = self.ecr_client.get_authorization_token()
except botocore.exceptions.ClientError as ex:
raise ECRAuthorizationError(
msg=ex.response["Error"]["Message"]) from ex
username, password = base64.b64decode(
token["authorizationData"][0]
["authorizationToken"]).decode().split(":")
registry = token["authorizationData"][0]["proxyEndpoint"]
try:
self.docker_client.login(username=ECR_USERNAME,
password=password,
registry=registry)
except APIError as ex:
raise DockerLoginFailedError(msg=str(ex)) from ex
self.auth_config = {"username": username, "password": password}
def upload(self, image):
"""
Uploads given local image to ECR.
:param image: locally tagged docker image that would be uploaded to ECR.
:return: remote ECR image path that has been uploaded.
"""
self.login()
try:
docker_img = self.docker_client.images.get(image)
_tag = tag_translation(image,
docker_image_id=docker_img.id,
gen_tag=self.tag)
docker_img.tag(repository=self.ecr_repo, tag=_tag)
push_logs = self.docker_client.api.push(
repository=self.ecr_repo,
tag=_tag,
auth_config=self.auth_config,
stream=True,
decode=True)
self._stream_progress(push_logs)
except (BuildError, APIError) as ex:
raise DockerPushFailedError(msg=str(ex)) from ex
return f"{self.ecr_repo}:{_tag}"
# TODO: move this to a generic class to allow for streaming logs back from docker.
def _stream_progress(self, logs):
"""
Stream progress from docker push logs and move the cursor based on the log id.
:param logs: generator from docker_clent.api.push
:return:
"""
ids = dict()
for log in logs:
_id = log.get("id", None)
status = log.get("status", None)
progress = log.get("progress", "")
error = log.get("error", "")
change_cursor_count = 0
if _id:
try:
curr_log_line_id = ids[_id]
change_cursor_count = len(ids) - curr_log_line_id
self.stream.write((cursor_up(change_cursor_count) +
cursor_left).encode())
except KeyError:
ids[_id] = len(ids)
else:
ids = dict()
self._stream_write(_id, status, progress, error)
if _id:
self.stream.write(
(cursor_down(change_cursor_count) + cursor_left).encode())
self.stream.write(os.linesep.encode())
def _stream_write(self, _id, status, progress, error):
"""
Write stream information to stderr, if the stream information contains a log id,
use the carraige return character to rewrite that particular line.
:param _id: docker log id
:param status: docker log status
:param progress: docker log progress
:param error: docker log error
"""
if error:
raise DockerPushFailedError(msg=error)
if not status:
return
# NOTE(sriram-mv): Required for the purposes of when the cursor overflows existing terminal buffer.
self.stream.write(os.linesep.encode())
self.stream.write((cursor_up() + cursor_left).encode())
self.stream.write(clear_line().encode())
if not _id:
self.stream.write(f"{status}{os.linesep}".encode())
else:
self.stream.write(f"\r{_id}: {status} {progress}".encode())
def _build_image(self, base_image, docker_tag, layers, is_debug_go, stream=None):
"""
Builds the image
Parameters
----------
base_image str
Base Image to use for the new image
docker_tag
Docker tag (REPOSITORY:TAG) to use when building the image
layers list(samcli.commands.local.lib.provider.Layer)
List of Layers to be use to mount in the image
Returns
-------
None
Raises
------
samcli.commands.local.cli_common.user_exceptions.ImageBuildException
When docker fails to build the image
"""
dockerfile_content = self._generate_dockerfile(base_image, layers, is_debug_go)
# Create dockerfile in the same directory of the layer cache
dockerfile_name = "dockerfile_" + str(uuid.uuid4())
full_dockerfile_path = Path(self.layer_downloader.layer_cache, dockerfile_name)
stream_writer = stream or StreamWriter(sys.stderr)
try:
with open(str(full_dockerfile_path), "w") as dockerfile:
dockerfile.write(dockerfile_content)
# add dockerfile and rapid source paths
tar_paths = {str(full_dockerfile_path): "Dockerfile", self._RAPID_SOURCE_PATH: "/init"}
if self._extensions_preview_enabled:
tar_paths = {str(full_dockerfile_path): "Dockerfile", self._RAPID_PREVIEW_SOURCE_PATH: "/init"}
if is_debug_go:
LOG.debug("Adding custom GO Bootstrap to support debugging")
tar_paths[self._GO_BOOTSTRAP_PATH] = "/aws-lambda-go"
for layer in layers:
tar_paths[layer.codeuri] = "/" + layer.name
# Set permission for all the files in the tarball to 500(Read and Execute Only)
# This is need for systems without unix like permission bits(Windows) while creating a unix image
# Without setting this explicitly, tar will default the permission to 666 which gives no execute permission
def set_item_permission(tar_info):
tar_info.mode = 0o500
return tar_info
# Set only on Windows, unix systems will preserve the host permission into the tarball
tar_filter = set_item_permission if platform.system().lower() == "windows" else None
with create_tarball(tar_paths, tar_filter=tar_filter) as tarballfile:
try:
resp_stream = self.docker_client.api.build(
fileobj=tarballfile, custom_context=True, rm=True, tag=docker_tag, pull=not self.skip_pull_image
)
for _ in resp_stream:
stream_writer.write(".")
stream_writer.flush()
stream_writer.write("\n")
except (docker.errors.BuildError, docker.errors.APIError) as ex:
stream_writer.write("\n")
LOG.exception("Failed to build Docker Image")
raise ImageBuildException("Building Image failed.") from ex
finally:
if full_dockerfile_path.exists():
full_dockerfile_path.unlink()
def __init__(self,
resources_to_build,
build_dir,
base_dir,
cache_dir,
cached=False,
is_building_specific_resource=False,
manifest_path_override=None,
container_manager=None,
parallel=False,
mode=None,
stream_writer=None,
docker_client=None):
"""
Initialize the class
Parameters
----------
resources_to_build: Iterator
Iterator that can vend out resources available in the SAM template
build_dir : str
Path to the directory where we will be storing built artifacts
base_dir : str
Path to a folder. Use this folder as the root to resolve relative source code paths against
cache_dir : str
Path to a the directory where we will be caching built artifacts
cached:
Optional. Set to True to build each function with cache to improve performance
is_building_specific_resource : boolean
Whether customer requested to build a specific resource alone in isolation,
by specifying function_identifier to the build command.
Ex: sam build MyServerlessFunction
container_manager : samcli.local.docker.manager.ContainerManager
Optional. If provided, we will attempt to build inside a Docker Container
parallel : bool
Optional. Set to True to build each function in parallel to improve performance
mode : str
Optional, name of the build mode to use ex: 'debug'
"""
self._resources_to_build = resources_to_build
self._build_dir = build_dir
self._base_dir = base_dir
self._cache_dir = cache_dir
self._cached = cached
self._manifest_path_override = manifest_path_override
self._is_building_specific_resource = is_building_specific_resource
self._container_manager = container_manager
self._parallel = parallel
self._mode = mode
self._stream_writer = stream_writer if stream_writer else StreamWriter(
osutils.stderr())
self._docker_client = docker_client if docker_client else docker.from_env(
)
self._deprecated_runtimes = {
"nodejs4.3", "nodejs6.10", "nodejs8.10", "dotnetcore2.0"
}
self._colored = Colored()
def _build_image(self,
base_image,
docker_tag,
layers,
is_debug_go,
stream=None):
"""
Builds the image
Parameters
----------
base_image str
Base Image to use for the new image
docker_tag
Docker tag (REPOSITORY:TAG) to use when building the image
layers list(samcli.commands.local.lib.provider.Layer)
List of Layers to be use to mount in the image
Returns
-------
None
Raises
------
samcli.commands.local.cli_common.user_exceptions.ImageBuildException
When docker fails to build the image
"""
dockerfile_content = self._generate_dockerfile(base_image, layers,
is_debug_go)
# Create dockerfile in the same directory of the layer cache
dockerfile_name = "dockerfile_" + str(uuid.uuid4())
full_dockerfile_path = Path(self.layer_downloader.layer_cache,
dockerfile_name)
stream_writer = stream or StreamWriter(sys.stderr)
try:
with open(str(full_dockerfile_path), "w") as dockerfile:
dockerfile.write(dockerfile_content)
# add dockerfile and rapid source paths
tar_paths = {
str(full_dockerfile_path): "Dockerfile",
self._RAPID_SOURCE_PATH: "/init"
}
if is_debug_go:
LOG.debug("Adding custom GO Bootstrap to support debugging")
tar_paths[self._GO_BOOTSTRAP_PATH] = "/aws-lambda-go"
for layer in layers:
tar_paths[layer.codeuri] = "/" + layer.name
with create_tarball(tar_paths) as tarballfile:
try:
resp_stream = self.docker_client.api.build(
fileobj=tarballfile,
custom_context=True,
rm=True,
tag=docker_tag,
pull=not self.skip_pull_image)
for _ in resp_stream:
stream_writer.write(".")
stream_writer.flush()
stream_writer.write("\n")
except (docker.errors.BuildError, docker.errors.APIError):
stream_writer.write("\n")
LOG.exception("Failed to build Docker Image")
raise ImageBuildException("Building Image failed.")
finally:
if full_dockerfile_path.exists():
full_dockerfile_path.unlink()
def build(self, runtime, packagetype, image, layers, stream=None):
"""
Build the image if one is not already on the system that matches the runtime and layers
Parameters
----------
runtime str
Name of the Lambda runtime
packagetype str
Packagetype for the Lambda
image str
Pre-defined invocation image.
layers list(samcli.commands.local.lib.provider.Layer)
List of layers
Returns
-------
str
The image to be used (REPOSITORY:TAG)
"""
image_name = None
if packagetype == IMAGE:
image_name = image
elif packagetype == ZIP:
image_name = f"{self._INVOKE_REPO_PREFIX}-{runtime}:latest"
if not image_name:
raise InvalidIntermediateImageError(
f"Invalid PackageType, PackageType needs to be one of [{ZIP}, {IMAGE}]"
)
if image:
self.skip_pull_image = True
# Default image tag to be the base image with a tag of 'rapid' instead of latest.
# If the image name had a digest, removing the @ so that a valid image name can be constructed
# to use for the local invoke image name.
image_tag = f"{image_name.split(':')[0].replace('@', '')}:rapid-{version}"
downloaded_layers = []
if layers and packagetype == ZIP:
downloaded_layers = self.layer_downloader.download_all(
layers, self.force_image_build)
docker_image_version = self._generate_docker_image_version(
downloaded_layers, runtime)
image_tag = f"{self._SAM_CLI_REPO_NAME}:{docker_image_version}"
image_not_found = False
# If we are not using layers, build anyways to ensure any updates to rapid get added
try:
self.docker_client.images.get(image_tag)
except docker.errors.ImageNotFound:
LOG.info("Image was not found.")
image_not_found = True
if (self.force_image_build or image_not_found
or any(layer.is_defined_within_template
for layer in downloaded_layers) or not runtime):
stream_writer = stream or StreamWriter(sys.stderr)
stream_writer.write("Building image...")
stream_writer.flush()
self._build_image(image if image else image_name,
image_tag,
downloaded_layers,
stream=stream_writer)
return image_tag