def test_must_flush_underlying_stream(self): stream_mock = Mock() writer = StreamWriter(stream_mock) writer.flush() stream_mock.flush.assert_called_once_with()
def test_auto_flush_must_be_off_by_default(self): stream_mock = Mock() writer = StreamWriter(stream_mock) writer.write("something") stream_mock.flush.assert_not_called()
def test_function_result_is_available_in_stdout_and_logs_in_stderr(self): # This is the JSON result from Lambda function # Convert to proper binary type to be compatible with Python 2 & 3 expected_output = b'{"a":"b"}' expected_stderr = b"**This string is printed from Lambda function**" layer_downloader = LayerDownloader("./", "./") image_builder = LambdaImage(layer_downloader, False, False) container = LambdaContainer(self.runtime, self.handler, self.code_dir, self.layers, image_builder) stdout_stream = io.BytesIO() stderr_stream = io.BytesIO() stdout_stream_writer = StreamWriter(stdout_stream) stderr_stream_writer = StreamWriter(stderr_stream) with self._create(container): container.start() container.wait_for_logs(stdout=stdout_stream_writer, stderr=stderr_stream_writer) function_output = stdout_stream.getvalue() function_stderr = stderr_stream.getvalue() self.assertEqual(function_output.strip(), expected_output) self.assertIn(expected_stderr, function_stderr)
def test_must_write_to_stream(self): buffer = "something" stream_mock = Mock() writer = StreamWriter(stream_mock) writer.write(buffer) stream_mock.write.assert_called_once_with(buffer)
def __init__(self, docker_client, ecr_client, ecr_repo, tag="latest", stream=stderr()): self.docker_client = docker_client if docker_client else docker.from_env( ) self.ecr_client = ecr_client self.ecr_repo = ecr_repo self.tag = tag self.auth_config = {} self.stream = StreamWriter(stream=stream, auto_flush=True)
def test_when_auto_flush_on_flush_after_each_write(self): stream_mock = Mock() flush_mock = Mock() stream_mock.flush = flush_mock lines = ["first", "second", "third"] writer = StreamWriter(stream_mock, True) for line in lines: writer.write(line) flush_mock.assert_called_once_with() flush_mock.reset_mock()
def test_must_invoke(self): input_event = '"some data"' expected_env_vars = { "var1": "override_value1", "var2": "shell_env_value2" } manager = ContainerManager() layer_downloader = LayerDownloader("./", "./") lambda_image = LambdaImage(layer_downloader, False, False) local_runtime = LambdaRuntime(manager, lambda_image) runner = LocalLambdaRunner(local_runtime, self.mock_function_provider, self.cwd, self.env_var_overrides, debug_context=None) # Append the real AWS credentials to the expected values. creds = runner.get_aws_creds() # default value for creds is not configured by the test. But coming from a downstream class expected_env_vars["AWS_SECRET_ACCESS_KEY"] = creds.get( "secret", "defaultsecret") expected_env_vars["AWS_ACCESS_KEY_ID"] = creds.get("key", "defaultkey") expected_env_vars["AWS_REGION"] = creds.get("region", "us-east-1") stdout_stream = io.BytesIO() stderr_stream = io.BytesIO() stdout_stream_writer = StreamWriter(stdout_stream) stderr_stream_writer = StreamWriter(stderr_stream) runner.invoke(self.function_name, input_event, stdout=stdout_stream_writer, stderr=stderr_stream_writer) # stderr is where the Lambda container runtime logs are available. It usually contains requestId, start time # etc. So it is non-zero in size self.assertGreater(len(stderr_stream.getvalue().strip()), 0, "stderr stream must contain data") # This should contain all the environment variables passed to the function actual_output = json.loads( stdout_stream.getvalue().strip().decode("utf-8")) for key, value in expected_env_vars.items(): self.assertTrue(key in actual_output, "Key '{}' must be in function output".format(key)) self.assertEqual(actual_output.get(key), value)
def test_check_environment_variables(self): variables = {"var1": "value1", "var2": "value2"} aws_creds = { "region": "ap-south-1", "key": "mykey", "secret": "mysecret" } timeout = 30 input_event = "" stdout_stream = io.BytesIO() stdout_stream_writer = StreamWriter(stdout_stream) expected_output = { "AWS_SAM_LOCAL": "true", "AWS_LAMBDA_FUNCTION_MEMORY_SIZE": "1024", "AWS_LAMBDA_FUNCTION_TIMEOUT": "30", "AWS_LAMBDA_FUNCTION_HANDLER": "index.handler", # Values coming from AWS Credentials "AWS_REGION": "ap-south-1", "AWS_DEFAULT_REGION": "ap-south-1", "AWS_ACCESS_KEY_ID": "mykey", "AWS_SECRET_ACCESS_KEY": "mysecret", # Custom environment variables "var1": "value1", "var2": "value2", } config = FunctionConfig( name="helloworld", runtime=RUNTIME, handler=HANDLER, code_abs_path=self.code_dir["envvar"], layers=[], memory=MEMORY, timeout=timeout, ) # Set the appropriate environment variables config.env_vars.variables = variables config.env_vars.aws_creds = aws_creds self.runtime.invoke(config, input_event, stdout=stdout_stream_writer) actual_output = json.loads(stdout_stream.getvalue().strip().decode( "utf-8")) # Output is a JSON String. Deserialize. # Make sure all key/value from expected_output is present in actual_output for key, value in expected_output.items(): # Do the key check first to print a nice error error message when it fails self.assertTrue( key in actual_output, "'{}' should be in environment variable output".format(key)) self.assertEqual( actual_output[key], expected_output[key], "Value of environment variable '{}' differs fromm expectation". format(key), )
def test_echo_function_with_zip_file(self, file_name_extension): timeout = 3 input_event = '"this input should be echoed"' expected_output = b'"this input should be echoed"' code_dir = self.code_dir["echo"] with make_zip(code_dir, file_name_extension) as code_zip_path: config = FunctionConfig( name="helloworld", runtime=RUNTIME, handler=HANDLER, code_abs_path=code_zip_path, layers=[], timeout=timeout, ) stdout_stream = io.BytesIO() stdout_stream_writer = StreamWriter(stdout_stream) self.runtime.invoke(config, input_event, stdout=stdout_stream_writer) actual_output = stdout_stream.getvalue() self.assertEqual(actual_output.strip(), expected_output)
def _invoke_sleep(self, timeout, sleep_duration, check_stdout, exceptions=None): name = "sleepfunction_timeout_{}_sleep_{}".format( timeout, sleep_duration) print("Invoking function " + name) try: stdout_stream = io.BytesIO() stdout_stream_writer = StreamWriter(stdout_stream) config = FunctionConfig(name=name, runtime=RUNTIME, handler=HANDLER, code_abs_path=self.code_dir, layers=[], memory=1024, timeout=timeout) self.runtime.invoke(config, sleep_duration, stdout=stdout_stream_writer) actual_output = stdout_stream.getvalue().strip( ) # Must output the sleep duration if check_stdout: self.assertEquals(actual_output.decode('utf-8'), str(sleep_duration)) except Exception as ex: if exceptions is not None: exceptions.append({"name": name, "error": ex}) else: raise
def test_function_timeout(self): """ Setup a short timeout and verify that the container is stopped """ stdout_stream = io.BytesIO() stdout_stream_writer = StreamWriter(stdout_stream) timeout = 1 # 1 second timeout sleep_seconds = 20 # Ask the function to sleep for 20 seconds config = FunctionConfig( name="sleep_timeout", runtime=RUNTIME, handler=HANDLER, code_abs_path=self.code_dir["sleep"], layers=[], timeout=timeout, ) # Measure the actual duration of execution start = timer() self.runtime.invoke(config, str(sleep_seconds), stdout=stdout_stream_writer) end = timer() # Make sure that the wall clock duration is around the ballpark of timeout value wall_clock_func_duration = end - start print ("Function completed in {} seconds".format(wall_clock_func_duration)) # The function should *not* preemptively stop self.assertGreater(wall_clock_func_duration, timeout - 1) # The function should not run for much longer than timeout. self.assertLess(wall_clock_func_duration, timeout + self.CONTAINER_STARTUP_OVERHEAD_SECONDS) # There should be no output from the function because timer was interrupted actual_output = stdout_stream.getvalue() self.assertEqual(actual_output.strip(), b"")
def build(self, runtime, layers, is_debug, stream=None): """ Build the image if one is not already on the system that matches the runtime and layers Parameters ---------- runtime str Name of the Lambda runtime layers list(samcli.commands.local.lib.provider.Layer) List of layers Returns ------- str The image to be used (REPOSITORY:TAG) """ base_image = f"{self._INVOKE_REPO_PREFIX}-{runtime}:latest" # Default image tag to be the base image with a tag of 'rapid' instead of latest image_tag = f"{self._INVOKE_REPO_PREFIX}-{runtime}:rapid-{version}" downloaded_layers = [] if layers: downloaded_layers = self.layer_downloader.download_all( layers, self.force_image_build) docker_image_version = self._generate_docker_image_version( downloaded_layers, runtime) image_tag = f"{self._SAM_CLI_REPO_NAME}:{docker_image_version}" image_not_found = False is_debug_go = runtime == "go1.x" and is_debug if is_debug_go: image_tag = f"{self._INVOKE_REPO_PREFIX}-{runtime}:debug-{version}" # If we are not using layers, build anyways to ensure any updates to rapid get added try: self.docker_client.images.get(image_tag) except docker.errors.ImageNotFound: LOG.info("Image was not found.") image_not_found = True if (self.force_image_build or image_not_found or any(layer.is_defined_within_template for layer in downloaded_layers)): stream_writer = stream or StreamWriter(sys.stderr) stream_writer.write("Building image...") stream_writer.flush() self._build_image(base_image, image_tag, downloaded_layers, is_debug_go, stream=stream_writer) return image_tag
def _request_handler(self, **kwargs): """ We handle all requests to the host:port. The general flow of handling a request is as follows * Fetch request from the Flask Global state. This is where Flask places the request and is per thread so multiple requests are still handled correctly * Find the Lambda function to invoke by doing a look up based on the request.endpoint and method * If we don't find the function, we will throw a 502 (just like the 404 and 405 responses we get from Flask. * Since we found a Lambda function to invoke, we construct the Lambda Event from the request * Then Invoke the Lambda function (docker container) * We then transform the response or errors we get from the Invoke and return the data back to the caller Parameters ---------- kwargs dict Keyword Args that are passed to the function from Flask. This happens when we have path parameters Returns ------- Response object """ route = self._get_current_route(request) try: event = self._construct_event(request, self.port, route.binary_types, route.stage_name, route.stage_variables) except UnicodeDecodeError: return ServiceErrorResponses.lambda_failure_response() stdout_stream = io.BytesIO() stdout_stream_writer = StreamWriter(stdout_stream, self.is_debugging) try: self.lambda_runner.invoke(route.function_name, event, stdout=stdout_stream_writer, stderr=self.stderr) except FunctionNotFound: return ServiceErrorResponses.lambda_not_found_response() lambda_response, lambda_logs, _ = LambdaOutputParser.get_lambda_output(stdout_stream) if self.stderr and lambda_logs: # Write the logs to stderr if available. self.stderr.write(lambda_logs) try: (status_code, headers, body) = self._parse_lambda_output(lambda_response, route.binary_types, request) except (KeyError, TypeError, ValueError): LOG.error("Function returned an invalid response (must include one of: body, headers, multiValueHeaders or " "statusCode in the response object). Response received: %s", lambda_response) return ServiceErrorResponses.lambda_failure_response() return self.service_response(body, headers, status_code)
def stderr(self) -> StreamWriter: """ Returns stream writer for stderr to output Lambda function errors to Returns ------- samcli.lib.utils.stream_writer.StreamWriter Stream writer for stderr """ stream = self._log_file_handle if self._log_file_handle else osutils.stderr() return StreamWriter(stream, self._is_debugging)
def pull_image(self, image_name, tag=None, stream=None): """ Ask Docker to pull the container image with given name. Parameters ---------- image_name str Name of the image stream samcli.lib.utils.stream_writer.StreamWriter Optional stream writer to output to. Defaults to stderr Raises ------ DockerImagePullFailedException If the Docker image was not available in the server """ if tag is None: tag = image_name.split(":")[1] if ":" in image_name else "latest" # use a global lock to get the image lock with self._lock: image_lock = self._lock_per_image.get(image_name) if not image_lock: image_lock = threading.Lock() self._lock_per_image[image_name] = image_lock # with specific image lock, pull this image only once # since there are different locks for each image, different images can be pulled in parallel with image_lock: stream_writer = stream or StreamWriter(sys.stderr) try: result_itr = self.docker_client.api.pull(image_name, tag=tag, stream=True, decode=True) except docker.errors.APIError as ex: LOG.debug("Failed to download image with name %s", image_name) raise DockerImagePullFailedException(str(ex)) from ex # io streams, especially StringIO, work only with unicode strings stream_writer.write( "\nFetching {} Docker container image...".format(image_name)) # Each line contains information on progress of the pull. Each line is a JSON string for _ in result_itr: # For every line, print a dot to show progress stream_writer.write(".") stream_writer.flush() # We are done. Go to the next line stream_writer.write("\n")
def _invoke_request_handler(self, function_name): """ Request Handler for the Local Lambda Invoke path. This method is responsible for understanding the incoming request and invoking the Local Lambda Function Parameters ---------- function_name str Name of the function to invoke Returns ------- A Flask Response response object as if it was returned from Lambda """ flask_request = request request_data = flask_request.get_data() if not request_data: request_data = b'{}' request_data = request_data.decode('utf-8') stdout_stream = io.BytesIO() stdout_stream_writer = StreamWriter(stdout_stream, self.is_debugging) try: self.lambda_runner.invoke(function_name, request_data, stdout=stdout_stream_writer, stderr=self.stderr) except FunctionNotFound: LOG.debug('%s was not found to invoke.', function_name) return LambdaErrorResponses.resource_not_found(function_name) lambda_response, lambda_logs, is_lambda_user_error_response = \ LambdaOutputParser.get_lambda_output(stdout_stream) if self.stderr and lambda_logs: # Write the logs to stderr if available. self.stderr.write(lambda_logs) if is_lambda_user_error_response: return self.service_response( lambda_response, { 'Content-Type': 'application/json', 'x-amz-function-error': 'Unhandled' }, 200) return self.service_response(lambda_response, {'Content-Type': 'application/json'}, 200)
def test_echo_function(self): timeout = 3 input_event = '{"a":"b"}' expected_output = b'{"a":"b"}' config = FunctionConfig(name="helloworld", runtime=RUNTIME, handler=HANDLER, code_abs_path=self.code_dir["echo"], layers=[], timeout=timeout) stdout_stream = io.BytesIO() stdout_stream_writer = StreamWriter(stdout_stream) self.runtime.invoke(config, input_event, stdout=stdout_stream_writer) actual_output = stdout_stream.getvalue() self.assertEquals(actual_output.strip(), expected_output)
def pull_image(self, image_name, stream=None): """ Ask Docker to pull the container image with given name. Parameters ---------- image_name str Name of the image stream samcli.lib.utils.stream_writer.StreamWriter Optional stream writer to output to. Defaults to stderr Raises ------ DockerImagePullFailedException If the Docker image was not available in the server """ stream_writer = stream or StreamWriter(sys.stderr) try: result_itr = self.docker_client.api.pull(image_name, stream=True, decode=True) except docker.errors.APIError as ex: LOG.debug("Failed to download image with name %s", image_name) raise DockerImagePullFailedException(str(ex)) # io streams, especially StringIO, work only with unicode strings stream_writer.write( "\nFetching {} Docker container image...".format(image_name)) # Each line contains information on progress of the pull. Each line is a JSON string for _ in result_itr: # For every line, print a dot to show progress stream_writer.write(".") stream_writer.flush() # We are done. Go to the next line stream_writer.write("\n")
def _request_handler(self, **kwargs): """ We handle all requests to the host:port. The general flow of handling a request is as follows * Fetch request from the Flask Global state. This is where Flask places the request and is per thread so multiple requests are still handled correctly * Find the Lambda function to invoke by doing a look up based on the request.endpoint and method * If we don't find the function, we will throw a 502 (just like the 404 and 405 responses we get from Flask. * Since we found a Lambda function to invoke, we construct the Lambda Event from the request * Then Invoke the Lambda function (docker container) * We then transform the response or errors we get from the Invoke and return the data back to the caller Parameters ---------- kwargs dict Keyword Args that are passed to the function from Flask. This happens when we have path parameters Returns ------- Response object """ route = self._get_current_route(request) cors_headers = Cors.cors_to_headers(self.api.cors) method, endpoint = self.get_request_methods_endpoints(request) if method == "OPTIONS" and self.api.cors: headers = Headers(cors_headers) return self.service_response("", headers, 200) try: # the Lambda Event 2.0 is only used for the HTTP API gateway with defined payload format version equal 2.0 # or none, as the default value to be used is 2.0 # https://docs.aws.amazon.com/apigatewayv2/latest/api-reference/apis-apiid-integrations.html#apis-apiid-integrations-prop-createintegrationinput-payloadformatversion if route.event_type == Route.HTTP and route.payload_format_version in [ None, "2.0" ]: route_key = self._v2_route_key(method, endpoint, route.is_default_route) event = self._construct_v_2_0_event_http( request, self.port, self.api.binary_media_types, self.api.stage_name, self.api.stage_variables, route_key, ) else: event = self._construct_v_1_0_event( request, self.port, self.api.binary_media_types, self.api.stage_name, self.api.stage_variables) except UnicodeDecodeError: return ServiceErrorResponses.lambda_failure_response() stdout_stream = io.BytesIO() stdout_stream_writer = StreamWriter(stdout_stream, self.is_debugging) try: self.lambda_runner.invoke(route.function_name, event, stdout=stdout_stream_writer, stderr=self.stderr) except FunctionNotFound: return ServiceErrorResponses.lambda_not_found_response() lambda_response, lambda_logs, _ = LambdaOutputParser.get_lambda_output( stdout_stream) if self.stderr and lambda_logs: # Write the logs to stderr if available. self.stderr.write(lambda_logs) try: if route.event_type == Route.HTTP and ( not route.payload_format_version or route.payload_format_version == "2.0"): (status_code, headers, body) = self._parse_v2_payload_format_lambda_output( lambda_response, self.api.binary_media_types, request) else: (status_code, headers, body) = self._parse_v1_payload_format_lambda_output( lambda_response, self.api.binary_media_types, request) except LambdaResponseParseException as ex: LOG.error("Invalid lambda response received: %s", ex) return ServiceErrorResponses.lambda_failure_response() return self.service_response(body, headers, status_code)
class ECRUploader: """ Class to upload Images to ECR. """ def __init__(self, docker_client, ecr_client, ecr_repo, tag="latest", stream=stderr()): self.docker_client = docker_client if docker_client else docker.from_env( ) self.ecr_client = ecr_client self.ecr_repo = ecr_repo self.tag = tag self.auth_config = {} self.stream = StreamWriter(stream=stream, auto_flush=True) def login(self): """ Logs into the supplied ECR with credentials. """ try: token = self.ecr_client.get_authorization_token() except botocore.exceptions.ClientError as ex: raise ECRAuthorizationError( msg=ex.response["Error"]["Message"]) from ex username, password = base64.b64decode( token["authorizationData"][0] ["authorizationToken"]).decode().split(":") registry = token["authorizationData"][0]["proxyEndpoint"] try: self.docker_client.login(username=ECR_USERNAME, password=password, registry=registry) except APIError as ex: raise DockerLoginFailedError(msg=str(ex)) from ex self.auth_config = {"username": username, "password": password} def upload(self, image): """ Uploads given local image to ECR. :param image: locally tagged docker image that would be uploaded to ECR. :return: remote ECR image path that has been uploaded. """ self.login() try: docker_img = self.docker_client.images.get(image) _tag = tag_translation(image, docker_image_id=docker_img.id, gen_tag=self.tag) docker_img.tag(repository=self.ecr_repo, tag=_tag) push_logs = self.docker_client.api.push( repository=self.ecr_repo, tag=_tag, auth_config=self.auth_config, stream=True, decode=True) self._stream_progress(push_logs) except (BuildError, APIError) as ex: raise DockerPushFailedError(msg=str(ex)) from ex return f"{self.ecr_repo}:{_tag}" # TODO: move this to a generic class to allow for streaming logs back from docker. def _stream_progress(self, logs): """ Stream progress from docker push logs and move the cursor based on the log id. :param logs: generator from docker_clent.api.push :return: """ ids = dict() for log in logs: _id = log.get("id", None) status = log.get("status", None) progress = log.get("progress", "") error = log.get("error", "") change_cursor_count = 0 if _id: try: curr_log_line_id = ids[_id] change_cursor_count = len(ids) - curr_log_line_id self.stream.write((cursor_up(change_cursor_count) + cursor_left).encode()) except KeyError: ids[_id] = len(ids) else: ids = dict() self._stream_write(_id, status, progress, error) if _id: self.stream.write( (cursor_down(change_cursor_count) + cursor_left).encode()) self.stream.write(os.linesep.encode()) def _stream_write(self, _id, status, progress, error): """ Write stream information to stderr, if the stream information contains a log id, use the carraige return character to rewrite that particular line. :param _id: docker log id :param status: docker log status :param progress: docker log progress :param error: docker log error """ if error: raise DockerPushFailedError(msg=error) if not status: return # NOTE(sriram-mv): Required for the purposes of when the cursor overflows existing terminal buffer. self.stream.write(os.linesep.encode()) self.stream.write((cursor_up() + cursor_left).encode()) self.stream.write(clear_line().encode()) if not _id: self.stream.write(f"{status}{os.linesep}".encode()) else: self.stream.write(f"\r{_id}: {status} {progress}".encode())
def _build_image(self, base_image, docker_tag, layers, is_debug_go, stream=None): """ Builds the image Parameters ---------- base_image str Base Image to use for the new image docker_tag Docker tag (REPOSITORY:TAG) to use when building the image layers list(samcli.commands.local.lib.provider.Layer) List of Layers to be use to mount in the image Returns ------- None Raises ------ samcli.commands.local.cli_common.user_exceptions.ImageBuildException When docker fails to build the image """ dockerfile_content = self._generate_dockerfile(base_image, layers, is_debug_go) # Create dockerfile in the same directory of the layer cache dockerfile_name = "dockerfile_" + str(uuid.uuid4()) full_dockerfile_path = Path(self.layer_downloader.layer_cache, dockerfile_name) stream_writer = stream or StreamWriter(sys.stderr) try: with open(str(full_dockerfile_path), "w") as dockerfile: dockerfile.write(dockerfile_content) # add dockerfile and rapid source paths tar_paths = {str(full_dockerfile_path): "Dockerfile", self._RAPID_SOURCE_PATH: "/init"} if self._extensions_preview_enabled: tar_paths = {str(full_dockerfile_path): "Dockerfile", self._RAPID_PREVIEW_SOURCE_PATH: "/init"} if is_debug_go: LOG.debug("Adding custom GO Bootstrap to support debugging") tar_paths[self._GO_BOOTSTRAP_PATH] = "/aws-lambda-go" for layer in layers: tar_paths[layer.codeuri] = "/" + layer.name # Set permission for all the files in the tarball to 500(Read and Execute Only) # This is need for systems without unix like permission bits(Windows) while creating a unix image # Without setting this explicitly, tar will default the permission to 666 which gives no execute permission def set_item_permission(tar_info): tar_info.mode = 0o500 return tar_info # Set only on Windows, unix systems will preserve the host permission into the tarball tar_filter = set_item_permission if platform.system().lower() == "windows" else None with create_tarball(tar_paths, tar_filter=tar_filter) as tarballfile: try: resp_stream = self.docker_client.api.build( fileobj=tarballfile, custom_context=True, rm=True, tag=docker_tag, pull=not self.skip_pull_image ) for _ in resp_stream: stream_writer.write(".") stream_writer.flush() stream_writer.write("\n") except (docker.errors.BuildError, docker.errors.APIError) as ex: stream_writer.write("\n") LOG.exception("Failed to build Docker Image") raise ImageBuildException("Building Image failed.") from ex finally: if full_dockerfile_path.exists(): full_dockerfile_path.unlink()
def __init__(self, resources_to_build, build_dir, base_dir, cache_dir, cached=False, is_building_specific_resource=False, manifest_path_override=None, container_manager=None, parallel=False, mode=None, stream_writer=None, docker_client=None): """ Initialize the class Parameters ---------- resources_to_build: Iterator Iterator that can vend out resources available in the SAM template build_dir : str Path to the directory where we will be storing built artifacts base_dir : str Path to a folder. Use this folder as the root to resolve relative source code paths against cache_dir : str Path to a the directory where we will be caching built artifacts cached: Optional. Set to True to build each function with cache to improve performance is_building_specific_resource : boolean Whether customer requested to build a specific resource alone in isolation, by specifying function_identifier to the build command. Ex: sam build MyServerlessFunction container_manager : samcli.local.docker.manager.ContainerManager Optional. If provided, we will attempt to build inside a Docker Container parallel : bool Optional. Set to True to build each function in parallel to improve performance mode : str Optional, name of the build mode to use ex: 'debug' """ self._resources_to_build = resources_to_build self._build_dir = build_dir self._base_dir = base_dir self._cache_dir = cache_dir self._cached = cached self._manifest_path_override = manifest_path_override self._is_building_specific_resource = is_building_specific_resource self._container_manager = container_manager self._parallel = parallel self._mode = mode self._stream_writer = stream_writer if stream_writer else StreamWriter( osutils.stderr()) self._docker_client = docker_client if docker_client else docker.from_env( ) self._deprecated_runtimes = { "nodejs4.3", "nodejs6.10", "nodejs8.10", "dotnetcore2.0" } self._colored = Colored()
def _build_image(self, base_image, docker_tag, layers, is_debug_go, stream=None): """ Builds the image Parameters ---------- base_image str Base Image to use for the new image docker_tag Docker tag (REPOSITORY:TAG) to use when building the image layers list(samcli.commands.local.lib.provider.Layer) List of Layers to be use to mount in the image Returns ------- None Raises ------ samcli.commands.local.cli_common.user_exceptions.ImageBuildException When docker fails to build the image """ dockerfile_content = self._generate_dockerfile(base_image, layers, is_debug_go) # Create dockerfile in the same directory of the layer cache dockerfile_name = "dockerfile_" + str(uuid.uuid4()) full_dockerfile_path = Path(self.layer_downloader.layer_cache, dockerfile_name) stream_writer = stream or StreamWriter(sys.stderr) try: with open(str(full_dockerfile_path), "w") as dockerfile: dockerfile.write(dockerfile_content) # add dockerfile and rapid source paths tar_paths = { str(full_dockerfile_path): "Dockerfile", self._RAPID_SOURCE_PATH: "/init" } if is_debug_go: LOG.debug("Adding custom GO Bootstrap to support debugging") tar_paths[self._GO_BOOTSTRAP_PATH] = "/aws-lambda-go" for layer in layers: tar_paths[layer.codeuri] = "/" + layer.name with create_tarball(tar_paths) as tarballfile: try: resp_stream = self.docker_client.api.build( fileobj=tarballfile, custom_context=True, rm=True, tag=docker_tag, pull=not self.skip_pull_image) for _ in resp_stream: stream_writer.write(".") stream_writer.flush() stream_writer.write("\n") except (docker.errors.BuildError, docker.errors.APIError): stream_writer.write("\n") LOG.exception("Failed to build Docker Image") raise ImageBuildException("Building Image failed.") finally: if full_dockerfile_path.exists(): full_dockerfile_path.unlink()
def build(self, runtime, packagetype, image, layers, stream=None): """ Build the image if one is not already on the system that matches the runtime and layers Parameters ---------- runtime str Name of the Lambda runtime packagetype str Packagetype for the Lambda image str Pre-defined invocation image. layers list(samcli.commands.local.lib.provider.Layer) List of layers Returns ------- str The image to be used (REPOSITORY:TAG) """ image_name = None if packagetype == IMAGE: image_name = image elif packagetype == ZIP: image_name = f"{self._INVOKE_REPO_PREFIX}-{runtime}:latest" if not image_name: raise InvalidIntermediateImageError( f"Invalid PackageType, PackageType needs to be one of [{ZIP}, {IMAGE}]" ) if image: self.skip_pull_image = True # Default image tag to be the base image with a tag of 'rapid' instead of latest. # If the image name had a digest, removing the @ so that a valid image name can be constructed # to use for the local invoke image name. image_tag = f"{image_name.split(':')[0].replace('@', '')}:rapid-{version}" downloaded_layers = [] if layers and packagetype == ZIP: downloaded_layers = self.layer_downloader.download_all( layers, self.force_image_build) docker_image_version = self._generate_docker_image_version( downloaded_layers, runtime) image_tag = f"{self._SAM_CLI_REPO_NAME}:{docker_image_version}" image_not_found = False # If we are not using layers, build anyways to ensure any updates to rapid get added try: self.docker_client.images.get(image_tag) except docker.errors.ImageNotFound: LOG.info("Image was not found.") image_not_found = True if (self.force_image_build or image_not_found or any(layer.is_defined_within_template for layer in downloaded_layers) or not runtime): stream_writer = stream or StreamWriter(sys.stderr) stream_writer.write("Building image...") stream_writer.flush() self._build_image(image if image else image_name, image_tag, downloaded_layers, stream=stream_writer) return image_tag