def test_must_add_multiple_items(self):
capability1 = Capability(language="a",
dependency_manager="b",
application_framework="c")
capability2 = Capability(language="d",
dependency_manager="e",
application_framework="f")
self.registry[capability1] = "some data"
self.registry[capability2] = "some other data"
self.assertEquals(len(self.registry), 2)
self.assertTrue(capability1 in self.registry)
self.assertTrue(capability2 in self.registry)
def __init__(self, language, dependency_manager, application_framework, supported_workflows=None):
"""
Initialize the builder.
:type supported_workflows: list
:param supported_workflows:
Optional list of workflow modules that should be loaded. By default we load all the workflows bundled
with this library. This property is primarily used for testing. But in future it could be used to
dynamically load user defined workflows.
If set to None, we will load the default workflow modules.
If set to empty list, we will **not** load any modules. Pass an empty list if the workflows
were already loaded by the time this class is instantiated.
:raises lambda_builders.exceptions.WorkflowNotFoundError: If a workflow for given capabilities is not found
"""
# Load defaults if necessary. We check for `None` explicitly because callers could pass an empty list
# if they do not want to load any modules. This supports the case where workflows are already loaded and
# don't need to be loaded again.
self.supported_workflows = _SUPPORTED_WORKFLOWS if supported_workflows is None else supported_workflows
for workflow_module in self.supported_workflows:
LOG.debug("Loading workflow module '%s'", workflow_module)
# If a module is already loaded, this call is pretty much a no-op. So it is okay to keep loading again.
importlib.import_module(workflow_module)
self.capability = Capability(
language=language, dependency_manager=dependency_manager, application_framework=application_framework
)
self.selected_workflow_cls = get_workflow(self.capability)
LOG.debug("Found workflow '%s' to support capabilities '%s'", self.selected_workflow_cls.NAME, self.capability)
class MyWorkflow(BaseWorkflow):
NAME = "MyWorkflow"
CAPABILITY = Capability(language=self.lang,
dependency_manager=self.lang_framework,
application_framework=self.app_framework)
def __init__(
self,
source_dir,
artifacts_dir,
scratch_dir,
manifest_path,
runtime=None,
optimizations=None,
options=None,
executable_search_paths=None,
mode=None,
):
super(MyWorkflow, self).__init__(
source_dir,
artifacts_dir,
scratch_dir,
manifest_path,
runtime=runtime,
optimizations=optimizations,
options=options,
executable_search_paths=executable_search_paths,
mode=mode,
)
class GoModulesWorkflow(BaseWorkflow):
NAME = "GoModulesBuilder"
CAPABILITY = Capability(language="go", dependency_manager="modules", application_framework=None)
def __init__(
self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtime=None, osutils=None, mode=None, **kwargs
):
super(GoModulesWorkflow, self).__init__(
source_dir, artifacts_dir, scratch_dir, manifest_path, runtime=runtime, **kwargs
)
if osutils is None:
osutils = OSUtils()
options = kwargs.get("options") or {}
handler = options.get("artifact_executable_name", None)
output_path = osutils.joinpath(artifacts_dir, handler)
builder = GoModulesBuilder(osutils, binaries=self.binaries, mode=mode)
self.actions = [GoModulesBuildAction(source_dir, output_path, builder)]
def get_validators(self):
return [GoRuntimeValidator(runtime=self.runtime)]
def test_must_add_item(self):
capability = Capability(language="a",
dependency_manager="b",
application_framework="c")
self.registry[capability] = self.workflow_data
self.assertEquals(self.workflow_data, self.registry[capability])
class JavaGradleWorkflow(BaseWorkflow):
"""
A Lambda builder workflow that knows how to build Java projects using Gradle.
"""
NAME = "JavaGradleWorkflow"
CAPABILITY = Capability(language="java",
dependency_manager="gradle",
application_framework=None)
INIT_FILE = "lambda-build-init.gradle"
def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path,
**kwargs):
super(JavaGradleWorkflow,
self).__init__(source_dir, artifacts_dir, scratch_dir,
manifest_path, **kwargs)
self.os_utils = OSUtils()
self.build_dir = None
subprocess_gradle = SubprocessGradle(
gradle_binary=self.binaries["gradle"], os_utils=self.os_utils)
self.actions = [
JavaGradleBuildAction(source_dir, manifest_path, subprocess_gradle,
scratch_dir, self.os_utils),
JavaGradleCopyArtifactsAction(source_dir, artifacts_dir,
self.build_output_dir,
self.os_utils),
]
def get_resolvers(self):
return [
GradleResolver(
executable_search_paths=self.executable_search_paths)
]
def get_validators(self):
return [GradleValidator(self.runtime, self.os_utils)]
@property
def build_output_dir(self):
if self.build_dir is None:
self.build_dir = os.path.join(self.scratch_dir,
self._compute_scratch_subdir())
return self.build_dir
def _compute_scratch_subdir(self):
"""
Compute where the init script will instruct Gradle to place the built artifacts for the lambda within
`scratch_dir`; i.e. the that it will set for 'project.buildDir`.
:return: The path of the buildDir used for building the lambda.
"""
sha1 = hashlib.sha1()
sha1.update(os.path.abspath(self.source_dir).encode("utf8"))
return sha1.hexdigest()
def test_fail_on_duplciate_entry(self):
capability = Capability(language="a",
dependency_manager="b",
application_framework="c")
self.registry[capability] = self.workflow_data
self.assertEquals(self.workflow_data, self.registry[capability])
with self.assertRaises(KeyError):
self.registry[capability] = "some other data"
def test_must_clear_entries(self):
capability = Capability(language="a",
dependency_manager="b",
application_framework="c")
self.registry[capability] = self.workflow_data
self.assertEquals(len(self.registry), 1)
self.registry.clear()
self.assertEquals(len(self.registry), 0)
def setUp(self):
self.mock_lock = Mock()
self.registry = Registry(write_lock=self.mock_lock)
self.capability = Capability(language="a",
dependency_manager="b",
application_framework="c")
self.workflow_data = "fake workflow"
# Always must call acquire() first before release()
self.expected_lock_call_order = [call.acquire(), call.release()]
class WriteHelloWorkflow(BaseWorkflow):
NAME = "WriteHelloWorkflow"
CAPABILITY = Capability(language="python",
dependency_manager="test",
application_framework="test")
def __init__(self, source_dir, artifacts_dir, *args, **kwargs):
super(WriteHelloWorkflow, self).__init__(source_dir, artifacts_dir,
*args, **kwargs)
self.actions = [WriteHelloAction(artifacts_dir)]
class NodejsNpmWorkflow(BaseWorkflow):
"""
A Lambda builder workflow that knows how to pack
NodeJS projects using NPM.
"""
NAME = "NodejsNpmBuilder"
CAPABILITY = Capability(language="nodejs",
dependency_manager="npm",
application_framework=None)
EXCLUDED_FILES = (".aws-sam")
def __init__(self,
source_dir,
artifacts_dir,
scratch_dir,
manifest_path,
runtime=None,
osutils=None,
**kwargs):
super(NodejsNpmWorkflow, self).__init__(source_dir,
artifacts_dir,
scratch_dir,
manifest_path,
runtime=runtime,
**kwargs)
if osutils is None:
osutils = OSUtils()
subprocess_npm = SubprocessNpm(osutils)
tar_dest_dir = osutils.joinpath(scratch_dir, 'unpacked')
tar_package_dir = osutils.joinpath(tar_dest_dir, 'package')
npm_pack = NodejsNpmPackAction(tar_dest_dir,
scratch_dir,
manifest_path,
osutils=osutils,
subprocess_npm=subprocess_npm)
npm_install = NodejsNpmInstallAction(artifacts_dir,
subprocess_npm=subprocess_npm)
self.actions = [
npm_pack,
CopySourceAction(tar_package_dir,
artifacts_dir,
excludes=self.EXCLUDED_FILES),
npm_install,
]
class GoDepWorkflow(BaseWorkflow):
"""
A Lambda builder workflow that knows how to build
Go projects using `dep`
"""
NAME = "GoDepBuilder"
CAPABILITY = Capability(language="go",
dependency_manager="dep",
application_framework=None)
EXCLUDED_FILES = ".aws-sam"
def __init__(self,
source_dir,
artifacts_dir,
scratch_dir,
manifest_path,
runtime=None,
osutils=None,
**kwargs):
super(GoDepWorkflow, self).__init__(source_dir,
artifacts_dir,
scratch_dir,
manifest_path,
runtime=runtime,
**kwargs)
options = kwargs["options"] if "options" in kwargs else {}
handler = options.get("artifact_executable_name", None)
if osutils is None:
osutils = OSUtils()
# project base name, where the Gopkg.toml and vendor dir are.
base_dir = osutils.abspath(osutils.dirname(manifest_path))
output_path = osutils.joinpath(osutils.abspath(artifacts_dir), handler)
subprocess_dep = SubprocessExec(osutils, "dep")
subprocess_go = SubprocessExec(osutils, "go")
self.actions = [
DepEnsureAction(base_dir, subprocess_dep),
GoBuildAction(base_dir,
osutils.abspath(source_dir),
output_path,
subprocess_go,
env=osutils.environ),
]
class DotnetCliPackageWorkflow(BaseWorkflow):
"""
A Lambda builder workflow that knows to build and package .NET Core Lambda functions
"""
NAME = "DotnetCliPackageBuilder"
CAPABILITY = Capability(language="dotnet",
dependency_manager="cli-package",
application_framework=None)
def __init__(self,
source_dir,
artifacts_dir,
scratch_dir,
manifest_path,
runtime=None,
mode=None,
**kwargs):
super(DotnetCliPackageWorkflow, self).__init__(source_dir,
artifacts_dir,
scratch_dir,
manifest_path,
runtime=runtime,
mode=mode,
**kwargs)
options = kwargs["options"] if "options" in kwargs else {}
subprocess_dotnetcli = SubprocessDotnetCLI(os_utils=OSUtils())
dotnetcli_install = GlobalToolInstallAction(
subprocess_dotnet=subprocess_dotnetcli)
dotnetcli_deployment = RunPackageAction(
source_dir,
subprocess_dotnet=subprocess_dotnetcli,
artifacts_dir=artifacts_dir,
options=options,
mode=mode)
self.actions = [
dotnetcli_install,
dotnetcli_deployment,
]
def get_resolvers(self):
return [
DotnetCliResolver(
executable_search_paths=self.executable_search_paths)
]
def test_must_load_all_default_workflows(self, get_workflow_mock, importlib_mock):
# instantiate
builder = LambdaBuilder(self.lang, self.lang_framework, self.app_framework)
self.assertEqual(builder.supported_workflows, [self.DEFAULT_WORKFLOW_MODULE])
# First check if the module was loaded
importlib_mock.import_module.assert_called_once_with(self.DEFAULT_WORKFLOW_MODULE)
# then check if we tried to get a workflow for given capability
get_workflow_mock.assert_called_with(
Capability(
language=self.lang, dependency_manager=self.lang_framework, application_framework=self.app_framework
)
)
class JavaMavenWorkflow(BaseWorkflow):
"""
A Lambda builder workflow that knows how to build Java projects using Maven.
"""
NAME = "JavaMavenWorkflow"
CAPABILITY = Capability(language="java", dependency_manager="maven", application_framework=None)
EXCLUDED_FILES = (".aws-sam", ".git")
def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, **kwargs):
super(JavaMavenWorkflow, self).__init__(source_dir, artifacts_dir, scratch_dir, manifest_path, **kwargs)
self.os_utils = OSUtils()
# Assuming root_dir is the same as source_dir for now
root_dir = source_dir
subprocess_maven = SubprocessMaven(
maven_binary=self.binaries["mvn"],
os_utils=self.os_utils,
)
copy_artifacts_action = JavaMavenCopyArtifactsAction(scratch_dir, artifacts_dir, self.os_utils)
if self.is_building_layer:
copy_artifacts_action = JavaMavenCopyLayerArtifactsAction(scratch_dir, artifacts_dir, self.os_utils)
self.actions = [
CopySourceAction(root_dir, scratch_dir, excludes=self.EXCLUDED_FILES),
JavaMavenBuildAction(scratch_dir, subprocess_maven),
JavaMavenCopyDependencyAction(scratch_dir, subprocess_maven),
copy_artifacts_action,
]
if self.dependencies_dir:
# clean up the dependencies first
self.actions.append(CleanUpAction(self.dependencies_dir))
if self.combine_dependencies:
self.actions.append(JavaCopyDependenciesAction(artifacts_dir, self.dependencies_dir, self.os_utils))
else:
self.actions.append(JavaMoveDependenciesAction(artifacts_dir, self.dependencies_dir, self.os_utils))
def get_resolvers(self):
return [MavenResolver(executable_search_paths=self.executable_search_paths)]
def get_validators(self):
return [MavenValidator(self.runtime, self.architecture, self.os_utils)]
class JavaMavenWorkflow(BaseWorkflow):
"""
A Lambda builder workflow that knows how to build Java projects using Maven.
"""
NAME = "JavaMavenWorkflow"
CAPABILITY = Capability(language="java",
dependency_manager="maven",
application_framework=None)
EXCLUDED_FILES = (".aws-sam", ".git")
def __init__(self,
source_dir,
artifacts_dir,
scratch_dir,
manifest_path,
**kwargs):
super(JavaMavenWorkflow, self).__init__(source_dir,
artifacts_dir,
scratch_dir,
manifest_path,
**kwargs)
self.os_utils = OSUtils()
# Assuming root_dir is the same as source_dir for now
root_dir = source_dir
subprocess_maven = SubprocessMaven(maven_binary=self.binaries['mvn'], os_utils=self.os_utils)
self.actions = [
CopySourceAction(root_dir, scratch_dir, excludes=self.EXCLUDED_FILES),
JavaMavenBuildAction(scratch_dir,
subprocess_maven),
JavaMavenCopyDependencyAction(scratch_dir,
subprocess_maven),
JavaMavenCopyArtifactsAction(scratch_dir,
artifacts_dir,
self.os_utils)
]
def get_resolvers(self):
return [MavenResolver(executable_search_paths=self.executable_search_paths)]
def get_validators(self):
return [MavenValidator(self.runtime, self.os_utils)]
class RubyBundlerWorkflow(BaseWorkflow):
"""
A Lambda builder workflow that knows how to build
Ruby projects using Bundler.
"""
NAME = "RubyBundlerBuilder"
CAPABILITY = Capability(language="ruby",
dependency_manager="bundler",
application_framework=None)
EXCLUDED_FILES = (".aws-sam", ".git")
def __init__(self,
source_dir,
artifacts_dir,
scratch_dir,
manifest_path,
runtime=None,
osutils=None,
**kwargs):
super(RubyBundlerWorkflow, self).__init__(source_dir,
artifacts_dir,
scratch_dir,
manifest_path,
runtime=runtime,
**kwargs)
if osutils is None:
osutils = OSUtils()
subprocess_bundler = SubprocessBundler(osutils)
bundle_install = RubyBundlerInstallAction(
artifacts_dir, subprocess_bundler=subprocess_bundler)
bundle_deployment = RubyBundlerVendorAction(
artifacts_dir, subprocess_bundler=subprocess_bundler)
self.actions = [
CopySourceAction(source_dir,
artifacts_dir,
excludes=self.EXCLUDED_FILES),
bundle_install,
bundle_deployment,
]
class RubyBundlerWorkflow(BaseWorkflow):
"""
A Lambda builder workflow that knows how to build
Ruby projects using Bundler.
"""
NAME = "RubyBundlerBuilder"
CAPABILITY = Capability(language="ruby",
dependency_manager="bundler",
application_framework=None)
EXCLUDED_FILES = (".aws-sam", ".git")
def __init__(self,
source_dir,
artifacts_dir,
scratch_dir,
manifest_path,
runtime=None,
osutils=None,
**kwargs):
super(RubyBundlerWorkflow, self).__init__(source_dir,
artifacts_dir,
scratch_dir,
manifest_path,
runtime=runtime,
**kwargs)
if osutils is None:
osutils = OSUtils()
self.actions = [
CopySourceAction(source_dir,
artifacts_dir,
excludes=self.EXCLUDED_FILES)
]
if self.download_dependencies:
# installed the dependencies into artifact folder
subprocess_bundler = SubprocessBundler(osutils)
bundle_install = RubyBundlerInstallAction(
artifacts_dir, subprocess_bundler=subprocess_bundler)
bundle_deployment = RubyBundlerVendorAction(
artifacts_dir, subprocess_bundler=subprocess_bundler)
self.actions.append(bundle_install)
self.actions.append(bundle_deployment)
# if dependencies folder exists, copy dependencies into dependencies into dependencies folder
if self.dependencies_dir:
# clean up the dependencies first
self.actions.append(CleanUpAction(self.dependencies_dir))
self.actions.append(
CopyDependenciesAction(source_dir, artifacts_dir,
self.dependencies_dir))
else:
# if dependencies folder exists and not download dependencies, simply copy the dependencies from the
# dependencies folder to artifact folder
if self.dependencies_dir:
self.actions.append(
CopySourceAction(self.dependencies_dir, artifacts_dir))
else:
LOG.info(
"download_dependencies is False and dependencies_dir is None. Copying the source files into the "
"artifacts directory. ")
class PythonPipWorkflow(BaseWorkflow):
NAME = "PythonPipBuilder"
CAPABILITY = Capability(language="python",
dependency_manager="pip",
application_framework=None)
# Common source files to exclude from build artifacts output
# Trimmed version of https://github.com/github/gitignore/blob/master/Python.gitignore
EXCLUDED_FILES = (
".aws-sam",
".chalice",
".git",
".gitignore",
# Compiled files
"*.pyc",
"__pycache__",
"*.so",
# Distribution / packaging
".Python",
"*.egg-info",
"*.egg",
# Installer logs
"pip-log.txt",
"pip-delete-this-directory.txt",
# Unit test / coverage reports
"htmlcov",
".tox",
".nox",
".coverage",
".cache",
".pytest_cache",
# pyenv
".python-version",
# mypy, Pyre
".mypy_cache",
".dmypy.json",
".pyre",
# environments
".env",
".venv",
"venv",
"venv.bak",
"env.bak",
"ENV",
"env",
# Editors
# TODO: Move the commonly ignored files to base class
".vscode",
".idea",
)
def __init__(self,
source_dir,
artifacts_dir,
scratch_dir,
manifest_path,
runtime=None,
osutils=None,
**kwargs):
super(PythonPipWorkflow, self).__init__(source_dir,
artifacts_dir,
scratch_dir,
manifest_path,
runtime=runtime,
**kwargs)
if osutils is None:
osutils = OSUtils()
if not self.download_dependencies and not self.dependencies_dir:
LOG.info(
"download_dependencies is False and dependencies_dir is None. Copying the source files into the "
"artifacts directory. ")
self.actions = []
if not osutils.file_exists(manifest_path):
LOG.warning(
"requirements.txt file not found. Continuing the build without dependencies."
)
self.actions.append(
CopySourceAction(source_dir,
artifacts_dir,
excludes=self.EXCLUDED_FILES))
return
# If a requirements.txt exists, run pip builder before copy action.
if self.download_dependencies:
if self.dependencies_dir:
# clean up the dependencies folder before installing
self.actions.append(CleanUpAction(self.dependencies_dir))
self.actions.append(
PythonPipBuildAction(
artifacts_dir,
scratch_dir,
manifest_path,
runtime,
self.dependencies_dir,
binaries=self.binaries,
architecture=self.architecture,
))
# if dependencies folder is provided, copy dependencies from dependencies folder to build folder
# if combine_dependencies is false, will not copy the dependencies from dependencies folder to artifact
# folder
if self.dependencies_dir and self.combine_dependencies:
# when copying downloaded dependencies back to artifacts folder, don't exclude anything
self.actions.append(
CopySourceAction(self.dependencies_dir, artifacts_dir))
self.actions.append(
CopySourceAction(source_dir,
artifacts_dir,
excludes=self.EXCLUDED_FILES))
def get_validators(self):
return [
PythonRuntimeValidator(runtime=self.runtime,
architecture=self.architecture)
]
class TestRegisteringWorkflows(TestCase):
CAPABILITY1 = Capability(language="test",
dependency_manager="testframework",
application_framework="appframework")
CAPABILITY2 = Capability(language="test2",
dependency_manager="testframework2",
application_framework="appframework2")
def tearDown(self):
DEFAULT_REGISTRY.clear()
def test_must_register_one_workflow(self):
# Just loading the classes will register them to default registry
class TestWorkflow(BaseWorkflow):
NAME = "TestWorkflow"
CAPABILITY = self.CAPABILITY1
result_cls = get_workflow(self.CAPABILITY1)
self.assertEquals(len(DEFAULT_REGISTRY), 1)
self.assertEquals(result_cls, TestWorkflow)
def test_must_register_two_workflows(self):
class TestWorkflow1(BaseWorkflow):
NAME = "TestWorkflow"
CAPABILITY = self.CAPABILITY1
class TestWorkflow2(BaseWorkflow):
NAME = "TestWorkflow2"
CAPABILITY = self.CAPABILITY2
self.assertEquals(len(DEFAULT_REGISTRY), 2)
self.assertEquals(get_workflow(self.CAPABILITY1), TestWorkflow1)
self.assertEquals(get_workflow(self.CAPABILITY2), TestWorkflow2)
def test_must_fail_if_name_not_present(self):
with self.assertRaises(ValueError) as ctx:
class TestWorkflow1(BaseWorkflow):
CAPABILITY = self.CAPABILITY1
self.assertEquals(len(DEFAULT_REGISTRY), 0)
self.assertEquals(str(ctx.exception),
"Workflow must provide a valid name")
def test_must_fail_if_capabilities_not_present(self):
with self.assertRaises(ValueError) as ctx:
class TestWorkflow1(BaseWorkflow):
NAME = "somename"
self.assertEquals(len(DEFAULT_REGISTRY), 0)
self.assertEquals(
str(ctx.exception),
"Workflow 'somename' must register valid capabilities")
def test_must_fail_if_capabilities_is_wrong_type(self):
with self.assertRaises(ValueError) as ctx:
class TestWorkflow1(BaseWorkflow):
NAME = "somename"
CAPABILITY = "wrong data type"
self.assertEquals(len(DEFAULT_REGISTRY), 0)
self.assertEquals(
str(ctx.exception),
"Workflow 'somename' must register valid capabilities")
class CustomMakeWorkflow(BaseWorkflow):
"""
A Lambda builder workflow for provided runtimes based on make.
"""
NAME = "CustomMakeBuilder"
CAPABILITY = Capability(language="provided",
dependency_manager=None,
application_framework=None)
EXCLUDED_FILES = (".aws-sam", ".git")
def __init__(self,
source_dir,
artifacts_dir,
scratch_dir,
manifest_path,
runtime=None,
osutils=None,
**kwargs):
super(CustomMakeWorkflow, self).__init__(source_dir,
artifacts_dir,
scratch_dir,
manifest_path,
runtime=runtime,
**kwargs)
self.os_utils = OSUtils()
# Find the logical id of the function to be built.
options = kwargs.get("options") or {}
build_logical_id = options.get("build_logical_id", None)
if not build_logical_id:
raise WorkflowFailedError(
workflow_name=self.NAME,
action_name=None,
reason="Build target {} is not found!".format(
build_logical_id),
)
subprocess_make = SubProcessMake(
make_exe=self.binaries["make"].binary_path, osutils=self.os_utils)
make_action = CustomMakeAction(
artifacts_dir,
scratch_dir,
manifest_path,
osutils=self.os_utils,
subprocess_make=subprocess_make,
build_logical_id=build_logical_id,
)
self.actions = [
CopySourceAction(source_dir,
scratch_dir,
excludes=self.EXCLUDED_FILES), make_action
]
def get_resolvers(self):
return [
PathResolver(runtime="provided",
binary="make",
executable_search_paths=self.executable_search_paths)
]
class PythonPipWorkflow(BaseWorkflow):
NAME = "PythonPipBuilder"
CAPABILITY = Capability(language="python",
dependency_manager="pip",
application_framework=None)
# Common source files to exclude from build artifacts output
# Trimmed version of https://github.com/github/gitignore/blob/master/Python.gitignore
EXCLUDED_FILES = (
".aws-sam",
".chalice",
".git",
# Compiled files
"*.pyc",
"__pycache__",
"*.so",
# Distribution / packaging
".Python",
"*.egg-info",
"*.egg",
# Installer logs
"pip-log.txt",
"pip-delete-this-directory.txt",
# Unit test / coverage reports
"htmlcov",
".tox",
".nox",
".coverage",
".cache",
".pytest_cache",
# pyenv
".python-version",
# mypy, Pyre
".mypy_cache",
".dmypy.json",
".pyre",
# environments
".env",
".venv",
"venv",
"venv.bak",
"env.bak",
"ENV",
# Editors
# TODO: Move the commonly ignored files to base class
".vscode",
".idea")
def __init__(self,
source_dir,
artifacts_dir,
scratch_dir,
manifest_path,
runtime=None,
**kwargs):
super(PythonPipWorkflow, self).__init__(source_dir,
artifacts_dir,
scratch_dir,
manifest_path,
runtime=runtime,
**kwargs)
self.actions = [
PythonPipBuildAction(artifacts_dir,
scratch_dir,
manifest_path,
runtime,
binaries=self.binaries),
CopySourceAction(source_dir,
artifacts_dir,
excludes=self.EXCLUDED_FILES),
]
def get_validators(self):
return [PythonRuntimeValidator(runtime=self.runtime)]
class NodejsNpmWorkflow(BaseWorkflow):
"""
A Lambda builder workflow that knows how to pack
NodeJS projects using NPM.
"""
NAME = "NodejsNpmBuilder"
CAPABILITY = Capability(language="nodejs",
dependency_manager="npm",
application_framework=None)
EXCLUDED_FILES = (".aws-sam", ".git")
CONFIG_PROPERTY = "aws_sam"
def __init__(self,
source_dir,
artifacts_dir,
scratch_dir,
manifest_path,
runtime=None,
osutils=None,
**kwargs):
super(NodejsNpmWorkflow, self).__init__(source_dir,
artifacts_dir,
scratch_dir,
manifest_path,
runtime=runtime,
**kwargs)
if osutils is None:
osutils = OSUtils()
if not osutils.file_exists(manifest_path):
LOG.warning(
"package.json file not found. Continuing the build without dependencies."
)
self.actions = [
CopySourceAction(source_dir,
artifacts_dir,
excludes=self.EXCLUDED_FILES)
]
return
subprocess_npm = SubprocessNpm(osutils)
self.actions = self.actions_without_bundler(source_dir, artifacts_dir,
scratch_dir, manifest_path,
osutils, subprocess_npm)
def actions_without_bundler(self, source_dir, artifacts_dir, scratch_dir,
manifest_path, osutils, subprocess_npm):
"""
Generate a list of Nodejs build actions without a bundler
:type source_dir: str
:param source_dir: an existing (readable) directory containing source files
:type artifacts_dir: str
:param artifacts_dir: an existing (writable) directory where to store the output.
:type scratch_dir: str
:param scratch_dir: an existing (writable) directory for temporary files
:type manifest_path: str
:param manifest_path: path to package.json of an NPM project with the source to pack
:type osutils: aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils
:param osutils: An instance of OS Utilities for file manipulation
:type subprocess_npm: aws_lambda_builders.workflows.nodejs_npm.npm.SubprocessNpm
:param subprocess_npm: An instance of the NPM process wrapper
:rtype: list
:return: List of build actions to execute
"""
tar_dest_dir = osutils.joinpath(scratch_dir, "unpacked")
tar_package_dir = osutils.joinpath(tar_dest_dir, "package")
npm_pack = NodejsNpmPackAction(tar_dest_dir,
scratch_dir,
manifest_path,
osutils=osutils,
subprocess_npm=subprocess_npm)
npm_copy_npmrc_and_lockfile = NodejsNpmrcAndLockfileCopyAction(
tar_package_dir, source_dir, osutils=osutils)
actions = [
npm_pack,
npm_copy_npmrc_and_lockfile,
CopySourceAction(tar_package_dir,
artifacts_dir,
excludes=self.EXCLUDED_FILES),
]
if self.download_dependencies:
# installed the dependencies into artifact folder
install_action = NodejsNpmWorkflow.get_install_action(
source_dir, artifacts_dir, subprocess_npm, osutils,
self.options)
actions.append(install_action)
# if dependencies folder exists, copy or move dependencies from artifact folder to dependencies folder
# depends on the combine_dependencies flag
if self.dependencies_dir:
# clean up the dependencies folder first
actions.append(CleanUpAction(self.dependencies_dir))
# if combine_dependencies is set, we should keep dependencies and source code in the artifact folder
# while copying the dependencies. Otherwise we should separate the dependencies and source code
if self.combine_dependencies:
actions.append(
CopyDependenciesAction(source_dir, artifacts_dir,
self.dependencies_dir))
else:
actions.append(
MoveDependenciesAction(source_dir, artifacts_dir,
self.dependencies_dir))
else:
# if dependencies folder exists and not download dependencies, simply copy the dependencies from the
# dependencies folder to artifact folder
if self.dependencies_dir and self.combine_dependencies:
actions.append(
CopySourceAction(self.dependencies_dir, artifacts_dir))
else:
LOG.info(
"download_dependencies is False and dependencies_dir is None. Copying the source files into the "
"artifacts directory. ")
actions.append(NodejsNpmrcCleanUpAction(artifacts_dir,
osutils=osutils))
actions.append(
NodejsNpmLockFileCleanUpAction(artifacts_dir, osutils=osutils))
if self.dependencies_dir:
actions.append(
NodejsNpmLockFileCleanUpAction(self.dependencies_dir,
osutils=osutils))
return actions
def get_resolvers(self):
"""
specialized path resolver that just returns the list of executable for the runtime on the path.
"""
return [PathResolver(runtime=self.runtime, binary="npm")]
@staticmethod
def get_install_action(source_dir,
artifacts_dir,
subprocess_npm,
osutils,
build_options,
is_production=True):
"""
Get the install action used to install dependencies at artifacts_dir
:type source_dir: str
:param source_dir: an existing (readable) directory containing source files
:type artifacts_dir: str
:param artifacts_dir: Dependencies will be installed in this directory.
:type osutils: aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils
:param osutils: An instance of OS Utilities for file manipulation
:type subprocess_npm: aws_lambda_builders.workflows.nodejs_npm.npm.SubprocessNpm
:param subprocess_npm: An instance of the NPM process wrapper
:type build_options: Dict
:param build_options: Object containing build options configurations
:type is_production: bool
:param is_production: NPM installation mode is production (eg --production=false to force dev dependencies)
:rtype: BaseAction
:return: Install action to use
"""
lockfile_path = osutils.joinpath(source_dir, "package-lock.json")
shrinkwrap_path = osutils.joinpath(source_dir, "npm-shrinkwrap.json")
npm_ci_option = False
if build_options and isinstance(build_options, dict):
npm_ci_option = build_options.get("use_npm_ci", False)
if (osutils.file_exists(lockfile_path)
or osutils.file_exists(shrinkwrap_path)) and npm_ci_option:
return NodejsNpmCIAction(artifacts_dir,
subprocess_npm=subprocess_npm)
return NodejsNpmInstallAction(artifacts_dir,
subprocess_npm=subprocess_npm,
is_production=is_production)
class TestRegistryEndToEnd(TestCase):
def setUp(self):
self.registry = Registry()
# Since the registry does not validate whether we register a valid workflow object, we can register any fake
# data
self.workflow_data = "fake workflow"
def test_must_add_item(self):
capability = Capability(language="a",
dependency_manager="b",
application_framework="c")
self.registry[capability] = self.workflow_data
self.assertEquals(self.workflow_data, self.registry[capability])
@parameterized.expand([
(Capability(language=None,
dependency_manager="b",
application_framework="c"), ),
(Capability(language="a",
dependency_manager=None,
application_framework="c"), ),
(Capability(language="a",
dependency_manager=None,
application_framework=None), ),
])
def test_must_add_item_with_optional_capabilities(self, capability):
self.registry[capability] = self.workflow_data
self.assertEquals(self.workflow_data, self.registry[capability])
def test_must_add_multiple_items(self):
capability1 = Capability(language="a",
dependency_manager="b",
application_framework="c")
capability2 = Capability(language="d",
dependency_manager="e",
application_framework="f")
self.registry[capability1] = "some data"
self.registry[capability2] = "some other data"
self.assertEquals(len(self.registry), 2)
self.assertTrue(capability1 in self.registry)
self.assertTrue(capability2 in self.registry)
def test_fail_on_duplciate_entry(self):
capability = Capability(language="a",
dependency_manager="b",
application_framework="c")
self.registry[capability] = self.workflow_data
self.assertEquals(self.workflow_data, self.registry[capability])
with self.assertRaises(KeyError):
self.registry[capability] = "some other data"
def test_must_clear_entries(self):
capability = Capability(language="a",
dependency_manager="b",
application_framework="c")
self.registry[capability] = self.workflow_data
self.assertEquals(len(self.registry), 1)
self.registry.clear()
self.assertEquals(len(self.registry), 0)
class MyWorkflow(BaseWorkflow):
__TESTING__ = True
NAME = "MyWorkflow"
CAPABILITY = Capability(language="test",
dependency_manager="testframework",
application_framework="appframework")
class NodejsNpmWorkflow(BaseWorkflow):
"""
A Lambda builder workflow that knows how to pack
NodeJS projects using NPM.
"""
NAME = "NodejsNpmBuilder"
CAPABILITY = Capability(language="nodejs",
dependency_manager="npm",
application_framework=None)
EXCLUDED_FILES = (".aws-sam")
def __init__(self,
source_dir,
artifacts_dir,
scratch_dir,
manifest_path,
runtime=None,
osutils=None,
**kwargs):
super(NodejsNpmWorkflow, self).__init__(source_dir,
artifacts_dir,
scratch_dir,
manifest_path,
runtime=runtime,
**kwargs)
if osutils is None:
osutils = OSUtils()
subprocess_npm = SubprocessNpm(osutils)
tar_dest_dir = osutils.joinpath(scratch_dir, 'unpacked')
tar_package_dir = osutils.joinpath(tar_dest_dir, 'package')
npm_pack = NodejsNpmPackAction(tar_dest_dir,
scratch_dir,
manifest_path,
osutils=osutils,
subprocess_npm=subprocess_npm)
npm_install = NodejsNpmInstallAction(artifacts_dir,
subprocess_npm=subprocess_npm)
npm_copy_npmrc = NodejsNpmrcCopyAction(tar_package_dir,
source_dir,
osutils=osutils)
self.actions = [
npm_pack, npm_copy_npmrc,
CopySourceAction(tar_package_dir,
artifacts_dir,
excludes=self.EXCLUDED_FILES), npm_install,
NodejsNpmrcCleanUpAction(artifacts_dir, osutils=osutils)
]
def get_resolvers(self):
"""
specialized path resolver that just returns the list of executable for the runtime on the path.
"""
return [PathResolver(runtime=self.runtime, binary="npm")]
def setUp(self):
self.registry = Registry()
self.capability = Capability(language="a",
dependency_manager="b",
application_framework="c")
self.workflow_data = "some workflow data"
class NodejsNpmEsbuildWorkflow(BaseWorkflow):
"""
A Lambda builder workflow that uses esbuild to bundle Node.js and transpile TS
NodeJS projects using NPM with esbuild.
"""
NAME = "NodejsNpmEsbuildBuilder"
CAPABILITY = Capability(language="nodejs",
dependency_manager="npm-esbuild",
application_framework=None)
EXCLUDED_FILES = (".aws-sam", ".git")
CONFIG_PROPERTY = "aws_sam"
def __init__(self,
source_dir,
artifacts_dir,
scratch_dir,
manifest_path,
runtime=None,
osutils=None,
**kwargs):
super(NodejsNpmEsbuildWorkflow, self).__init__(source_dir,
artifacts_dir,
scratch_dir,
manifest_path,
runtime=runtime,
**kwargs)
if osutils is None:
osutils = OSUtils()
subprocess_npm = SubprocessNpm(osutils)
subprocess_esbuild = self._get_esbuild_subprocess(
subprocess_npm, scratch_dir, osutils)
bundler_config = self.get_build_properties()
if not osutils.file_exists(manifest_path):
LOG.warning(
"package.json file not found. Bundling source without dependencies."
)
self.actions = [
EsbuildBundleAction(source_dir, artifacts_dir, bundler_config,
osutils, subprocess_esbuild)
]
return
if not is_experimental_esbuild_scope(self.experimental_flags):
raise EsbuildExecutionError(
message="Feature flag must be enabled to use this workflow")
self.actions = self.actions_with_bundler(source_dir, scratch_dir,
artifacts_dir, bundler_config,
osutils, subprocess_npm,
subprocess_esbuild)
def actions_with_bundler(self, source_dir, scratch_dir, artifacts_dir,
bundler_config, osutils, subprocess_npm,
subprocess_esbuild) -> List[BaseAction]:
"""
Generate a list of Nodejs build actions with a bundler
:type source_dir: str
:param source_dir: an existing (readable) directory containing source files
:type scratch_dir: str
:param scratch_dir: an existing (writable) directory for temporary files
:type artifacts_dir: str
:param artifacts_dir: an existing (writable) directory where to store the output.
:type bundler_config: dict
:param bundler_config: configurations for the bundler action
:type osutils: aws_lambda_builders.workflows.nodejs_npm.utils.OSUtils
:param osutils: An instance of OS Utilities for file manipulation
:type subprocess_npm: aws_lambda_builders.workflows.nodejs_npm.npm.SubprocessNpm
:param subprocess_npm: An instance of the NPM process wrapper
:type subprocess_esbuild: aws_lambda_builders.workflows.nodejs_npm_esbuild.esbuild.SubprocessEsbuild
:param subprocess_esbuild: An instance of the esbuild process wrapper
:rtype: list
:return: List of build actions to execute
"""
actions: List[BaseAction] = [
CopySourceAction(source_dir,
scratch_dir,
excludes=self.EXCLUDED_FILES +
tuple(["node_modules"]))
]
subprocess_node = SubprocessNodejs(osutils,
self.executable_search_paths,
which=which)
# Bundle dependencies separately in a dependency layer. We need to check the esbuild
# version here to ensure that it supports skipping dependency bundling
esbuild_no_deps = [
EsbuildCheckVersionAction(scratch_dir, subprocess_esbuild),
EsbuildBundleAction(
scratch_dir,
artifacts_dir,
bundler_config,
osutils,
subprocess_esbuild,
subprocess_node,
skip_deps=True,
),
]
esbuild_with_deps = EsbuildBundleAction(scratch_dir, artifacts_dir,
bundler_config, osutils,
subprocess_esbuild)
install_action = NodejsNpmWorkflow.get_install_action(
source_dir,
scratch_dir,
subprocess_npm,
osutils,
self.options,
is_production=False)
if self.download_dependencies and not self.dependencies_dir:
return actions + [install_action, esbuild_with_deps]
return self._accelerate_workflow_actions(source_dir, scratch_dir,
actions, install_action,
esbuild_with_deps,
esbuild_no_deps)
def _accelerate_workflow_actions(self, source_dir, scratch_dir, actions,
install_action, esbuild_with_deps,
esbuild_no_deps):
"""
Generate a list of Nodejs build actions for incremental build and auto dependency layer
:type source_dir: str
:param source_dir: an existing (readable) directory containing source files
:type scratch_dir: str
:param scratch_dir: an existing (writable) directory for temporary files
:type actions: List[BaseAction]
:param actions: List of existing actions
:type install_action: BaseAction
:param install_action: Installation action for npm
:type esbuild_with_deps: BaseAction
:param esbuild_with_deps: Standard esbuild action bundling source with deps
:type esbuild_no_deps: List[BaseAction]
:param esbuild_no_deps: esbuild action not including dependencies in the bundled artifacts
:rtype: list
:return: List of build actions to execute
"""
if self.download_dependencies:
actions += [install_action, CleanUpAction(self.dependencies_dir)]
if self.combine_dependencies:
# Auto dependency layer disabled, first build
actions += [
esbuild_with_deps,
CopyDependenciesAction(source_dir, scratch_dir,
self.dependencies_dir)
]
else:
# Auto dependency layer enabled, first build
actions += esbuild_no_deps + [
MoveDependenciesAction(source_dir, scratch_dir,
self.dependencies_dir)
]
else:
if self.dependencies_dir:
actions.append(
CopySourceAction(self.dependencies_dir, scratch_dir))
if self.combine_dependencies:
# Auto dependency layer disabled, subsequent builds
actions += [esbuild_with_deps]
else:
# Auto dependency layer enabled, subsequent builds
actions += esbuild_no_deps
else:
# Invalid workflow, can't have no dependency dir and no installation
raise EsbuildExecutionError(
message="Lambda Builders encountered and invalid workflow")
return actions
def get_build_properties(self):
"""
Get the aws_sam specific properties from the manifest, if they exist.
:rtype: dict
:return: Dict with aws_sam specific bundler configs
"""
if self.options and isinstance(self.options, dict):
LOG.debug(
"Lambda Builders found the following esbuild properties:\n%s",
json.dumps(self.options))
return self.options
return {}
def get_resolvers(self):
"""
specialized path resolver that just returns the list of executable for the runtime on the path.
"""
return [PathResolver(runtime=self.runtime, binary="npm")]
def _get_esbuild_subprocess(self, subprocess_npm, scratch_dir,
osutils) -> SubprocessEsbuild:
npm_bin_path = subprocess_npm.run(["bin"], cwd=scratch_dir)
executable_search_paths = [npm_bin_path]
if self.executable_search_paths is not None:
executable_search_paths = executable_search_paths + self.executable_search_paths
return SubprocessEsbuild(osutils, executable_search_paths, which=which)
