def _build_function_in_process(self, config, source_dir, artifacts_dir,
scratch_dir, manifest_path, runtime,
options):
builder = LambdaBuilder(
language=config.language,
dependency_manager=config.dependency_manager,
application_framework=config.application_framework,
)
runtime = runtime.replace(".al2", "")
try:
builder.build(
source_dir,
artifacts_dir,
scratch_dir,
manifest_path,
runtime=runtime,
executable_search_paths=config.executable_search_paths,
mode=self._mode,
options=options,
)
except LambdaBuilderError as ex:
raise BuildError(wrapped_from=ex.__class__.__name__,
msg=str(ex)) from ex
return artifacts_dir
def setUp(self):
self.source_dir = self.TEST_DATA_FOLDER
self.artifacts_dir = tempfile.mkdtemp()
self.scratch_dir = tempfile.mkdtemp()
self.manifest_path_valid = os.path.join(self.TEST_DATA_FOLDER,
"requirements-numpy.txt")
self.manifest_path_invalid = os.path.join(self.TEST_DATA_FOLDER,
"requirements-invalid.txt")
self.test_data_files = {
"__init__.py", "main.py", "requirements-invalid.txt",
"requirements-numpy.txt"
}
self.builder = LambdaBuilder(language="python",
dependency_manager="pip",
application_framework=None)
self.runtime = "{language}{major}.{minor}".format(
language=self.builder.capability.language,
major=sys.version_info.major,
minor=sys.version_info.minor)
self.runtime_mismatch = {
"python3.6": "python2.7",
"python3.7": "python2.7",
"python2.7": "python3.8",
"python3.8": "python2.7",
}
class TestDotnetDep(TestCase):
TEST_DATA_FOLDER = os.path.join(os.path.dirname(__file__), "testdata")
def setUp(self):
self.artifacts_dir = tempfile.mkdtemp()
self.scratch_dir = tempfile.mkdtemp()
self.builder = LambdaBuilder(language="dotnet",
dependency_manager="cli-package",
application_framework=None)
self.runtime = "dotnetcore2.1"
def tearDown(self):
shutil.rmtree(self.artifacts_dir)
shutil.rmtree(self.scratch_dir)
def test_with_defaults_file(self):
source_dir = os.path.join(self.TEST_DATA_FOLDER, "WithDefaultsFile")
self.builder.build(source_dir,
self.artifacts_dir,
self.scratch_dir,
source_dir,
runtime=self.runtime)
expected_files = {
"Amazon.Lambda.Core.dll", "Amazon.Lambda.Serialization.Json.dll",
"Newtonsoft.Json.dll", "WithDefaultsFile.deps.json",
"WithDefaultsFile.dll", "WithDefaultsFile.pdb",
"WithDefaultsFile.runtimeconfig.json"
}
output_files = set(os.listdir(self.artifacts_dir))
self.assertEquals(expected_files, output_files)
def test_require_parameters(self):
source_dir = os.path.join(self.TEST_DATA_FOLDER, "RequireParameters")
self.builder.build(source_dir,
self.artifacts_dir,
self.scratch_dir,
source_dir,
runtime=self.runtime,
options={
"--framework": "netcoreapp2.1",
"--configuration": "Debug"
})
expected_files = {
"Amazon.Lambda.Core.dll", "Amazon.Lambda.Serialization.Json.dll",
"Newtonsoft.Json.dll", "RequireParameters.deps.json",
"RequireParameters.dll", "RequireParameters.pdb",
"RequireParameters.runtimeconfig.json"
}
output_files = set(os.listdir(self.artifacts_dir))
self.assertEquals(expected_files, output_files)
def test_with_mocks(self, get_workflow_mock, importlib_mock):
workflow_cls = Mock()
workflow_instance = workflow_cls.return_value = Mock()
get_workflow_mock.return_value = workflow_cls
with patch.object(LambdaBuilder, "_validate_runtime"):
builder = LambdaBuilder(self.lang,
self.lang_framework,
self.app_framework,
supported_workflows=[])
builder.build("source_dir",
"artifacts_dir",
"scratch_dir",
"manifest_path",
runtime="runtime",
optimizations="optimizations",
options="options")
workflow_cls.assert_called_with("source_dir",
"artifacts_dir",
"scratch_dir",
"manifest_path",
runtime="runtime",
optimizations="optimizations",
options="options")
workflow_instance.run.assert_called_once()
def setUp(self):
self.artifacts_dir = tempfile.mkdtemp()
self.scratch_dir = tempfile.mkdtemp()
self.builder = LambdaBuilder(language='java',
dependency_manager='gradle',
application_framework=None)
self.runtime = 'java11'
def build_lambda(source_dir, output_dir, runtime, archive_name):
"""
Build a Lambda Function zip using a builder from aws-lambda-builders
:param source_dir: Source Directory
:param output_dir: Output Directory
:param runtime: Lambda Runtime
:param archive_name: Archive name (optional)
:return: Path to generated zip file
"""
config = get_config(runtime, source_dir)
builder = LambdaBuilder(config.language, config.dependency_manager, None)
manifest_path = os.path.join(source_dir, config.manifest_name)
archive_name = archive_name if archive_name else os.path.basename(os.path.normpath(source_dir))
info(f'\nBuilding {runtime} Lambda function from {source_dir}\n')
with tempfile.TemporaryDirectory() as artifacts_dir:
with tempfile.TemporaryDirectory() as scratch_dir:
try:
builder.build(source_dir, artifacts_dir, scratch_dir, manifest_path, runtime)
zip_file = make_archive(os.path.join(output_dir, archive_name), 'zip', artifacts_dir)
info(f'\nBuilt Lambda Archive {zip_file}')
return zip_file
except LambdaBuilderError as e:
raise PackagingError(e)
def setUp(self):
self.artifacts_dir = tempfile.mkdtemp()
self.scratch_dir = tempfile.mkdtemp()
self.builder = LambdaBuilder(language="go",
dependency_manager="modules",
application_framework=None)
self.runtime = "go1.x"
def setUp(self):
self.artifacts_dir = tempfile.mkdtemp()
self.scratch_dir = tempfile.mkdtemp()
self.no_deps = os.path.join(self.TEST_DATA_FOLDER, "no-deps")
self.builder = LambdaBuilder(language="ruby",
dependency_manager="bundler",
application_framework=None)
self.runtime = "ruby2.5"
def setUp(self):
self.artifacts_dir = tempfile.mkdtemp()
self.scratch_dir = tempfile.mkdtemp()
self.dependencies_dir = tempfile.mkdtemp()
self.no_deps = os.path.join(self.TEST_DATA_FOLDER, "no-deps")
self.builder = LambdaBuilder(language="nodejs", dependency_manager="npm", application_framework=None)
def setUp(self):
self.artifacts_dir = tempfile.mkdtemp()
self.scratch_dir = tempfile.mkdtemp()
self.builder = LambdaBuilder(language="dotnet",
dependency_manager="cli-package",
application_framework=None)
self.runtime = "dotnetcore2.1"
def setUp(self):
self.artifacts_dir = tempfile.mkdtemp()
self.scratch_dir = tempfile.mkdtemp()
os.environ["GOPATH"] = self.TEST_DATA_FOLDER
self.no_deps = os.path.join(self.TEST_DATA_FOLDER, "src", "nodeps")
self.builder = LambdaBuilder(language="go", dependency_manager="dep", application_framework=None)
self.runtime = "go1.x"
class TestBuilderWithHelloWorkflow(TestCase):
HELLO_WORKFLOW_MODULE = "hello_workflow.write_hello"
TEST_WORKFLOWS_FOLDER = os.path.join(os.path.dirname(__file__), "testdata",
"workflows")
def setUp(self):
# Temporarily add the testdata folder to PYTHOHNPATH
sys.path.append(self.TEST_WORKFLOWS_FOLDER)
self.source_dir = tempfile.mkdtemp()
self.artifacts_dir = tempfile.mkdtemp()
self.scratch_dir = os.path.join(tempfile.mkdtemp(), "scratch")
self.hello_builder = LambdaBuilder(
language="python",
dependency_manager="test",
application_framework="test",
supported_workflows=[self.HELLO_WORKFLOW_MODULE],
)
# The builder should write a file called hello.txt with contents "Hello World"
self.expected_filename = os.path.join(self.artifacts_dir, "hello.txt")
self.expected_contents = "Hello World"
def tearDown(self):
self.hello_builder._clear_workflows()
shutil.rmtree(self.source_dir)
shutil.rmtree(self.artifacts_dir)
shutil.rmtree(self.scratch_dir)
# Remove the workflows folder from PYTHONPATH
sys.path.remove(self.TEST_WORKFLOWS_FOLDER)
def test_run_hello_workflow_with_exec_paths(self):
self.hello_builder.build(
self.source_dir,
self.artifacts_dir,
self.scratch_dir,
"/ignored",
"python3.8",
executable_search_paths=[str(pathlib.Path(sys.executable).parent)],
)
self.assertTrue(os.path.exists(self.expected_filename))
contents = ""
with open(self.expected_filename, "r") as fp:
contents = fp.read()
self.assertEqual(contents, self.expected_contents)
def setUp(self):
# Temporarily add the testdata folder to PYTHOHNPATH
sys.path.append(self.TEST_WORKFLOWS_FOLDER)
self.source_dir = tempfile.mkdtemp()
self.artifacts_dir = tempfile.mkdtemp()
self.hello_builder = LambdaBuilder(
language="test",
dependency_manager="test",
application_framework="test",
supported_workflows=[self.HELLO_WORKFLOW_MODULE])
# The builder should write a file called hello.txt with contents "Hello World"
self.expected_filename = os.path.join(self.artifacts_dir, 'hello.txt')
self.expected_contents = "Hello World"
def test_with_real_workflow_class(self):
"""Define a real workflow class and try to fetch it. This ensures the workflow registration actually works.
"""
# Declare my test workflow.
class MyWorkflow(BaseWorkflow):
NAME = "MyWorkflow"
CAPABILITY = Capability(language=self.lang,
dependency_manager=self.lang_framework,
application_framework=self.app_framework)
def __init__(self,
source_dir,
artifacts_dir,
scratch_dir,
manifest_path,
runtime=None,
optimizations=None,
options=None):
super(MyWorkflow, self).__init__(source_dir,
artifacts_dir,
scratch_dir,
manifest_path,
runtime=runtime,
optimizations=optimizations,
options=options)
# Don't load any other workflows. The above class declaration will automatically load the workflow into registry
builder = LambdaBuilder(self.lang,
self.lang_framework,
self.app_framework,
supported_workflows=[])
# Make sure this workflow is selected
self.assertEqual(builder.selected_workflow_cls, MyWorkflow)
def setUp(self):
self.source_dir = self.TEST_DATA_FOLDER
self.artifacts_dir = tempfile.mkdtemp()
self.scratch_dir = tempfile.mkdtemp()
self.manifest_path_valid = os.path.join(self.TEST_DATA_FOLDER,
"Makefile")
self.test_data_files = {
"__init__.py", "main.py", "requirements-requests.txt"
}
self.builder = LambdaBuilder(language="provided",
dependency_manager=None,
application_framework=None)
self.runtime = "provided"
class TestRubyWorkflow(TestCase):
"""
Verifies that `ruby` workflow works by building a Lambda using Bundler
"""
TEST_DATA_FOLDER = os.path.join(os.path.dirname(__file__), "testdata")
def setUp(self):
self.artifacts_dir = tempfile.mkdtemp()
self.scratch_dir = tempfile.mkdtemp()
self.no_deps = os.path.join(self.TEST_DATA_FOLDER, "no-deps")
self.builder = LambdaBuilder(language="ruby", dependency_manager="bundler", application_framework=None)
self.runtime = "ruby2.5"
def tearDown(self):
shutil.rmtree(self.artifacts_dir)
shutil.rmtree(self.scratch_dir)
def test_builds_project_without_dependencies(self):
source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-deps")
self.builder.build(
source_dir, self.artifacts_dir, self.scratch_dir, os.path.join(source_dir, "Gemfile"), runtime=self.runtime
)
expected_files = {"handler.rb", "Gemfile", "Gemfile.lock", ".bundle", "vendor"}
output_files = set(os.listdir(self.artifacts_dir))
self.assertEqual(expected_files, output_files)
def test_builds_project_with_dependencies(self):
source_dir = os.path.join(self.TEST_DATA_FOLDER, "with-deps")
self.builder.build(
source_dir, self.artifacts_dir, self.scratch_dir, os.path.join(source_dir, "Gemfile"), runtime=self.runtime
)
expected_files = {"handler.rb", "Gemfile", "Gemfile.lock", ".bundle", "vendor"}
output_files = set(os.listdir(self.artifacts_dir))
self.assertEqual(expected_files, output_files)
def test_builds_project_and_ignores_excluded_files(self):
source_dir = os.path.join(self.TEST_DATA_FOLDER, "excluded-files")
self.builder.build(
source_dir, self.artifacts_dir, self.scratch_dir, os.path.join(source_dir, "Gemfile"), runtime=self.runtime
)
expected_files = {"handler.rb", "Gemfile", "Gemfile.lock", ".bundle", "vendor"}
output_files = set(os.listdir(self.artifacts_dir))
self.assertEqual(expected_files, output_files)
def test_fails_if_bundler_cannot_resolve_dependencies(self):
source_dir = os.path.join(self.TEST_DATA_FOLDER, "broken-deps")
with self.assertRaises(WorkflowFailedError) as ctx:
self.builder.build(
source_dir,
self.artifacts_dir,
self.scratch_dir,
os.path.join(source_dir, "Gemfile"),
runtime=self.runtime,
)
self.assertIn("RubyBundlerBuilder:RubyBundle - Bundler Failed: ", str(ctx.exception))
def test_must_not_load_any_workflows(self, get_workflow_mock, importlib_mock):
modules = [] # Load no modules
builder = LambdaBuilder(self.lang, self.lang_framework, self.app_framework, supported_workflows=modules)
self.assertEqual(builder.supported_workflows, [])
# Make sure the modules are loaded in same order as passed
importlib_mock.import_module.assert_not_called()
def _build_function_in_process(self, config, source_dir, artifacts_dir,
scratch_dir, manifest_path, runtime):
builder = LambdaBuilder(
language=config.language,
dependency_manager=config.dependency_manager,
application_framework=config.application_framework)
try:
builder.build(source_dir,
artifacts_dir,
scratch_dir,
manifest_path,
runtime=runtime)
except LambdaBuilderError as ex:
raise BuildError(str(ex))
return artifacts_dir
def test_with_mocks(self, scratch_dir_exists, get_workflow_mock,
importlib_mock, os_mock):
workflow_cls = Mock()
workflow_instance = workflow_cls.return_value = Mock()
os_mock.path.exists.return_value = scratch_dir_exists
get_workflow_mock.return_value = workflow_cls
builder = LambdaBuilder(self.lang,
self.lang_framework,
self.app_framework,
supported_workflows=[])
builder.build(
"source_dir",
"artifacts_dir",
"scratch_dir",
"manifest_path",
runtime="runtime",
optimizations="optimizations",
options="options",
executable_search_paths="executable_search_paths",
mode=None,
)
workflow_cls.assert_called_with(
"source_dir",
"artifacts_dir",
"scratch_dir",
"manifest_path",
runtime="runtime",
optimizations="optimizations",
options="options",
executable_search_paths="executable_search_paths",
mode=None,
)
workflow_instance.run.assert_called_once()
os_mock.path.exists.assert_called_once_with("scratch_dir")
if scratch_dir_exists:
os_mock.makedirs.not_called()
else:
os_mock.makedirs.assert_called_once_with("scratch_dir")
def test_must_support_loading_custom_workflows(self, get_workflow_mock, importlib_mock):
modules = ["a.b.c", "c.d", "e.f", "z.k"]
# instantiate
builder = LambdaBuilder(self.lang, self.lang_framework, self.app_framework, supported_workflows=modules)
self.assertEqual(builder.supported_workflows, modules)
# Make sure the modules are loaded in same order as passed
importlib_mock.import_module.assert_has_calls([call(m) for m in modules], any_order=False)
class TestGoWorkflow(TestCase):
"""
Verifies that `go` workflow works by building a Lambda using Go Modules
"""
TEST_DATA_FOLDER = os.path.join(os.path.dirname(__file__), "testdata")
def setUp(self):
self.artifacts_dir = tempfile.mkdtemp()
self.scratch_dir = tempfile.mkdtemp()
self.builder = LambdaBuilder(language="go", dependency_manager="modules", application_framework=None)
self.runtime = "go1.x"
def tearDown(self):
shutil.rmtree(self.artifacts_dir)
shutil.rmtree(self.scratch_dir)
def test_builds_project_without_dependencies(self):
source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-deps")
self.builder.build(
source_dir,
self.artifacts_dir,
self.scratch_dir,
os.path.join(source_dir, "go.mod"),
runtime=self.runtime,
options={"artifact_executable_name": "no-deps-main"},
)
expected_files = {"no-deps-main"}
output_files = set(os.listdir(self.artifacts_dir))
print(output_files)
self.assertEqual(expected_files, output_files)
def test_builds_project_with_dependencies(self):
source_dir = os.path.join(self.TEST_DATA_FOLDER, "with-deps")
self.builder.build(
source_dir,
self.artifacts_dir,
self.scratch_dir,
os.path.join(source_dir, "go.mod"),
runtime=self.runtime,
options={"artifact_executable_name": "with-deps-main"},
)
expected_files = {"with-deps-main"}
output_files = set(os.listdir(self.artifacts_dir))
self.assertEqual(expected_files, output_files)
def test_fails_if_modules_cannot_resolve_dependencies(self):
source_dir = os.path.join(self.TEST_DATA_FOLDER, "broken-deps")
with self.assertRaises(WorkflowFailedError) as ctx:
self.builder.build(
source_dir,
self.artifacts_dir,
self.scratch_dir,
os.path.join(source_dir, "go.mod"),
runtime=self.runtime,
options={"artifact_executable_name": "failed"},
)
self.assertIn("GoModulesBuilder:Build - Builder Failed: ", str(ctx.exception))
class TestJavaMaven(TestCase):
SINGLE_BUILD_TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "testdata", "single-build")
def setUp(self):
self.artifacts_dir = tempfile.mkdtemp()
self.scratch_dir = tempfile.mkdtemp()
self.builder = LambdaBuilder(language="java", dependency_manager="maven", application_framework=None)
self.runtime = "java8"
def tearDown(self):
shutil.rmtree(self.artifacts_dir)
shutil.rmtree(self.scratch_dir)
def test_build_single_build_with_deps_resources_exclude_test_jars(self):
source_dir = os.path.join(self.SINGLE_BUILD_TEST_DATA_DIR, "with-deps")
manifest_path = os.path.join(source_dir, "pom.xml")
self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, manifest_path, runtime=self.runtime)
expected_files = [
p("aws", "lambdabuilders", "Main.class"),
p("some_data.txt"),
p("lib", "software.amazon.awssdk.annotations-2.1.0.jar"),
]
self.assert_artifact_contains_files(expected_files)
self.assert_artifact_not_contains_file(p("lib", "junit-4.12.jar"))
self.assert_src_dir_not_touched(source_dir)
def test_build_single_build_no_deps(self):
source_dir = os.path.join(self.SINGLE_BUILD_TEST_DATA_DIR, "no-deps")
manifest_path = os.path.join(source_dir, "pom.xml")
self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, manifest_path, runtime=self.runtime)
expected_files = [p("aws", "lambdabuilders", "Main.class"), p("some_data.txt")]
self.assert_artifact_contains_files(expected_files)
self.assert_artifact_not_contains_file(p("lib"))
self.assert_src_dir_not_touched(source_dir)
def test_build_single_build_with_deps_broken(self):
source_dir = os.path.join(self.SINGLE_BUILD_TEST_DATA_DIR, "with-deps-broken")
manifest_path = os.path.join(source_dir, "pom.xml")
with self.assertRaises(WorkflowFailedError) as raised:
self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, manifest_path, runtime=self.runtime)
self.assertTrue(raised.exception.args[0].startswith("JavaMavenWorkflow:MavenBuild - Maven Failed"))
self.assert_src_dir_not_touched(source_dir)
def assert_artifact_contains_files(self, files):
for f in files:
self.assert_artifact_contains_file(f)
def assert_artifact_contains_file(self, p):
self.assertTrue(os.path.exists(os.path.join(self.artifacts_dir, p)))
def assert_artifact_not_contains_file(self, p):
self.assertFalse(os.path.exists(os.path.join(self.artifacts_dir, p)))
def assert_src_dir_not_touched(self, source_dir):
self.assertFalse(os.path.exists(os.path.join(source_dir, "target")))
def test_must_load_all_default_workflows(self, get_workflow_mock, importlib_mock):
# instantiate
builder = LambdaBuilder(self.lang, self.lang_framework, self.app_framework)
self.assertEqual(builder.supported_workflows, [self.DEFAULT_WORKFLOW_MODULE])
# First check if the module was loaded
importlib_mock.import_module.assert_called_once_with(self.DEFAULT_WORKFLOW_MODULE)
# then check if we tried to get a workflow for given capability
get_workflow_mock.assert_called_with(
Capability(
language=self.lang, dependency_manager=self.lang_framework, application_framework=self.app_framework
)
)
class TestJavaMaven(TestCase):
SINGLE_BUILD_TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "testdata", "single-build")
def setUp(self):
self.artifacts_dir = tempfile.mkdtemp()
self.scratch_dir = tempfile.mkdtemp()
self.builder = LambdaBuilder(language='java', dependency_manager='maven', application_framework=None)
self.runtime = 'java8'
def tearDown(self):
shutil.rmtree(self.artifacts_dir)
shutil.rmtree(self.scratch_dir)
def test_build_single_build_with_deps_resources_exclude_test_jars(self):
source_dir = os.path.join(self.SINGLE_BUILD_TEST_DATA_DIR, 'with-deps')
manifest_path = os.path.join(source_dir, 'pom.xml')
self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, manifest_path, runtime=self.runtime)
expected_files = [p('aws', 'lambdabuilders', 'Main.class'), p('some_data.txt'),
p('lib', 'annotations-2.1.0.jar')]
self.assert_artifact_contains_files(expected_files)
self.assert_artifact_not_contains_file(p('lib', 'junit-4.12.jar'))
self.assert_src_dir_not_touched(source_dir)
def test_build_single_build_no_deps(self):
source_dir = os.path.join(self.SINGLE_BUILD_TEST_DATA_DIR, 'no-deps')
manifest_path = os.path.join(source_dir, 'pom.xml')
self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, manifest_path, runtime=self.runtime)
expected_files = [p('aws', 'lambdabuilders', 'Main.class'), p('some_data.txt')]
self.assert_artifact_contains_files(expected_files)
self.assert_artifact_not_contains_file(p('lib'))
self.assert_src_dir_not_touched(source_dir)
def test_build_single_build_with_deps_broken(self):
source_dir = os.path.join(self.SINGLE_BUILD_TEST_DATA_DIR, 'with-deps-broken')
manifest_path = os.path.join(source_dir, 'pom.xml')
with self.assertRaises(WorkflowFailedError) as raised:
self.builder.build(source_dir, self.artifacts_dir, self.scratch_dir, manifest_path, runtime=self.runtime)
self.assertTrue(raised.exception.args[0].startswith('JavaMavenWorkflow:MavenBuild - Maven Failed'))
self.assert_src_dir_not_touched(source_dir)
def assert_artifact_contains_files(self, files):
for f in files:
self.assert_artifact_contains_file(f)
def assert_artifact_contains_file(self, p):
self.assertTrue(os.path.exists(os.path.join(self.artifacts_dir, p)))
def assert_artifact_not_contains_file(self, p):
self.assertFalse(os.path.exists(os.path.join(self.artifacts_dir, p)))
def assert_src_dir_not_touched(self, source_dir):
self.assertFalse(os.path.exists(os.path.join(source_dir, 'target')))
class TestJavaGradle(TestCase):
SINGLE_BUILD_TEST_DATA_DIR = os.path.join(os.path.dirname(__file__),
"testdata", "single-build")
MULTI_BUILD_TEST_DATA_DIR = os.path.join(os.path.dirname(__file__),
"testdata", "multi-build")
def setUp(self):
self.artifacts_dir = tempfile.mkdtemp()
self.scratch_dir = tempfile.mkdtemp()
self.builder = LambdaBuilder(language='java',
dependency_manager='gradle',
application_framework=None)
self.runtime = 'java11'
def tearDown(self):
shutil.rmtree(self.artifacts_dir)
shutil.rmtree(self.scratch_dir)
def test_build_single_build_with_deps(self):
source_dir = os.path.join(self.SINGLE_BUILD_TEST_DATA_DIR, 'with-deps')
manifest_path = os.path.join(source_dir, 'build.gradle')
self.builder.build(source_dir,
self.artifacts_dir,
self.scratch_dir,
manifest_path,
runtime=self.runtime)
expected_files = [
p('aws', 'lambdabuilders', 'Main.class'),
p('lib', 'annotations-2.1.0.jar')
]
self.assert_artifact_contains_files(expected_files)
def test_build_single_build_with_resources(self):
source_dir = os.path.join(self.SINGLE_BUILD_TEST_DATA_DIR,
'with-resources')
manifest_path = os.path.join(source_dir, 'build.gradle')
self.builder.build(source_dir,
self.artifacts_dir,
self.scratch_dir,
manifest_path,
runtime=self.runtime)
expected_files = [
p('aws', 'lambdabuilders', 'Main.class'),
p('some_data.txt'),
p('lib', 'annotations-2.1.0.jar')
]
self.assert_artifact_contains_files(expected_files)
def test_build_single_build_with_test_deps_test_jars_not_included(self):
source_dir = os.path.join(self.SINGLE_BUILD_TEST_DATA_DIR,
'with-test-deps')
manifest_path = os.path.join(source_dir, 'build.gradle')
self.builder.build(source_dir,
self.artifacts_dir,
self.scratch_dir,
manifest_path,
runtime=self.runtime)
expected_files = [
p('aws', 'lambdabuilders', 'Main.class'),
p('lib', 'annotations-2.1.0.jar')
]
self.assert_artifact_contains_files(expected_files)
self.assert_artifact_not_contains_file(p('lib', 's3-2.1.0.jar'))
def test_build_single_build_with_deps_gradlew(self):
source_dir = os.path.join(self.SINGLE_BUILD_TEST_DATA_DIR,
'with-deps-gradlew')
manifest_path = os.path.join(source_dir, 'build.gradle')
self.builder.build(source_dir,
self.artifacts_dir,
self.scratch_dir,
manifest_path,
runtime=self.runtime,
executable_search_paths=[source_dir])
expected_files = [
p('aws', 'lambdabuilders', 'Main.class'),
p('lib', 'annotations-2.1.0.jar')
]
self.assert_artifact_contains_files(expected_files)
def test_build_multi_build_with_deps_lambda1(self):
parent_dir = os.path.join(self.MULTI_BUILD_TEST_DATA_DIR, 'with-deps')
manifest_path = os.path.join(parent_dir, 'lambda1', 'build.gradle')
lambda1_source = os.path.join(parent_dir, 'lambda1')
self.builder.build(lambda1_source,
self.artifacts_dir,
self.scratch_dir,
manifest_path,
runtime=self.runtime)
lambda1_expected_files = [
p('aws', 'lambdabuilders', 'Lambda1_Main.class'),
p('lib', 'annotations-2.1.0.jar')
]
self.assert_artifact_contains_files(lambda1_expected_files)
def test_build_multi_build_with_deps_lambda2(self):
parent_dir = os.path.join(self.MULTI_BUILD_TEST_DATA_DIR, 'with-deps')
manifest_path = os.path.join(parent_dir, 'lambda2', 'build.gradle')
lambda2_source = os.path.join(parent_dir, 'lambda2')
self.builder.build(lambda2_source,
self.artifacts_dir,
self.scratch_dir,
manifest_path,
runtime=self.runtime)
lambda2_expected_files = [
p('aws', 'lambdabuilders', 'Lambda2_Main.class'),
p('lib', 'annotations-2.1.0.jar')
]
self.assert_artifact_contains_files(lambda2_expected_files)
def test_build_multi_build_with_deps_inter_module(self):
parent_dir = os.path.join(self.MULTI_BUILD_TEST_DATA_DIR,
'with-deps-inter-module')
manifest_path = os.path.join(parent_dir, 'lambda1', 'build.gradle')
lambda1_source = os.path.join(parent_dir, 'lambda1')
self.builder.build(lambda1_source,
self.artifacts_dir,
self.scratch_dir,
manifest_path,
runtime=self.runtime)
lambda1_expected_files = [
p('aws', 'lambdabuilders', 'Lambda1_Main.class'),
p('lib', 'annotations-2.1.0.jar'),
p('lib', 'common.jar')
]
self.assert_artifact_contains_files(lambda1_expected_files)
def test_build_single_build_with_deps_broken(self):
source_dir = os.path.join(self.SINGLE_BUILD_TEST_DATA_DIR,
'with-deps-broken')
manifest_path = os.path.join(source_dir, 'build.gradle')
with self.assertRaises(WorkflowFailedError) as raised:
self.builder.build(source_dir,
self.artifacts_dir,
self.scratch_dir,
manifest_path,
runtime=self.runtime)
self.assertTrue(raised.exception.args[0].startswith(
'JavaGradleWorkflow:GradleBuild - Gradle Failed'))
def assert_artifact_contains_files(self, files):
for f in files:
self.assert_artifact_contains_file(f)
def assert_artifact_contains_file(self, p):
self.assertTrue(os.path.exists(os.path.join(self.artifacts_dir, p)))
def assert_artifact_not_contains_file(self, p):
self.assertFalse(os.path.exists(os.path.join(self.artifacts_dir, p)))
def assert_zip_contains(self, zip_path, files):
with ZipFile(zip_path) as z:
zip_names = set(z.namelist())
self.assertTrue(set(files).issubset(zip_names))
class TestGoDep(TestCase):
TEST_DATA_FOLDER = os.path.join(os.path.dirname(__file__), "data")
def setUp(self):
self.artifacts_dir = tempfile.mkdtemp()
self.scratch_dir = tempfile.mkdtemp()
os.environ["GOPATH"] = self.TEST_DATA_FOLDER
self.no_deps = os.path.join(self.TEST_DATA_FOLDER, "src", "nodeps")
self.builder = LambdaBuilder(language="go", dependency_manager="dep", application_framework=None)
self.runtime = "go1.x"
def tearDown(self):
shutil.rmtree(self.artifacts_dir)
shutil.rmtree(self.scratch_dir)
def test_builds_project_with_no_deps(self):
source_dir = os.path.join(self.TEST_DATA_FOLDER, "src", "nodeps")
self.builder.build(
source_dir,
self.artifacts_dir,
self.scratch_dir,
os.path.join(source_dir, "Gopkg.toml"),
runtime=self.runtime,
options={"artifact_executable_name": "main"},
)
expected_files = {"main"}
output_files = set(os.listdir(self.artifacts_dir))
self.assertEqual(expected_files, output_files)
def test_builds_project_and_excludes_hidden_aws_sam(self):
source_dir = os.path.join(self.TEST_DATA_FOLDER, "src", "excluded-files")
self.builder.build(
source_dir,
self.artifacts_dir,
self.scratch_dir,
os.path.join(source_dir, "Gopkg.toml"),
runtime=self.runtime,
options={"artifact_executable_name": "main"},
)
expected_files = {"main"}
output_files = set(os.listdir(self.artifacts_dir))
self.assertEqual(expected_files, output_files)
def test_builds_project_with_no_gopkg_file(self):
source_dir = os.path.join(self.TEST_DATA_FOLDER, "src", "no-gopkg")
with self.assertRaises(WorkflowFailedError) as ex:
self.builder.build(
source_dir,
self.artifacts_dir,
self.scratch_dir,
os.path.join(source_dir, "Gopkg.toml"),
runtime=self.runtime,
options={"artifact_executable_name": "main"},
)
self.assertEqual(
"GoDepBuilder:DepEnsure - Exec Failed: could not find project Gopkg.toml,"
+ " use dep init to initiate a manifest",
str(ex.exception),
)
def test_builds_project_with_remote_deps(self):
source_dir = os.path.join(self.TEST_DATA_FOLDER, "src", "remote-deps")
self.builder.build(
source_dir,
self.artifacts_dir,
self.scratch_dir,
os.path.join(source_dir, "Gopkg.toml"),
runtime=self.runtime,
options={"artifact_executable_name": "main"},
)
expected_files = {"main"}
output_files = set(os.listdir(self.artifacts_dir))
self.assertEqual(expected_files, output_files)
def test_builds_project_with_failed_remote_deps(self):
source_dir = os.path.join(self.TEST_DATA_FOLDER, "src", "failed-remote")
with self.assertRaises(WorkflowFailedError) as ex:
self.builder.build(
source_dir,
self.artifacts_dir,
self.scratch_dir,
os.path.join(source_dir, "Gopkg.toml"),
runtime=self.runtime,
options={"artifact_executable_name": "main"},
)
# The full message is super long, so part of it is fine.
self.assertNotEqual(str(ex.exception).find("unable to deduce repository and source type for"), -1)
class TestCustomMakeWorkflow(TestCase):
"""
Verifies that `custom_make` workflow works by building a Lambda that requires Numpy
"""
MAKEFILE_DIRECTORY = "makefile-root"
TEST_DATA_FOLDER = os.path.join(os.path.dirname(__file__), "testdata",
MAKEFILE_DIRECTORY)
def setUp(self):
self.source_dir = self.TEST_DATA_FOLDER
self.artifacts_dir = tempfile.mkdtemp()
self.scratch_dir = tempfile.mkdtemp()
self.manifest_path_valid = os.path.join(self.TEST_DATA_FOLDER,
"Makefile")
self.test_data_files = {
"__init__.py", "main.py", "requirements-requests.txt"
}
self.builder = LambdaBuilder(language="provided",
dependency_manager=None,
application_framework=None)
self.runtime = "provided"
def tearDown(self):
shutil.rmtree(self.artifacts_dir)
shutil.rmtree(self.scratch_dir)
def test_must_build_python_project_through_makefile(self):
self.builder.build(
self.source_dir,
self.artifacts_dir,
self.scratch_dir,
self.manifest_path_valid,
runtime=self.runtime,
options={"build_logical_id": "HelloWorldFunction"},
)
dependencies_installed = {
"chardet",
"urllib3",
"idna",
"urllib3-1.25.9.dist-info",
"chardet-3.0.4.dist-info",
"certifi-2020.4.5.2.dist-info",
"certifi",
"idna-2.10.dist-info",
"requests",
"requests-2.23.0.dist-info",
}
expected_files = self.test_data_files.union(dependencies_installed)
output_files = set(os.listdir(self.artifacts_dir))
self.assertEquals(expected_files, output_files)
def test_must_build_python_project_through_makefile_unknown_target(self):
with self.assertRaises(WorkflowFailedError):
self.builder.build(
self.source_dir,
self.artifacts_dir,
self.scratch_dir,
self.manifest_path_valid,
runtime=self.runtime,
options={"build_logical_id": "HelloWorldFunction2"},
)
class TestPythonPipWorkflow(TestCase):
"""
Verifies that `python_pip` workflow works by building a Lambda that requires Numpy
"""
TEST_DATA_FOLDER = os.path.join(os.path.dirname(__file__), "testdata")
def setUp(self):
self.source_dir = self.TEST_DATA_FOLDER
self.artifacts_dir = tempfile.mkdtemp()
self.scratch_dir = tempfile.mkdtemp()
self.manifest_path_valid = os.path.join(self.TEST_DATA_FOLDER,
"requirements-numpy.txt")
self.manifest_path_invalid = os.path.join(self.TEST_DATA_FOLDER,
"requirements-invalid.txt")
self.test_data_files = {
"__init__.py", "main.py", "requirements-invalid.txt",
"requirements-numpy.txt"
}
self.builder = LambdaBuilder(language="python",
dependency_manager="pip",
application_framework=None)
self.runtime = "{language}{major}.{minor}".format(
language=self.builder.capability.language,
major=sys.version_info.major,
minor=sys.version_info.minor)
self.runtime_mismatch = {
"python3.6": "python2.7",
"python3.7": "python2.7",
"python2.7": "python3.8",
"python3.8": "python2.7",
}
def tearDown(self):
shutil.rmtree(self.artifacts_dir)
shutil.rmtree(self.scratch_dir)
def test_must_build_python_project(self):
self.builder.build(self.source_dir,
self.artifacts_dir,
self.scratch_dir,
self.manifest_path_valid,
runtime=self.runtime)
if self.runtime == "python2.7":
expected_files = self.test_data_files.union(
{"numpy", "numpy-1.15.4.data", "numpy-1.15.4.dist-info"})
elif self.runtime == "python3.6":
expected_files = self.test_data_files.union(
{"numpy", "numpy-1.17.4.dist-info"})
else:
expected_files = self.test_data_files.union(
{"numpy", "numpy-1.20.3.dist-info", "numpy.libs"})
output_files = set(os.listdir(self.artifacts_dir))
self.assertEqual(expected_files, output_files)
def test_mismatch_runtime_python_project(self):
# NOTE : Build still works if other versions of python are accessible on the path. eg: /usr/bin/python2.7
# is still accessible within a python 3 virtualenv.
try:
self.builder.build(
self.source_dir,
self.artifacts_dir,
self.scratch_dir,
self.manifest_path_valid,
runtime=self.runtime_mismatch[self.runtime],
)
except WorkflowFailedError as ex:
self.assertIn("Binary validation failed", str(ex))
def test_runtime_validate_python_project_fail_open_unsupported_runtime(
self):
with self.assertRaises(WorkflowFailedError):
self.builder.build(self.source_dir,
self.artifacts_dir,
self.scratch_dir,
self.manifest_path_valid,
runtime="python2.8")
def test_must_fail_to_resolve_dependencies(self):
with self.assertRaises(WorkflowFailedError) as ctx:
self.builder.build(self.source_dir,
self.artifacts_dir,
self.scratch_dir,
self.manifest_path_invalid,
runtime=self.runtime)
# In Python2 a 'u' is now added to the exception string. To account for this, we see if either one is in the
# output
message_in_exception = "Invalid requirement: 'adfasf=1.2.3'" in str(
ctx.exception) or "Invalid requirement: u'adfasf=1.2.3'" in str(
ctx.exception)
self.assertTrue(message_in_exception)
def test_must_fail_if_requirements_not_found(self):
with self.assertRaises(WorkflowFailedError) as ctx:
self.builder.build(
self.source_dir,
self.artifacts_dir,
self.scratch_dir,
os.path.join("non", "existent", "manifest"),
runtime=self.runtime,
)
self.builder.build(
self.source_dir,
self.artifacts_dir,
self.scratch_dir,
os.path.join("non", "existent", "manifest"),
runtime=self.runtime,
)
self.assertIn("Requirements file not found", str(ctx.exception))
def main(): # pylint: disable=too-many-statements
"""
Implementation of CLI Interface. Handles only one JSON-RPC method at a time and responds with data
Input is passed as JSON string either through stdin or as the first argument to the command. Output is always
printed to stdout.
"""
# For now the request is not validated
if len(sys.argv) > 1:
request_str = sys.argv[1]
LOG.debug("Using the request object from command line argument")
else:
LOG.debug("Reading the request object from stdin")
request_str = sys.stdin.read()
request = json.loads(request_str)
request_id = request["id"]
params = request["params"]
# Currently, this is the only supported method
if request["method"] != "LambdaBuilder.build":
response = _error_response(request_id, -32601, "Method unavailable")
return _write_response(response, 1)
try:
protocol_version = _parse_version(params.get("__protocol_version"))
version_compatibility_check(protocol_version)
except ValueError:
response = _error_response(request_id, 505, "Unsupported Protocol Version")
return _write_response(response, 1)
capabilities = params["capability"]
supported_workflows = params.get("supported_workflows")
exit_code = 0
response = None
try:
builder = LambdaBuilder(
language=capabilities["language"],
dependency_manager=capabilities["dependency_manager"],
application_framework=capabilities["application_framework"],
supported_workflows=supported_workflows,
)
artifacts_dir = params["artifacts_dir"]
builder.build(
params["source_dir"],
params["artifacts_dir"],
params["scratch_dir"],
params["manifest_path"],
executable_search_paths=params.get("executable_search_paths", None),
runtime=params["runtime"],
optimizations=params["optimizations"],
options=params["options"],
mode=params.get("mode", None),
download_dependencies=params.get("download_dependencies", True),
dependencies_dir=params.get("dependencies_dir", None),
combine_dependencies=params.get("combine_dependencies", True),
architecture=params.get("architecture", X86_64),
is_building_layer=params.get("is_building_layer", False),
experimental_flags=params.get("experimental_flags", []),
)
# Return a success response
response = _success_response(request_id, artifacts_dir)
except (WorkflowNotFoundError, WorkflowUnknownError, WorkflowFailedError) as ex:
LOG.debug("Builder workflow failed", exc_info=ex)
exit_code = 1
response = _error_response(request_id, 400, str(ex))
except Exception as ex:
LOG.debug("Builder crashed", exc_info=ex)
exit_code = 1
response = _error_response(request_id, 500, str(ex))
_write_response(response, exit_code)
class TestNodejsNpmWorkflow(TestCase):
"""
Verifies that `nodejs_npm` workflow works by building a Lambda using NPM
"""
TEST_DATA_FOLDER = os.path.join(os.path.dirname(__file__), "testdata")
def setUp(self):
self.artifacts_dir = tempfile.mkdtemp()
self.scratch_dir = tempfile.mkdtemp()
self.no_deps = os.path.join(self.TEST_DATA_FOLDER, "no-deps")
self.builder = LambdaBuilder(language="nodejs",
dependency_manager="npm",
application_framework=None)
self.runtime = "nodejs8.10"
def tearDown(self):
shutil.rmtree(self.artifacts_dir)
shutil.rmtree(self.scratch_dir)
def test_builds_project_without_dependencies(self):
source_dir = os.path.join(self.TEST_DATA_FOLDER, "no-deps")
self.builder.build(source_dir,
self.artifacts_dir,
self.scratch_dir,
os.path.join(source_dir, "package.json"),
runtime=self.runtime)
expected_files = {"package.json", "included.js"}
output_files = set(os.listdir(self.artifacts_dir))
self.assertEquals(expected_files, output_files)
def test_builds_project_with_remote_dependencies(self):
source_dir = os.path.join(self.TEST_DATA_FOLDER, "npm-deps")
self.builder.build(source_dir,
self.artifacts_dir,
self.scratch_dir,
os.path.join(source_dir, "package.json"),
runtime=self.runtime)
expected_files = {"package.json", "included.js", "node_modules"}
output_files = set(os.listdir(self.artifacts_dir))
self.assertEquals(expected_files, output_files)
expected_modules = {"minimal-request-promise"}
output_modules = set(
os.listdir(os.path.join(self.artifacts_dir, "node_modules")))
self.assertEquals(expected_modules, output_modules)
def test_fails_if_npm_cannot_resolve_dependencies(self):
source_dir = os.path.join(self.TEST_DATA_FOLDER, "broken-deps")
with self.assertRaises(WorkflowFailedError) as ctx:
self.builder.build(source_dir,
self.artifacts_dir,
self.scratch_dir,
os.path.join(source_dir, "package.json"),
runtime=self.runtime)
self.assertIn(
"No matching version found for [email protected]_EXISTENT",
str(ctx.exception))
def test_fails_if_package_json_is_broken(self):
source_dir = os.path.join(self.TEST_DATA_FOLDER, "broken-package")
with self.assertRaises(WorkflowFailedError) as ctx:
self.builder.build(source_dir,
self.artifacts_dir,
self.scratch_dir,
os.path.join(source_dir, "package.json"),
runtime=self.runtime)
self.assertIn("Unexpected end of JSON input", str(ctx.exception))
