def test_eq(self) -> None:
path = os.path.join(self.manifests_path, "1.0.0",
"opensearch-1.0.0.yml")
manifest1 = InputManifest.from_path(path)
manifest2 = InputManifest.from_path(path)
self.assertEqual(manifest1, manifest1)
self.assertEqual(manifest1, manifest2)
def test_neq(self) -> None:
path1 = os.path.join(self.manifests_path, "1.0.0",
"opensearch-1.0.0.yml")
path2 = os.path.join(self.manifests_path, "1.1.0",
"opensearch-1.1.0.yml")
manifest1 = InputManifest.from_path(path1)
manifest2 = InputManifest.from_path(path2)
self.assertNotEqual(manifest1, manifest2)
def test_invalid_ref(self) -> None:
data_path = os.path.join(os.path.dirname(__file__), "data")
manifest_path = os.path.join(data_path, "invalid-ref.yml")
with self.assertRaises(Exception) as context:
InputManifest.from_path(manifest_path)
self.assertTrue(
str(context.exception).startswith(
"Invalid manifest schema: {'components': "))
def test_1_1(self) -> None:
path = os.path.join(self.manifests_path, "1.1.0",
"opensearch-1.1.0.yml")
manifest = InputManifest.from_path(path)
self.assertEqual(manifest.version, "1.0")
self.assertEqual(manifest.build.name, "OpenSearch")
self.assertEqual(manifest.build.filename, "opensearch")
self.assertEqual(manifest.build.version, "1.1.0")
self.assertEqual(
len(list(manifest.components.select(focus=["common-utils"]))), 1)
# opensearch component
opensearch_component: InputComponentFromSource = manifest.components[
"OpenSearch"] # type: ignore[assignment]
self.assertEqual(opensearch_component.name, "OpenSearch")
self.assertEqual(
opensearch_component.repository,
"https://github.com/opensearch-project/OpenSearch.git",
)
self.assertEqual(opensearch_component.ref, "tags/1.1.0")
# components
for component in manifest.components.values():
self.assertIsInstance(component, InputComponentFromSource)
# alerting component checks
alerting_component: InputComponentFromSource = manifest.components[
"alerting"] # type: ignore[assignment]
self.assertIsNotNone(alerting_component)
self.assertEqual(len(alerting_component.checks), 2)
for check in alerting_component.checks:
self.assertIsInstance(check, Check)
self.assertIsNone(alerting_component.checks[0].args)
self.assertEqual(alerting_component.checks[1].args, "alerting")
def test_1_1_1_dist(self) -> None:
data_path = os.path.realpath(
os.path.join(os.path.dirname(__file__), "data"))
path = os.path.join(data_path,
"opensearch-dashboards-from-dist-1.1.1.yml")
manifest = InputManifest.from_path(path)
self.assertEqual(manifest.version, "1.0")
self.assertEqual(manifest.build.name, "OpenSearch Dashboards")
self.assertEqual(manifest.build.filename, "opensearch-dashboards")
self.assertEqual(manifest.build.version, "1.1.1")
self.assertEqual(
len(list(
manifest.components.select(focus=["alertingDashboards"]))), 1)
opensearch_component: InputComponentFromDist = manifest.components[
"OpenSearch-Dashboards"] # type: ignore[assignment]
self.assertIsInstance(opensearch_component, InputComponentFromDist)
self.assertEqual(opensearch_component.name, "OpenSearch-Dashboards")
self.assertEqual(
opensearch_component.dist,
"https://ci.opensearch.org/ci/dbc/bundle-build-dashboards/1.1.0/20210930",
)
for component in manifest.components.values():
if component.name in [
"reportsDashboards", "functionalTestDashboards"
]:
self.assertIsInstance(component, InputComponentFromSource)
else:
self.assertIsInstance(component, InputComponentFromDist)
def test_to_dict(self) -> None:
path = os.path.join(self.manifests_path, "1.1.0",
"opensearch-1.1.0.yml")
manifest = InputManifest.from_path(path)
data = manifest.to_dict()
with open(path) as f:
self.assertEqual(yaml.safe_load(f), data)
def test_stable(self, mock_output: Mock) -> None:
mock_output.return_value.decode.return_value = "updated\tHEAD"
path = os.path.join(self.manifests_path, "1.1.0",
"opensearch-1.1.0.yml")
manifest = InputManifest.from_path(path).stable()
opensearch: InputComponentFromSource = manifest.components[
"OpenSearch"] # type: ignore[assignment]
self.assertEqual(opensearch.ref, "updated")
def test_neq_update(self) -> None:
path = os.path.join(self.manifests_path, "1.0.0",
"opensearch-1.0.0.yml")
manifest1 = InputManifest.from_path(path)
manifest2 = copy.deepcopy(manifest1)
self.assertEqual(manifest1, manifest2)
manifest2.components["name"] = InputComponentFromDist({
"name": "name",
"dist": "dist"
})
self.assertNotEqual(manifest1, manifest2)
def test_select(self) -> None:
path = os.path.join(self.manifests_path, "1.1.0",
"opensearch-1.1.0.yml")
manifest = InputManifest.from_path(path)
self.assertEqual(
len(list(manifest.components.select(focus="common-utils"))), 1)
self.assertNotEqual(
len(list(manifest.components.select(platform="windows"))), 0)
self.assertEqual(
len(
list(manifest.components.select(focus="k-NN",
platform="linux"))), 1)
def main():
args = BuildArgs()
console.configure(level=args.logging_level)
manifest = InputManifest.from_file(args.manifest)
if args.ref_manifest:
manifest = manifest.stable()
if os.path.exists(args.ref_manifest):
if manifest == InputManifest.from_path(args.ref_manifest):
logging.info(f"No changes since {args.ref_manifest}")
else:
logging.info(f"Updating {args.ref_manifest}")
manifest.to_file(args.ref_manifest)
else:
logging.info(f"Creating {args.ref_manifest}")
manifest.to_file(args.ref_manifest)
exit(0)
output_dir = BuildOutputDir(manifest.build.filename).dir
with TemporaryDirectory(keep=args.keep, chdir=True) as work_dir:
logging.info(f"Building in {work_dir.name}")
target = BuildTarget(
name=manifest.build.name,
version=manifest.build.version,
patches=manifest.build.patches,
snapshot=args.snapshot if args.snapshot is not None else manifest.build.snapshot,
output_dir=output_dir,
platform=args.platform or manifest.build.platform,
architecture=args.architecture or manifest.build.architecture,
)
build_recorder = BuildRecorder(target)
logging.info(f"Building {manifest.build.name} ({target.architecture}) into {target.output_dir}")
for component in manifest.components.select(focus=args.component, platform=target.platform):
logging.info(f"Building {component.name}")
builder = Builders.builder_from(component, target)
try:
builder.checkout(work_dir.name)
builder.build(build_recorder)
builder.export_artifacts(build_recorder)
except:
logging.error(f"Error building {component.name}, retry with: {args.component_command(component.name)}")
raise
build_recorder.write_manifest()
logging.info("Done.")
def test_select_none(self) -> None:
path = os.path.join(self.manifests_path, "1.1.0",
"opensearch-1.1.0.yml")
manifest = InputManifest.from_path(path)
with self.assertRaises(ValueError) as ctx:
self.assertEqual(
len(
list(
manifest.components.select(focus=["k-NN"],
platform="windows"))), 0)
self.assertEqual(
str(ctx.exception),
"No components matched focus=k-NN, platform=windows.")
def test_1_0(self) -> None:
path = os.path.join(self.manifests_path, "1.0.0",
"opensearch-1.0.0.yml")
manifest = InputManifest.from_path(path)
self.assertEqual(manifest.version, "1.0")
self.assertEqual(manifest.build.name, "OpenSearch")
self.assertEqual(manifest.build.filename, "opensearch")
self.assertEqual(manifest.build.version, "1.0.0")
self.assertEqual(
len(list(manifest.components.select(focus=["common-utils"]))), 1)
opensearch_component: InputComponentFromSource = manifest.components[
"OpenSearch"] # type: ignore[assignment]
self.assertIsInstance(opensearch_component, InputComponentFromSource)
self.assertEqual(opensearch_component.name, "OpenSearch")
self.assertEqual(
opensearch_component.repository,
"https://github.com/opensearch-project/OpenSearch.git",
)
self.assertEqual(opensearch_component.ref, "tags/1.0.0")
for component in manifest.components.values():
self.assertIsInstance(component, InputComponentFromSource)
def test_1_2(self) -> None:
data_path = os.path.realpath(
os.path.join(os.path.dirname(__file__), "data"))
path = os.path.join(data_path, "opensearch-1.2.0.yml")
manifest = InputManifest.from_path(path)
self.assertEqual(manifest.version, "1.0")
self.assertEqual(manifest.build.name, "OpenSearch")
self.assertEqual(manifest.build.filename, "opensearch")
self.assertEqual(manifest.build.version, "1.2.0")
self.assertEqual(
manifest.ci.image.name,
"opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028"
)
self.assertEqual(manifest.ci.image.args,
"-e JAVA_HOME=/usr/lib/jvm/adoptopenjdk-14-hotspot")
self.assertNotEqual(len(manifest.components), 0)
self.assertEqual(
len(list(manifest.components.select(focus=["common-utils"]))), 1)
# opensearch component
opensearch_component: InputComponentFromSource = manifest.components[
"OpenSearch"] # type: ignore[assignment]
self.assertEqual(opensearch_component.name, "OpenSearch")
self.assertEqual(
opensearch_component.repository,
"https://github.com/opensearch-project/OpenSearch.git",
)
self.assertEqual(opensearch_component.ref, "tags/1.2.0")
# components
for component in manifest.components.values():
self.assertIsInstance(component, InputComponentFromSource)
# alerting component checks
alerting_component = manifest.components["alerting"]
self.assertIsNotNone(alerting_component)
self.assertEqual(len(alerting_component.checks), 2)
for check in alerting_component.checks:
self.assertIsInstance(check, Check)
self.assertIsNone(alerting_component.checks[0].args)
self.assertEqual(alerting_component.checks[1].args, "alerting")
class TestRunBuild(unittest.TestCase):
@pytest.fixture(autouse=True)
def capfd(self, capfd):
self.capfd = capfd
@patch("argparse._sys.argv", ["run_build.py", "--help"])
def test_usage(self):
with self.assertRaises(SystemExit):
main()
out, _ = self.capfd.readouterr()
self.assertTrue(out.startswith("usage:"))
MANIFESTS = os.path.join(
os.path.dirname(__file__),
"..",
"manifests",
)
OPENSEARCH_MANIFEST = os.path.realpath(
os.path.join(MANIFESTS, "1.1.0", "opensearch-1.1.0.yml"))
OPENSEARCH_MANIFEST_1_2 = os.path.realpath(
os.path.join(MANIFESTS, "1.2.0", "opensearch-1.2.0.yml"))
@patch("argparse._sys.argv",
["run_build.py", OPENSEARCH_MANIFEST, "-p", "linux"])
@patch("run_build.Builders.builder_from", return_value=MagicMock())
@patch("run_build.BuildRecorder", return_value=MagicMock())
@patch("run_build.TemporaryDirectory")
def test_main_platform_linux(self, mock_temp, mock_recorder, mock_builder,
*mocks):
mock_temp.return_value.__enter__.return_value.name = tempfile.gettempdir(
)
main()
self.assertNotEqual(mock_builder.return_value.build.call_count, 0)
self.assertEqual(mock_builder.return_value.build.call_count,
mock_builder.return_value.export_artifacts.call_count)
mock_recorder.return_value.write_manifest.assert_called()
@patch("argparse._sys.argv",
["run_build.py", OPENSEARCH_MANIFEST, "-p", "darwin"])
@patch("run_build.Builders.builder_from", return_value=MagicMock())
@patch("run_build.BuildRecorder", return_value=MagicMock())
@patch("run_build.TemporaryDirectory")
def test_main_platform_darwin(self, mock_temp, mock_recorder, mock_builder,
*mocks):
mock_temp.return_value.__enter__.return_value.name = tempfile.gettempdir(
)
main()
self.assertNotEqual(mock_builder.return_value.build.call_count, 0)
self.assertEqual(mock_builder.return_value.build.call_count,
mock_builder.return_value.export_artifacts.call_count)
mock_recorder.return_value.write_manifest.assert_called()
@patch("argparse._sys.argv",
["run_build.py", OPENSEARCH_MANIFEST, "-p", "windows"])
@patch("run_build.Builders.builder_from", return_value=MagicMock())
@patch("run_build.BuildRecorder", return_value=MagicMock())
@patch("run_build.TemporaryDirectory")
def test_main_platform_windows(self, mock_temp, mock_recorder,
mock_builder, *mocks):
mock_temp.return_value.__enter__.return_value.name = tempfile.gettempdir(
)
main()
# excludes performance analyzer and k-nn
for call_args in mock_builder.call_args:
if len(call_args) > 1:
component = call_args[0]
self.assertNotIn("k-nn", component.name.lower())
self.assertNotIn("analyzer", component.name.lower())
self.assertNotEqual(mock_builder.call_count, 0)
self.assertEqual(mock_builder.return_value.build.call_count,
mock_builder.call_count)
self.assertEqual(mock_builder.return_value.export_artifacts.call_count,
mock_builder.call_count)
mock_recorder.return_value.write_manifest.assert_called()
OPENSEARCH_DASHBOARDS_MANIFEST = os.path.realpath(
os.path.join(
os.path.dirname(__file__),
"..",
"manifests",
"1.1.0",
"opensearch-dashboards-1.1.0.yml",
))
@patch("argparse._sys.argv",
["run_build.py", OPENSEARCH_DASHBOARDS_MANIFEST, "-a", "x64"])
@patch("run_build.Builders.builder_from", return_value=MagicMock())
@patch("run_build.BuildRecorder", return_value=MagicMock())
@patch("run_build.TemporaryDirectory")
def test_main_with_architecture(self, mock_temp, mock_recorder,
mock_builder, *mocks):
mock_temp.return_value.__enter__.return_value.name = tempfile.gettempdir(
)
main()
self.assertEqual(mock_builder.return_value.build.call_count, 11)
self.assertEqual(mock_builder.return_value.export_artifacts.call_count,
11)
mock_recorder.return_value.write_manifest.assert_called()
@patch("argparse._sys.argv", [
"run_build.py", OPENSEARCH_DASHBOARDS_MANIFEST, "-p",
"invalidplatform", "-a", "x64"
])
def test_main_with_invalid_platform(self, *mocks):
with self.assertRaises(SystemExit):
main()
@patch("argparse._sys.argv", [
"run_build.py", OPENSEARCH_DASHBOARDS_MANIFEST, "-p", "linux", "-a",
"invalidarchitecture"
])
def test_main_with_invalid_architecture(self, *mocks):
with self.assertRaises(SystemExit):
main()
@patch("os.path.exists", return_value=True)
@patch("argparse._sys.argv",
["run_build.py", OPENSEARCH_MANIFEST, "--lock"])
@patch("run_build.InputManifest.from_path",
return_value=InputManifest.from_path(OPENSEARCH_MANIFEST))
@patch("run_build.InputManifest.stable",
return_value=InputManifest.from_path(OPENSEARCH_MANIFEST))
@patch("run_build.InputManifest.to_file")
@patch("logging.info")
def test_main_manifest_lock_without_changes(self, mock_logging,
mock_to_file, mock_stable,
*mocks):
with self.assertRaises(SystemExit):
main()
mock_stable.assert_called_with()
mock_to_file.assert_not_called()
mock_logging.assert_called_with(
f"No changes since {self.OPENSEARCH_MANIFEST}.lock")
@patch("os.path.exists", return_value=True)
@patch("argparse._sys.argv",
["run_build.py", OPENSEARCH_MANIFEST, "--lock"])
@patch("run_build.InputManifest.from_path",
return_value=InputManifest.from_path(OPENSEARCH_MANIFEST))
@patch("run_build.InputManifest.stable",
return_value=InputManifest.from_path(OPENSEARCH_MANIFEST_1_2))
@patch("run_build.InputManifest.to_file")
@patch("logging.info")
def test_main_manifest_lock_with_changes(self, mock_logging, mock_to_file,
mock_stable, *mocks):
with self.assertRaises(SystemExit):
main()
mock_stable.assert_called_with()
mock_to_file.assert_called_with(self.OPENSEARCH_MANIFEST + ".lock")
mock_logging.assert_called_with(
f"Updating {self.OPENSEARCH_MANIFEST}.lock")
@patch("os.path.exists", return_value=False)
@patch("argparse._sys.argv",
["run_build.py", OPENSEARCH_MANIFEST, "--lock"])
@patch("run_build.InputManifest.from_path",
return_value=InputManifest.from_path(OPENSEARCH_MANIFEST))
@patch("run_build.InputManifest.stable",
return_value=InputManifest.from_path(OPENSEARCH_MANIFEST_1_2))
@patch("run_build.InputManifest.to_file")
@patch("logging.info")
def test_main_manifest_new_lock(self, mock_logging, mock_to_file,
mock_stable, *mocks):
with self.assertRaises(SystemExit):
main()
mock_stable.assert_called_with()
mock_to_file.assert_called_with(self.OPENSEARCH_MANIFEST + ".lock")
mock_logging.assert_called_with(
f"Creating {self.OPENSEARCH_MANIFEST}.lock")
@patch("os.path.exists", return_value=False)
@patch("argparse._sys.argv", [
"run_build.py", OPENSEARCH_MANIFEST, "--lock", "--architecture",
"arm64", "--platform", "windows", "--snapshot"
])
@patch("run_build.InputManifest.from_path",
return_value=InputManifest.from_path(OPENSEARCH_MANIFEST))
@patch("run_build.InputManifest.stable",
return_value=InputManifest.from_path(OPENSEARCH_MANIFEST_1_2))
@patch("run_build.InputManifest.to_file")
@patch("logging.info")
def test_main_manifest_new_lock_with_overrides(self, mock_logging,
mock_to_file, mock_stable,
*mocks):
with self.assertRaises(SystemExit):
main()
mock_stable.assert_called_with()
mock_to_file.assert_called_with(self.OPENSEARCH_MANIFEST + ".lock")
mock_logging.assert_called_with(
f"Creating {self.OPENSEARCH_MANIFEST}.lock")