def test_eq(self) -> None: path = os.path.join(self.manifests_path, "1.0.0", "opensearch-1.0.0.yml") manifest1 = InputManifest.from_path(path) manifest2 = InputManifest.from_path(path) self.assertEqual(manifest1, manifest1) self.assertEqual(manifest1, manifest2)
def test_to_file_formatted(self) -> None: data_path = os.path.join(os.path.dirname(__file__), "data") manifest = InputManifest({ "schema-version": "1.0", "build": { "name": "OpenSearch", "version": "2.0.0" }, "ci": { "image": { "name": "image-name", "args": "-e JAVA_HOME=/opt/java/openjdk-11" } }, "components": [{ "name": "OpenSearch", "ref": "main", "repository": "https://github.com/opensearch-project/OpenSearch.git", "checks": ["gradle:publish", "gradle:properties:version"] }] }) with TemporaryDirectory() as path: output_path = os.path.join(path.name, "manifest.yml") manifest.to_file(output_path) with open(output_path) as f: written_manifest = f.read() with open(os.path.join(data_path, "formatted.yml")) as f: formatted_manifest = f.read() self.assertEqual(formatted_manifest, written_manifest)
def test_neq(self) -> None: path1 = os.path.join(self.manifests_path, "1.0.0", "opensearch-1.0.0.yml") path2 = os.path.join(self.manifests_path, "1.1.0", "opensearch-1.1.0.yml") manifest1 = InputManifest.from_path(path1) manifest2 = InputManifest.from_path(path2) self.assertNotEqual(manifest1, manifest2)
def test_invalid_ref(self) -> None: data_path = os.path.join(os.path.dirname(__file__), "data") manifest_path = os.path.join(data_path, "invalid-ref.yml") with self.assertRaises(Exception) as context: InputManifest.from_path(manifest_path) self.assertTrue( str(context.exception).startswith( "Invalid manifest schema: {'components': "))
def main(): args = BuildArgs() console.configure(level=args.logging_level) manifest = InputManifest.from_file(args.manifest) if args.ref_manifest: manifest = manifest.stable() if os.path.exists(args.ref_manifest): if manifest == InputManifest.from_path(args.ref_manifest): logging.info(f"No changes since {args.ref_manifest}") else: logging.info(f"Updating {args.ref_manifest}") manifest.to_file(args.ref_manifest) else: logging.info(f"Creating {args.ref_manifest}") manifest.to_file(args.ref_manifest) exit(0) output_dir = BuildOutputDir(manifest.build.filename).dir with TemporaryDirectory(keep=args.keep, chdir=True) as work_dir: logging.info(f"Building in {work_dir.name}") target = BuildTarget( name=manifest.build.name, version=manifest.build.version, patches=manifest.build.patches, snapshot=args.snapshot if args.snapshot is not None else manifest.build.snapshot, output_dir=output_dir, platform=args.platform or manifest.build.platform, architecture=args.architecture or manifest.build.architecture, ) build_recorder = BuildRecorder(target) logging.info(f"Building {manifest.build.name} ({target.architecture}) into {target.output_dir}") for component in manifest.components.select(focus=args.component, platform=target.platform): logging.info(f"Building {component.name}") builder = Builders.builder_from(component, target) try: builder.checkout(work_dir.name) builder.build(build_recorder) builder.export_artifacts(build_recorder) except: logging.error(f"Error building {component.name}, retry with: {args.component_command(component.name)}") raise build_recorder.write_manifest() logging.info("Done.")
def test_1_1(self) -> None: path = os.path.join(self.manifests_path, "1.1.0", "opensearch-1.1.0.yml") manifest = InputManifest.from_path(path) self.assertEqual(manifest.version, "1.0") self.assertEqual(manifest.build.name, "OpenSearch") self.assertEqual(manifest.build.filename, "opensearch") self.assertEqual(manifest.build.version, "1.1.0") self.assertEqual( len(list(manifest.components.select(focus=["common-utils"]))), 1) # opensearch component opensearch_component: InputComponentFromSource = manifest.components[ "OpenSearch"] # type: ignore[assignment] self.assertEqual(opensearch_component.name, "OpenSearch") self.assertEqual( opensearch_component.repository, "https://github.com/opensearch-project/OpenSearch.git", ) self.assertEqual(opensearch_component.ref, "tags/1.1.0") # components for component in manifest.components.values(): self.assertIsInstance(component, InputComponentFromSource) # alerting component checks alerting_component: InputComponentFromSource = manifest.components[ "alerting"] # type: ignore[assignment] self.assertIsNotNone(alerting_component) self.assertEqual(len(alerting_component.checks), 2) for check in alerting_component.checks: self.assertIsInstance(check, Check) self.assertIsNone(alerting_component.checks[0].args) self.assertEqual(alerting_component.checks[1].args, "alerting")
def create_manifest(self, version: str, components: List = []) -> InputManifest: image_map = { "opensearch": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2", "opensearch-dashboards": "opensearchstaging/ci-runner:ci-runner-centos7-opensearch-dashboards-build-v2" } data: Dict = { "schema-version": "1.0", "build": { "name": self.name, "version": version }, "ci": { "image": { "name": image_map[self.prefix] } }, "components": [], } for component in components: logging.info(f" Adding {component.name}") data["components"].append(component.to_dict()) return InputManifest(data)
def test_1_1_1_dist(self) -> None: data_path = os.path.realpath( os.path.join(os.path.dirname(__file__), "data")) path = os.path.join(data_path, "opensearch-dashboards-from-dist-1.1.1.yml") manifest = InputManifest.from_path(path) self.assertEqual(manifest.version, "1.0") self.assertEqual(manifest.build.name, "OpenSearch Dashboards") self.assertEqual(manifest.build.filename, "opensearch-dashboards") self.assertEqual(manifest.build.version, "1.1.1") self.assertEqual( len(list( manifest.components.select(focus=["alertingDashboards"]))), 1) opensearch_component: InputComponentFromDist = manifest.components[ "OpenSearch-Dashboards"] # type: ignore[assignment] self.assertIsInstance(opensearch_component, InputComponentFromDist) self.assertEqual(opensearch_component.name, "OpenSearch-Dashboards") self.assertEqual( opensearch_component.dist, "https://ci.opensearch.org/ci/dbc/bundle-build-dashboards/1.1.0/20210930", ) for component in manifest.components.values(): if component.name in [ "reportsDashboards", "functionalTestDashboards" ]: self.assertIsInstance(component, InputComponentFromSource) else: self.assertIsInstance(component, InputComponentFromDist)
def test_to_dict(self) -> None: path = os.path.join(self.manifests_path, "1.1.0", "opensearch-1.1.0.yml") manifest = InputManifest.from_path(path) data = manifest.to_dict() with open(path) as f: self.assertEqual(yaml.safe_load(f), data)
def test_stable(self, mock_output: Mock) -> None: mock_output.return_value.decode.return_value = "updated\tHEAD" path = os.path.join(self.manifests_path, "1.1.0", "opensearch-1.1.0.yml") manifest = InputManifest.from_path(path).stable() opensearch: InputComponentFromSource = manifest.components[ "OpenSearch"] # type: ignore[assignment] self.assertEqual(opensearch.ref, "updated")
def test_neq_update(self) -> None: path = os.path.join(self.manifests_path, "1.0.0", "opensearch-1.0.0.yml") manifest1 = InputManifest.from_path(path) manifest2 = copy.deepcopy(manifest1) self.assertEqual(manifest1, manifest2) manifest2.components["name"] = InputComponentFromDist({ "name": "name", "dist": "dist" }) self.assertNotEqual(manifest1, manifest2)
def test_select(self) -> None: path = os.path.join(self.manifests_path, "1.1.0", "opensearch-1.1.0.yml") manifest = InputManifest.from_path(path) self.assertEqual( len(list(manifest.components.select(focus="common-utils"))), 1) self.assertNotEqual( len(list(manifest.components.select(platform="windows"))), 0) self.assertEqual( len( list(manifest.components.select(focus="k-NN", platform="linux"))), 1)
def test_select_none(self) -> None: path = os.path.join(self.manifests_path, "1.1.0", "opensearch-1.1.0.yml") manifest = InputManifest.from_path(path) with self.assertRaises(ValueError) as ctx: self.assertEqual( len( list( manifest.components.select(focus=["k-NN"], platform="windows"))), 0) self.assertEqual( str(ctx.exception), "No components matched focus=k-NN, platform=windows.")
def main(): args = CheckoutArgs() console.configure(level=args.logging_level) manifest = InputManifest.from_file(args.manifest) with TemporaryDirectory(keep=True, chdir=True) as work_dir: logging.info(f"Checking out into {work_dir.name}") for component in manifest.components.select(): logging.info(f"Checking out {component.name}") with GitRepository( component.repository, component.ref, os.path.join(work_dir.name, component.name), component.working_directory, ) as repo: logging.debug(f"Checked out {component.name} into {repo.dir}") logging.info(f"Done, checked out into {work_dir.name}.")
def create_manifest(self, version, components=[]): data = { "schema-version": "1.0", "build": { "name": self.name, "version": version }, "ci": { "image": { "name": "opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028" } }, "components": [], } for component in components: logging.info(f" Adding {component.name}") data["components"].append(component.to_dict()) return InputManifest(data)
def test_1_0(self) -> None: path = os.path.join(self.manifests_path, "1.0.0", "opensearch-1.0.0.yml") manifest = InputManifest.from_path(path) self.assertEqual(manifest.version, "1.0") self.assertEqual(manifest.build.name, "OpenSearch") self.assertEqual(manifest.build.filename, "opensearch") self.assertEqual(manifest.build.version, "1.0.0") self.assertEqual( len(list(manifest.components.select(focus=["common-utils"]))), 1) opensearch_component: InputComponentFromSource = manifest.components[ "OpenSearch"] # type: ignore[assignment] self.assertIsInstance(opensearch_component, InputComponentFromSource) self.assertEqual(opensearch_component.name, "OpenSearch") self.assertEqual( opensearch_component.repository, "https://github.com/opensearch-project/OpenSearch.git", ) self.assertEqual(opensearch_component.ref, "tags/1.0.0") for component in manifest.components.values(): self.assertIsInstance(component, InputComponentFromSource)
def test_1_2(self) -> None: data_path = os.path.realpath( os.path.join(os.path.dirname(__file__), "data")) path = os.path.join(data_path, "opensearch-1.2.0.yml") manifest = InputManifest.from_path(path) self.assertEqual(manifest.version, "1.0") self.assertEqual(manifest.build.name, "OpenSearch") self.assertEqual(manifest.build.filename, "opensearch") self.assertEqual(manifest.build.version, "1.2.0") self.assertEqual( manifest.ci.image.name, "opensearchstaging/ci-runner:centos7-x64-arm64-jdkmulti-node10.24.1-cypress6.9.1-20211028" ) self.assertEqual(manifest.ci.image.args, "-e JAVA_HOME=/usr/lib/jvm/adoptopenjdk-14-hotspot") self.assertNotEqual(len(manifest.components), 0) self.assertEqual( len(list(manifest.components.select(focus=["common-utils"]))), 1) # opensearch component opensearch_component: InputComponentFromSource = manifest.components[ "OpenSearch"] # type: ignore[assignment] self.assertEqual(opensearch_component.name, "OpenSearch") self.assertEqual( opensearch_component.repository, "https://github.com/opensearch-project/OpenSearch.git", ) self.assertEqual(opensearch_component.ref, "tags/1.2.0") # components for component in manifest.components.values(): self.assertIsInstance(component, InputComponentFromSource) # alerting component checks alerting_component = manifest.components["alerting"] self.assertIsNotNone(alerting_component) self.assertEqual(len(alerting_component.checks), 2) for check in alerting_component.checks: self.assertIsInstance(check, Check) self.assertIsNone(alerting_component.checks[0].args) self.assertEqual(alerting_component.checks[1].args, "alerting")
def __init__(self, file, args): super().__init__(InputManifest.from_file(file), args)
def __init__(self, file: TextIOWrapper, args: CiArgs) -> None: super().__init__(InputManifest.from_file(file), args)
class TestRunBuild(unittest.TestCase): @pytest.fixture(autouse=True) def capfd(self, capfd): self.capfd = capfd @patch("argparse._sys.argv", ["run_build.py", "--help"]) def test_usage(self): with self.assertRaises(SystemExit): main() out, _ = self.capfd.readouterr() self.assertTrue(out.startswith("usage:")) MANIFESTS = os.path.join( os.path.dirname(__file__), "..", "manifests", ) OPENSEARCH_MANIFEST = os.path.realpath( os.path.join(MANIFESTS, "1.1.0", "opensearch-1.1.0.yml")) OPENSEARCH_MANIFEST_1_2 = os.path.realpath( os.path.join(MANIFESTS, "1.2.0", "opensearch-1.2.0.yml")) @patch("argparse._sys.argv", ["run_build.py", OPENSEARCH_MANIFEST, "-p", "linux"]) @patch("run_build.Builders.builder_from", return_value=MagicMock()) @patch("run_build.BuildRecorder", return_value=MagicMock()) @patch("run_build.TemporaryDirectory") def test_main_platform_linux(self, mock_temp, mock_recorder, mock_builder, *mocks): mock_temp.return_value.__enter__.return_value.name = tempfile.gettempdir( ) main() self.assertNotEqual(mock_builder.return_value.build.call_count, 0) self.assertEqual(mock_builder.return_value.build.call_count, mock_builder.return_value.export_artifacts.call_count) mock_recorder.return_value.write_manifest.assert_called() @patch("argparse._sys.argv", ["run_build.py", OPENSEARCH_MANIFEST, "-p", "darwin"]) @patch("run_build.Builders.builder_from", return_value=MagicMock()) @patch("run_build.BuildRecorder", return_value=MagicMock()) @patch("run_build.TemporaryDirectory") def test_main_platform_darwin(self, mock_temp, mock_recorder, mock_builder, *mocks): mock_temp.return_value.__enter__.return_value.name = tempfile.gettempdir( ) main() self.assertNotEqual(mock_builder.return_value.build.call_count, 0) self.assertEqual(mock_builder.return_value.build.call_count, mock_builder.return_value.export_artifacts.call_count) mock_recorder.return_value.write_manifest.assert_called() @patch("argparse._sys.argv", ["run_build.py", OPENSEARCH_MANIFEST, "-p", "windows"]) @patch("run_build.Builders.builder_from", return_value=MagicMock()) @patch("run_build.BuildRecorder", return_value=MagicMock()) @patch("run_build.TemporaryDirectory") def test_main_platform_windows(self, mock_temp, mock_recorder, mock_builder, *mocks): mock_temp.return_value.__enter__.return_value.name = tempfile.gettempdir( ) main() # excludes performance analyzer and k-nn for call_args in mock_builder.call_args: if len(call_args) > 1: component = call_args[0] self.assertNotIn("k-nn", component.name.lower()) self.assertNotIn("analyzer", component.name.lower()) self.assertNotEqual(mock_builder.call_count, 0) self.assertEqual(mock_builder.return_value.build.call_count, mock_builder.call_count) self.assertEqual(mock_builder.return_value.export_artifacts.call_count, mock_builder.call_count) mock_recorder.return_value.write_manifest.assert_called() OPENSEARCH_DASHBOARDS_MANIFEST = os.path.realpath( os.path.join( os.path.dirname(__file__), "..", "manifests", "1.1.0", "opensearch-dashboards-1.1.0.yml", )) @patch("argparse._sys.argv", ["run_build.py", OPENSEARCH_DASHBOARDS_MANIFEST, "-a", "x64"]) @patch("run_build.Builders.builder_from", return_value=MagicMock()) @patch("run_build.BuildRecorder", return_value=MagicMock()) @patch("run_build.TemporaryDirectory") def test_main_with_architecture(self, mock_temp, mock_recorder, mock_builder, *mocks): mock_temp.return_value.__enter__.return_value.name = tempfile.gettempdir( ) main() self.assertEqual(mock_builder.return_value.build.call_count, 11) self.assertEqual(mock_builder.return_value.export_artifacts.call_count, 11) mock_recorder.return_value.write_manifest.assert_called() @patch("argparse._sys.argv", [ "run_build.py", OPENSEARCH_DASHBOARDS_MANIFEST, "-p", "invalidplatform", "-a", "x64" ]) def test_main_with_invalid_platform(self, *mocks): with self.assertRaises(SystemExit): main() @patch("argparse._sys.argv", [ "run_build.py", OPENSEARCH_DASHBOARDS_MANIFEST, "-p", "linux", "-a", "invalidarchitecture" ]) def test_main_with_invalid_architecture(self, *mocks): with self.assertRaises(SystemExit): main() @patch("os.path.exists", return_value=True) @patch("argparse._sys.argv", ["run_build.py", OPENSEARCH_MANIFEST, "--lock"]) @patch("run_build.InputManifest.from_path", return_value=InputManifest.from_path(OPENSEARCH_MANIFEST)) @patch("run_build.InputManifest.stable", return_value=InputManifest.from_path(OPENSEARCH_MANIFEST)) @patch("run_build.InputManifest.to_file") @patch("logging.info") def test_main_manifest_lock_without_changes(self, mock_logging, mock_to_file, mock_stable, *mocks): with self.assertRaises(SystemExit): main() mock_stable.assert_called_with() mock_to_file.assert_not_called() mock_logging.assert_called_with( f"No changes since {self.OPENSEARCH_MANIFEST}.lock") @patch("os.path.exists", return_value=True) @patch("argparse._sys.argv", ["run_build.py", OPENSEARCH_MANIFEST, "--lock"]) @patch("run_build.InputManifest.from_path", return_value=InputManifest.from_path(OPENSEARCH_MANIFEST)) @patch("run_build.InputManifest.stable", return_value=InputManifest.from_path(OPENSEARCH_MANIFEST_1_2)) @patch("run_build.InputManifest.to_file") @patch("logging.info") def test_main_manifest_lock_with_changes(self, mock_logging, mock_to_file, mock_stable, *mocks): with self.assertRaises(SystemExit): main() mock_stable.assert_called_with() mock_to_file.assert_called_with(self.OPENSEARCH_MANIFEST + ".lock") mock_logging.assert_called_with( f"Updating {self.OPENSEARCH_MANIFEST}.lock") @patch("os.path.exists", return_value=False) @patch("argparse._sys.argv", ["run_build.py", OPENSEARCH_MANIFEST, "--lock"]) @patch("run_build.InputManifest.from_path", return_value=InputManifest.from_path(OPENSEARCH_MANIFEST)) @patch("run_build.InputManifest.stable", return_value=InputManifest.from_path(OPENSEARCH_MANIFEST_1_2)) @patch("run_build.InputManifest.to_file") @patch("logging.info") def test_main_manifest_new_lock(self, mock_logging, mock_to_file, mock_stable, *mocks): with self.assertRaises(SystemExit): main() mock_stable.assert_called_with() mock_to_file.assert_called_with(self.OPENSEARCH_MANIFEST + ".lock") mock_logging.assert_called_with( f"Creating {self.OPENSEARCH_MANIFEST}.lock") @patch("os.path.exists", return_value=False) @patch("argparse._sys.argv", [ "run_build.py", OPENSEARCH_MANIFEST, "--lock", "--architecture", "arm64", "--platform", "windows", "--snapshot" ]) @patch("run_build.InputManifest.from_path", return_value=InputManifest.from_path(OPENSEARCH_MANIFEST)) @patch("run_build.InputManifest.stable", return_value=InputManifest.from_path(OPENSEARCH_MANIFEST_1_2)) @patch("run_build.InputManifest.to_file") @patch("logging.info") def test_main_manifest_new_lock_with_overrides(self, mock_logging, mock_to_file, mock_stable, *mocks): with self.assertRaises(SystemExit): main() mock_stable.assert_called_with() mock_to_file.assert_called_with(self.OPENSEARCH_MANIFEST + ".lock") mock_logging.assert_called_with( f"Creating {self.OPENSEARCH_MANIFEST}.lock")