def setUp(self) -> None:
self.data_path = os.path.realpath(
os.path.join(os.path.dirname(__file__), "data"))
self.manifest_filename = os.path.join(self.data_path,
"opensearch-build-1.1.0.yml")
self.manifest = BuildManifest.from_path(self.manifest_filename)
self.manifest_filename_distribution = os.path.join(
self.data_path, "opensearch-build-1.3.0.yml")
self.manifest_distribution = BuildManifest.from_path(
self.manifest_filename_distribution)
def setUp(self) -> None:
self.dists = Dists
self.manifest_tar = BuildManifest.from_path(
os.path.join(os.path.dirname(__file__),
"data/opensearch-build-linux-1.1.1.yml"))
self.manifest_zip = BuildManifest.from_path(
os.path.join(os.path.dirname(__file__),
"data/opensearch-build-windows-1.3.0.yml"))
self.manifest_rpm = BuildManifest.from_path(
os.path.join(os.path.dirname(__file__),
"data/opensearch-build-rpm-1.3.0.yml"))
def test_install_maven_dependencies_remote(self, mock_request, mock_copyfile, mock_makedirs):
counter = ThreadSafeCounter()
mock_request.side_effect = counter.thread_safe_count
dependency_installer = DependencyInstallerOpenSearch(
"https://ci.opensearch.org/x/y",
BuildManifest.from_path(self.BUILD_MANIFEST),
BundleManifest.from_path(self.DIST_MANIFEST_REMOTE)
)
dependency_installer.install_maven_dependencies()
self.assertEqual(counter.call_count, 2375)
mock_makedirs.assert_called_with(
os.path.realpath(
os.path.join(
dependency_installer.maven_local_path, "org", "opensearch", "notification"
)
),
exist_ok=True
)
mock_copyfile.assert_not_called()
mock_request.assert_has_calls([
call(
"https://ci.opensearch.org/x/y/builds/opensearch/maven/org/opensearch/notification/alerting-notification-1.2.0.0.jar",
os.path.realpath(os.path.join(dependency_installer.maven_local_path, "org", "opensearch", "notification", "alerting-notification-1.2.0.0.jar"))
)
])
def test_bundle_install_min(self) -> None:
manifest_path = os.path.join(
os.path.dirname(__file__),
"data/opensearch-dashboards-build-1.1.0.yml")
artifacts_path = os.path.join(os.path.dirname(__file__),
"data/artifacts")
bundle = BundleOpenSearchDashboards(
BuildManifest.from_path(manifest_path), artifacts_path,
MagicMock())
with patch("subprocess.check_call") as mock_check_call:
bundle.install_min()
self.assertEqual(mock_check_call.call_count, 1)
mock_check_call.assert_has_calls([
call(
" ".join([
"bash",
ScriptFinder.find_install_script(
"OpenSearch-Dashboards"),
"-v 1.1.0",
"-p linux",
"-a x64",
"-f",
artifacts_path,
"-o",
bundle.min_dist.archive_path,
]),
cwd=bundle.min_dist.archive_path,
shell=True,
),
])
def test_missing_component(self, mock_manifest: Mock,
find_build_root: Mock) -> None:
find_build_root.return_value = "url/linux/x64/builds/opensearch"
check = CiCheckManifestComponent(
InputComponentFromDist({
"name": "does-not-exist",
"dist": "url"
}),
CiTarget(version="1.1.0",
name="opensearch",
qualifier=None,
snapshot=True))
mock_manifest.from_url.return_value = BuildManifest.from_path(
self.BUILD_MANIFEST)
with self.assertRaises(
CiCheckManifestComponent.MissingComponentError) as ctx:
check.check()
self.assertEqual(
str(ctx.exception),
"Missing does-not-exist in url/linux/x64/builds/opensearch/manifest.yml."
)
find_build_root.assert_called()
def test_retrieves_manifests(self, mock_manifest: Mock,
find_build_root: Mock) -> None:
find_build_root.return_value = "url/linux/ARCH/builds/opensearch"
check = CiCheckManifestComponent(
InputComponentFromDist({
"name": "common-utils",
"dist": "url"
}),
CiTarget(version="1.1.0",
name="opensearch",
qualifier=None,
snapshot=True))
mock_manifest.from_url.return_value = BuildManifest.from_path(
self.BUILD_MANIFEST)
check.check()
mock_manifest.from_url.assert_has_calls([
call("url/linux/ARCH/builds/opensearch/manifest.yml"),
call("url/linux/ARCH/builds/opensearch/manifest.yml"),
])
find_build_root.assert_has_calls([
call("url", "linux", "x64", "opensearch"),
call("url", "linux", "arm64", "opensearch"),
])
def test_bundle_install_min_patch(self, mock_os_rename: Mock,
mock_check_call: Mock) -> None:
manifest_path = os.path.join(os.path.dirname(__file__), "data",
"opensearch-build-linux-1.1.1.yml")
artifacts_path = os.path.join(os.path.dirname(__file__), "data",
"artifacts")
bundle = BundleOpenSearch(BuildManifest.from_path(manifest_path),
artifacts_path, MagicMock())
bundle.install_min()
self.assertEqual(mock_check_call.call_count, 1)
mock_os_rename.assert_called_with(
os.path.join(bundle.tmp_dir.name, "opensearch-1.1.0"),
os.path.join(bundle.tmp_dir.name, "opensearch-1.1.1"))
mock_check_call.assert_has_calls([
call(
" ".join([
"bash",
ScriptFinder.find_install_script("OpenSearch"),
"-v 1.1.1-SNAPSHOT",
"-p linux",
"-a x64",
"-f",
artifacts_path,
"-o",
bundle.min_dist.archive_path,
]),
cwd=bundle.min_dist.archive_path,
shell=True,
),
])
def test_install_maven_dependencies_local(self, mock_request: Mock,
mock_copyfile: Mock,
mock_makedirs: Mock) -> None:
counter = ThreadSafeCounter()
mock_copyfile.side_effect = counter.thread_safe_count
dependency_installer = DependencyInstallerOpenSearch(
self.DATA, BuildManifest.from_path(self.BUILD_MANIFEST),
BundleManifest.from_path(self.DIST_MANIFEST_LOCAL))
dependency_installer.install_maven_dependencies()
mock_makedirs.assert_called_with(os.path.realpath(
os.path.join(dependency_installer.maven_local_path, "org",
"opensearch", "notification")),
exist_ok=True)
mock_request.assert_not_called()
self.assertEqual(counter.call_count, 2375)
mock_copyfile.assert_has_calls([
call(
os.path.join(self.DATA, "builds", "opensearch", "maven", "org",
"opensearch", "notification",
"alerting-notification-1.2.0.0.jar"),
os.path.realpath(
os.path.join(dependency_installer.maven_local_path, "org",
"opensearch", "notification",
"alerting-notification-1.2.0.0.jar")))
])
def test_versions(self) -> None:
self.assertTrue(len(BuildManifest.VERSIONS))
for version in BuildManifest.VERSIONS:
manifest = BuildManifest.from_path(
os.path.join(self.data_path, "build",
f"opensearch-build-schema-version-{version}.yml"))
self.assertEqual(version, manifest.version)
self.assertIsNotNone(any(manifest.components))
def test_select_none(self) -> None:
path = os.path.join(self.data_path, "build",
"opensearch-build-schema-version-1.2.yml")
manifest = BuildManifest.from_path(path)
with self.assertRaises(ValueError) as ctx:
self.assertEqual(len(list(manifest.components.select(focus="x"))),
0)
self.assertEqual(str(ctx.exception), "No components matched focus=x.")
def setUp(self) -> None:
self.maxDiff = None
manifest_path = os.path.join(os.path.dirname(__file__), "data",
"opensearch-build-linux-1.1.0.yml")
self.manifest = BuildManifest.from_path(manifest_path)
self.bundle_recorder = BundleRecorder(
self.manifest.build, "output_dir", "artifacts_dir",
BundleFileLocation("bundle_output_dir", "opensearch", "tar"))
manifest_distribution_path = os.path.join(
os.path.dirname(__file__), "data",
"opensearch-build-windows-1.3.0.yml")
self.manifest_distribution = BuildManifest.from_path(
manifest_distribution_path)
self.bundle_recorder_distribution = BundleRecorder(
self.manifest_distribution.build, "output_dir", "artifacts_dir",
BundleFileLocation("bundle_output_dir", "opensearch", "tar"))
def test_select_two_are_unknown(self) -> None:
path = os.path.join(self.data_path, "build",
"opensearch-build-schema-version-1.2.yml")
manifest = BuildManifest.from_path(path)
with self.assertRaises(ValueError) as ctx:
self.assertEqual(
len(list(manifest.components.select(focus=["x", "y"]))), 0)
self.assertEqual(str(ctx.exception), "Unknown components=x,y.")
def setUp(self) -> None:
manifest_path = os.path.join(os.path.dirname(__file__), "data", "opensearch-dashboards-build-1.1.0.yml")
self.manifest = BuildManifest.from_path(manifest_path)
self.bundle_recorder = BundleRecorder(
self.manifest.build,
"output_dir",
"artifacts_dir",
BundleFileLocation("bundle_output_dir", "opensearch-dashboards")
)
def setUp(self) -> None:
os.chdir(os.path.dirname(__file__))
self.bundle_manifest = BundleManifest.from_path(self.BUNDLE_MANIFEST)
self.build_manifest = BuildManifest.from_path(self.BUILD_MANIFEST)
self.test_manifest = TestManifest.from_path(self.TEST_MANIFEST)
self.work_dir = Path("test_dir")
self.test_recorder = MagicMock()
self.save_logs = MagicMock()
self.test_recorder.test_results_logs = self.save_logs
def setUp(self) -> None:
self.artifacts_path = os.path.join(os.path.dirname(__file__), "data/artifacts/dist/")
self.manifest = BuildManifest.from_path(os.path.join(os.path.dirname(__file__), "data/opensearch-build-rpm-1.3.0.yml"))
self.distTar = DistTar(
"OpenSearch",
self.artifacts_path + "opensearch-min-1.3.0-linux-x64.tar.gz",
"opensearch-1.3.0",
self.manifest.build
)
def setUp(self) -> None:
self.package_name = 'opensearch-1.3.0-1.x86_64.rpm'
self.artifacts_path = os.path.join(os.path.dirname(__file__),
"data/artifacts/dist")
self.package_path = os.path.join(self.artifacts_path,
self.package_name)
self.bundle_rpm = BundleRpm('opensearch', self.package_path,
'opensearch-1.3.0')
self.manifest_rpm = BuildManifest.from_path(
os.path.join(os.path.dirname(__file__),
"data/opensearch-build-rpm-1.3.0.yml"))
self.bundle_rpm_qualifier = BundleRpm('opensearch', self.package_path,
'opensearch-2.0.0-alpha1')
self.manifest_rpm_qualifier = BuildManifest.from_path(
os.path.join(os.path.dirname(__file__),
"data/opensearch-build-rpm-2.0.0-alpha1.yml"))
def test_select(self) -> None:
path = os.path.join(self.data_path, "build",
"opensearch-build-schema-version-1.2.yml")
manifest = BuildManifest.from_path(path)
self.assertEqual(
len(list(manifest.components.select(focus=["common-utils"]))), 1)
self.assertEqual(
len(
list(
manifest.components.select(
focus=["common-utils", "job-scheduler"]))), 2)
def test_bundle_install_min_no_patch(self, mock_os_rename: Mock,
mock_check_call: Mock) -> None:
manifest_path = os.path.join(os.path.dirname(__file__), "data",
"opensearch-build-linux-1.1.0.yml")
artifacts_path = os.path.join(os.path.dirname(__file__), "data",
"artifacts")
bundle = BundleOpenSearch(BuildManifest.from_path(manifest_path),
artifacts_path, MagicMock())
bundle.install_min()
mock_os_rename.assert_not_called()
mock_check_call.assert_called()
def test_bundle_does_not_exist_raises_error(self) -> None:
manifest_path = os.path.join(os.path.dirname(__file__),
"data/opensearch-build-linux-1.1.0.yml")
with self.assertRaises(FileNotFoundError) as ctx:
self.DummyBundle(
BuildManifest.from_path(manifest_path),
os.path.join(os.path.dirname(__file__), "data",
"does-not-exist"),
MagicMock(),
)
self.assertTrue(
"opensearch-min-1.1.0-linux-x64.tar.gz" in str(ctx.exception))
def test_bundle_install_components(self,
bundle_install_plugin: Mock) -> None:
manifest_path = os.path.join(os.path.dirname(__file__), "data",
"opensearch-build-linux-1.1.0.yml")
bundle = BundleOpenSearch(
BuildManifest.from_path(manifest_path),
os.path.join(os.path.dirname(__file__), "data", "artifacts"),
MagicMock(),
)
bundle.install_components()
self.assertEqual(bundle_install_plugin.call_count, 12)
def test_install_build_dependencies_remote(self, mock_request, mock_copyfile, mock_makedirs):
dependency_installer = DependencyInstallerOpenSearch(
"https://ci.opensearch.org/x/y", BuildManifest.from_path(self.BUILD_MANIFEST), BundleManifest.from_path(self.DIST_MANIFEST_REMOTE)
)
dependencies = dict({"opensearch-job-scheduler": "1.1.0.0"})
dependency_installer.install_build_dependencies(dependencies, os.path.dirname(__file__))
mock_makedirs.assert_called_with(os.path.dirname(__file__), exist_ok=True)
mock_copyfile.assert_not_called()
mock_request.assert_called_once_with(
"https://ci.opensearch.org/x/y/builds/opensearch/plugins/opensearch-job-scheduler-1.1.0.0.zip",
os.path.realpath(os.path.join(os.path.dirname(__file__), "opensearch-job-scheduler-1.1.0.0.zip")),
)
def test_check(self, mock_manifest_from_url: Mock, find_build_root: Mock):
mock_manifest_from_url.return_value = BuildManifest.from_path(
self.BUILD_MANIFEST)
component = InputComponentFromDist({
"name": "common-utils",
"dist": "url",
"checks": ["manifest:component"]
})
list = CiCheckListDist(
component,
CiTarget(version="1.1.0", name="opensearch", snapshot=True))
list.check()
mock_manifest_from_url.assert_called()
find_build_root.assert_called()
def test_install_build_dependencies_local(self, mock_request, mock_copyfile, mock_makedirs):
dependency_installer = DependencyInstallerOpenSearch(
self.DATA,
BuildManifest.from_path(self.BUILD_MANIFEST),
BundleManifest.from_path(self.DIST_MANIFEST_LOCAL)
)
dependencies = dict({"opensearch-job-scheduler": "1.1.0.0"})
dependency_installer.install_build_dependencies(dependencies, os.path.dirname(__file__))
mock_makedirs.assert_called_with(os.path.dirname(__file__), exist_ok=True)
mock_request.assert_not_called()
mock_copyfile.assert_called_once_with(
os.path.join(self.DATA, "builds", "opensearch", "plugins", "opensearch-job-scheduler-1.1.0.0.zip"),
os.path.realpath(os.path.join(os.path.dirname(__file__), "opensearch-job-scheduler-1.1.0.0.zip")),
)
def test_install_maven_dependencies_remote_failure(self, mock_request, mock_copyfile, mock_makedirs):
def mock_retrieve(source, dest):
raise HTTPError(url=source, hdrs={}, fp=None, msg="Not Found", code=404)
mock_request.side_effect = mock_retrieve
dependency_installer = DependencyInstallerOpenSearch(
"https://ci.opensearch.org/x/y",
BuildManifest.from_path(self.BUILD_MANIFEST),
BundleManifest.from_path(self.DIST_MANIFEST_REMOTE)
)
with self.assertRaises(HTTPError) as ctx:
dependency_installer.install_maven_dependencies()
self.assertEqual(str(ctx.exception), "HTTP Error 404: Not Found")
def test_execute_with_missing_job_scheduler(
self, mock_test_recorder, mock_install_build_dependencies, *mock):
invalid_build_manifest = BuildManifest.from_path(
"data/build_manifest_missing_components.yml")
test_config, component = self.__get_test_config_and_bundle_component(
"index-management")
dependency_installer = MagicMock()
integ_test_suite = IntegTestSuiteOpenSearch(
dependency_installer, component, test_config, self.bundle_manifest,
invalid_build_manifest, "tmpdir", mock_test_recorder)
with self.assertRaises(KeyError) as ctx:
integ_test_suite.execute_tests()
self.assertEqual(str(ctx.exception), "'job-scheduler'")
dependency_installer.install_build_dependencies.assert_not_called()
def test_bundle_opensearch(self) -> None:
manifest_path = os.path.join(os.path.dirname(__file__), "data",
"opensearch-build-linux-1.1.0.yml")
artifacts_path = os.path.join(os.path.dirname(__file__), "data",
"artifacts")
bundle = BundleOpenSearch(BuildManifest.from_path(manifest_path),
artifacts_path, MagicMock())
self.assertEqual(bundle.min_dist.name, "OpenSearch")
self.assertEqual(len(bundle.components), 15)
self.assertEqual(bundle.artifacts_dir, artifacts_path)
self.assertIsNotNone(bundle.bundle_recorder)
self.assertEqual(bundle.installed_plugins, [])
self.assertTrue(
bundle.min_dist.path.endswith(
"opensearch-min-1.1.0-linux-x64.tar.gz"))
self.assertIsNotNone(bundle.min_dist.archive_path)
def test_bundle(self, dist_extract: Mock) -> None:
manifest_path = os.path.join(os.path.dirname(__file__),
"data/opensearch-build-linux-1.1.0.yml")
artifacts_path = os.path.join(os.path.dirname(__file__), "data",
"artifacts")
bundle = self.DummyBundle(BuildManifest.from_path(manifest_path),
artifacts_path, MagicMock())
self.assertEqual(bundle.min_dist.name, "OpenSearch")
self.assertEqual(len(bundle.plugins), 12)
self.assertEqual(bundle.artifacts_dir, artifacts_path)
self.assertIsNotNone(bundle.bundle_recorder)
self.assertEqual(bundle.installed_plugins, [])
self.assertTrue(
bundle.min_dist.path.endswith(
"opensearch-min-1.1.0-linux-x64.tar.gz"))
dist_extract.assert_called_once()
def test_bundle_install_plugin(self, path_isfile: Mock) -> None:
manifest_path = os.path.join(
os.path.dirname(__file__),
"data/opensearch-dashboards-build-1.1.0.yml")
artifacts_path = os.path.join(os.path.dirname(__file__), "data",
"artifacts")
bundle = BundleOpenSearchDashboards(
BuildManifest.from_path(manifest_path), artifacts_path,
MagicMock())
plugin = bundle.components['alertingDashboards']
with patch("shutil.copyfile") as mock_copyfile:
with patch("subprocess.check_call") as mock_check_call:
bundle.install_plugin(plugin)
self.assertEqual(mock_copyfile.call_count, 1)
self.assertEqual(mock_check_call.call_count, 2)
install_plugin_bin = os.path.join(
bundle.min_dist.archive_path, "bin",
"opensearch-dashboards-plugin")
mock_check_call.assert_has_calls([
call(
f'{install_plugin_bin} --allow-root install file:{os.path.join(bundle.tmp_dir.name, "alertingDashboards-1.1.0.zip")}',
cwd=bundle.min_dist.archive_path,
shell=True,
),
call(
" ".join([
"bash",
ScriptFinder.find_install_script(
"alertingDashboards"),
"-v 1.1.0",
"-p linux",
"-a x64",
"-f",
artifacts_path,
"-o",
bundle.min_dist.archive_path,
]),
cwd=bundle.min_dist.archive_path,
shell=True,
),
])
self.assertEqual(path_isfile.call_count, 2)
def test_bundle_install_plugin(self, path_isfile: Mock) -> None:
manifest_path = os.path.join(os.path.dirname(__file__), "data",
"opensearch-build-linux-1.1.0.yml")
artifacts_path = os.path.join(os.path.dirname(__file__), "data",
"artifacts")
bundle = BundleOpenSearch(BuildManifest.from_path(manifest_path),
artifacts_path, MagicMock())
plugin = bundle.components['job-scheduler']
with patch("shutil.copyfile") as mock_copyfile:
with patch("subprocess.check_call") as mock_check_call:
bundle.install_plugin(plugin)
self.assertEqual(mock_copyfile.call_count, 1)
self.assertEqual(mock_check_call.call_count, 2)
script = "opensearch-plugin.bat" if current_platform(
) == "windows" else "opensearch-plugin"
install_plugin_bin = os.path.join(bundle.min_dist.archive_path,
"bin", script)
mock_check_call.assert_has_calls([
call(
f'{install_plugin_bin} install --batch file:{os.path.join(bundle.tmp_dir.name, "opensearch-job-scheduler-1.1.0.0.zip")}',
cwd=bundle.min_dist.archive_path,
shell=True,
),
call(
" ".join([
"bash",
ScriptFinder.find_install_script(
"opensearch-job-scheduler"),
"-v 1.1.0",
"-p linux",
"-a x64",
"-f",
artifacts_path,
"-o",
bundle.min_dist.archive_path,
]),
cwd=bundle.min_dist.archive_path,
shell=True,
),
])
self.assertEqual(path_isfile.call_count, 2)
def test_bundle_distribution(self, dist_extract: Mock) -> None:
manifest_path = os.path.join(
os.path.dirname(__file__),
"data/opensearch-build-windows-1.3.0.yml")
artifacts_path = os.path.join(os.path.dirname(__file__), "data",
"artifacts")
bundle = self.DummyBundle(BuildManifest.from_path(manifest_path),
artifacts_path, MagicMock())
self.assertEqual(bundle.min_dist.name, "OpenSearch")
self.assertEqual(bundle.min_dist.__class__.__name__, "DistZip")
self.assertEqual(bundle.build.distribution, "zip")
self.assertEqual(bundle.build.platform, "windows")
self.assertEqual(bundle.artifacts_dir, artifacts_path)
self.assertIsNotNone(bundle.bundle_recorder)
self.assertEqual(bundle.installed_plugins, [])
self.assertTrue(
bundle.min_dist.path.endswith(
"opensearch-min-1.3.0-windows-x64.zip"))
dist_extract.assert_called_once()