def setUp(self) -> None:
self.data_path = os.path.realpath(
os.path.join(os.path.dirname(__file__), "data"))
self.manifest_filename = os.path.join(self.data_path,
"opensearch-build-1.1.0.yml")
self.manifest = BuildManifest.from_path(self.manifest_filename)
self.manifest_filename_distribution = os.path.join(
self.data_path, "opensearch-build-1.3.0.yml")
self.manifest_distribution = BuildManifest.from_path(
self.manifest_filename_distribution)
def setUp(self) -> None:
self.dists = Dists
self.manifest_tar = BuildManifest.from_path(
os.path.join(os.path.dirname(__file__),
"data/opensearch-build-linux-1.1.1.yml"))
self.manifest_zip = BuildManifest.from_path(
os.path.join(os.path.dirname(__file__),
"data/opensearch-build-windows-1.3.0.yml"))
self.manifest_rpm = BuildManifest.from_path(
os.path.join(os.path.dirname(__file__),
"data/opensearch-build-rpm-1.3.0.yml"))
def test_bundle_install_min_patch(self, mock_os_rename: Mock,
mock_check_call: Mock) -> None:
manifest_path = os.path.join(os.path.dirname(__file__), "data",
"opensearch-build-linux-1.1.1.yml")
artifacts_path = os.path.join(os.path.dirname(__file__), "data",
"artifacts")
bundle = BundleOpenSearch(BuildManifest.from_path(manifest_path),
artifacts_path, MagicMock())
bundle.install_min()
self.assertEqual(mock_check_call.call_count, 1)
mock_os_rename.assert_called_with(
os.path.join(bundle.tmp_dir.name, "opensearch-1.1.0"),
os.path.join(bundle.tmp_dir.name, "opensearch-1.1.1"))
mock_check_call.assert_has_calls([
call(
" ".join([
"bash",
ScriptFinder.find_install_script("OpenSearch"),
"-v 1.1.1-SNAPSHOT",
"-p linux",
"-a x64",
"-f",
artifacts_path,
"-o",
bundle.min_dist.archive_path,
]),
cwd=bundle.min_dist.archive_path,
shell=True,
),
])
def test_missing_component(self, mock_manifest: Mock,
find_build_root: Mock) -> None:
find_build_root.return_value = "url/linux/x64/builds/opensearch"
check = CiCheckManifestComponent(
InputComponentFromDist({
"name": "does-not-exist",
"dist": "url"
}),
CiTarget(version="1.1.0",
name="opensearch",
qualifier=None,
snapshot=True))
mock_manifest.from_url.return_value = BuildManifest.from_path(
self.BUILD_MANIFEST)
with self.assertRaises(
CiCheckManifestComponent.MissingComponentError) as ctx:
check.check()
self.assertEqual(
str(ctx.exception),
"Missing does-not-exist in url/linux/x64/builds/opensearch/manifest.yml."
)
find_build_root.assert_called()
def test_bundle_install_min(self) -> None:
manifest_path = os.path.join(
os.path.dirname(__file__),
"data/opensearch-dashboards-build-1.1.0.yml")
artifacts_path = os.path.join(os.path.dirname(__file__),
"data/artifacts")
bundle = BundleOpenSearchDashboards(
BuildManifest.from_path(manifest_path), artifacts_path,
MagicMock())
with patch("subprocess.check_call") as mock_check_call:
bundle.install_min()
self.assertEqual(mock_check_call.call_count, 1)
mock_check_call.assert_has_calls([
call(
" ".join([
"bash",
ScriptFinder.find_install_script(
"OpenSearch-Dashboards"),
"-v 1.1.0",
"-p linux",
"-a x64",
"-f",
artifacts_path,
"-o",
bundle.min_dist.archive_path,
]),
cwd=bundle.min_dist.archive_path,
shell=True,
),
])
def test_install_maven_dependencies_local(self, mock_request: Mock,
mock_copyfile: Mock,
mock_makedirs: Mock) -> None:
counter = ThreadSafeCounter()
mock_copyfile.side_effect = counter.thread_safe_count
dependency_installer = DependencyInstallerOpenSearch(
self.DATA, BuildManifest.from_path(self.BUILD_MANIFEST),
BundleManifest.from_path(self.DIST_MANIFEST_LOCAL))
dependency_installer.install_maven_dependencies()
mock_makedirs.assert_called_with(os.path.realpath(
os.path.join(dependency_installer.maven_local_path, "org",
"opensearch", "notification")),
exist_ok=True)
mock_request.assert_not_called()
self.assertEqual(counter.call_count, 2375)
mock_copyfile.assert_has_calls([
call(
os.path.join(self.DATA, "builds", "opensearch", "maven", "org",
"opensearch", "notification",
"alerting-notification-1.2.0.0.jar"),
os.path.realpath(
os.path.join(dependency_installer.maven_local_path, "org",
"opensearch", "notification",
"alerting-notification-1.2.0.0.jar")))
])
def main() -> int:
args = AssembleArgs()
console.configure(level=args.logging_level)
build_manifest = BuildManifest.from_file(args.manifest)
build = build_manifest.build
artifacts_dir = os.path.dirname(os.path.realpath(args.manifest.name))
output_dir = AssembleOutputDir(build.filename).dir
logging.info(f"Bundling {build.name} ({build.architecture}) on {build.platform} into {output_dir} ...")
bundle_recorder = BundleRecorder(
build,
output_dir,
artifacts_dir,
BundleLocations.from_path(args.base_url, os.getcwd(), build.filename)
)
with Bundles.create(build_manifest, artifacts_dir, bundle_recorder, args.keep) as bundle:
bundle.install_min()
bundle.install_plugins()
logging.info(f"Installed plugins: {bundle.installed_plugins}")
# Save a copy of the manifest inside of the tar
bundle_recorder.write_manifest(bundle.min_dist.archive_path)
bundle.package(output_dir)
bundle_recorder.write_manifest(output_dir)
logging.info("Done.")
return 0
def test_retrieves_manifests(self, mock_manifest: Mock,
find_build_root: Mock) -> None:
find_build_root.return_value = "url/linux/ARCH/builds/opensearch"
check = CiCheckManifestComponent(
InputComponentFromDist({
"name": "common-utils",
"dist": "url"
}),
CiTarget(version="1.1.0",
name="opensearch",
qualifier=None,
snapshot=True))
mock_manifest.from_url.return_value = BuildManifest.from_path(
self.BUILD_MANIFEST)
check.check()
mock_manifest.from_url.assert_has_calls([
call("url/linux/ARCH/builds/opensearch/manifest.yml"),
call("url/linux/ARCH/builds/opensearch/manifest.yml"),
])
find_build_root.assert_has_calls([
call("url", "linux", "x64", "opensearch"),
call("url", "linux", "arm64", "opensearch"),
])
def test_install_maven_dependencies_remote(self, mock_request, mock_copyfile, mock_makedirs):
counter = ThreadSafeCounter()
mock_request.side_effect = counter.thread_safe_count
dependency_installer = DependencyInstallerOpenSearch(
"https://ci.opensearch.org/x/y",
BuildManifest.from_path(self.BUILD_MANIFEST),
BundleManifest.from_path(self.DIST_MANIFEST_REMOTE)
)
dependency_installer.install_maven_dependencies()
self.assertEqual(counter.call_count, 2375)
mock_makedirs.assert_called_with(
os.path.realpath(
os.path.join(
dependency_installer.maven_local_path, "org", "opensearch", "notification"
)
),
exist_ok=True
)
mock_copyfile.assert_not_called()
mock_request.assert_has_calls([
call(
"https://ci.opensearch.org/x/y/builds/opensearch/maven/org/opensearch/notification/alerting-notification-1.2.0.0.jar",
os.path.realpath(os.path.join(dependency_installer.maven_local_path, "org", "opensearch", "notification", "alerting-notification-1.2.0.0.jar"))
)
])
def __init__(self, path: str, build_dir: str, bundle_dir: str) -> None:
self.path = path
self.build_dir = build_dir
self.bundle_dir = bundle_dir
self.bundle_manifest = BundleManifest.from_urlpath("/".join([self.path.rstrip("/"), self.bundle_dir]))
self.build_manifest = BuildManifest.from_urlpath("/".join([self.path.rstrip("/"), self.build_dir]))
def test_legacy_from_url(self, urlopen: Mock) -> None:
# Fake the dashboard file to look like its coming from the url, but its really in the data directory
cm = MagicMock()
self.manifest_filename = os.path.join(
self.data_path, "opensearch-dashboards-build-1.1.0.yml")
cm.read.return_value.decode.return_value = open(
self.manifest_filename, 'r').read()
cm.__enter__.return_value = cm
urlopen.return_value = cm
self.manifest = BuildManifest.from_url('http://fakeurl')
self.assertEqual(self.manifest.version, "1.1")
self.assertEqual(self.manifest.build.name, "OpenSearch Dashboards")
self.assertEqual(self.manifest.build.filename, "opensearch-dashboards")
self.assertEqual(self.manifest.build.version, "1.1.0")
self.assertEqual(len(self.manifest.components), 10)
component = self.manifest.components['OpenSearch-Dashboards']
self.assertEqual(component.commit_id,
"44d2cb5b4f9a7c641c1fef32ec569bc48ec46979")
self.assertEqual(component.name, 'OpenSearch-Dashboards')
self.assertEqual(component.ref, "1.1")
self.assertEqual(
component.repository,
'https://github.com/opensearch-project/OpenSearch-Dashboards.git')
self.assertEqual(
component.artifacts['dist'],
['dist/opensearch-dashboards-min-1.1.0-linux-x64.tar.gz'])
urlopen.assert_called_once_with('http://fakeurl')
def __download_build_manifest(self) -> None:
self.distribution_url = manifests.distribution.find_build_root(
self.component.dist, self.target.platform,
self.target.architecture, self.target_name)
manifest_url = f"{self.distribution_url}/manifest.yml"
logging.info(f"Downloading {manifest_url} ...")
self.build_manifest = BuildManifest.from_url(manifest_url)
def test_select_two_are_unknown(self) -> None:
path = os.path.join(self.data_path, "build",
"opensearch-build-schema-version-1.2.yml")
manifest = BuildManifest.from_path(path)
with self.assertRaises(ValueError) as ctx:
self.assertEqual(
len(list(manifest.components.select(focus=["x", "y"]))), 0)
self.assertEqual(str(ctx.exception), "Unknown components=x,y.")
def test_versions(self) -> None:
self.assertTrue(len(BuildManifest.VERSIONS))
for version in BuildManifest.VERSIONS:
manifest = BuildManifest.from_path(
os.path.join(self.data_path, "build",
f"opensearch-build-schema-version-{version}.yml"))
self.assertEqual(version, manifest.version)
self.assertIsNotNone(any(manifest.components))
def test_select_none(self) -> None:
path = os.path.join(self.data_path, "build",
"opensearch-build-schema-version-1.2.yml")
manifest = BuildManifest.from_path(path)
with self.assertRaises(ValueError) as ctx:
self.assertEqual(len(list(manifest.components.select(focus="x"))),
0)
self.assertEqual(str(ctx.exception), "No components matched focus=x.")
def setUp(self) -> None:
self.maxDiff = None
manifest_path = os.path.join(os.path.dirname(__file__), "data",
"opensearch-build-linux-1.1.0.yml")
self.manifest = BuildManifest.from_path(manifest_path)
self.bundle_recorder = BundleRecorder(
self.manifest.build, "output_dir", "artifacts_dir",
BundleFileLocation("bundle_output_dir", "opensearch", "tar"))
manifest_distribution_path = os.path.join(
os.path.dirname(__file__), "data",
"opensearch-build-windows-1.3.0.yml")
self.manifest_distribution = BuildManifest.from_path(
manifest_distribution_path)
self.bundle_recorder_distribution = BundleRecorder(
self.manifest_distribution.build, "output_dir", "artifacts_dir",
BundleFileLocation("bundle_output_dir", "opensearch", "tar"))
def setUp(self) -> None:
os.chdir(os.path.dirname(__file__))
self.bundle_manifest = BundleManifest.from_path(self.BUNDLE_MANIFEST)
self.build_manifest = BuildManifest.from_path(self.BUILD_MANIFEST)
self.test_manifest = TestManifest.from_path(self.TEST_MANIFEST)
self.work_dir = Path("test_dir")
self.test_recorder = MagicMock()
self.save_logs = MagicMock()
self.test_recorder.test_results_logs = self.save_logs
def setUp(self) -> None:
manifest_path = os.path.join(os.path.dirname(__file__), "data", "opensearch-dashboards-build-1.1.0.yml")
self.manifest = BuildManifest.from_path(manifest_path)
self.bundle_recorder = BundleRecorder(
self.manifest.build,
"output_dir",
"artifacts_dir",
BundleFileLocation("bundle_output_dir", "opensearch-dashboards")
)
def setUp(self) -> None:
self.package_name = 'opensearch-1.3.0-1.x86_64.rpm'
self.artifacts_path = os.path.join(os.path.dirname(__file__),
"data/artifacts/dist")
self.package_path = os.path.join(self.artifacts_path,
self.package_name)
self.bundle_rpm = BundleRpm('opensearch', self.package_path,
'opensearch-1.3.0')
self.manifest_rpm = BuildManifest.from_path(
os.path.join(os.path.dirname(__file__),
"data/opensearch-build-rpm-1.3.0.yml"))
self.bundle_rpm_qualifier = BundleRpm('opensearch', self.package_path,
'opensearch-2.0.0-alpha1')
self.manifest_rpm_qualifier = BuildManifest.from_path(
os.path.join(os.path.dirname(__file__),
"data/opensearch-build-rpm-2.0.0-alpha1.yml"))
def setUp(self) -> None:
self.artifacts_path = os.path.join(os.path.dirname(__file__), "data/artifacts/dist/")
self.manifest = BuildManifest.from_path(os.path.join(os.path.dirname(__file__), "data/opensearch-build-rpm-1.3.0.yml"))
self.distTar = DistTar(
"OpenSearch",
self.artifacts_path + "opensearch-min-1.3.0-linux-x64.tar.gz",
"opensearch-1.3.0",
self.manifest.build
)
def test_select(self) -> None:
path = os.path.join(self.data_path, "build",
"opensearch-build-schema-version-1.2.yml")
manifest = BuildManifest.from_path(path)
self.assertEqual(
len(list(manifest.components.select(focus=["common-utils"]))), 1)
self.assertEqual(
len(
list(
manifest.components.select(
focus=["common-utils", "job-scheduler"]))), 2)
def test_bundle_install_min_no_patch(self, mock_os_rename: Mock,
mock_check_call: Mock) -> None:
manifest_path = os.path.join(os.path.dirname(__file__), "data",
"opensearch-build-linux-1.1.0.yml")
artifacts_path = os.path.join(os.path.dirname(__file__), "data",
"artifacts")
bundle = BundleOpenSearch(BuildManifest.from_path(manifest_path),
artifacts_path, MagicMock())
bundle.install_min()
mock_os_rename.assert_not_called()
mock_check_call.assert_called()
def __sign__(self) -> None:
manifest = BuildManifest.from_file(self.target.open("r"))
basepath = self.target.parent
for component in manifest.components.select(focus=self.components):
logging.info(f"Signing {component.name}")
for component_artifact_type in component.artifacts:
if self.artifact_type and self.artifact_type != component_artifact_type:
continue
super().__sign_artifacts__(component.artifacts[component_artifact_type], basepath)
def test_bundle_does_not_exist_raises_error(self) -> None:
manifest_path = os.path.join(os.path.dirname(__file__),
"data/opensearch-build-linux-1.1.0.yml")
with self.assertRaises(FileNotFoundError) as ctx:
self.DummyBundle(
BuildManifest.from_path(manifest_path),
os.path.join(os.path.dirname(__file__), "data",
"does-not-exist"),
MagicMock(),
)
self.assertTrue(
"opensearch-min-1.1.0-linux-x64.tar.gz" in str(ctx.exception))
def test_bundle_install_components(self,
bundle_install_plugin: Mock) -> None:
manifest_path = os.path.join(os.path.dirname(__file__), "data",
"opensearch-build-linux-1.1.0.yml")
bundle = BundleOpenSearch(
BuildManifest.from_path(manifest_path),
os.path.join(os.path.dirname(__file__), "data", "artifacts"),
MagicMock(),
)
bundle.install_components()
self.assertEqual(bundle_install_plugin.call_count, 12)
def test_install_build_dependencies_remote(self, mock_request, mock_copyfile, mock_makedirs):
dependency_installer = DependencyInstallerOpenSearch(
"https://ci.opensearch.org/x/y", BuildManifest.from_path(self.BUILD_MANIFEST), BundleManifest.from_path(self.DIST_MANIFEST_REMOTE)
)
dependencies = dict({"opensearch-job-scheduler": "1.1.0.0"})
dependency_installer.install_build_dependencies(dependencies, os.path.dirname(__file__))
mock_makedirs.assert_called_with(os.path.dirname(__file__), exist_ok=True)
mock_copyfile.assert_not_called()
mock_request.assert_called_once_with(
"https://ci.opensearch.org/x/y/builds/opensearch/plugins/opensearch-job-scheduler-1.1.0.0.zip",
os.path.realpath(os.path.join(os.path.dirname(__file__), "opensearch-job-scheduler-1.1.0.0.zip")),
)
def check(self) -> None:
for architecture in BuildArgs.SUPPORTED_ARCHITECTURES:
# Since we only have 'linux' builds now we hard code it to 'linux'
# Once we have all platform builds on S3 we can then add a second loop for 'BuildArgs.SUPPORTED_PLATFORMS'
distribution_url = distribution.find_build_root(
self.component.dist, "linux", architecture, self.target.name)
manifest_url = f"{distribution_url}/manifest.yml"
self.build_manifest = BuildManifest.from_url(manifest_url)
if self.component.name in self.build_manifest.components:
logging.info(f"Found {self.component.name} in {manifest_url}.")
else:
raise CiCheckManifestComponent.MissingComponentError(
self.component.name, manifest_url)
def test_check(self, mock_manifest_from_url: Mock, find_build_root: Mock):
mock_manifest_from_url.return_value = BuildManifest.from_path(
self.BUILD_MANIFEST)
component = InputComponentFromDist({
"name": "common-utils",
"dist": "url",
"checks": ["manifest:component"]
})
list = CiCheckListDist(
component,
CiTarget(version="1.1.0", name="opensearch", snapshot=True))
list.check()
mock_manifest_from_url.assert_called()
find_build_root.assert_called()
def test_install_build_dependencies_local(self, mock_request, mock_copyfile, mock_makedirs):
dependency_installer = DependencyInstallerOpenSearch(
self.DATA,
BuildManifest.from_path(self.BUILD_MANIFEST),
BundleManifest.from_path(self.DIST_MANIFEST_LOCAL)
)
dependencies = dict({"opensearch-job-scheduler": "1.1.0.0"})
dependency_installer.install_build_dependencies(dependencies, os.path.dirname(__file__))
mock_makedirs.assert_called_with(os.path.dirname(__file__), exist_ok=True)
mock_request.assert_not_called()
mock_copyfile.assert_called_once_with(
os.path.join(self.DATA, "builds", "opensearch", "plugins", "opensearch-job-scheduler-1.1.0.0.zip"),
os.path.realpath(os.path.join(os.path.dirname(__file__), "opensearch-job-scheduler-1.1.0.0.zip")),
)
def test_install_maven_dependencies_remote_failure(self, mock_request, mock_copyfile, mock_makedirs):
def mock_retrieve(source, dest):
raise HTTPError(url=source, hdrs={}, fp=None, msg="Not Found", code=404)
mock_request.side_effect = mock_retrieve
dependency_installer = DependencyInstallerOpenSearch(
"https://ci.opensearch.org/x/y",
BuildManifest.from_path(self.BUILD_MANIFEST),
BundleManifest.from_path(self.DIST_MANIFEST_REMOTE)
)
with self.assertRaises(HTTPError) as ctx:
dependency_installer.install_maven_dependencies()
self.assertEqual(str(ctx.exception), "HTTP Error 404: Not Found")