def test_to_file_formatted(self) -> None:
data_path = os.path.join(os.path.dirname(__file__), "data")
manifest = InputManifest({
"schema-version":
"1.0",
"build": {
"name": "OpenSearch",
"version": "2.0.0"
},
"ci": {
"image": {
"name": "image-name",
"args": "-e JAVA_HOME=/opt/java/openjdk-11"
}
},
"components": [{
"name":
"OpenSearch",
"ref":
"main",
"repository":
"https://github.com/opensearch-project/OpenSearch.git",
"checks": ["gradle:publish", "gradle:properties:version"]
}]
})
with TemporaryDirectory() as path:
output_path = os.path.join(path.name, "manifest.yml")
manifest.to_file(output_path)
with open(output_path) as f:
written_manifest = f.read()
with open(os.path.join(data_path, "formatted.yml")) as f:
formatted_manifest = f.read()
self.assertEqual(formatted_manifest, written_manifest)
def __init__(self,
url: str,
ref: str,
directory: str = None,
working_subdirectory: str = None) -> None:
self.url = url
self.ref = ref
if directory is None:
self.temp_dir = TemporaryDirectory()
self.dir = os.path.realpath(self.temp_dir.name)
else:
self.temp_dir = None
self.dir = directory
os.makedirs(self.dir, exist_ok=False)
self.working_subdirectory = working_subdirectory
self.__checkout__()
def test_checkout(self):
with TemporaryDirectory() as tmpdir:
subdir = os.path.join(tmpdir.name, ".github")
repo = self.test_component.checkout(subdir)
self.assertEqual(repo.url,
"https://github.com/opensearch-project/.github")
self.assertEqual(repo.ref,
"8ac515431bf24caf92fea9d9b0af3b8f10b88453")
def main():
args = TestArgs()
console.configure(level=args.logging_level)
with TemporaryDirectory(keep=args.keep) as work_dir:
bundle_manifest = BundleManifest.from_urlpath(
args.paths.get("opensearch", os.getcwd()))
BwcTestSuite(bundle_manifest, work_dir.name, args.component,
args.keep).execute()
def test_checkout(self, mock_repo: Mock) -> None:
with TemporaryDirectory() as tmpdir:
subdir = os.path.join(tmpdir.name, ".github")
repo = self.test_component.checkout(subdir)
self.assertEqual(repo, mock_repo.return_value)
mock_repo.assert_called_with(
"https://github.com/opensearch-project/.github",
"8ac515431bf24caf92fea9d9b0af3b8f10b88453", subdir)
def test_write_manifest(self) -> None:
with TemporaryDirectory() as dest_dir:
self.bundle_recorder.write_manifest(dest_dir.name)
manifest_path = os.path.join(dest_dir.name, "manifest.yml")
self.assertTrue(os.path.isfile(manifest_path))
data = self.bundle_recorder.get_manifest().to_dict()
with open(manifest_path) as f:
self.assertEqual(yaml.safe_load(f), data)
def test_to_file_formatted(self) -> None:
manifest_path = os.path.join(self.data_path, "min.yml")
manifest = TestManifest.SampleManifest.from_path(manifest_path)
with TemporaryDirectory() as path:
output_path = os.path.join(path.name, "manifest.yml")
manifest.to_file(output_path)
with open(output_path) as f:
written_manifest = f.read()
self.assertEqual("---\nschema-version: '1.0'\n", written_manifest)
def test_to_file(self) -> None:
manifest_path = os.path.join(self.data_path, "min.yml")
manifest = TestManifest.SampleManifest.from_path(manifest_path)
with TemporaryDirectory() as path:
output_path = os.path.join(path.name, "manifest.yml")
manifest.to_file(output_path)
self.assertTrue(os.path.isfile(manifest_path))
with open(output_path) as f:
self.assertEqual(yaml.safe_load(f), manifest.to_dict())
def test_remove_readonly(self):
with TemporaryDirectory() as work_dir:
filename = os.path.join(work_dir.name, "test.txt")
with open(filename, "w+") as f:
f.write("This is intentionally left blank.")
mode = os.stat(filename)[stat.ST_MODE]
os.chmod(filename, mode & ~stat.S_IWUSR & ~stat.S_IWGRP & ~stat.S_IWOTH)
self.assertTrue(os.path.exists(filename))
self.assertFalse(os.path.exists(work_dir.name))
self.assertFalse(os.path.exists(filename))
def test_write_manifest(self) -> None:
with TemporaryDirectory() as dest_dir:
mock = self.__mock(snapshot=False)
mock.target.output_dir = dest_dir.name
mock.write_manifest()
manifest_path = os.path.join(dest_dir.name, "manifest.yml")
self.assertTrue(os.path.isfile(manifest_path))
data = mock.get_manifest().to_dict()
with open(manifest_path) as f:
self.assertEqual(yaml.safe_load(f), data)
def main():
args = BuildArgs()
console.configure(level=args.logging_level)
manifest = InputManifest.from_file(args.manifest)
if args.ref_manifest:
manifest = manifest.stable()
if os.path.exists(args.ref_manifest):
if manifest == InputManifest.from_path(args.ref_manifest):
logging.info(f"No changes since {args.ref_manifest}")
else:
logging.info(f"Updating {args.ref_manifest}")
manifest.to_file(args.ref_manifest)
else:
logging.info(f"Creating {args.ref_manifest}")
manifest.to_file(args.ref_manifest)
exit(0)
output_dir = BuildOutputDir(manifest.build.filename).dir
with TemporaryDirectory(keep=args.keep, chdir=True) as work_dir:
logging.info(f"Building in {work_dir.name}")
target = BuildTarget(
name=manifest.build.name,
version=manifest.build.version,
patches=manifest.build.patches,
snapshot=args.snapshot if args.snapshot is not None else manifest.build.snapshot,
output_dir=output_dir,
platform=args.platform or manifest.build.platform,
architecture=args.architecture or manifest.build.architecture,
)
build_recorder = BuildRecorder(target)
logging.info(f"Building {manifest.build.name} ({target.architecture}) into {target.output_dir}")
for component in manifest.components.select(focus=args.component, platform=target.platform):
logging.info(f"Building {component.name}")
builder = Builders.builder_from(component, target)
try:
builder.checkout(work_dir.name)
builder.build(build_recorder)
builder.export_artifacts(build_recorder)
except:
logging.error(f"Error building {component.name}, retry with: {args.component_command(component.name)}")
raise
build_recorder.write_manifest()
logging.info("Done.")
def test_checkout_into_dir(self, *mocks: Any) -> None:
with TemporaryDirectory() as tmpdir:
subdir = os.path.join(tmpdir.name, ".github")
repo = GitRepository(
url="https://github.com/opensearch-project/.github",
ref="8ac515431bf24caf92fea9d9b0af3b8f10b88453",
directory=subdir,
)
self.assertEqual(repo.url,
"https://github.com/opensearch-project/.github")
self.assertEqual(repo.ref,
"8ac515431bf24caf92fea9d9b0af3b8f10b88453")
self.assertEqual(repo.sha,
"8ac515431bf24caf92fea9d9b0af3b8f10b88453")
self.assertIsNone(repo.temp_dir)
self.assertEqual(repo.dir, subdir)
def test_extractall_preserves_permissions(self):
with TemporaryDirectory() as tmp:
temp_file = os.path.join(tmp.name, "test.zip")
with ZipFile(temp_file, "w", zipfile.ZIP_DEFLATED) as zip:
zip.write(os.path.join(self.data_path, "executable.sh"),
"executable.sh")
zip.write(__file__, "regular.py")
with ZipFile(temp_file, "r") as zip:
zip.extractall(tmp.name)
executable_file = os.path.join(tmp.name, "executable.sh")
self.assertTrue(os.path.exists(executable_file))
self.assertTrue(os.access(executable_file, os.X_OK))
regular_file = os.path.join(tmp.name, "regular.py")
self.assertTrue(os.path.exists(regular_file))
def __init__(self,
build_manifest: BuildManifest,
artifacts_dir: str,
bundle_recorder: BundleRecorder,
keep: bool = False) -> None:
"""
Construct a new Bundle instance.
:param build_manifest: A BuildManifest created from the build workflow.
:param artifacts_dir: Dir location where build artifacts can be found locally
:param bundle_recorder: The bundle recorder that will capture and build a BundleManifest
"""
self.build = build_manifest.build
self.plugins = self.__get_plugins(build_manifest.components)
self.artifacts_dir = artifacts_dir
self.bundle_recorder = bundle_recorder
self.tmp_dir = TemporaryDirectory(keep=keep)
self.min_dist = self.__get_min_dist(build_manifest.components)
self.installed_plugins: List[str] = []
def __check__(self):
target = CiTarget(version=self.manifest.build.version,
name=self.manifest.build.filename,
snapshot=self.args.snapshot)
with TemporaryDirectory(keep=self.args.keep, chdir=True) as work_dir:
logging.info(f"Sanity-testing in {work_dir.name}")
logging.info(f"Sanity testing {self.manifest.build.name}")
for component in self.manifest.components.select(
focus=self.args.component):
logging.info(f"Sanity testing {component.name}")
ci_check_list = CiCheckLists.from_component(component, target)
ci_check_list.checkout(work_dir.name)
ci_check_list.check()
logging.info("Done.")
def main():
args = CheckoutArgs()
console.configure(level=args.logging_level)
manifest = InputManifest.from_file(args.manifest)
with TemporaryDirectory(keep=True, chdir=True) as work_dir:
logging.info(f"Checking out into {work_dir.name}")
for component in manifest.components.select():
logging.info(f"Checking out {component.name}")
with GitRepository(
component.repository,
component.ref,
os.path.join(work_dir.name, component.name),
component.working_directory,
) as repo:
logging.debug(f"Checked out {component.name} into {repo.dir}")
logging.info(f"Done, checked out into {work_dir.name}.")
def run_tests(self) -> None:
config = yaml.safe_load(self.args.config)
with TemporaryDirectory(keep=self.args.keep, chdir=True) as work_dir:
current_workspace = os.path.join(work_dir.name, "infra")
logging.info("current_workspace is " + str(current_workspace))
with GitRepository(self.get_infra_repo_url(), "main",
current_workspace):
with WorkingDirectory(current_workspace):
with PerfSingleNodeCluster.create(
self.test_manifest, config, self.args.stack,
PerfTestClusterConfig(self.security),
current_workspace) as test_cluster:
perf_test_suite = PerfTestSuite(
self.test_manifest,
test_cluster.endpoint_with_port, self.security,
current_workspace, self.tests_dir, self.args)
retry_call(perf_test_suite.execute,
tries=3,
delay=60,
backoff=2)
def main():
"""
Entry point for Performance Test with bundle manifest, config file containing the required arguments for running
rally test and the stack name for the cluster. Will call out in test.sh with perf as argument
"""
parser = argparse.ArgumentParser(description="Test an OpenSearch Bundle")
parser.add_argument("--bundle-manifest",
type=argparse.FileType("r"),
help="Bundle Manifest file.",
required=True)
parser.add_argument("--stack",
dest="stack",
help="Stack name for performance test")
parser.add_argument("--config",
type=argparse.FileType("r"),
help="Config file.",
required=True)
parser.add_argument("--keep",
dest="keep",
action="store_true",
help="Do not delete the working temporary directory.")
args = parser.parse_args()
manifest = BundleManifest.from_file(args.bundle_manifest)
config = yaml.safe_load(args.config)
with TemporaryDirectory(keep=args.keep, chdir=True) as work_dir:
current_workspace = os.path.join(work_dir.name, "infra")
with GitRepository(get_infra_repo_url(), "main", current_workspace):
security = "security" in manifest.components
with WorkingDirectory(current_workspace):
with PerfTestCluster.create(
manifest, config, args.stack, security,
current_workspace) as (test_cluster_endpoint,
test_cluster_port):
perf_test_suite = PerfTestSuite(manifest,
test_cluster_endpoint,
security,
current_workspace)
perf_test_suite.execute()
def run(self):
with TemporaryDirectory(keep=self.args.keep, chdir=True) as work_dir:
all_results = TestSuiteResults()
for component in self.components.select(focus=self.args.component):
if component.name in self.test_manifest.components:
test_config = self.test_manifest.components[component.name]
if test_config.integ_test:
test_suite = self.__create_test_suite__(
component, test_config, work_dir)
test_results = test_suite.execute_tests()
all_results.append(component.name, test_results)
else:
logging.info(
f"Skipping integ-tests for {component.name}, as it is currently not supported"
)
else:
logging.info(
f"Skipping integ-tests for {component.name}, as it is currently not declared in the test manifest"
)
return all_results
def run_tests(self) -> None:
with TemporaryDirectory(keep=self.args.keep, chdir=True) as work_dir:
current_workspace = os.path.join(work_dir.name, self.args.component)
with GitRepository(self.get_plugin_repo_url(), "main", current_workspace):
with WorkingDirectory(current_workspace):
subprocess.check_call(f"{self.command}", cwd=os.getcwd(), shell=True)
def test_keep_true(self):
with TemporaryDirectory(keep=True) as work_dir:
self.assertTrue(os.path.exists(work_dir.name))
self.assertTrue(os.path.exists(work_dir.name))
def update(self, min_klass, component_klass, keep=False):
known_versions = self.versions
logging.info(f"Known versions: {known_versions}")
main_versions = {}
with TemporaryDirectory(keep=keep, chdir=True) as work_dir:
logging.info(f"Checking out components into {work_dir.name}")
# check out and build #main, 1.x, etc.
branches = min_klass.branches()
logging.info(f"Checking {self.name} {branches} branches")
for branch in branches:
c = min_klass.checkout(
path=os.path.join(work_dir.name,
self.name.replace(" ", ""), branch),
branch=branch,
)
version = c.version
logging.info(f"{self.name}#{branch} is version {version}")
if version not in main_versions.keys():
main_versions[version] = [c]
if component_klass is not None:
# components can increment their own version first without incrementing min
manifest = self.latest
logging.info(
f"Examining components in the latest manifest of {manifest.build.name} ({manifest.build.version})"
)
for component in manifest.components.values():
if component.name == self.name:
continue
logging.info(f"Checking out {component.name}#main")
component = component_klass.checkout(
name=component.name,
path=os.path.join(work_dir.name, component.name),
version=manifest.build.version,
branch="main",
)
component_version = component.version
if component_version:
release_version = ".".join(
component_version.split(".")[:3])
if release_version not in main_versions.keys():
main_versions[release_version] = []
main_versions[release_version].append(component)
logging.info(
f"{component.name}#main is version {release_version} (from {component_version})"
)
# summarize
logging.info("Found versions on main:")
for main_version in main_versions.keys():
for component in main_versions[main_version]:
logging.info(f" {component.name}={main_version}")
# generate new manifests
for release_version in sorted(main_versions.keys() -
known_versions):
self.write_manifest(release_version,
main_versions[release_version])
self.add_to_cron(release_version)
def test_path(self) -> None:
with TemporaryDirectory() as work_dir:
self.assertIsInstance(work_dir.path, Path)
self.assertTrue(work_dir.path.exists())
def test_chdir_false(self) -> None:
before_dir = os.getcwd()
with TemporaryDirectory():
self.assertEqual(before_dir, os.getcwd())
self.assertEqual(before_dir, os.getcwd())
def test_chdir_true(self) -> None:
before_dir = os.getcwd()
with TemporaryDirectory(chdir=True):
self.assertNotEqual(before_dir, os.getcwd())
self.assertEqual(before_dir, os.getcwd())
def test_keep_false(self) -> None:
with TemporaryDirectory() as work_dir:
self.assertTrue(os.path.exists(work_dir.name))
self.assertFalse(os.path.exists(work_dir.name))
class GitRepository:
dir: str
"""
This class checks out a Git repository at a particular ref into an empty named directory (or temporary a directory if no named directory is given).
Temporary directories will be automatically deleted when the GitRepository object goes out of scope; named directories will be left alone.
Clients can obtain the actual commit ID by querying the "sha" attribute, and the temp directory name with "dir".
"""
def __init__(self,
url: str,
ref: str,
directory: str = None,
working_subdirectory: str = None) -> None:
self.url = url
self.ref = ref
if directory is None:
self.temp_dir = TemporaryDirectory()
self.dir = os.path.realpath(self.temp_dir.name)
else:
self.temp_dir = None
self.dir = directory
os.makedirs(self.dir, exist_ok=False)
self.working_subdirectory = working_subdirectory
self.__checkout__()
def __enter__(self) -> 'GitRepository':
return self
def __exit__(self, exc_type: Any, exc_value: Any,
exc_traceback: Any) -> None:
if self.temp_dir:
self.temp_dir.__exit__(exc_type, exc_value, exc_traceback)
def __checkout__(self) -> None:
self.execute_silent("git init", self.dir)
self.execute_silent(f"git remote add origin {self.url}", self.dir)
self.execute_silent(f"git fetch --depth 1 origin {self.ref}", self.dir)
self.execute_silent("git checkout FETCH_HEAD", self.dir)
self.sha = self.output("git rev-parse HEAD", self.dir)
logging.info(
f"Checked out {self.url}@{self.ref} into {self.dir} at {self.sha}")
@property
def working_directory(self) -> str:
if self.working_subdirectory:
return os.path.join(self.dir, self.working_subdirectory)
else:
return self.dir
@classmethod
def stable_ref(self, url: str, ref: str) -> List[str]:
results = subprocess.check_output(
f"git ls-remote {url} {ref}",
shell=True).decode().strip().split("\t")
return results if len(results) > 1 else [ref, ref]
def execute_silent(self, command: str, cwd: str = None) -> None:
cwd = cwd or self.working_directory
logging.info(f'Executing "{command}" in {cwd}')
subprocess.check_call(
command,
cwd=cwd,
shell=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
def output(self, command: str, cwd: str = None) -> str:
cwd = cwd or self.working_directory
logging.info(f'Executing "{command}" in {cwd}')
return subprocess.check_output(command, cwd=cwd,
shell=True).decode().strip()
def execute(self, command: str, cwd: str = None) -> None:
cwd = cwd or self.working_directory
logging.info(f'Executing "{command}" in {cwd}')
subprocess.check_call(command, cwd=cwd, shell=True)
def path(self, subdirname: str = None) -> Path:
dirname = self.dir
if subdirname:
dirname = os.path.join(self.dir, subdirname)
return Path(dirname)