def checkout(self, work_dir: str) -> None:
self.git_repo = GitRepository(
self.component.repository,
self.component.ref,
os.path.join(work_dir, self.component.name),
self.component.working_directory,
)
class BuilderFromSource(Builder):
def checkout(self, work_dir: str) -> None:
self.git_repo = GitRepository(
self.component.repository,
self.component.ref,
os.path.join(work_dir, self.component.name),
self.component.working_directory,
)
def build(self, build_recorder: BuildRecorder) -> None:
# List of components whose build scripts support `-d` parameter
# Bundled plugins do not need `-d` as they are java based zips
DISTRIBUTION_SUPPORTED_COMPONENTS = [
"OpenSearch", "OpenSearch-Dashboards"
]
build_script = ScriptFinder.find_build_script(
self.target.name, self.component.name,
self.git_repo.working_directory)
build_command = " ".join(
filter(None, [
"bash",
build_script,
f"-v {self.target.version}",
f"-q {self.target.qualifier}"
if self.target.qualifier else None,
f"-p {self.target.platform}",
f"-a {self.target.architecture}",
f"-d {self.target.distribution}" if self.component.name
in DISTRIBUTION_SUPPORTED_COMPONENTS else None,
f"-s {str(self.target.snapshot).lower()}",
f"-o {self.output_path}",
]))
self.git_repo.execute(build_command)
build_recorder.record_component(self.component.name, self.git_repo)
def export_artifacts(self, build_recorder: BuildRecorder) -> None:
artifacts_path = os.path.join(self.git_repo.working_directory,
self.output_path)
for artifact_type in [
"maven", "dist", "plugins", "libs", "core-plugins"
]:
for dir, _, files in os.walk(
os.path.join(artifacts_path, artifact_type)):
for file_name in files:
absolute_path = os.path.join(dir, file_name)
relative_path = os.path.relpath(absolute_path,
artifacts_path)
build_recorder.record_artifact(self.component.name,
artifact_type,
relative_path,
absolute_path)
class Signer:
git_repo: GitRepository
ACCEPTED_FILE_TYPES = [".zip", ".jar", ".war", ".pom", ".module", ".tar.gz", ".whl", ".crate", ".rpm"]
def __init__(self) -> None:
self.git_repo = GitRepository(self.get_repo_url(), "HEAD", working_subdirectory="src")
self.git_repo.execute("./bootstrap")
self.git_repo.execute("rm config.cfg")
def sign_artifact(self, artifact: str, basepath: Path, signature_type: str) -> None:
if not self.is_valid_file_type(artifact):
logging.info(f"Skipping signing of file {artifact}")
return
self.generate_signature_and_verify(artifact, basepath, signature_type)
def sign_artifacts(self, artifacts: List[str], basepath: Path, signature_type: str) -> None:
for artifact in artifacts:
if not self.is_valid_file_type(artifact):
logging.info(f"Skipping signing of file {artifact}")
continue
self.generate_signature_and_verify(artifact, basepath, signature_type)
def generate_signature_and_verify(self, artifact: str, basepath: Path, signature_type: str) -> None:
location = os.path.join(basepath, artifact)
self.sign(location, signature_type)
self.verify(location + signature_type)
def is_valid_file_type(self, file_name: str) -> bool:
return any(
file_name.endswith(x) for x in Signer.ACCEPTED_FILE_TYPES
)
def get_repo_url(self) -> str:
if "GITHUB_TOKEN" in os.environ:
return "https://${GITHUB_TOKEN}@github.com/opensearch-project/opensearch-signer-client.git"
return "https://github.com/opensearch-project/opensearch-signer-client.git"
def __remove_existing_signature__(self, signature_file: str) -> None:
if os.path.exists(signature_file):
logging.warning(f"Removing existing signature file {signature_file}")
os.remove(signature_file)
def sign(self, filename: str, signature_type: str) -> None:
signature_file = filename + signature_type
self.__remove_existing_signature__(signature_file)
signing_cmd = [
"./opensearch-signer-client",
"-i",
filename,
"-o",
signature_file,
"-p",
"pgp",
]
self.git_repo.execute(" ".join(signing_cmd))
def verify(self, filename: str) -> None:
verify_cmd = ["gpg", "--verify-files", filename]
self.git_repo.execute(" ".join(verify_cmd))
class BuilderFromSource(Builder):
def checkout(self, work_dir: str) -> None:
self.git_repo = GitRepository(
self.component.repository,
self.component.ref,
os.path.join(work_dir, self.component.name),
self.component.working_directory,
)
def build(self, build_recorder: BuildRecorder) -> None:
build_script = ScriptFinder.find_build_script(
self.target.name, self.component.name,
self.git_repo.working_directory)
build_command = " ".join([
"bash",
build_script,
"-v",
self.target.version,
"-p",
self.target.platform,
"-a",
self.target.architecture,
"-s",
str(self.target.snapshot).lower(),
"-o",
self.output_path,
])
self.git_repo.execute(build_command)
build_recorder.record_component(self.component.name, self.git_repo)
def export_artifacts(self, build_recorder: BuildRecorder) -> None:
artifacts_path = os.path.join(self.git_repo.working_directory,
self.output_path)
for artifact_type in [
"maven", "dist", "plugins", "libs", "core-plugins"
]:
for dir, _, files in os.walk(
os.path.join(artifacts_path, artifact_type)):
for file_name in files:
absolute_path = os.path.join(dir, file_name)
relative_path = os.path.relpath(absolute_path,
artifacts_path)
build_recorder.record_artifact(self.component.name,
artifact_type,
relative_path,
absolute_path)
def __init__(
self,
work_dir,
component,
test_config,
test_recorder,
dependency_installer,
bundle_manifest,
build_manifest
):
self.work_dir = work_dir
self.component = component
self.test_config = test_config
self.test_recorder = test_recorder
self.dependency_installer = dependency_installer
self.bundle_manifest = bundle_manifest
self.build_manifest = build_manifest
self.repo = GitRepository(
self.component.repository,
self.component.commit_id,
os.path.join(self.work_dir, self.component.name),
test_config.working_directory
)
self.save_logs = test_recorder.test_results_logs
self.additional_cluster_config = None
def test_checkout_into_dir(self, *mocks: Any) -> None:
with GitRepository(
url="https://github.com/opensearch-project/.github",
ref="163b5acaf6c7d220f800684801bbf2e12f99c797",
working_subdirectory="ISSUE_TEMPLATE",
) as repo:
working_directory = os.path.join(repo.dir, "ISSUE_TEMPLATE")
self.assertEqual(repo.working_directory, working_directory)
self.assertFalse(os.path.exists(repo.dir))
class TestGitRepository(unittest.TestCase):
def setUp(self):
self.repo = GitRepository(
url="https://github.com/opensearch-project/.github",
ref="8ac515431bf24caf92fea9d9b0af3b8f10b88453",
)
def test_checkout(self):
self.assertEqual(self.repo.url,
"https://github.com/opensearch-project/.github")
self.assertEqual(self.repo.ref,
"8ac515431bf24caf92fea9d9b0af3b8f10b88453")
self.assertEqual(self.repo.sha,
"8ac515431bf24caf92fea9d9b0af3b8f10b88453")
self.assertIs(type(self.repo.temp_dir), TemporaryDirectory)
self.assertEqual(self.repo.dir,
os.path.realpath(self.repo.temp_dir.name))
self.assertTrue(
os.path.isfile(os.path.join(self.repo.dir, "CODE_OF_CONDUCT.md")))
# was added in the next commit
self.assertFalse(
os.path.exists(os.path.join(self.repo.dir, "CONTRIBUTING.md")))
def test_execute(self):
self.repo.execute("echo $PWD > created.txt")
self.assertTrue(
os.path.isfile(os.path.join(self.repo.dir, "created.txt")))
def test_execute_in_dir(self):
self.repo.execute("echo $PWD > created.txt",
os.path.join(self.repo.dir, "ISSUE_TEMPLATE"))
self.assertFalse(
os.path.isfile(os.path.join(self.repo.dir, "created.txt")))
self.assertTrue(
os.path.isfile(
os.path.join(self.repo.dir, "ISSUE_TEMPLATE", "created.txt")))
@patch("subprocess.check_call")
def test_execute_silent(self, mock_subprocess):
self.repo.execute_silent("echo .")
subprocess.check_call.assert_called_with(
"echo .",
cwd=self.repo.dir,
shell=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
@patch("subprocess.check_output")
def test_output(self, mock_subprocess):
self.repo.output("echo hello")
subprocess.check_output.assert_called_with("echo hello",
cwd=self.repo.dir,
shell=True)
def checkout(self, path: str, branch: str = "main", snapshot: bool = False) -> 'ComponentOpenSearchDashboardsMin':
with GitRepository(
"https://github.com/opensearch-project/OpenSearch-Dashboards.git",
branch,
path,
) as repo:
return ComponentOpenSearchDashboardsMin(
repo,
snapshot,
)
def checkout(self,
path: str,
branch: str = "main",
snapshot: bool = False) -> 'ComponentOpenSearchMin':
return ComponentOpenSearchMin(
GitRepository(
"https://github.com/opensearch-project/OpenSearch.git", branch,
path),
snapshot,
)
class TestGitRepository(unittest.TestCase):
@patch('subprocess.check_call', return_value=0)
@patch('subprocess.check_output',
return_value='8ac515431bf24caf92fea9d9b0af3b8f10b88453'.encode())
def setUp(self, *mocks: Any) -> None:
self.repo = GitRepository(
url="https://github.com/opensearch-project/.github",
ref="8ac515431bf24caf92fea9d9b0af3b8f10b88453",
)
def test_checkout(self) -> None:
self.assertEqual(self.repo.url,
"https://github.com/opensearch-project/.github")
self.assertEqual(self.repo.ref,
"8ac515431bf24caf92fea9d9b0af3b8f10b88453")
self.assertEqual(self.repo.sha,
"8ac515431bf24caf92fea9d9b0af3b8f10b88453")
self.assertIs(type(self.repo.temp_dir), TemporaryDirectory)
self.assertEqual(self.repo.dir,
os.path.realpath(self.repo.temp_dir.name))
def test_execute(self) -> None:
self.repo.execute("echo $PWD > created.txt")
self.assertTrue(
os.path.isfile(os.path.join(self.repo.dir, "created.txt")))
@patch('subprocess.check_call', return_value=0)
def test_execute_in_subdir(self, mock_check_call: Mock) -> None:
subdir = os.path.join(self.repo.dir, "ISSUE_TEMPLATE")
self.repo.execute("echo $PWD > created.txt", subdir)
mock_check_call.assert_called_with('echo $PWD > created.txt',
cwd=subdir,
shell=True)
@patch("subprocess.check_call")
def test_execute_silent(self, mock_subprocess: Mock) -> None:
self.repo.execute_silent("echo .")
mock_subprocess.assert_called_with(
"echo .",
cwd=self.repo.dir,
shell=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
@patch("subprocess.check_output")
def test_output(self, mock_subprocess: Any) -> None:
self.repo.output("echo hello")
mock_subprocess.assert_called_with("echo hello",
cwd=self.repo.dir,
shell=True)
def __init__(self, work_dir: Path, component: Any, test_config: Any,
test_recorder: TestRecorder,
manifest: BundleManifest) -> None:
self.work_dir = work_dir
self.component = component
self.test_config = test_config
self.test_recorder = test_recorder
self.manifest = manifest
self.repo = GitRepository(
self.component.repository, self.component.commit_id,
os.path.join(self.work_dir, self.component.name),
test_config.working_directory)
self.save_logs = test_recorder.test_results_logs
def test_checkout_into_dir(self, *mocks: Any) -> None:
with TemporaryDirectory() as tmpdir:
subdir = os.path.join(tmpdir.name, ".github")
repo = GitRepository(
url="https://github.com/opensearch-project/.github",
ref="8ac515431bf24caf92fea9d9b0af3b8f10b88453",
directory=subdir,
)
self.assertEqual(repo.url,
"https://github.com/opensearch-project/.github")
self.assertEqual(repo.ref,
"8ac515431bf24caf92fea9d9b0af3b8f10b88453")
self.assertEqual(repo.sha,
"8ac515431bf24caf92fea9d9b0af3b8f10b88453")
self.assertIsNone(repo.temp_dir)
self.assertEqual(repo.dir, subdir)
def main():
args = CheckoutArgs()
console.configure(level=args.logging_level)
manifest = InputManifest.from_file(args.manifest)
with TemporaryDirectory(keep=True, chdir=True) as work_dir:
logging.info(f"Checking out into {work_dir.name}")
for component in manifest.components.select():
logging.info(f"Checking out {component.name}")
with GitRepository(
component.repository,
component.ref,
os.path.join(work_dir.name, component.name),
component.working_directory,
) as repo:
logging.debug(f"Checked out {component.name} into {repo.dir}")
logging.info(f"Done, checked out into {work_dir.name}.")
def run_tests(self) -> None:
config = yaml.safe_load(self.args.config)
with TemporaryDirectory(keep=self.args.keep, chdir=True) as work_dir:
current_workspace = os.path.join(work_dir.name, "infra")
logging.info("current_workspace is " + str(current_workspace))
with GitRepository(self.get_infra_repo_url(), "main",
current_workspace):
with WorkingDirectory(current_workspace):
with PerfSingleNodeCluster.create(
self.test_manifest, config, self.args.stack,
PerfTestClusterConfig(self.security),
current_workspace) as test_cluster:
perf_test_suite = PerfTestSuite(
self.test_manifest,
test_cluster.endpoint_with_port, self.security,
current_workspace, self.tests_dir, self.args)
retry_call(perf_test_suite.execute,
tries=3,
delay=60,
backoff=2)
def checkout(
self,
name,
path,
opensearch_version,
branch="main",
snapshot=False,
working_directory=None,
):
with GitRepository(
f"https://github.com/opensearch-project/{name}.git",
branch,
path,
working_directory,
) as repo:
return ComponentOpenSearch(
name,
repo,
opensearch_version,
snapshot,
)
def main():
"""
Entry point for Performance Test with bundle manifest, config file containing the required arguments for running
rally test and the stack name for the cluster. Will call out in test.sh with perf as argument
"""
parser = argparse.ArgumentParser(description="Test an OpenSearch Bundle")
parser.add_argument("--bundle-manifest",
type=argparse.FileType("r"),
help="Bundle Manifest file.",
required=True)
parser.add_argument("--stack",
dest="stack",
help="Stack name for performance test")
parser.add_argument("--config",
type=argparse.FileType("r"),
help="Config file.",
required=True)
parser.add_argument("--keep",
dest="keep",
action="store_true",
help="Do not delete the working temporary directory.")
args = parser.parse_args()
manifest = BundleManifest.from_file(args.bundle_manifest)
config = yaml.safe_load(args.config)
with TemporaryDirectory(keep=args.keep, chdir=True) as work_dir:
current_workspace = os.path.join(work_dir.name, "infra")
with GitRepository(get_infra_repo_url(), "main", current_workspace):
security = "security" in manifest.components
with WorkingDirectory(current_workspace):
with PerfTestCluster.create(
manifest, config, args.stack, security,
current_workspace) as (test_cluster_endpoint,
test_cluster_port):
perf_test_suite = PerfTestSuite(manifest,
test_cluster_endpoint,
security,
current_workspace)
perf_test_suite.execute()
def __stabilize__(self) -> None:
ref, name = GitRepository.stable_ref(self.repository, self.ref)
logging.info(f"Updating ref for {self.repository} from {self.ref} to {ref} ({name})")
self.ref = ref
def checkout(self, directory: str) -> GitRepository:
return GitRepository(self.repository, self.commit_id, directory)
def __init__(self) -> None:
self.git_repo = GitRepository(self.get_repo_url(), "HEAD", working_subdirectory="src")
self.git_repo.execute("./bootstrap")
self.git_repo.execute("rm config.cfg")
def setUp(self, *mocks: Any) -> None:
self.repo = GitRepository(
url="https://github.com/opensearch-project/.github",
ref="8ac515431bf24caf92fea9d9b0af3b8f10b88453",
)
def test_stable_ref_none(self, mock_output: Mock) -> None:
ref, name = GitRepository.stable_ref(
"https://github.com/opensearch-project/OpenSearch", "sha")
self.assertEqual(ref, "sha")
self.assertEqual(name, "sha")
def test_stable_ref_none(self, mock_output):
mock_output.return_value.decode.return_value = ""
ref, name = GitRepository.stable_ref(
"https://github.com/opensearch-project/OpenSearch", "sha")
self.assertEqual(ref, "sha")
self.assertEqual(name, "sha")
def run_tests(self) -> None:
with TemporaryDirectory(keep=self.args.keep, chdir=True) as work_dir:
current_workspace = os.path.join(work_dir.name, self.args.component)
with GitRepository(self.get_plugin_repo_url(), "main", current_workspace):
with WorkingDirectory(current_workspace):
subprocess.check_call(f"{self.command}", cwd=os.getcwd(), shell=True)