def compile_all( genny_repo_root: str, workspace_root: str, build_system: str, os_family: str, linux_distro: str, ignore_toolchain_version: bool, ): toolchain_info = toolchain.toolchain_info( genny_repo_root=genny_repo_root, workspace_root=workspace_root, os_family=os_family, linux_distro=linux_distro, ignore_toolchain_version=ignore_toolchain_version, ) cmd_base = [build_system] if build_system == "make": cmd_base.append("-j8") compile_cmd = [*cmd_base, "-C", "build"] run_command( cmd=compile_cmd, env=toolchain_info.toolchain_env, cwd=genny_repo_root, capture=False, check=True, )
def _fetch_and_install_impl(self) -> None: tarball = os.path.join(self._install_dir, self._name + ".tgz") # if os.path.isfile(tarball): # SLOG.info("Skipping downloading since already exists", tarball=tarball) # else: url = self._get_url() SLOG.debug("Downloading", name=self._name, url=url) urllib.request.urlretrieve(url, tarball) SLOG.debug("Finished Downloading", name=self._name, tarball=tarball) SLOG.debug("Extracting", name=self._name, into=self.result_dir) shutil.rmtree(self.result_dir, ignore_errors=True) os.makedirs(self.result_dir, exist_ok=True) # use tar(1) because python's TarFile was inexplicably truncating the tarball run_command( cmd=["tar", "-xzf", tarball, "-C", self.result_dir], capture=False, check=True, # The cwd doesn't actually matter here since we do 'tar -C' cwd=self._workspace_root, ) SLOG.info("Downloaded and installed.", name=self._name, into=self.result_dir) # Get space back. os.remove(tarball)
def run_generate_uuid_tag(genny_repo_root: str): path = os.path.join(genny_repo_root, "src", "lamplib", "src", "genny", "tasks", "generate-uuid-tag.sh") cmd_runner.run_command( cmd=[path], cwd=genny_repo_root, capture=False, check=True, )
def cmake( genny_repo_root: str, workspace_root: str, build_system: str, os_family: str, linux_distro: str, ignore_toolchain_version: bool, sanitizer: str, cmake_args: List[str], ): toolchain_info = toolchain.toolchain_info( workspace_root=workspace_root, genny_repo_root=genny_repo_root, os_family=os_family, linux_distro=linux_distro, ignore_toolchain_version=ignore_toolchain_version, ) generators = {"make": "Unix Makefiles", "ninja": "Ninja"} cmake_cmd = ["cmake", "-B", "build", "-G", generators[build_system]] # We set both the prefix path and the toolchain file here as a hack to allow cmake # to find both shared and static libraries. vcpkg doesn't natively support a project # using both. cmake_prefix_paths = [ os.path.join( toolchain_info.toolchain_dir, f"installed/x64-{toolchain_info.triplet_os}-dynamic", ), os.path.join( toolchain_info.toolchain_dir, f"installed/x64-{toolchain_info.triplet_os}", ), ] cmake_toolchain_file = os.path.join(toolchain_info.toolchain_dir, "scripts/buildsystems/vcpkg.cmake") cmake_cmd += [ "-DGENNY_WORKSPACE_ROOT={}".format(workspace_root), # "-DGENNY_REPO_ROOT={}".format(genny_repo_root), # Not needed (yet). "-DCMAKE_PREFIX_PATH={}".format(";".join(cmake_prefix_paths)), "-DCMAKE_TOOLCHAIN_FILE={}".format(cmake_toolchain_file), "-DCMAKE_EXPORT_COMPILE_COMMANDS=1", f"-DVCPKG_TARGET_TRIPLET=x64-{toolchain_info.triplet_os}", ] cmake_cmd += _sanitizer_flags(sanitizer) cmake_cmd += cmake_args run_command( cwd=genny_repo_root, cmd=cmake_cmd, env=toolchain_info.toolchain_env, capture=False, check=True, )
def run_create_new_actor(genny_repo_root: str, actor_name: str): path = os.path.join(genny_repo_root, "src", "lamplib", "src", "genny", "tasks", "create-new-actor.sh") cmd_runner.run_command( cmd=[path, actor_name], cwd=genny_repo_root, capture=False, check=True, )
def run_resmoke() -> None: # See if we can put this in the resmoke suite def or something? env["CTEST_OUTPUT_ON_FAILURE"] = "1" cmd_runner.run_command( cmd=cmd, cwd=workspace_root, env=env, capture=False, # If we're create_new_actor_test we don't want # to barf when resmoke fails. We expect it to fail. check=False if is_cnats else True, # `not is_cnats` was hard to read. )
def modified_workload_files(self) -> Set[str]: """Relies on git to find files in src/workloads modified versus origin/master""" src_path = os.path.join(self.workspace_root, "src") all_repo_directories = { path for path in os.listdir(src_path) if os.path.isdir(os.path.join(src_path, path)) } command = ("git diff --name-only --diff-filter=AMR " "$(git merge-base HEAD origin) -- src/workloads/") modified_workloads = set() for repo_directory in all_repo_directories: repo_path = os.path.join(src_path, repo_directory) cmd = run_command(cmd=[command], cwd=repo_path, shell=True, check=True) if cmd.returncode != 0: SLOG.fatal("Failed to compare workload directory to origin.", *cmd) raise RuntimeError( "Failed to compare workload directory to origin: stdout: {cmd.stdout} stderr: {cmd.stderr}" ) lines = cmd.stdout modified_workloads.update({ os.path.join(repo_path, line) for line in lines if line.endswith(".yml") }) return modified_workloads
def main_genny_runner(genny_args: List[str], genny_repo_root: str, cleanup_metrics: bool, workspace_root: str): """ Intended to be the main entry point for running Genny. """ with poplar_grpc( cleanup_metrics=cleanup_metrics, workspace_root=workspace_root, genny_repo_root=genny_repo_root, ): path = os.path.join(genny_repo_root, "dist", "bin", "genny_core") if not os.path.exists(path): SLOG.error("genny_core not found. Run install first.", path=path) raise Exception(f"genny_core not found at {path}.") cmd = [path, *genny_args] preprocessed_dir = os.path.join(workspace_root, "build/WorkloadOutput/workload") os.makedirs(preprocessed_dir, exist_ok=True) # Intercept the workload given to the core binary. index = -1 if "-w" in cmd: index = cmd.index("-w") + 1 elif "--workload-file" in cmd: index = cmd.index("--workload-file") + 1 elif "dry-run" in cmd: index = cmd.index("dry-run") + 1 if index >= 0: workload_path = cmd[index] smoke = "-s" in cmd or "--smoke-test" in cmd temp_workload = os.path.join(preprocessed_dir, os.path.basename(workload_path)) with open(temp_workload, "w") as f: preprocess.preprocess(workload_path=workload_path, smoke=smoke, output_file=f) cmd[index] = temp_workload run_command( cmd=cmd, capture=False, check=True, cwd=workspace_root, )
def cmd_func() -> bool: output: cmd_runner.RunCommandOutput = cmd_runner.run_command( cmd=ctest_cmd, cwd=workdir, env=info.toolchain_env, capture=False, check=True) return output.returncode == 0
def modified_workload_files(self) -> Set[str]: """Relies on git to find files in src/workloads modified versus origin/master""" command = ( "git diff --name-only --diff-filter=AMR " "$(git merge-base HEAD origin/master) -- src/workloads/" ) lines = run_command(cmd=[command], cwd=self.genny_repo_root, shell=True, check=True).stdout return {os.path.join(self.genny_repo_root, line) for line in lines if line.endswith(".yml")}
def main_genny_runner(genny_args: List[str], genny_repo_root: str, cleanup_metrics: bool, workspace_root: str): """ Intended to be the main entry point for running Genny. """ with poplar_grpc( cleanup_metrics=cleanup_metrics, workspace_root=workspace_root, genny_repo_root=genny_repo_root, ): path = os.path.join(genny_repo_root, "dist", "bin", "genny_core") if not os.path.exists(path): SLOG.error("genny_core not found. Run install first.", path=path) raise Exception(f"genny_core not found at {path}.") cmd = [path, *genny_args] run_command( cmd=cmd, capture=False, check=True, cwd=workspace_root, )
def install( genny_repo_root: str, workspace_root: str, build_system: str, os_family: str, linux_distro: str, ignore_toolchain_version: bool, ): toolchain_info = toolchain.toolchain_info( genny_repo_root=genny_repo_root, workspace_root=workspace_root, os_family=os_family, linux_distro=linux_distro, ignore_toolchain_version=ignore_toolchain_version, ) install_cmd = [build_system, "-C", "build", "install"] run_command( cmd=install_cmd, env=toolchain_info.toolchain_env, cwd=genny_repo_root, capture=False, check=True, )
def _can_ignore(self): curator = _find_curator(workspace_root=self._workspace_root, genny_repo_root=self._genny_repo_root) if curator is None: return False res: RunCommandOutput = run_command( cmd=[curator, "-v"], check=True, cwd=self._workspace_root, ) installed_version = "".join(res.stdout).strip() wanted_version = f"curator version {CuratorDownloader.CURATOR_VERSION}" SLOG.debug("Comparing curator versions", wanted=wanted_version, installed=installed_version) return installed_version == wanted_version
def modified_workload_files(self) -> Set[str]: """Relies on git to find files in src/workloads modified versus origin/master""" src_path = os.path.join(self.workspace_root, "src") all_repo_directories = { path for path in os.listdir(src_path) if os.path.isdir(os.path.join(src_path, path)) } command = ("git diff --name-only --diff-filter=AMR " "$(git merge-base HEAD origin/master) -- src/workloads/") modified_workloads = set() for repo_directory in all_repo_directories: repo_path = os.path.join(src_path, repo_directory) lines = run_command(cmd=[command], cwd=repo_path, shell=True, check=True).stdout modified_workloads.update({ os.path.join(repo_path, line) for line in lines if line.endswith(".yml") }) return modified_workloads
def _check_toolchain_githash(self): res = "".join( run_command(cmd=["git", "rev-parse", "HEAD"], cwd=self.result_dir, check=True).stdout) return res.strip() == ToolchainDownloader.TOOLCHAIN_GIT_HASH
def main_genny_runner( genny_args: List[str], genny_repo_root: str, cleanup_metrics: bool, workspace_root: str, hang: bool = False, ): """ Intended to be the main entry point for running Genny. """ with poplar_grpc( cleanup_metrics=cleanup_metrics, workspace_root=workspace_root, genny_repo_root=genny_repo_root, ): path = os.path.join(genny_repo_root, "dist", "bin", "genny_core") if not os.path.exists(path): SLOG.error("genny_core not found. Run install first.", path=path) raise Exception(f"genny_core not found at {path}.") cmd = [path, *genny_args] preprocessed_dir = os.path.join(workspace_root, "build/WorkloadOutput/workload") os.makedirs(preprocessed_dir, exist_ok=True) # Intercept the workload given to the core binary. index = -1 if "-w" in cmd: index = cmd.index("-w") + 1 elif "--workload-file" in cmd: index = cmd.index("--workload-file") + 1 elif "dry-run" in cmd: index = cmd.index("dry-run") + 1 if index >= 0: workload_path = cmd[index] smoke = "-s" in cmd or "--smoke-test" in cmd temp_workload = os.path.join(preprocessed_dir, os.path.basename(workload_path)) with open(temp_workload, "w") as f: preprocess.preprocess(workload_path=workload_path, smoke=smoke, output_file=f) cmd[index] = temp_workload if hang: import time import shlex SLOG.info( "Debug mode. Poplar is running. " "Start genny_core (./build/src/driver/genny_core or ./dist/bin/genny_core) " "on your own with the fully processed workload file." f"\n\n {shlex.join(cmd)}\n\n" "Ctrl+C here when done.") while True: time.sleep(10) run_command( cmd=cmd, capture=False, check=True, cwd=workspace_root, )
def _setup_resmoke( workspace_root: str, genny_repo_root: str, mongo_dir: Optional[str], mongodb_archive_url: Optional[str], ): if mongo_dir is not None: mongo_repo_path = mongo_dir else: evergreen_mongo_repo = os.path.join(workspace_root, "src", "mongo") if os.path.exists(evergreen_mongo_repo): mongo_repo_path = evergreen_mongo_repo else: mongo_repo_path = os.path.join(genny_repo_root, "build", "resmoke-mongo") xunit_xml_path = os.path.join(workspace_root, "build", "XUnitXML") os.makedirs(xunit_xml_path, exist_ok=True) SLOG.info("Created xunit result dir", path=xunit_xml_path) resmoke_venv: str = os.path.join(mongo_repo_path, "resmoke_venv") resmoke_python: str = os.path.join(resmoke_venv, "bin", "python3") # Clone repo unless exists if not os.path.exists(mongo_repo_path): SLOG.info("Mongo repo doesn't exist. Checking it out.", mongo_repo_path=mongo_repo_path) cmd_runner.run_command( cmd=[ "git", "clone", "[email protected]:mongodb/mongo.git", mongo_repo_path ], cwd=workspace_root, check=True, capture=False, ) cmd_runner.run_command( # If changing this sha, you may need to use later binaries # in the _canned_artifacts dict. cmd=[ "git", "checkout", "298d4d6bbb9980b74bded06241067fe6771bef68" ], cwd=mongo_repo_path, check=True, capture=False, ) else: SLOG.info("Using existing mongo repo checkout", mongo_repo_path=mongo_repo_path) cmd_runner.run_command( cmd=["git", "rev-parse", "HEAD"], check=False, cwd=mongo_repo_path, capture=False, ) # Look for mongod in # build/opt/mongo/db/mongod # build/install/bin/mongod # bin/ opt = os.path.join(mongo_repo_path, "build", "opt", "mongo", "db", "mongod") install = os.path.join(mongo_repo_path, "build", "install", "bin", "mongod") from_tarball = os.path.join(mongo_repo_path, "bin", "mongod") if os.path.exists(opt): mongod = opt elif os.path.exists(install): mongod = install elif os.path.exists(from_tarball): mongod = from_tarball else: mongod = None if mongod is not None and mongodb_archive_url is not None: SLOG.info( "Found existing mongod so will not download artifacts.", existing_mongod=mongod, wont_download_artifacts_from=mongodb_archive_url, ) if mongod is None: SLOG.info( "Couldn't find pre-build monogod. Fetching and installing.", looked_at=(opt, install, from_tarball), fetching=mongodb_archive_url, ) if mongodb_archive_url is None: info = toolchain.toolchain_info(genny_repo_root=genny_repo_root, workspace_root=workspace_root) if info.is_darwin: artifact_key = "osx" elif info.linux_distro == "amazon2": artifact_key = "amazon2" else: raise Exception( f"No pre-built artifacts for distro {info.linux_distro}. You can either:" f"1. compile/install a local mongo checkout in ./src/mongo." f"2. Modify the _canned_artifacts dict in the genny python to include an artifact from a waterfall build." f"3. Pass in the --mongodb-archive-url parameter to force a canned artifact." ) mongodb_archive_url = _canned_artifacts[artifact_key] cmd_runner.run_command( cmd=["curl", "-LSs", mongodb_archive_url, "-o", "mongodb.tgz"], cwd=mongo_repo_path, capture=False, check=True, ) cmd_runner.run_command( cmd=["tar", "--strip-components=1", "-zxf", "mongodb.tgz"], cwd=mongo_repo_path, capture=False, check=True, ) mongod = from_tarball bin_dir = os.path.dirname(mongod) # Setup resmoke venv unless exists resmoke_setup_sentinel = os.path.join(resmoke_venv, "setup-done") if not os.path.exists(resmoke_setup_sentinel): SLOG.info("Resmoke venv doesn't exist. Creating.", resmoke_venv=resmoke_venv) shutil.rmtree(resmoke_venv, ignore_errors=True) import venv venv.create(env_dir=resmoke_venv, with_pip=True, symlinks=True) reqs_file = os.path.join(mongo_repo_path, "etc", "pip", "evgtest-requirements.txt") cmd = [resmoke_python, "-mpip", "install", "-r", reqs_file] cmd_runner.run_command( cmd=cmd, cwd=workspace_root, capture=False, check=True, ) open(resmoke_setup_sentinel, "w") return resmoke_python, mongo_repo_path, bin_dir
def main_genny_runner( workload_yaml_path, mongo_uri, verbosity, override, dry_run, smoke_test, genny_repo_root: str, cleanup_metrics: bool, workspace_root: str, hang: bool = False, ): """ Intended to be the main entry point for running Genny. """ with poplar_grpc( cleanup_metrics=cleanup_metrics, workspace_root=workspace_root, genny_repo_root=genny_repo_root, ): path = os.path.join(genny_repo_root, "dist", "bin", "genny_core") if not os.path.exists(path): SLOG.error("genny_core not found. Run install first.", path=path) raise Exception(f"genny_core not found at {path}.") cmd = [path] if dry_run: cmd.append("dry-run") else: cmd.append("run") cmd.append("--verbosity") cmd.append(verbosity) preprocessed_dir = os.path.join(workspace_root, "build/WorkloadOutput/workload") os.makedirs(preprocessed_dir, exist_ok=True) processed_workload = os.path.join(preprocessed_dir, os.path.basename(workload_yaml_path)) with open(processed_workload, "w") as f: preprocess.preprocess( workload_path=workload_yaml_path, default_uri=mongo_uri, smoke=smoke_test, output_file=f, override_file_path=override, ) cmd.append("--workload-file") cmd.append(processed_workload) if hang: import time import shlex SLOG.info( "Debug mode. Poplar is running. " "Start genny_core (./build/src/driver/genny_core or ./dist/bin/genny_core) " "on your own with the fully processed workload file." f"\n\n {shlex.join(cmd)}\n\n" "Ctrl+C here when done.") while True: time.sleep(10) run_command( cmd=cmd, capture=False, check=True, cwd=workspace_root, )
def _run_command(cmd): run_command(cmd=cmd, cwd=genny_repo_root, capture=False, check=True)