def compile_all(
genny_repo_root: str,
workspace_root: str,
build_system: str,
os_family: str,
linux_distro: str,
ignore_toolchain_version: bool,
):
toolchain_info = toolchain.toolchain_info(
genny_repo_root=genny_repo_root,
workspace_root=workspace_root,
os_family=os_family,
linux_distro=linux_distro,
ignore_toolchain_version=ignore_toolchain_version,
)
cmd_base = [build_system]
if build_system == "make":
cmd_base.append("-j8")
compile_cmd = [*cmd_base, "-C", "build"]
run_command(
cmd=compile_cmd,
env=toolchain_info.toolchain_env,
cwd=genny_repo_root,
capture=False,
check=True,
)
def _fetch_and_install_impl(self) -> None:
tarball = os.path.join(self._install_dir, self._name + ".tgz")
# if os.path.isfile(tarball):
# SLOG.info("Skipping downloading since already exists", tarball=tarball)
# else:
url = self._get_url()
SLOG.debug("Downloading", name=self._name, url=url)
urllib.request.urlretrieve(url, tarball)
SLOG.debug("Finished Downloading", name=self._name, tarball=tarball)
SLOG.debug("Extracting", name=self._name, into=self.result_dir)
shutil.rmtree(self.result_dir, ignore_errors=True)
os.makedirs(self.result_dir, exist_ok=True)
# use tar(1) because python's TarFile was inexplicably truncating the tarball
run_command(
cmd=["tar", "-xzf", tarball, "-C", self.result_dir],
capture=False,
check=True,
# The cwd doesn't actually matter here since we do 'tar -C'
cwd=self._workspace_root,
)
SLOG.info("Downloaded and installed.",
name=self._name,
into=self.result_dir)
# Get space back.
os.remove(tarball)
def run_generate_uuid_tag(genny_repo_root: str):
path = os.path.join(genny_repo_root, "src", "lamplib", "src", "genny",
"tasks", "generate-uuid-tag.sh")
cmd_runner.run_command(
cmd=[path],
cwd=genny_repo_root,
capture=False,
check=True,
)
def cmake(
genny_repo_root: str,
workspace_root: str,
build_system: str,
os_family: str,
linux_distro: str,
ignore_toolchain_version: bool,
sanitizer: str,
cmake_args: List[str],
):
toolchain_info = toolchain.toolchain_info(
workspace_root=workspace_root,
genny_repo_root=genny_repo_root,
os_family=os_family,
linux_distro=linux_distro,
ignore_toolchain_version=ignore_toolchain_version,
)
generators = {"make": "Unix Makefiles", "ninja": "Ninja"}
cmake_cmd = ["cmake", "-B", "build", "-G", generators[build_system]]
# We set both the prefix path and the toolchain file here as a hack to allow cmake
# to find both shared and static libraries. vcpkg doesn't natively support a project
# using both.
cmake_prefix_paths = [
os.path.join(
toolchain_info.toolchain_dir,
f"installed/x64-{toolchain_info.triplet_os}-dynamic",
),
os.path.join(
toolchain_info.toolchain_dir,
f"installed/x64-{toolchain_info.triplet_os}",
),
]
cmake_toolchain_file = os.path.join(toolchain_info.toolchain_dir,
"scripts/buildsystems/vcpkg.cmake")
cmake_cmd += [
"-DGENNY_WORKSPACE_ROOT={}".format(workspace_root),
# "-DGENNY_REPO_ROOT={}".format(genny_repo_root), # Not needed (yet).
"-DCMAKE_PREFIX_PATH={}".format(";".join(cmake_prefix_paths)),
"-DCMAKE_TOOLCHAIN_FILE={}".format(cmake_toolchain_file),
"-DCMAKE_EXPORT_COMPILE_COMMANDS=1",
f"-DVCPKG_TARGET_TRIPLET=x64-{toolchain_info.triplet_os}",
]
cmake_cmd += _sanitizer_flags(sanitizer)
cmake_cmd += cmake_args
run_command(
cwd=genny_repo_root,
cmd=cmake_cmd,
env=toolchain_info.toolchain_env,
capture=False,
check=True,
)
def run_create_new_actor(genny_repo_root: str, actor_name: str):
path = os.path.join(genny_repo_root, "src", "lamplib", "src", "genny",
"tasks", "create-new-actor.sh")
cmd_runner.run_command(
cmd=[path, actor_name],
cwd=genny_repo_root,
capture=False,
check=True,
)
def run_resmoke() -> None:
# See if we can put this in the resmoke suite def or something?
env["CTEST_OUTPUT_ON_FAILURE"] = "1"
cmd_runner.run_command(
cmd=cmd,
cwd=workspace_root,
env=env,
capture=False,
# If we're create_new_actor_test we don't want
# to barf when resmoke fails. We expect it to fail.
check=False
if is_cnats else True, # `not is_cnats` was hard to read.
)
def modified_workload_files(self) -> Set[str]:
"""Relies on git to find files in src/workloads modified versus origin/master"""
src_path = os.path.join(self.workspace_root, "src")
all_repo_directories = {
path
for path in os.listdir(src_path)
if os.path.isdir(os.path.join(src_path, path))
}
command = ("git diff --name-only --diff-filter=AMR "
"$(git merge-base HEAD origin) -- src/workloads/")
modified_workloads = set()
for repo_directory in all_repo_directories:
repo_path = os.path.join(src_path, repo_directory)
cmd = run_command(cmd=[command],
cwd=repo_path,
shell=True,
check=True)
if cmd.returncode != 0:
SLOG.fatal("Failed to compare workload directory to origin.",
*cmd)
raise RuntimeError(
"Failed to compare workload directory to origin: stdout: {cmd.stdout} stderr: {cmd.stderr}"
)
lines = cmd.stdout
modified_workloads.update({
os.path.join(repo_path, line)
for line in lines if line.endswith(".yml")
})
return modified_workloads
def main_genny_runner(genny_args: List[str], genny_repo_root: str,
cleanup_metrics: bool, workspace_root: str):
"""
Intended to be the main entry point for running Genny.
"""
with poplar_grpc(
cleanup_metrics=cleanup_metrics,
workspace_root=workspace_root,
genny_repo_root=genny_repo_root,
):
path = os.path.join(genny_repo_root, "dist", "bin", "genny_core")
if not os.path.exists(path):
SLOG.error("genny_core not found. Run install first.", path=path)
raise Exception(f"genny_core not found at {path}.")
cmd = [path, *genny_args]
preprocessed_dir = os.path.join(workspace_root,
"build/WorkloadOutput/workload")
os.makedirs(preprocessed_dir, exist_ok=True)
# Intercept the workload given to the core binary.
index = -1
if "-w" in cmd:
index = cmd.index("-w") + 1
elif "--workload-file" in cmd:
index = cmd.index("--workload-file") + 1
elif "dry-run" in cmd:
index = cmd.index("dry-run") + 1
if index >= 0:
workload_path = cmd[index]
smoke = "-s" in cmd or "--smoke-test" in cmd
temp_workload = os.path.join(preprocessed_dir,
os.path.basename(workload_path))
with open(temp_workload, "w") as f:
preprocess.preprocess(workload_path=workload_path,
smoke=smoke,
output_file=f)
cmd[index] = temp_workload
run_command(
cmd=cmd,
capture=False,
check=True,
cwd=workspace_root,
)
def cmd_func() -> bool:
output: cmd_runner.RunCommandOutput = cmd_runner.run_command(
cmd=ctest_cmd,
cwd=workdir,
env=info.toolchain_env,
capture=False,
check=True)
return output.returncode == 0
def modified_workload_files(self) -> Set[str]:
"""Relies on git to find files in src/workloads modified versus origin/master"""
command = (
"git diff --name-only --diff-filter=AMR "
"$(git merge-base HEAD origin/master) -- src/workloads/"
)
lines = run_command(cmd=[command], cwd=self.genny_repo_root, shell=True, check=True).stdout
return {os.path.join(self.genny_repo_root, line) for line in lines if line.endswith(".yml")}
def main_genny_runner(genny_args: List[str], genny_repo_root: str,
cleanup_metrics: bool, workspace_root: str):
"""
Intended to be the main entry point for running Genny.
"""
with poplar_grpc(
cleanup_metrics=cleanup_metrics,
workspace_root=workspace_root,
genny_repo_root=genny_repo_root,
):
path = os.path.join(genny_repo_root, "dist", "bin", "genny_core")
if not os.path.exists(path):
SLOG.error("genny_core not found. Run install first.", path=path)
raise Exception(f"genny_core not found at {path}.")
cmd = [path, *genny_args]
run_command(
cmd=cmd,
capture=False,
check=True,
cwd=workspace_root,
)
def install(
genny_repo_root: str,
workspace_root: str,
build_system: str,
os_family: str,
linux_distro: str,
ignore_toolchain_version: bool,
):
toolchain_info = toolchain.toolchain_info(
genny_repo_root=genny_repo_root,
workspace_root=workspace_root,
os_family=os_family,
linux_distro=linux_distro,
ignore_toolchain_version=ignore_toolchain_version,
)
install_cmd = [build_system, "-C", "build", "install"]
run_command(
cmd=install_cmd,
env=toolchain_info.toolchain_env,
cwd=genny_repo_root,
capture=False,
check=True,
)
def _can_ignore(self):
curator = _find_curator(workspace_root=self._workspace_root,
genny_repo_root=self._genny_repo_root)
if curator is None:
return False
res: RunCommandOutput = run_command(
cmd=[curator, "-v"],
check=True,
cwd=self._workspace_root,
)
installed_version = "".join(res.stdout).strip()
wanted_version = f"curator version {CuratorDownloader.CURATOR_VERSION}"
SLOG.debug("Comparing curator versions",
wanted=wanted_version,
installed=installed_version)
return installed_version == wanted_version
def modified_workload_files(self) -> Set[str]:
"""Relies on git to find files in src/workloads modified versus origin/master"""
src_path = os.path.join(self.workspace_root, "src")
all_repo_directories = {
path
for path in os.listdir(src_path)
if os.path.isdir(os.path.join(src_path, path))
}
command = ("git diff --name-only --diff-filter=AMR "
"$(git merge-base HEAD origin/master) -- src/workloads/")
modified_workloads = set()
for repo_directory in all_repo_directories:
repo_path = os.path.join(src_path, repo_directory)
lines = run_command(cmd=[command],
cwd=repo_path,
shell=True,
check=True).stdout
modified_workloads.update({
os.path.join(repo_path, line)
for line in lines if line.endswith(".yml")
})
return modified_workloads
def _check_toolchain_githash(self):
res = "".join(
run_command(cmd=["git", "rev-parse", "HEAD"],
cwd=self.result_dir,
check=True).stdout)
return res.strip() == ToolchainDownloader.TOOLCHAIN_GIT_HASH
def main_genny_runner(
genny_args: List[str],
genny_repo_root: str,
cleanup_metrics: bool,
workspace_root: str,
hang: bool = False,
):
"""
Intended to be the main entry point for running Genny.
"""
with poplar_grpc(
cleanup_metrics=cleanup_metrics,
workspace_root=workspace_root,
genny_repo_root=genny_repo_root,
):
path = os.path.join(genny_repo_root, "dist", "bin", "genny_core")
if not os.path.exists(path):
SLOG.error("genny_core not found. Run install first.", path=path)
raise Exception(f"genny_core not found at {path}.")
cmd = [path, *genny_args]
preprocessed_dir = os.path.join(workspace_root,
"build/WorkloadOutput/workload")
os.makedirs(preprocessed_dir, exist_ok=True)
# Intercept the workload given to the core binary.
index = -1
if "-w" in cmd:
index = cmd.index("-w") + 1
elif "--workload-file" in cmd:
index = cmd.index("--workload-file") + 1
elif "dry-run" in cmd:
index = cmd.index("dry-run") + 1
if index >= 0:
workload_path = cmd[index]
smoke = "-s" in cmd or "--smoke-test" in cmd
temp_workload = os.path.join(preprocessed_dir,
os.path.basename(workload_path))
with open(temp_workload, "w") as f:
preprocess.preprocess(workload_path=workload_path,
smoke=smoke,
output_file=f)
cmd[index] = temp_workload
if hang:
import time
import shlex
SLOG.info(
"Debug mode. Poplar is running. "
"Start genny_core (./build/src/driver/genny_core or ./dist/bin/genny_core) "
"on your own with the fully processed workload file."
f"\n\n {shlex.join(cmd)}\n\n"
"Ctrl+C here when done.")
while True:
time.sleep(10)
run_command(
cmd=cmd,
capture=False,
check=True,
cwd=workspace_root,
)
def _setup_resmoke(
workspace_root: str,
genny_repo_root: str,
mongo_dir: Optional[str],
mongodb_archive_url: Optional[str],
):
if mongo_dir is not None:
mongo_repo_path = mongo_dir
else:
evergreen_mongo_repo = os.path.join(workspace_root, "src", "mongo")
if os.path.exists(evergreen_mongo_repo):
mongo_repo_path = evergreen_mongo_repo
else:
mongo_repo_path = os.path.join(genny_repo_root, "build",
"resmoke-mongo")
xunit_xml_path = os.path.join(workspace_root, "build", "XUnitXML")
os.makedirs(xunit_xml_path, exist_ok=True)
SLOG.info("Created xunit result dir", path=xunit_xml_path)
resmoke_venv: str = os.path.join(mongo_repo_path, "resmoke_venv")
resmoke_python: str = os.path.join(resmoke_venv, "bin", "python3")
# Clone repo unless exists
if not os.path.exists(mongo_repo_path):
SLOG.info("Mongo repo doesn't exist. Checking it out.",
mongo_repo_path=mongo_repo_path)
cmd_runner.run_command(
cmd=[
"git", "clone", "[email protected]:mongodb/mongo.git",
mongo_repo_path
],
cwd=workspace_root,
check=True,
capture=False,
)
cmd_runner.run_command(
# If changing this sha, you may need to use later binaries
# in the _canned_artifacts dict.
cmd=[
"git", "checkout", "298d4d6bbb9980b74bded06241067fe6771bef68"
],
cwd=mongo_repo_path,
check=True,
capture=False,
)
else:
SLOG.info("Using existing mongo repo checkout",
mongo_repo_path=mongo_repo_path)
cmd_runner.run_command(
cmd=["git", "rev-parse", "HEAD"],
check=False,
cwd=mongo_repo_path,
capture=False,
)
# Look for mongod in
# build/opt/mongo/db/mongod
# build/install/bin/mongod
# bin/
opt = os.path.join(mongo_repo_path, "build", "opt", "mongo", "db",
"mongod")
install = os.path.join(mongo_repo_path, "build", "install", "bin",
"mongod")
from_tarball = os.path.join(mongo_repo_path, "bin", "mongod")
if os.path.exists(opt):
mongod = opt
elif os.path.exists(install):
mongod = install
elif os.path.exists(from_tarball):
mongod = from_tarball
else:
mongod = None
if mongod is not None and mongodb_archive_url is not None:
SLOG.info(
"Found existing mongod so will not download artifacts.",
existing_mongod=mongod,
wont_download_artifacts_from=mongodb_archive_url,
)
if mongod is None:
SLOG.info(
"Couldn't find pre-build monogod. Fetching and installing.",
looked_at=(opt, install, from_tarball),
fetching=mongodb_archive_url,
)
if mongodb_archive_url is None:
info = toolchain.toolchain_info(genny_repo_root=genny_repo_root,
workspace_root=workspace_root)
if info.is_darwin:
artifact_key = "osx"
elif info.linux_distro == "amazon2":
artifact_key = "amazon2"
else:
raise Exception(
f"No pre-built artifacts for distro {info.linux_distro}. You can either:"
f"1. compile/install a local mongo checkout in ./src/mongo."
f"2. Modify the _canned_artifacts dict in the genny python to include an artifact from a waterfall build."
f"3. Pass in the --mongodb-archive-url parameter to force a canned artifact."
)
mongodb_archive_url = _canned_artifacts[artifact_key]
cmd_runner.run_command(
cmd=["curl", "-LSs", mongodb_archive_url, "-o", "mongodb.tgz"],
cwd=mongo_repo_path,
capture=False,
check=True,
)
cmd_runner.run_command(
cmd=["tar", "--strip-components=1", "-zxf", "mongodb.tgz"],
cwd=mongo_repo_path,
capture=False,
check=True,
)
mongod = from_tarball
bin_dir = os.path.dirname(mongod)
# Setup resmoke venv unless exists
resmoke_setup_sentinel = os.path.join(resmoke_venv, "setup-done")
if not os.path.exists(resmoke_setup_sentinel):
SLOG.info("Resmoke venv doesn't exist. Creating.",
resmoke_venv=resmoke_venv)
shutil.rmtree(resmoke_venv, ignore_errors=True)
import venv
venv.create(env_dir=resmoke_venv, with_pip=True, symlinks=True)
reqs_file = os.path.join(mongo_repo_path, "etc", "pip",
"evgtest-requirements.txt")
cmd = [resmoke_python, "-mpip", "install", "-r", reqs_file]
cmd_runner.run_command(
cmd=cmd,
cwd=workspace_root,
capture=False,
check=True,
)
open(resmoke_setup_sentinel, "w")
return resmoke_python, mongo_repo_path, bin_dir
def main_genny_runner(
workload_yaml_path,
mongo_uri,
verbosity,
override,
dry_run,
smoke_test,
genny_repo_root: str,
cleanup_metrics: bool,
workspace_root: str,
hang: bool = False,
):
"""
Intended to be the main entry point for running Genny.
"""
with poplar_grpc(
cleanup_metrics=cleanup_metrics,
workspace_root=workspace_root,
genny_repo_root=genny_repo_root,
):
path = os.path.join(genny_repo_root, "dist", "bin", "genny_core")
if not os.path.exists(path):
SLOG.error("genny_core not found. Run install first.", path=path)
raise Exception(f"genny_core not found at {path}.")
cmd = [path]
if dry_run:
cmd.append("dry-run")
else:
cmd.append("run")
cmd.append("--verbosity")
cmd.append(verbosity)
preprocessed_dir = os.path.join(workspace_root,
"build/WorkloadOutput/workload")
os.makedirs(preprocessed_dir, exist_ok=True)
processed_workload = os.path.join(preprocessed_dir,
os.path.basename(workload_yaml_path))
with open(processed_workload, "w") as f:
preprocess.preprocess(
workload_path=workload_yaml_path,
default_uri=mongo_uri,
smoke=smoke_test,
output_file=f,
override_file_path=override,
)
cmd.append("--workload-file")
cmd.append(processed_workload)
if hang:
import time
import shlex
SLOG.info(
"Debug mode. Poplar is running. "
"Start genny_core (./build/src/driver/genny_core or ./dist/bin/genny_core) "
"on your own with the fully processed workload file."
f"\n\n {shlex.join(cmd)}\n\n"
"Ctrl+C here when done.")
while True:
time.sleep(10)
run_command(
cmd=cmd,
capture=False,
check=True,
cwd=workspace_root,
)
def _run_command(cmd):
run_command(cmd=cmd, cwd=genny_repo_root, capture=False, check=True)
