def cmd_itest_clean(args, bazel_args, mode_args):
_raise_on_glob_target(args.target)
itest_target = _get_itest_target(args.bazel_path, args.target)
container_name = _get_container_name_for_target(itest_target.name)
containers = _get_all_containers(args.docker_path)
if container_name in containers:
message = """Refusing to remove data directory because container {name} is still running. Try:
Stop and remove this container:
bzl itest-stop {target}""".format(name=container_name,
target=itest_target.name)
sys.exit(message)
host_data_dir = os.path.join(HOST_DATA_DIR_PREFIX, container_name)
if not os.path.exists(host_data_dir):
sys.exit("Data directory {} does not exist".format(host_data_dir))
if _should_remove_container_dir(args, host_data_dir):
exec_wrapper.execv("/usr/bin/sudo",
["/usr/bin/sudo", "rm", "-rf", host_data_dir])
else:
print(
"WARN: Skipping containers with persistent data",
host_data_dir,
file=sys.stderr,
)
print(
" bzl itest-clean --expunge %s - remove persistent data" %
args.target,
file=sys.stderr,
)
def run_build_tool(bazel_path, target, targets, squelch_output=False):
workspace = find_workspace()
if not workspace:
return
try:
# If we can bootstrap a new version, do it once.
with metrics.create_and_register_timer("bzl_bootstrap_ms") as t:
bzl_script = build_tool(bazel_path,
target,
targets,
squelch_output=squelch_output)
bzl_script_path = os.path.join(workspace, bzl_script)
argv = [bzl_script_path] + list(sys.argv[1:])
os.environ["BZL_SKIP_BOOTSTRAP"] = "1"
os.environ["BZL_BOOTSTRAP_MS"] = str(t.get_interval_ms())
os.environ["BZL_RUNNING_REBUILT_BZL"] = "1"
exec_wrapper.execv(bzl_script_path, argv)
except subprocess.CalledProcessError:
print(
"WARN: Failed to build %s, continuing without self-update." %
target,
file=sys.stderr,
)
# If something goes wrong during rebuild, just run this version.
pass
def cmd_itest_clean_all(args, bazel_args, mode_args):
# By default, clean all transient container storage.
# If there are persistent_test_tmpdir subdirectories, show a warning.
disk_container_names = frozenset(os.listdir(HOST_DATA_DIR_PREFIX))
running_container_names = frozenset(_get_all_containers(args.docker_path))
idle_container_names = disk_container_names - running_container_names
if running_container_names:
print >> sys.stderr, "WARN: Skipping running containers", ", ".join(
sorted(running_container_names))
print >> sys.stderr, " Run bzl itest-stop-all to stop all of them."
delete_dir_list = []
skipped_persistent_dirs = []
for name in idle_container_names:
host_data_dir = os.path.join(HOST_DATA_DIR_PREFIX, name)
if _should_remove_container_dir(args, host_data_dir):
delete_dir_list.append(host_data_dir)
else:
skipped_persistent_dirs.append(host_data_dir)
if skipped_persistent_dirs:
print >> sys.stderr, "WARN: Skipping containers with persistent data", ", ".join(
sorted(skipped_persistent_dirs))
print >> sys.stderr, " bzl itest-clean-all --expunge will remove persistent data."
if delete_dir_list:
delete_dir_list.sort()
exec_wrapper.execv("/usr/bin/sudo",
["/usr/bin/sudo", "rm", "-rf"] + delete_dir_list)
def cmd_gen_as_tool(args, bazel_args, mode_args):
workspace_dir = bazel_utils.find_workspace()
bzl_gen = bazel_utils.build_tool(args.bazel_path,
"@dbx_build_tools//build_tools:bzl-gen")
bzl_gen_path = os.path.join(workspace_dir, bzl_gen)
argv = [os.path.basename(bzl_gen_path)] + mode_args
exec_wrapper.execv(bzl_gen_path, argv)
def cmd_itest_stop(args, bazel_args, mode_args):
_raise_on_glob_target(args.target)
itest_target = _get_itest_target(args.bazel_path, args.target)
container_name = _get_container_name_for_target(itest_target.name)
_verify_args(args, itest_target, container_should_be_running=True)
exec_wrapper.execv(args.docker_path,
[args.docker_path, "rm", "-f", container_name])
def cmd_itest_exec(args, bazel_args, mode_args):
_raise_on_glob_target(args.target)
itest_target = _get_itest_target(args.bazel_path, args.target)
container_name = _get_container_name_for_target(itest_target.name)
_verify_args(args, itest_target, container_should_be_running=True)
docker_exec_args = [args.docker_path, "exec"]
if sys.stdin.isatty():
docker_exec_args += ["--interactive", "--tty"]
docker_exec_args += [container_name]
exec_wrapper.execv(args.docker_path, docker_exec_args + args.cmd)
def cmd_itest_stop_all(args, bazel_args, mode_args):
containers = _get_all_containers(args.docker_path)
if containers:
if not args.force:
print("""Will stop the following containers:
{}
""".format("\n".join(containers)))
reply = input("Continue? [y/N] ")
if reply.strip().lower() not in ("y", "yes"):
sys.exit("Aborted.")
exec_wrapper.execv(args.docker_path,
[args.docker_path, "rm", "-f"] + containers)
else:
print("No containers to stop")
def cmd_itest_run(args, bazel_args, mode_args):
_raise_on_glob_target(args.target)
_build_target(args, bazel_args, mode_args, args.target)
itest_target = _get_itest_target(args.bazel_path,
args.target,
use_implicit_output=True)
container_name = _get_container_name_for_target(itest_target.name)
_verify_args(args, itest_target, container_should_be_running=False)
tmpdir_name = "test_tmpdir"
if args.persist_tmpdir:
tmpdir_name = "persistent_test_tmpdir"
host_data_dir = os.path.join(HOST_DATA_DIR_PREFIX, container_name)
host_tmpdir = os.path.join(host_data_dir, tmpdir_name)
for dirname in [host_tmpdir, HOST_HOME_DIR]:
if not os.path.exists(dirname):
os.makedirs(dirname)
container_tmpdir = os.path.join(IN_CONTAINER_DATA_DIR, tmpdir_name)
workspace = bazel_utils.find_workspace()
cwd = workspace
# order matters here. The last command shows up as the last thing the user ran, i.e.
# the first command they see when they hit "up"
history_cmds = [] # type: ignore[var-annotated]
if itest_target.has_services:
history_cmds = [
"svcctl --help",
"svcctl status",
'svcctl status -format "{{.CPUTime}} {{.Name}}" | sort -rgb | head',
]
test_bin = os.path.join(host_data_dir, RUN_TEST_BIN_NAME)
with open(test_bin, "w") as f:
f.write("""#!/bin/bash -eu
cd {cwd}
exec {test} "$@"
""".format(
cwd=itest_target.executable_path + ".runfiles/__main__",
test=" ".join(itest_target.test_cmd),
))
os.chmod(test_bin, 0o755)
test_cmd_str = " ".join(
pipes.quote(x)
for x in [os.path.join(IN_CONTAINER_DATA_DIR, RUN_TEST_BIN_NAME)] +
args.test_arg)
history_cmds.append(test_cmd_str)
launch_cmd = itest_target.service_launch_cmd
if args.verbose:
launch_cmd += ["--svc.verbose"]
default_paths = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin".split(
":")
itest_paths = [
os.path.join(
workspace,
os.path.dirname(bazel_utils.executable_for_label(SVCCTL_TARGET))),
os.path.join(workspace, "build_tools/itest"),
]
env = {
"DROPBOX_SERVER_TEST":
"1",
"PATH":
":".join(itest_paths + default_paths),
"TEST_TMPDIR":
container_tmpdir,
"HOST_TEST_TMPDIR":
host_tmpdir,
"HOME":
IN_CONTAINER_HOME_DIR, # override HOME since we can't readily edit /etc/passwd
"LAUNCH_CMD":
" ".join(launch_cmd),
"TEST_CMD":
test_cmd_str,
# Set how much directory to clean up on startup. Pass this into the container so it gets
# cleaned up as root.
"CLEANDIR":
os.path.join(container_tmpdir, "logs")
if args.persist_tmpdir else container_tmpdir,
}
history_file = os.path.join(HOST_HOME_DIR, ".bash_history")
bash_history.merge_history([history_file], history_cmds, history_file)
bashrc_file_src = runfiles.data_path(
"@dbx_build_tools//build_tools/bzl_lib/itest/bashrc")
if args.build_image:
docker_image = args.build_image
else:
docker_image = os.path.join(args.docker_registry, DEFAULT_IMAGE)
init_cmd_args = [
runfiles.data_path(
"@dbx_build_tools//build_tools/bzl_lib/itest/bzl-itest-init")
]
# Set a fail-safe limit for an itest container to keep it from detonating the whole
# machine. RSS limits are a funny thing in docker. Most likely the oom-killer will
# start killing things inside the container rendering it unstable.
# FIXME(msolo) It would be nice to teardown the container on out-of-memory and leave
# some sort of note.
mem_limit_kb = _guess_mem_limit_kb()
docker_run_args = [
args.docker_path,
"run",
"--net=host",
"--name",
container_name,
"--workdir",
cwd,
"--detach",
"--memory",
"%dK" % mem_limit_kb,
# Swap is disabled anyway, so squelch a spurious warning.
"--memory-swap",
"-1",
# Store target name in config, to be able to reload it later.
"--label",
"itest-target=%s" % args.target,
]
if args.privileged:
docker_run_args += ["--privileged"]
# set env variables. This will also set it for subsequent `docker exec` commands
for k, v in env.items():
docker_run_args += ["-e", "{}={}".format(k, v)]
with metrics.create_and_register_timer("bazel_info_ms"):
with open(os.devnull, "w") as dev_null:
output_base = subprocess.check_output(
[args.bazel_path, "info", "output_base"],
stderr=dev_null).strip()
install_base = subprocess.check_output(
[args.bazel_path, "info", "install_base"],
stderr=dev_null).strip()
mounts = [
(os.fsencode(workspace), b"ro"),
(b"/sqpkg", b"ro"),
(output_base, b"ro"),
(install_base, b"ro"),
(b"/etc/ssl", b"ro"),
(b"/usr/share/ca-certificates", b"ro"),
# We bind mount /run/dropbox/sock-drawer/ as read-write so that services outside
# itest (ie ULXC jails) can publish sockets here that can be used from the inside
# (bind mount), and so that services inside itest (ie RivieraFS) can publish
# sockets here (read-write) that can be used from the outside
(DEFAULT_SOCKET_DIRECTORY_PATH, b"rw"),
]
for path, perms in mounts:
# Docker will happily create a mount source that is nonexistent, but it may not have the
# right permissions. Better to just mount nothing.
if not os.path.exists(path):
print("missing mount point:", path, file=sys.stderr)
continue
src = os.path.realpath(path)
docker_run_args += ["-v", b"%s:%s:%s" % (src, path, perms)]
# Allow bzl itest containers to observe external changes to the mount table.
if os.path.exists("/mnt/sqpkg"):
docker_run_args += ["-v", "/mnt/sqpkg:/mnt/sqpkg:rslave"]
if sys.stdin.isatty():
# otherwise text wrapping on subsequent shells is messed up
docker_run_args += ["--tty"]
docker_run_args += [
"-v", "{}:{}:rw".format(host_data_dir, IN_CONTAINER_DATA_DIR)
]
docker_run_args += [
"-v", "{}:{}:rw".format(HOST_HOME_DIR, IN_CONTAINER_HOME_DIR)
]
docker_run_args += [
"-v", "{}:{}:ro".format(bashrc_file_src, "/etc/bash.bashrc")
]
docker_run_args += [docker_image]
docker_run_args += init_cmd_args
with metrics.create_and_register_timer("services_start_ms"):
with open(os.devnull, "w") as f:
subprocess.check_call(docker_run_args, stdout=f)
docker_exec_args = [args.docker_path, "exec"]
if sys.stdin.isatty():
docker_exec_args += ["--interactive", "--tty"]
docker_exec_args += [container_name]
exit_code = subprocess.call(docker_exec_args + [
runfiles.data_path(
"@dbx_build_tools//build_tools/bzl_lib/itest/bzl-itest-wait")
])
if exit_code == 0:
# run the test command
with metrics.create_and_register_timer("test_ms"):
# NOT check_call. Even if this script doesn't exit with 0 (e.g. test fails),
# we want to keep going
subprocess.call(docker_exec_args +
["/bin/bash", "-c", test_cmd_str])
if itest_target.has_services:
services_started = (subprocess.check_output(
[
args.docker_path,
"exec",
container_name,
"svcctl",
"status",
"--all",
"--format={{.Name}}",
],
universal_newlines=True,
).strip().split("\n"))
metrics.set_extra_attributes("services_started",
",".join(services_started))
metrics.set_gauge("services_started_count", len(services_started))
# report metrics now, instead of after the interactive session since
# we don't want to measure that
metrics.report_metrics()
if args.detach:
# display message of the day then exit
exec_wrapper.execv(args.docker_path,
docker_exec_args + ["cat", "/etc/motd"])
else:
exit_code = subprocess.call(docker_exec_args + ["/bin/bash"])
with open(os.devnull, "w") as devnull:
subprocess.check_call(
[args.docker_path, "rm", "-f", container_name],
stdout=devnull,
stderr=devnull,
)
sys.exit(exit_code)
def cmd_gen_as_tool(args, bazel_args, mode_args):
# type: (argparse.Namespace, List[str], List[str]) -> None
bzl_gen_path = _get_bzl_gen_path(args.bazel_path)
bazel_path_args = ["--bazel-path", args.bazel_path]
argv = [os.path.basename(bzl_gen_path)] + bazel_path_args + mode_args
exec_wrapper.execv(bzl_gen_path, argv)
