def _wait_until_alive(self, client: EdenClient) -> None:
def is_alive() -> Optional[bool]:
if client.getStatus() == fb303_status.ALIVE:
return True
return None
poll_until(is_alive, timeout=60)
def test_start_blocked_mount_init(self) -> None:
self.eden.shutdown()
self.eden.spawn_nowait(
extra_args=["--enable_fault_injection", "--fault_injection_block_mounts"]
)
# Wait for eden to report the mount point in the listMounts() output
def is_initializing() -> Optional[bool]:
try:
with self.eden.get_thrift_client() as client:
if self.eden.get_mount_state(Path(self.mount), client) is not None:
return True
assert self.eden._process is not None
if self.eden._process.poll():
self.fail("eden exited before becoming healthy")
return None
except (EdenNotRunningError, TException):
return None
poll_until(is_initializing, timeout=60)
with self.eden.get_thrift_client() as client:
# Since we blocked mount initialization the mount should still
# report as INITIALIZING, and edenfs should report itself STARTING
self.assertEqual({self.mount: "INITIALIZING"}, self.eden.list_cmd_simple())
self.assertEqual(fb_status.STARTING, client.getStatus())
# Unblock mounting and wait for the mount to transition to running
client.unblockFault(UnblockFaultArg(keyClass="mount", keyValueRegex=".*"))
self._wait_for_mount_running(client)
self.assertEqual(fb_status.ALIVE, client.getStatus())
self.assertEqual({self.mount: "RUNNING"}, self.eden.list_cmd_simple())
def _wait_for_mount_running(self, client: EdenClient) -> None:
def mount_running() -> Optional[bool]:
if (self.eden.get_mount_state(Path(self.mount),
client) == MountState.RUNNING):
return True
return None
poll_until(mount_running, timeout=60)
def _wait_for_mount_running(self, client: EdenClient) -> None:
def mount_running() -> Optional[bool]:
if (
self.eden.get_mount_state(Path(self.mount), client)
== MountState.RUNNING
):
return True
return None
poll_until(mount_running, timeout=60)
def _wait_for_mount_running(
self, client: EdenClient, path: Optional[Path] = None
) -> None:
mount_path = path if path is not None else Path(self.mount)
def mount_running() -> Optional[bool]:
if self.eden.get_mount_state(mount_path, client) == MountState.RUNNING:
return True
return None
poll_until(mount_running, timeout=60)
def test_async_stop_stops_daemon_eventually(self) -> None:
with self.spawn_fake_edenfs(self.eden_dir) as daemon_pid:
stop_process = self.spawn_stop(["--timeout", "0"])
stop_process.expect_exact("Sent async shutdown request to edenfs.")
self.assert_process_exit_code(
stop_process, SHUTDOWN_EXIT_CODE_REQUESTED_SHUTDOWN)
def daemon_exited() -> typing.Optional[bool]:
if did_process_exit(daemon_pid):
return True
else:
return None
poll_until(daemon_exited, timeout=10)
def test_async_stop_stops_daemon_eventually(self) -> None:
with self.spawn_fake_edenfs(self.eden_dir) as daemon_pid:
stop_process = self.spawn_stop(["--timeout", "0"])
stop_process.expect_exact("Sent async shutdown request to edenfs.")
self.assert_process_exit_code(
stop_process, SHUTDOWN_EXIT_CODE_REQUESTED_SHUTDOWN
)
def daemon_exited() -> typing.Optional[bool]:
if did_process_exit(daemon_pid):
return True
else:
return None
poll_until(daemon_exited, timeout=10)
def test_mount_init_state(self) -> None:
self.eden.run_cmd("unmount", self.mount)
self.assertEqual({self.mount: "NOT_RUNNING"},
self.eden.list_cmd_simple())
with self.eden.get_thrift_client() as client:
fault = FaultDefinition(keyClass="mount",
keyValueRegex=".*",
block=True)
client.injectFault(fault)
# Run the "eden mount" CLI command.
# This won't succeed until we unblock the mount.
mount_cmd = self.eden.get_eden_cli_args("mount", self.mount)
mount_proc = subprocess.Popen(mount_cmd)
# Wait for the new mount to be reported by edenfs
def mount_started() -> Optional[bool]:
if self.eden.get_mount_state(Path(self.mount),
client) is not None:
return True
if mount_proc.poll() is not None:
raise Exception(
f"eden mount command finished (with status "
f"{mount_proc.returncode}) while mounting was "
f"still blocked")
return None
poll_until(mount_started, timeout=30)
self.assertEqual({self.mount: "INITIALIZING"},
self.eden.list_cmd_simple())
# Most thrift calls to access the mount should be disallowed while it is
# still initializing.
self._assert_thrift_calls_fail_during_mount_init(client)
# Unblock mounting and wait for the mount to transition to running
client.unblockFault(
UnblockFaultArg(keyClass="mount", keyValueRegex=".*"))
self._wait_for_mount_running(client)
self.assertEqual({self.mount: "RUNNING"},
self.eden.list_cmd_simple())
def shutdown(self) -> None:
'''
Run "eden shutdown" to stop the eden daemon.
'''
assert self._process is not None
# We need to take care here: the normal `eden shutdown` command will
# wait for eden to successfully finish by repeatedly testing to see
# whether the process is alive. However, running as root we don't
# spawn an intermediate process and this results in our child process
# (attached to self._process) to land in a defunct state such that
# the `kill(pid, 0)` test in the `eden shutdown` command still considers
# the process alive. To avoid this situation we ask the shutdown
# command not to wait and instead perform our own polling here against
# the real process handle.
self.run_cmd('shutdown', '-t', '0')
util.poll_until(lambda: self._process.poll(), timeout=15)
return_code = self._process.wait()
self._process = None
if return_code != 0:
raise Exception('eden exited unsuccessfully with status {}'.format(
return_code))
def _wait_until_initializing(self, num_mounts: int = 1) -> None:
"""Wait until EdenFS is initializing mount points.
This is primarily intended to be used to wait until the mount points are
initializing when starting EdenFS with --fault_injection_block_mounts.
"""
def is_initializing() -> Optional[bool]:
try:
with self.eden.get_thrift_client() as client:
# Return successfully when listMounts() reports the number of
# mounts that we expect.
mounts = client.listMounts()
if len(mounts) == num_mounts:
return True
edenfs_process = self.eden._process
assert edenfs_process is not None
if edenfs_process.poll():
self.fail("eden exited before becoming healthy")
return None
except (EdenNotRunningError, TException):
return None
poll_until(is_initializing, timeout=60)
def test_mount_init_state(self) -> None:
self.eden.run_cmd("unmount", self.mount)
self.assertEqual({self.mount: "NOT_RUNNING"}, self.eden.list_cmd_simple())
with self.eden.get_thrift_client() as client:
fault = FaultDefinition(keyClass="mount", keyValueRegex=".*", block=True)
client.injectFault(fault)
# Run the "eden mount" CLI command.
# This won't succeed until we unblock the mount.
mount_cmd = self.eden.get_eden_cli_args("mount", self.mount)
mount_proc = subprocess.Popen(mount_cmd)
# Wait for the new mount to be reported by edenfs
def mount_started() -> Optional[bool]:
if self.eden.get_mount_state(Path(self.mount), client) is not None:
return True
if mount_proc.poll() is not None:
raise Exception(
f"eden mount command finished (with status "
f"{mount_proc.returncode}) while mounting was "
f"still blocked"
)
return None
poll_until(mount_started, timeout=30)
self.assertEqual({self.mount: "INITIALIZING"}, self.eden.list_cmd_simple())
# Most thrift calls to access the mount should be disallowed while it is
# still initializing.
self._assert_thrift_calls_fail_during_mount_init(client)
# Unblock mounting and wait for the mount to transition to running
client.unblockFault(UnblockFaultArg(keyClass="mount", keyValueRegex=".*"))
self._wait_for_mount_running(client)
self.assertEqual({self.mount: "RUNNING"}, self.eden.list_cmd_simple())
def wait_until_process_is_zombie(process: subprocess.Popen) -> None:
def is_zombie() -> typing.Optional[bool]:
return True if is_zombie_process(process.pid) else None
poll_until(is_zombie, timeout=3)
def test_local_store_stats(self) -> None:
# Update the config to tell the local store to updates its stats frequently
# and also check if it needs to reload the config file frequently.
initial_config = """\
[config]
reload-interval = "100ms"
[store]
stats-interval = "100ms"
"""
self.eden.user_rc_path.write_text(initial_config)
counter_regex = r"local_store\..*"
with self.get_thrift_client() as client:
# Makes sure that EdenFS picks up our updated config,
# since we wrote it out after EdenFS started.
client.reloadConfig()
# Get the local store counters
# Assert that the exist and are greater than 0.
# (Since we include memtable sizes in the values these are currently always
# reported as taking up at least a small amount of space.)
initial_counters = client.getRegexCounters(counter_regex)
self.assertGreater(initial_counters.get("local_store.blob.size"),
0)
self.assertGreater(
initial_counters.get("local_store.blobmeta.size"), 0)
self.assertGreater(initial_counters.get("local_store.tree.size"),
0)
self.assertGreater(
initial_counters.get("local_store.hgcommit2tree.size"), 0)
self.assertGreater(
initial_counters.get("local_store.hgproxyhash.size"), 0)
self.assertGreater(
initial_counters.get("local_store.ephemeral.total_size"), 0)
self.assertGreater(
initial_counters.get("local_store.persistent.total_size"), 0)
# Make sure the counters are less than 500MB, just as a sanity check
self.assertLess(
initial_counters.get("local_store.ephemeral.total_size"),
500_000_000)
self.assertLess(
initial_counters.get("local_store.persistent.total_size"),
500_000_000)
# Read back several files
self.assertEqual((self.mount_path / "a/dir/foo.txt").read_text(),
"foo\n")
self.assertEqual((self.mount_path / "a/dir/bar.txt").read_text(),
"bar\n")
self.assertEqual(
(self.mount_path / "a/another_dir/hello.txt").read_text(),
"hola\n")
# The tree store size should be larger now after reading these files.
# The counters won't be updated until the store.stats-interval expires.
# Wait for this to happen.
def tree_size_incremented() -> Optional[bool]:
tree_size = client.getCounter("local_store.tree.size")
initial_tree_size = initial_counters.get(
"local_store.tree.size")
assert initial_tree_size is not None
if tree_size > initial_tree_size:
return True
return None
poll_until(tree_size_incremented, timeout=1, interval=0.1)
# EdenFS should not import blobs to local store
self.assertEqual(
initial_counters.get("local_store.blob.size"),
client.getCounter("local_store.blob.size"),
)
# Update the config file with a very small GC limit that will force GC to be
# triggered
self.eden.user_rc_path.write_text(initial_config + """
blob-size-limit = "1"
blobmeta-size-limit = "1"
tree-size-limit = "1"
hgcommit2tree-size-limit = "1"
""")
# Wait until a GC run has completed.
def gc_run_succeeded() -> Optional[Dict[str, int]]:
counters = client.getRegexCounters(counter_regex)
if counters.get(
"local_store.auto_gc.last_run_succeeded") is not None:
return counters
return None
counters = poll_until(gc_run_succeeded, timeout=5, interval=0.05)
# Check the local_store.auto_gc counters
self.assertEqual(
counters.get("local_store.auto_gc.last_run_succeeded"), 1)
self.assertGreater(counters.get("local_store.auto_gc.success"), 0)
self.assertEqual(counters.get("local_store.auto_gc.failure", 0), 0)
self.assertGreaterEqual(
counters.get("local_store.auto_gc.last_duration_ms"), 0)
# Run "eden stats local-store" and check the output
stats_output = self.eden.run_cmd("stats", "local-store")
print(stats_output)
m = re.search(r"Successful Auto-GC Runs:\s+(\d+)", stats_output)
self.assertIsNotNone(m)
assert m is not None # make the type checker happy
self.assertGreater(int(m.group(1)), 0)
self.assertRegex(stats_output, r"Last Auto-GC Result:\s+Success")
self.assertRegex(stats_output, r"Failed Auto-GC Runs:\s+0")
self.assertRegex(stats_output, r"Total Ephemeral Size:")
self.assertRegex(stats_output, r"Total Persistent Size:")
def poll_until_inactive(self, timeout: float) -> None:
def check_inactive() -> typing.Optional[bool]:
return True if self.query_active_state() == "inactive" else None
poll_until(check_inactive, timeout=timeout)
def poll_until_inactive(self, timeout: float) -> None:
def check_inactive() -> typing.Optional[bool]:
return True if self.query_active_state() == "inactive" else None
poll_until(check_inactive, timeout=timeout)
