def _wait_until_alive(self, client: EdenClient) -> None: def is_alive() -> Optional[bool]: if client.getStatus() == fb303_status.ALIVE: return True return None poll_until(is_alive, timeout=60)
def test_start_blocked_mount_init(self) -> None: self.eden.shutdown() self.eden.spawn_nowait( extra_args=["--enable_fault_injection", "--fault_injection_block_mounts"] ) # Wait for eden to report the mount point in the listMounts() output def is_initializing() -> Optional[bool]: try: with self.eden.get_thrift_client() as client: if self.eden.get_mount_state(Path(self.mount), client) is not None: return True assert self.eden._process is not None if self.eden._process.poll(): self.fail("eden exited before becoming healthy") return None except (EdenNotRunningError, TException): return None poll_until(is_initializing, timeout=60) with self.eden.get_thrift_client() as client: # Since we blocked mount initialization the mount should still # report as INITIALIZING, and edenfs should report itself STARTING self.assertEqual({self.mount: "INITIALIZING"}, self.eden.list_cmd_simple()) self.assertEqual(fb_status.STARTING, client.getStatus()) # Unblock mounting and wait for the mount to transition to running client.unblockFault(UnblockFaultArg(keyClass="mount", keyValueRegex=".*")) self._wait_for_mount_running(client) self.assertEqual(fb_status.ALIVE, client.getStatus()) self.assertEqual({self.mount: "RUNNING"}, self.eden.list_cmd_simple())
def _wait_for_mount_running(self, client: EdenClient) -> None: def mount_running() -> Optional[bool]: if (self.eden.get_mount_state(Path(self.mount), client) == MountState.RUNNING): return True return None poll_until(mount_running, timeout=60)
def _wait_for_mount_running(self, client: EdenClient) -> None: def mount_running() -> Optional[bool]: if ( self.eden.get_mount_state(Path(self.mount), client) == MountState.RUNNING ): return True return None poll_until(mount_running, timeout=60)
def _wait_for_mount_running( self, client: EdenClient, path: Optional[Path] = None ) -> None: mount_path = path if path is not None else Path(self.mount) def mount_running() -> Optional[bool]: if self.eden.get_mount_state(mount_path, client) == MountState.RUNNING: return True return None poll_until(mount_running, timeout=60)
def test_async_stop_stops_daemon_eventually(self) -> None: with self.spawn_fake_edenfs(self.eden_dir) as daemon_pid: stop_process = self.spawn_stop(["--timeout", "0"]) stop_process.expect_exact("Sent async shutdown request to edenfs.") self.assert_process_exit_code( stop_process, SHUTDOWN_EXIT_CODE_REQUESTED_SHUTDOWN) def daemon_exited() -> typing.Optional[bool]: if did_process_exit(daemon_pid): return True else: return None poll_until(daemon_exited, timeout=10)
def test_async_stop_stops_daemon_eventually(self) -> None: with self.spawn_fake_edenfs(self.eden_dir) as daemon_pid: stop_process = self.spawn_stop(["--timeout", "0"]) stop_process.expect_exact("Sent async shutdown request to edenfs.") self.assert_process_exit_code( stop_process, SHUTDOWN_EXIT_CODE_REQUESTED_SHUTDOWN ) def daemon_exited() -> typing.Optional[bool]: if did_process_exit(daemon_pid): return True else: return None poll_until(daemon_exited, timeout=10)
def test_mount_init_state(self) -> None: self.eden.run_cmd("unmount", self.mount) self.assertEqual({self.mount: "NOT_RUNNING"}, self.eden.list_cmd_simple()) with self.eden.get_thrift_client() as client: fault = FaultDefinition(keyClass="mount", keyValueRegex=".*", block=True) client.injectFault(fault) # Run the "eden mount" CLI command. # This won't succeed until we unblock the mount. mount_cmd = self.eden.get_eden_cli_args("mount", self.mount) mount_proc = subprocess.Popen(mount_cmd) # Wait for the new mount to be reported by edenfs def mount_started() -> Optional[bool]: if self.eden.get_mount_state(Path(self.mount), client) is not None: return True if mount_proc.poll() is not None: raise Exception( f"eden mount command finished (with status " f"{mount_proc.returncode}) while mounting was " f"still blocked") return None poll_until(mount_started, timeout=30) self.assertEqual({self.mount: "INITIALIZING"}, self.eden.list_cmd_simple()) # Most thrift calls to access the mount should be disallowed while it is # still initializing. self._assert_thrift_calls_fail_during_mount_init(client) # Unblock mounting and wait for the mount to transition to running client.unblockFault( UnblockFaultArg(keyClass="mount", keyValueRegex=".*")) self._wait_for_mount_running(client) self.assertEqual({self.mount: "RUNNING"}, self.eden.list_cmd_simple())
def shutdown(self) -> None: ''' Run "eden shutdown" to stop the eden daemon. ''' assert self._process is not None # We need to take care here: the normal `eden shutdown` command will # wait for eden to successfully finish by repeatedly testing to see # whether the process is alive. However, running as root we don't # spawn an intermediate process and this results in our child process # (attached to self._process) to land in a defunct state such that # the `kill(pid, 0)` test in the `eden shutdown` command still considers # the process alive. To avoid this situation we ask the shutdown # command not to wait and instead perform our own polling here against # the real process handle. self.run_cmd('shutdown', '-t', '0') util.poll_until(lambda: self._process.poll(), timeout=15) return_code = self._process.wait() self._process = None if return_code != 0: raise Exception('eden exited unsuccessfully with status {}'.format( return_code))
def _wait_until_initializing(self, num_mounts: int = 1) -> None: """Wait until EdenFS is initializing mount points. This is primarily intended to be used to wait until the mount points are initializing when starting EdenFS with --fault_injection_block_mounts. """ def is_initializing() -> Optional[bool]: try: with self.eden.get_thrift_client() as client: # Return successfully when listMounts() reports the number of # mounts that we expect. mounts = client.listMounts() if len(mounts) == num_mounts: return True edenfs_process = self.eden._process assert edenfs_process is not None if edenfs_process.poll(): self.fail("eden exited before becoming healthy") return None except (EdenNotRunningError, TException): return None poll_until(is_initializing, timeout=60)
def test_mount_init_state(self) -> None: self.eden.run_cmd("unmount", self.mount) self.assertEqual({self.mount: "NOT_RUNNING"}, self.eden.list_cmd_simple()) with self.eden.get_thrift_client() as client: fault = FaultDefinition(keyClass="mount", keyValueRegex=".*", block=True) client.injectFault(fault) # Run the "eden mount" CLI command. # This won't succeed until we unblock the mount. mount_cmd = self.eden.get_eden_cli_args("mount", self.mount) mount_proc = subprocess.Popen(mount_cmd) # Wait for the new mount to be reported by edenfs def mount_started() -> Optional[bool]: if self.eden.get_mount_state(Path(self.mount), client) is not None: return True if mount_proc.poll() is not None: raise Exception( f"eden mount command finished (with status " f"{mount_proc.returncode}) while mounting was " f"still blocked" ) return None poll_until(mount_started, timeout=30) self.assertEqual({self.mount: "INITIALIZING"}, self.eden.list_cmd_simple()) # Most thrift calls to access the mount should be disallowed while it is # still initializing. self._assert_thrift_calls_fail_during_mount_init(client) # Unblock mounting and wait for the mount to transition to running client.unblockFault(UnblockFaultArg(keyClass="mount", keyValueRegex=".*")) self._wait_for_mount_running(client) self.assertEqual({self.mount: "RUNNING"}, self.eden.list_cmd_simple())
def wait_until_process_is_zombie(process: subprocess.Popen) -> None: def is_zombie() -> typing.Optional[bool]: return True if is_zombie_process(process.pid) else None poll_until(is_zombie, timeout=3)
def test_local_store_stats(self) -> None: # Update the config to tell the local store to updates its stats frequently # and also check if it needs to reload the config file frequently. initial_config = """\ [config] reload-interval = "100ms" [store] stats-interval = "100ms" """ self.eden.user_rc_path.write_text(initial_config) counter_regex = r"local_store\..*" with self.get_thrift_client() as client: # Makes sure that EdenFS picks up our updated config, # since we wrote it out after EdenFS started. client.reloadConfig() # Get the local store counters # Assert that the exist and are greater than 0. # (Since we include memtable sizes in the values these are currently always # reported as taking up at least a small amount of space.) initial_counters = client.getRegexCounters(counter_regex) self.assertGreater(initial_counters.get("local_store.blob.size"), 0) self.assertGreater( initial_counters.get("local_store.blobmeta.size"), 0) self.assertGreater(initial_counters.get("local_store.tree.size"), 0) self.assertGreater( initial_counters.get("local_store.hgcommit2tree.size"), 0) self.assertGreater( initial_counters.get("local_store.hgproxyhash.size"), 0) self.assertGreater( initial_counters.get("local_store.ephemeral.total_size"), 0) self.assertGreater( initial_counters.get("local_store.persistent.total_size"), 0) # Make sure the counters are less than 500MB, just as a sanity check self.assertLess( initial_counters.get("local_store.ephemeral.total_size"), 500_000_000) self.assertLess( initial_counters.get("local_store.persistent.total_size"), 500_000_000) # Read back several files self.assertEqual((self.mount_path / "a/dir/foo.txt").read_text(), "foo\n") self.assertEqual((self.mount_path / "a/dir/bar.txt").read_text(), "bar\n") self.assertEqual( (self.mount_path / "a/another_dir/hello.txt").read_text(), "hola\n") # The tree store size should be larger now after reading these files. # The counters won't be updated until the store.stats-interval expires. # Wait for this to happen. def tree_size_incremented() -> Optional[bool]: tree_size = client.getCounter("local_store.tree.size") initial_tree_size = initial_counters.get( "local_store.tree.size") assert initial_tree_size is not None if tree_size > initial_tree_size: return True return None poll_until(tree_size_incremented, timeout=1, interval=0.1) # EdenFS should not import blobs to local store self.assertEqual( initial_counters.get("local_store.blob.size"), client.getCounter("local_store.blob.size"), ) # Update the config file with a very small GC limit that will force GC to be # triggered self.eden.user_rc_path.write_text(initial_config + """ blob-size-limit = "1" blobmeta-size-limit = "1" tree-size-limit = "1" hgcommit2tree-size-limit = "1" """) # Wait until a GC run has completed. def gc_run_succeeded() -> Optional[Dict[str, int]]: counters = client.getRegexCounters(counter_regex) if counters.get( "local_store.auto_gc.last_run_succeeded") is not None: return counters return None counters = poll_until(gc_run_succeeded, timeout=5, interval=0.05) # Check the local_store.auto_gc counters self.assertEqual( counters.get("local_store.auto_gc.last_run_succeeded"), 1) self.assertGreater(counters.get("local_store.auto_gc.success"), 0) self.assertEqual(counters.get("local_store.auto_gc.failure", 0), 0) self.assertGreaterEqual( counters.get("local_store.auto_gc.last_duration_ms"), 0) # Run "eden stats local-store" and check the output stats_output = self.eden.run_cmd("stats", "local-store") print(stats_output) m = re.search(r"Successful Auto-GC Runs:\s+(\d+)", stats_output) self.assertIsNotNone(m) assert m is not None # make the type checker happy self.assertGreater(int(m.group(1)), 0) self.assertRegex(stats_output, r"Last Auto-GC Result:\s+Success") self.assertRegex(stats_output, r"Failed Auto-GC Runs:\s+0") self.assertRegex(stats_output, r"Total Ephemeral Size:") self.assertRegex(stats_output, r"Total Persistent Size:")
def poll_until_inactive(self, timeout: float) -> None: def check_inactive() -> typing.Optional[bool]: return True if self.query_active_state() == "inactive" else None poll_until(check_inactive, timeout=timeout)