def test_eden_start_launches_separate_processes_for_separate_eden_dirs( self) -> None: eden_dir_1 = self.eden_dir eden_dir_2 = pathlib.Path(self.make_temporary_directory()) start_1_process = self.spawn_start(eden_dir=eden_dir_1) self.assert_process_succeeds(start_1_process) start_2_process = self.spawn_start(eden_dir=eden_dir_2) self.assert_process_succeeds(start_2_process) instance_1_health: HealthStatus = EdenInstance( str(eden_dir_1), etc_eden_dir=None, home_dir=None).check_health() self.assertEqual( instance_1_health.status, fb303_status.ALIVE, f"First edenfs process should be healthy, but it isn't: " f"{instance_1_health}", ) instance_2_health: HealthStatus = EdenInstance( str(eden_dir_2), etc_eden_dir=None, home_dir=None).check_health() self.assertEqual( instance_2_health.status, fb303_status.ALIVE, f"Second edenfs process should be healthy, but it isn't: " f"{instance_2_health}", ) self.assertNotEqual( instance_1_health.pid, instance_2_health.pid, f"The edenfs process should have separate process IDs", )
def run_normal_checks( tracker: ProblemTracker, instance: EdenInstance, out: ui.Output, mount_table: mtab.MountTable, fs_util: filesystem.FsUtil, ) -> None: checkouts: Dict[Path, CheckoutInfo] = {} # Get information about the checkouts currently known to the running edenfs process with instance.get_thrift_client() as client: for mount in client.listMounts(): # Old versions of edenfs did not return a mount state field. # These versions only listed running mounts, so treat the mount state # as running in this case. mount_state = mount.state if mount.state is not None else MountState.RUNNING path = Path(os.fsdecode(mount.mountPoint)) checkout = CheckoutInfo( instance, path, running_state_dir=Path(os.fsdecode(mount.edenClientPath)), state=mount_state, ) checkouts[path] = checkout # Get information about the checkouts listed in the config file for configured_checkout in instance.get_checkouts(): checkout_info = checkouts.get(configured_checkout.path, None) if checkout_info is None: checkout_info = CheckoutInfo(instance, configured_checkout.path) checkout_info.configured_state_dir = configured_checkout.state_dir checkouts[checkout_info.path] = checkout_info checkout_info.configured_state_dir = configured_checkout.state_dir check_filesystems.check_eden_directory(tracker, instance) check_stale_mounts.check_for_stale_mounts(tracker, mount_table) check_edenfs_version(tracker, instance) check_filesystems.check_disk_usage(tracker, list(instance.get_mount_paths()), instance, fs_util=fs_util) watchman_info = check_watchman.pre_check() for path, checkout in sorted(checkouts.items()): out.writeln(f"Checking {path}") try: check_mount(tracker, checkout, mount_table, fs_util, watchman_info) except Exception as ex: tracker.add_problem( Problem(f"unexpected error while checking {path}: {ex}"))
def run_normal_checks( tracker: ProblemTracker, instance: EdenInstance, out: ui.Output, mount_table: mtab.MountTable, fs_util: filesystem.FsUtil, ) -> None: checkouts: Dict[Path, CheckoutInfo] = {} # Get information about the checkouts currently known to the running edenfs process with instance.get_thrift_client() as client: for mount in client.listMounts(): # Old versions of edenfs did not return a mount state field. # These versions only listed running mounts, so treat the mount state # as running in this case. mount_state = mount.state if mount.state is not None else MountState.RUNNING path = Path(os.fsdecode(mount.mountPoint)) checkout = CheckoutInfo( instance, path, running_state_dir=Path(os.fsdecode(mount.edenClientPath)), state=mount_state, ) checkouts[path] = checkout # Get information about the checkouts listed in the config file for configured_checkout in instance.get_checkouts(): checkout_info = checkouts.get(configured_checkout.path, None) if checkout_info is None: checkout_info = CheckoutInfo(instance, configured_checkout.path) checkout_info.configured_state_dir = configured_checkout.state_dir checkouts[checkout_info.path] = checkout_info checkout_info.configured_state_dir = configured_checkout.state_dir check_filesystems.check_eden_directory(tracker, instance) check_stale_mounts.check_for_stale_mounts(tracker, mount_table) check_edenfs_version(tracker, instance) check_filesystems.check_disk_usage( tracker, list(instance.get_mount_paths()), instance ) watchman_info = check_watchman.pre_check() for path, checkout in sorted(checkouts.items()): out.writeln(f"Checking {path}") try: check_mount(tracker, checkout, mount_table, fs_util, watchman_info) except Exception as ex: tracker.add_problem( Problem(f"unexpected error while checking {path}: {ex}") )
def check_edenfs_version(tracker: ProblemTracker, instance: EdenInstance) -> None: rver, release = instance.get_running_version_parts() if not rver or not release: # This could be a dev build that returns the empty # string for both of these values. return running_version = version.format_eden_version((rver, release)) installed_version = version.get_installed_eden_rpm_version() if running_version == installed_version: return tracker.add_problem( Problem( dedent(f"""\ The version of Eden that is installed on your machine is: fb-eden-{installed_version}.x86_64 but the version of Eden that is currently running is: fb-eden-{running_version}.x86_64 Consider running `eden restart` to migrate to the newer version, which may have important bug fixes or performance improvements. """), severity=ProblemSeverity.ADVICE, ))
def _os_is_bad_release(instance: EdenInstance, release: str) -> bool: known_bad_kernel_versions = instance.get_config_value( "doctor.known-bad-kernel-versions", default="") if not known_bad_kernel_versions: return False for regex in known_bad_kernel_versions.split(","): if re.search(regex, release): return True # matched known bad release return False # no match to bad release
def _os_is_kernel_version_too_old(instance: EdenInstance, release: str) -> bool: min_kernel_version = instance.get_config_value( "doctor.minimum-kernel-version", default="") if not min_kernel_version: return False try: return _parse_os_kernel_version(release) < _parse_os_kernel_version( min_kernel_version) except ValueError: # If the kernel version failed to parse because one of the # components wasn't an int, whatever. return False
def check_disk_usage( tracker: ProblemTracker, mount_paths: List[str], instance: EdenInstance, fs_util: FsUtil, ) -> None: prob_advice_space_used_ratio_threshold = 0.90 prob_error_absolute_space_used_threshold = 1024 * 1024 * 1024 # 1GB eden_mount_pts_set = get_mount_pts_set(tracker, mount_paths, instance) for eden_mount_pt in eden_mount_pts_set: if eden_mount_pt and os.path.exists(eden_mount_pt): disk_status = fs_util.statvfs(eden_mount_pt) avail = disk_status.f_frsize * disk_status.f_bavail size = disk_status.f_frsize * disk_status.f_blocks if size == 0: continue used = size - avail used_percent = float(used) / size message = ( "Eden lazily loads your files and needs enough disk space to " "store these files when loaded." ) extra_message = instance.get_config_value( "doctor.low-disk-space-message", "" ) if extra_message: message = f"{message} {extra_message}" if avail <= prob_error_absolute_space_used_threshold: tracker.add_problem( Problem( f"{eden_mount_pt} " f"has only {str(avail)} bytes available. " f"{message}", severity=ProblemSeverity.ERROR, ) ) elif used_percent >= prob_advice_space_used_ratio_threshold: tracker.add_problem( Problem( f"{eden_mount_pt} " f"is {used_percent:.2%} full. " f"{message}", severity=ProblemSeverity.ADVICE, ) )
def run_edenfs_not_healthy_checks( tracker: ProblemTracker, instance: EdenInstance, out: ui.Output, status: config_mod.HealthStatus, mount_table: mtab.MountTable, fs_util: filesystem.FsUtil, ) -> None: check_stale_mounts.check_for_stale_mounts(tracker, mount_table) configured_mounts = instance.get_mount_paths() check_filesystems.check_disk_usage(tracker, list(configured_mounts), instance) if configured_mounts: tracker.add_problem(EdenfsNotHealthy())
def run_edenfs_not_healthy_checks( tracker: ProblemTracker, instance: EdenInstance, out: ui.Output, status: config_mod.HealthStatus, mount_table: mtab.MountTable, fs_util: filesystem.FsUtil, ) -> None: check_stale_mounts.check_for_stale_mounts(tracker, mount_table) configured_mounts = instance.get_mount_paths() check_filesystems.check_disk_usage(tracker, list(configured_mounts), instance) if configured_mounts: tracker.add_problem(EdenfsNotHealthy())
def check_disk_usage( tracker: ProblemTracker, mount_paths: List[str], instance: EdenInstance ) -> None: prob_advice_space_used_ratio_threshold = 0.90 prob_error_absolute_space_used_threshold = 1024 * 1024 * 1024 # 1GB eden_mount_pts_set = get_mount_pts_set(tracker, mount_paths, instance) for eden_mount_pt in eden_mount_pts_set: if eden_mount_pt and os.path.exists(eden_mount_pt): disk_status = os.statvfs(eden_mount_pt) avail = disk_status.f_frsize * disk_status.f_bavail size = disk_status.f_frsize * disk_status.f_blocks if size == 0: continue used = size - avail used_percent = float(used) / size message = ( "Eden lazily loads your files and needs enough disk space to " "store these files when loaded." ) extra_message = instance.get_config_value( "doctor.low-disk-space-message", "" ) if extra_message: message = f"{message} {extra_message}" if avail <= prob_error_absolute_space_used_threshold: tracker.add_problem( Problem( f"{eden_mount_pt} " f"has only {str(avail)} bytes available. " f"{message}", severity=ProblemSeverity.ERROR, ) ) elif used_percent >= prob_advice_space_used_ratio_threshold: tracker.add_problem( Problem( f"{eden_mount_pt} " f"is {used_percent:.2%} full. " f"{message}", severity=ProblemSeverity.ADVICE, ) )
def test_eden_start_fails_if_service_is_running(self) -> None: with self.spawn_fake_edenfs(self.eden_dir): # Make fake_edenfs inaccessible and undetectable (without talking to # systemd), but keep the systemd service alive. (self.eden_dir / "lock").unlink() (self.eden_dir / "socket").unlink() health: HealthStatus = EdenInstance(str(self.eden_dir), etc_eden_dir=None, home_dir=None).check_health() self.assertEqual(health.status, fb303_status.DEAD) service = self.get_edenfs_systemd_service(eden_dir=self.eden_dir) self.assertEqual(service.query_active_state(), "active") start_process = self.spawn_start() start_process.expect_exact( f"error: edenfs systemd service is already running") # edenfsctl should show the output of 'systemctl status'. start_process.expect(r"\bfb-edenfs@.*?\.service\b") start_process.expect(r"Active:[^\n]*active \(running\)") self.assert_process_fails(start_process, 1)
def _check_edenfs_health(self) -> HealthStatus: instance = EdenInstance(str(self.eden_dir), etc_eden_dir=None, home_dir=None) return instance.check_health()
def cure_what_ails_you( instance: EdenInstance, dry_run: bool, mount_table: mtab.MountTable, fs_util: filesystem.FsUtil, process_finder: process_finder.ProcessFinder, out: Optional[ui.Output] = None, ) -> int: if out is None: out = ui.get_output() if not dry_run: fixer = ProblemFixer(out) else: fixer = DryRunFixer(out) check_working_directory(fixer) # check OS type, kernel version etc. check_os.run_operating_system_checks(fixer, instance, out) # check multiple edenfs running with some rogue stale PIDs check_rogue_edenfs.check_many_edenfs_are_running(fixer, process_finder) status = instance.check_health() if status.status == fb_status.ALIVE: run_normal_checks(fixer, instance, out, mount_table, fs_util) elif status.status == fb_status.STARTING: fixer.add_problem(EdenfsStarting()) elif status.status == fb_status.STOPPING: fixer.add_problem(EdenfsStopping()) elif status.status == fb_status.DEAD: run_edenfs_not_healthy_checks(fixer, instance, out, status, mount_table, fs_util) if fixer.num_problems == 0: out.writeln("Eden is not in use.") return 0 else: fixer.add_problem(EdenfsUnexpectedStatus(status)) if fixer.num_problems == 0: out.writeln("No issues detected.", fg=out.GREEN) return 0 def problem_count(num: int) -> str: if num == 1: return "1 problem" return f"{num} problems" if dry_run: out.writeln( f"Discovered {problem_count(fixer.num_problems)} during --dry-run", fg=out.YELLOW, ) return 1 if fixer.num_fixed_problems: out.writeln( f"Successfully fixed {problem_count(fixer.num_fixed_problems)}.", fg=out.YELLOW, ) if fixer.num_failed_fixes: out.writeln(f"Failed to fix {problem_count(fixer.num_failed_fixes)}.", fg=out.RED) if fixer.num_manual_fixes: if fixer.num_manual_fixes == 1: msg = f"1 issue requires manual attention." else: msg = f"{fixer.num_manual_fixes} issues require manual attention." out.writeln(msg, fg=out.YELLOW) if fixer.num_fixed_problems == fixer.num_problems: return 0 out.write( "Ask in the Eden Users group if you need help fixing issues with Eden:\n" "https://fb.facebook.com/groups/eden.users/\n") return 1
def test_list_mounts(self): self.maxDiff = None thrift_mounts = [ MountInfo( mountPoint=b"/data/users/johndoe/mercurial", edenClientPath=b"/home/johndoe/.eden/clients/mercurial", state=MountState.RUNNING, ), MountInfo( mountPoint=b"/data/users/johndoe/git", edenClientPath=b"/home/johndoe/.eden/clients/git", state=MountState.SHUTTING_DOWN, ), MountInfo( mountPoint=b"/data/users/johndoe/apache", edenClientPath=b"/home/johndoe/.eden/clients/apache", state=MountState.RUNNING, ), MountInfo( mountPoint=b"/data/users/johndoe/configs", edenClientPath=b"/home/johndoe/.eden/clients/configs", state=MountState.INITIALIZING, ), MountInfo( mountPoint=b"/data/users/johndoe/repos/linux", edenClientPath=b"/home/johndoe/.eden/clients/linux", state=MountState.RUNNING, ), MountInfo( mountPoint=b"/data/users/johndoe/other_repos/linux", edenClientPath=b"/home/johndoe/.eden/clients/linux2", state=MountState.RUNNING, ), ] instance = EdenInstance( config_dir="/home/johndoe/.eden", etc_eden_dir="/etc/eden", home_dir="/home/johndoe", ) config_checkouts = [ EdenCheckout( instance, Path("/data/users/johndoe/mercurial"), Path("/home/johndoe/.eden/clients/mercurial"), ), EdenCheckout( instance, Path("/data/users/johndoe/git"), Path("/home/johndoe/.eden/clients/git"), ), EdenCheckout( instance, Path("/data/users/johndoe/repos/linux"), Path("/home/johndoe/.eden/clients/linux"), ), EdenCheckout( instance, Path("/data/users/johndoe/other_repos/linux"), Path("/home/johndoe/.eden/clients/linux2"), ), EdenCheckout( instance, Path("/data/users/johndoe/www"), Path("/home/johndoe/.eden/clients/www"), ), ] mounts = main_mod.ListCmd.combine_mount_info(thrift_mounts, config_checkouts) normal_out = TestOutput() main_mod.ListCmd.print_mounts(normal_out, mounts) self.assertEqual( """\ /data/users/johndoe/apache (unconfigured) /data/users/johndoe/configs (INITIALIZING) (unconfigured) /data/users/johndoe/git (SHUTTING_DOWN) /data/users/johndoe/mercurial /data/users/johndoe/other_repos/linux /data/users/johndoe/repos/linux /data/users/johndoe/www (not mounted) """, normal_out.getvalue(), ) json_out = TestOutput() main_mod.ListCmd.print_mounts_json(json_out, mounts) self.assertEqual( """\ { "/data/users/johndoe/apache": { "configured": false, "data_dir": "/home/johndoe/.eden/clients/apache", "state": "RUNNING" }, "/data/users/johndoe/configs": { "configured": false, "data_dir": "/home/johndoe/.eden/clients/configs", "state": "INITIALIZING" }, "/data/users/johndoe/git": { "configured": true, "data_dir": "/home/johndoe/.eden/clients/git", "state": "SHUTTING_DOWN" }, "/data/users/johndoe/mercurial": { "configured": true, "data_dir": "/home/johndoe/.eden/clients/mercurial", "state": "RUNNING" }, "/data/users/johndoe/other_repos/linux": { "configured": true, "data_dir": "/home/johndoe/.eden/clients/linux2", "state": "RUNNING" }, "/data/users/johndoe/repos/linux": { "configured": true, "data_dir": "/home/johndoe/.eden/clients/linux", "state": "RUNNING" }, "/data/users/johndoe/www": { "configured": true, "data_dir": "/home/johndoe/.eden/clients/www", "state": "NOT_RUNNING" } } """, json_out.getvalue(), )
def test_list_mounts_old_thrift(self): self.maxDiff = None # Simulate an older edenfs daemon that does not send the "state" field thrift_mounts = [ MountInfo( mountPoint=b"/data/users/johndoe/mercurial", edenClientPath=b"/home/johndoe/.eden/clients/mercurial", ), MountInfo( mountPoint=b"/data/users/johndoe/git", edenClientPath=b"/home/johndoe/.eden/clients/git", ), MountInfo( mountPoint=b"/data/users/johndoe/configs", edenClientPath=b"/home/johndoe/.eden/clients/configs", ), ] instance = EdenInstance( config_dir="/home/johndoe/.eden", etc_eden_dir="/etc/eden", home_dir="/home/johndoe", ) config_checkouts = [ EdenCheckout( instance, Path("/data/users/johndoe/mercurial"), Path("/home/johndoe/.eden/clients/mercurial"), ), EdenCheckout( instance, Path("/data/users/johndoe/git"), Path("/home/johndoe/.eden/clients/git"), ), EdenCheckout( instance, Path("/data/users/johndoe/www"), Path("/home/johndoe/.eden/clients/www"), ), ] mounts = main_mod.ListCmd.combine_mount_info(thrift_mounts, config_checkouts) normal_out = TestOutput() main_mod.ListCmd.print_mounts(normal_out, mounts) self.assertEqual( """\ /data/users/johndoe/configs (unconfigured) /data/users/johndoe/git /data/users/johndoe/mercurial /data/users/johndoe/www (not mounted) """, normal_out.getvalue(), ) json_out = TestOutput() main_mod.ListCmd.print_mounts_json(json_out, mounts) self.assertEqual( """\ { "/data/users/johndoe/configs": { "configured": false, "data_dir": "/home/johndoe/.eden/clients/configs", "state": "RUNNING" }, "/data/users/johndoe/git": { "configured": true, "data_dir": "/home/johndoe/.eden/clients/git", "state": "RUNNING" }, "/data/users/johndoe/mercurial": { "configured": true, "data_dir": "/home/johndoe/.eden/clients/mercurial", "state": "RUNNING" }, "/data/users/johndoe/www": { "configured": true, "data_dir": "/home/johndoe/.eden/clients/www", "state": "NOT_RUNNING" } } """, json_out.getvalue(), )
def cure_what_ails_you( instance: EdenInstance, dry_run: bool, mount_table: mtab.MountTable, fs_util: filesystem.FsUtil, process_finder: process_finder.ProcessFinder, out: Optional[ui.Output] = None, ) -> int: if out is None: out = ui.get_output() if not dry_run: fixer = ProblemFixer(out) else: fixer = DryRunFixer(out) check_working_directory(fixer) # check OS type, kernel version etc. check_os.run_operating_system_checks(fixer, instance, out) # check multiple edenfs running with some rogue stale PIDs check_rogue_edenfs.check_many_edenfs_are_running(fixer, process_finder) status = instance.check_health() if status.status == fb_status.ALIVE: run_normal_checks(fixer, instance, out, mount_table, fs_util) elif status.status == fb_status.STARTING: fixer.add_problem(EdenfsStarting()) elif status.status == fb_status.STOPPING: fixer.add_problem(EdenfsStopping()) elif status.status == fb_status.DEAD: run_edenfs_not_healthy_checks( fixer, instance, out, status, mount_table, fs_util ) if fixer.num_problems == 0: out.writeln("Eden is not in use.") return 0 else: fixer.add_problem(EdenfsUnexpectedStatus(status)) if fixer.num_problems == 0: out.writeln("No issues detected.", fg=out.GREEN) return 0 def problem_count(num: int) -> str: if num == 1: return "1 problem" return f"{num} problems" if dry_run: out.writeln( f"Discovered {problem_count(fixer.num_problems)} during --dry-run", fg=out.YELLOW, ) return 1 if fixer.num_fixed_problems: out.writeln( f"Successfully fixed {problem_count(fixer.num_fixed_problems)}.", fg=out.YELLOW, ) if fixer.num_failed_fixes: out.writeln( f"Failed to fix {problem_count(fixer.num_failed_fixes)}.", fg=out.RED ) if fixer.num_manual_fixes: if fixer.num_manual_fixes == 1: msg = f"1 issue requires manual attention." else: msg = f"{fixer.num_manual_fixes} issues require manual attention." out.writeln(msg, fg=out.YELLOW) if fixer.num_fixed_problems == fixer.num_problems: return 0 out.write( "Ask in the Eden Users group if you need help fixing issues with Eden:\n" "https://fb.facebook.com/groups/eden.users/\n" ) return 1
def _check_edenfs_health(self) -> HealthStatus: instance = EdenInstance(str(self.eden_dir), etc_eden_dir=None, home_dir=None) return instance.check_health()