def test_eden_start_launches_separate_processes_for_separate_eden_dirs(
self) -> None:
eden_dir_1 = self.eden_dir
eden_dir_2 = pathlib.Path(self.make_temporary_directory())
start_1_process = self.spawn_start(eden_dir=eden_dir_1)
self.assert_process_succeeds(start_1_process)
start_2_process = self.spawn_start(eden_dir=eden_dir_2)
self.assert_process_succeeds(start_2_process)
instance_1_health: HealthStatus = EdenInstance(
str(eden_dir_1), etc_eden_dir=None, home_dir=None).check_health()
self.assertEqual(
instance_1_health.status,
fb303_status.ALIVE,
f"First edenfs process should be healthy, but it isn't: "
f"{instance_1_health}",
)
instance_2_health: HealthStatus = EdenInstance(
str(eden_dir_2), etc_eden_dir=None, home_dir=None).check_health()
self.assertEqual(
instance_2_health.status,
fb303_status.ALIVE,
f"Second edenfs process should be healthy, but it isn't: "
f"{instance_2_health}",
)
self.assertNotEqual(
instance_1_health.pid,
instance_2_health.pid,
f"The edenfs process should have separate process IDs",
)
def run_normal_checks(
tracker: ProblemTracker,
instance: EdenInstance,
out: ui.Output,
mount_table: mtab.MountTable,
fs_util: filesystem.FsUtil,
) -> None:
checkouts: Dict[Path, CheckoutInfo] = {}
# Get information about the checkouts currently known to the running edenfs process
with instance.get_thrift_client() as client:
for mount in client.listMounts():
# Old versions of edenfs did not return a mount state field.
# These versions only listed running mounts, so treat the mount state
# as running in this case.
mount_state = mount.state if mount.state is not None else MountState.RUNNING
path = Path(os.fsdecode(mount.mountPoint))
checkout = CheckoutInfo(
instance,
path,
running_state_dir=Path(os.fsdecode(mount.edenClientPath)),
state=mount_state,
)
checkouts[path] = checkout
# Get information about the checkouts listed in the config file
for configured_checkout in instance.get_checkouts():
checkout_info = checkouts.get(configured_checkout.path, None)
if checkout_info is None:
checkout_info = CheckoutInfo(instance, configured_checkout.path)
checkout_info.configured_state_dir = configured_checkout.state_dir
checkouts[checkout_info.path] = checkout_info
checkout_info.configured_state_dir = configured_checkout.state_dir
check_filesystems.check_eden_directory(tracker, instance)
check_stale_mounts.check_for_stale_mounts(tracker, mount_table)
check_edenfs_version(tracker, instance)
check_filesystems.check_disk_usage(tracker,
list(instance.get_mount_paths()),
instance,
fs_util=fs_util)
watchman_info = check_watchman.pre_check()
for path, checkout in sorted(checkouts.items()):
out.writeln(f"Checking {path}")
try:
check_mount(tracker, checkout, mount_table, fs_util, watchman_info)
except Exception as ex:
tracker.add_problem(
Problem(f"unexpected error while checking {path}: {ex}"))
def run_normal_checks(
tracker: ProblemTracker,
instance: EdenInstance,
out: ui.Output,
mount_table: mtab.MountTable,
fs_util: filesystem.FsUtil,
) -> None:
checkouts: Dict[Path, CheckoutInfo] = {}
# Get information about the checkouts currently known to the running edenfs process
with instance.get_thrift_client() as client:
for mount in client.listMounts():
# Old versions of edenfs did not return a mount state field.
# These versions only listed running mounts, so treat the mount state
# as running in this case.
mount_state = mount.state if mount.state is not None else MountState.RUNNING
path = Path(os.fsdecode(mount.mountPoint))
checkout = CheckoutInfo(
instance,
path,
running_state_dir=Path(os.fsdecode(mount.edenClientPath)),
state=mount_state,
)
checkouts[path] = checkout
# Get information about the checkouts listed in the config file
for configured_checkout in instance.get_checkouts():
checkout_info = checkouts.get(configured_checkout.path, None)
if checkout_info is None:
checkout_info = CheckoutInfo(instance, configured_checkout.path)
checkout_info.configured_state_dir = configured_checkout.state_dir
checkouts[checkout_info.path] = checkout_info
checkout_info.configured_state_dir = configured_checkout.state_dir
check_filesystems.check_eden_directory(tracker, instance)
check_stale_mounts.check_for_stale_mounts(tracker, mount_table)
check_edenfs_version(tracker, instance)
check_filesystems.check_disk_usage(
tracker, list(instance.get_mount_paths()), instance
)
watchman_info = check_watchman.pre_check()
for path, checkout in sorted(checkouts.items()):
out.writeln(f"Checking {path}")
try:
check_mount(tracker, checkout, mount_table, fs_util, watchman_info)
except Exception as ex:
tracker.add_problem(
Problem(f"unexpected error while checking {path}: {ex}")
)
def check_edenfs_version(tracker: ProblemTracker,
instance: EdenInstance) -> None:
rver, release = instance.get_running_version_parts()
if not rver or not release:
# This could be a dev build that returns the empty
# string for both of these values.
return
running_version = version.format_eden_version((rver, release))
installed_version = version.get_installed_eden_rpm_version()
if running_version == installed_version:
return
tracker.add_problem(
Problem(
dedent(f"""\
The version of Eden that is installed on your machine is:
fb-eden-{installed_version}.x86_64
but the version of Eden that is currently running is:
fb-eden-{running_version}.x86_64
Consider running `eden restart` to migrate to the newer version, which
may have important bug fixes or performance improvements.
"""),
severity=ProblemSeverity.ADVICE,
))
def _os_is_bad_release(instance: EdenInstance, release: str) -> bool:
known_bad_kernel_versions = instance.get_config_value(
"doctor.known-bad-kernel-versions", default="")
if not known_bad_kernel_versions:
return False
for regex in known_bad_kernel_versions.split(","):
if re.search(regex, release):
return True # matched known bad release
return False # no match to bad release
def _os_is_kernel_version_too_old(instance: EdenInstance,
release: str) -> bool:
min_kernel_version = instance.get_config_value(
"doctor.minimum-kernel-version", default="")
if not min_kernel_version:
return False
try:
return _parse_os_kernel_version(release) < _parse_os_kernel_version(
min_kernel_version)
except ValueError:
# If the kernel version failed to parse because one of the
# components wasn't an int, whatever.
return False
def check_disk_usage(
tracker: ProblemTracker,
mount_paths: List[str],
instance: EdenInstance,
fs_util: FsUtil,
) -> None:
prob_advice_space_used_ratio_threshold = 0.90
prob_error_absolute_space_used_threshold = 1024 * 1024 * 1024 # 1GB
eden_mount_pts_set = get_mount_pts_set(tracker, mount_paths, instance)
for eden_mount_pt in eden_mount_pts_set:
if eden_mount_pt and os.path.exists(eden_mount_pt):
disk_status = fs_util.statvfs(eden_mount_pt)
avail = disk_status.f_frsize * disk_status.f_bavail
size = disk_status.f_frsize * disk_status.f_blocks
if size == 0:
continue
used = size - avail
used_percent = float(used) / size
message = (
"Eden lazily loads your files and needs enough disk space to "
"store these files when loaded."
)
extra_message = instance.get_config_value(
"doctor.low-disk-space-message", ""
)
if extra_message:
message = f"{message} {extra_message}"
if avail <= prob_error_absolute_space_used_threshold:
tracker.add_problem(
Problem(
f"{eden_mount_pt} "
f"has only {str(avail)} bytes available. "
f"{message}",
severity=ProblemSeverity.ERROR,
)
)
elif used_percent >= prob_advice_space_used_ratio_threshold:
tracker.add_problem(
Problem(
f"{eden_mount_pt} "
f"is {used_percent:.2%} full. "
f"{message}",
severity=ProblemSeverity.ADVICE,
)
)
def run_edenfs_not_healthy_checks(
tracker: ProblemTracker,
instance: EdenInstance,
out: ui.Output,
status: config_mod.HealthStatus,
mount_table: mtab.MountTable,
fs_util: filesystem.FsUtil,
) -> None:
check_stale_mounts.check_for_stale_mounts(tracker, mount_table)
configured_mounts = instance.get_mount_paths()
check_filesystems.check_disk_usage(tracker, list(configured_mounts), instance)
if configured_mounts:
tracker.add_problem(EdenfsNotHealthy())
def run_edenfs_not_healthy_checks(
tracker: ProblemTracker,
instance: EdenInstance,
out: ui.Output,
status: config_mod.HealthStatus,
mount_table: mtab.MountTable,
fs_util: filesystem.FsUtil,
) -> None:
check_stale_mounts.check_for_stale_mounts(tracker, mount_table)
configured_mounts = instance.get_mount_paths()
check_filesystems.check_disk_usage(tracker, list(configured_mounts),
instance)
if configured_mounts:
tracker.add_problem(EdenfsNotHealthy())
def check_disk_usage(
tracker: ProblemTracker, mount_paths: List[str], instance: EdenInstance
) -> None:
prob_advice_space_used_ratio_threshold = 0.90
prob_error_absolute_space_used_threshold = 1024 * 1024 * 1024 # 1GB
eden_mount_pts_set = get_mount_pts_set(tracker, mount_paths, instance)
for eden_mount_pt in eden_mount_pts_set:
if eden_mount_pt and os.path.exists(eden_mount_pt):
disk_status = os.statvfs(eden_mount_pt)
avail = disk_status.f_frsize * disk_status.f_bavail
size = disk_status.f_frsize * disk_status.f_blocks
if size == 0:
continue
used = size - avail
used_percent = float(used) / size
message = (
"Eden lazily loads your files and needs enough disk space to "
"store these files when loaded."
)
extra_message = instance.get_config_value(
"doctor.low-disk-space-message", ""
)
if extra_message:
message = f"{message} {extra_message}"
if avail <= prob_error_absolute_space_used_threshold:
tracker.add_problem(
Problem(
f"{eden_mount_pt} "
f"has only {str(avail)} bytes available. "
f"{message}",
severity=ProblemSeverity.ERROR,
)
)
elif used_percent >= prob_advice_space_used_ratio_threshold:
tracker.add_problem(
Problem(
f"{eden_mount_pt} "
f"is {used_percent:.2%} full. "
f"{message}",
severity=ProblemSeverity.ADVICE,
)
)
def test_eden_start_fails_if_service_is_running(self) -> None:
with self.spawn_fake_edenfs(self.eden_dir):
# Make fake_edenfs inaccessible and undetectable (without talking to
# systemd), but keep the systemd service alive.
(self.eden_dir / "lock").unlink()
(self.eden_dir / "socket").unlink()
health: HealthStatus = EdenInstance(str(self.eden_dir),
etc_eden_dir=None,
home_dir=None).check_health()
self.assertEqual(health.status, fb303_status.DEAD)
service = self.get_edenfs_systemd_service(eden_dir=self.eden_dir)
self.assertEqual(service.query_active_state(), "active")
start_process = self.spawn_start()
start_process.expect_exact(
f"error: edenfs systemd service is already running")
# edenfsctl should show the output of 'systemctl status'.
start_process.expect(r"\bfb-edenfs@.*?\.service\b")
start_process.expect(r"Active:[^\n]*active \(running\)")
self.assert_process_fails(start_process, 1)
def _check_edenfs_health(self) -> HealthStatus:
instance = EdenInstance(str(self.eden_dir), etc_eden_dir=None, home_dir=None)
return instance.check_health()
def cure_what_ails_you(
instance: EdenInstance,
dry_run: bool,
mount_table: mtab.MountTable,
fs_util: filesystem.FsUtil,
process_finder: process_finder.ProcessFinder,
out: Optional[ui.Output] = None,
) -> int:
if out is None:
out = ui.get_output()
if not dry_run:
fixer = ProblemFixer(out)
else:
fixer = DryRunFixer(out)
check_working_directory(fixer)
# check OS type, kernel version etc.
check_os.run_operating_system_checks(fixer, instance, out)
# check multiple edenfs running with some rogue stale PIDs
check_rogue_edenfs.check_many_edenfs_are_running(fixer, process_finder)
status = instance.check_health()
if status.status == fb_status.ALIVE:
run_normal_checks(fixer, instance, out, mount_table, fs_util)
elif status.status == fb_status.STARTING:
fixer.add_problem(EdenfsStarting())
elif status.status == fb_status.STOPPING:
fixer.add_problem(EdenfsStopping())
elif status.status == fb_status.DEAD:
run_edenfs_not_healthy_checks(fixer, instance, out, status,
mount_table, fs_util)
if fixer.num_problems == 0:
out.writeln("Eden is not in use.")
return 0
else:
fixer.add_problem(EdenfsUnexpectedStatus(status))
if fixer.num_problems == 0:
out.writeln("No issues detected.", fg=out.GREEN)
return 0
def problem_count(num: int) -> str:
if num == 1:
return "1 problem"
return f"{num} problems"
if dry_run:
out.writeln(
f"Discovered {problem_count(fixer.num_problems)} during --dry-run",
fg=out.YELLOW,
)
return 1
if fixer.num_fixed_problems:
out.writeln(
f"Successfully fixed {problem_count(fixer.num_fixed_problems)}.",
fg=out.YELLOW,
)
if fixer.num_failed_fixes:
out.writeln(f"Failed to fix {problem_count(fixer.num_failed_fixes)}.",
fg=out.RED)
if fixer.num_manual_fixes:
if fixer.num_manual_fixes == 1:
msg = f"1 issue requires manual attention."
else:
msg = f"{fixer.num_manual_fixes} issues require manual attention."
out.writeln(msg, fg=out.YELLOW)
if fixer.num_fixed_problems == fixer.num_problems:
return 0
out.write(
"Ask in the Eden Users group if you need help fixing issues with Eden:\n"
"https://fb.facebook.com/groups/eden.users/\n")
return 1
def test_list_mounts(self):
self.maxDiff = None
thrift_mounts = [
MountInfo(
mountPoint=b"/data/users/johndoe/mercurial",
edenClientPath=b"/home/johndoe/.eden/clients/mercurial",
state=MountState.RUNNING,
),
MountInfo(
mountPoint=b"/data/users/johndoe/git",
edenClientPath=b"/home/johndoe/.eden/clients/git",
state=MountState.SHUTTING_DOWN,
),
MountInfo(
mountPoint=b"/data/users/johndoe/apache",
edenClientPath=b"/home/johndoe/.eden/clients/apache",
state=MountState.RUNNING,
),
MountInfo(
mountPoint=b"/data/users/johndoe/configs",
edenClientPath=b"/home/johndoe/.eden/clients/configs",
state=MountState.INITIALIZING,
),
MountInfo(
mountPoint=b"/data/users/johndoe/repos/linux",
edenClientPath=b"/home/johndoe/.eden/clients/linux",
state=MountState.RUNNING,
),
MountInfo(
mountPoint=b"/data/users/johndoe/other_repos/linux",
edenClientPath=b"/home/johndoe/.eden/clients/linux2",
state=MountState.RUNNING,
),
]
instance = EdenInstance(
config_dir="/home/johndoe/.eden",
etc_eden_dir="/etc/eden",
home_dir="/home/johndoe",
)
config_checkouts = [
EdenCheckout(
instance,
Path("/data/users/johndoe/mercurial"),
Path("/home/johndoe/.eden/clients/mercurial"),
),
EdenCheckout(
instance,
Path("/data/users/johndoe/git"),
Path("/home/johndoe/.eden/clients/git"),
),
EdenCheckout(
instance,
Path("/data/users/johndoe/repos/linux"),
Path("/home/johndoe/.eden/clients/linux"),
),
EdenCheckout(
instance,
Path("/data/users/johndoe/other_repos/linux"),
Path("/home/johndoe/.eden/clients/linux2"),
),
EdenCheckout(
instance,
Path("/data/users/johndoe/www"),
Path("/home/johndoe/.eden/clients/www"),
),
]
mounts = main_mod.ListCmd.combine_mount_info(thrift_mounts,
config_checkouts)
normal_out = TestOutput()
main_mod.ListCmd.print_mounts(normal_out, mounts)
self.assertEqual(
"""\
/data/users/johndoe/apache (unconfigured)
/data/users/johndoe/configs (INITIALIZING) (unconfigured)
/data/users/johndoe/git (SHUTTING_DOWN)
/data/users/johndoe/mercurial
/data/users/johndoe/other_repos/linux
/data/users/johndoe/repos/linux
/data/users/johndoe/www (not mounted)
""",
normal_out.getvalue(),
)
json_out = TestOutput()
main_mod.ListCmd.print_mounts_json(json_out, mounts)
self.assertEqual(
"""\
{
"/data/users/johndoe/apache": {
"configured": false,
"data_dir": "/home/johndoe/.eden/clients/apache",
"state": "RUNNING"
},
"/data/users/johndoe/configs": {
"configured": false,
"data_dir": "/home/johndoe/.eden/clients/configs",
"state": "INITIALIZING"
},
"/data/users/johndoe/git": {
"configured": true,
"data_dir": "/home/johndoe/.eden/clients/git",
"state": "SHUTTING_DOWN"
},
"/data/users/johndoe/mercurial": {
"configured": true,
"data_dir": "/home/johndoe/.eden/clients/mercurial",
"state": "RUNNING"
},
"/data/users/johndoe/other_repos/linux": {
"configured": true,
"data_dir": "/home/johndoe/.eden/clients/linux2",
"state": "RUNNING"
},
"/data/users/johndoe/repos/linux": {
"configured": true,
"data_dir": "/home/johndoe/.eden/clients/linux",
"state": "RUNNING"
},
"/data/users/johndoe/www": {
"configured": true,
"data_dir": "/home/johndoe/.eden/clients/www",
"state": "NOT_RUNNING"
}
}
""",
json_out.getvalue(),
)
def test_list_mounts_old_thrift(self):
self.maxDiff = None
# Simulate an older edenfs daemon that does not send the "state" field
thrift_mounts = [
MountInfo(
mountPoint=b"/data/users/johndoe/mercurial",
edenClientPath=b"/home/johndoe/.eden/clients/mercurial",
),
MountInfo(
mountPoint=b"/data/users/johndoe/git",
edenClientPath=b"/home/johndoe/.eden/clients/git",
),
MountInfo(
mountPoint=b"/data/users/johndoe/configs",
edenClientPath=b"/home/johndoe/.eden/clients/configs",
),
]
instance = EdenInstance(
config_dir="/home/johndoe/.eden",
etc_eden_dir="/etc/eden",
home_dir="/home/johndoe",
)
config_checkouts = [
EdenCheckout(
instance,
Path("/data/users/johndoe/mercurial"),
Path("/home/johndoe/.eden/clients/mercurial"),
),
EdenCheckout(
instance,
Path("/data/users/johndoe/git"),
Path("/home/johndoe/.eden/clients/git"),
),
EdenCheckout(
instance,
Path("/data/users/johndoe/www"),
Path("/home/johndoe/.eden/clients/www"),
),
]
mounts = main_mod.ListCmd.combine_mount_info(thrift_mounts,
config_checkouts)
normal_out = TestOutput()
main_mod.ListCmd.print_mounts(normal_out, mounts)
self.assertEqual(
"""\
/data/users/johndoe/configs (unconfigured)
/data/users/johndoe/git
/data/users/johndoe/mercurial
/data/users/johndoe/www (not mounted)
""",
normal_out.getvalue(),
)
json_out = TestOutput()
main_mod.ListCmd.print_mounts_json(json_out, mounts)
self.assertEqual(
"""\
{
"/data/users/johndoe/configs": {
"configured": false,
"data_dir": "/home/johndoe/.eden/clients/configs",
"state": "RUNNING"
},
"/data/users/johndoe/git": {
"configured": true,
"data_dir": "/home/johndoe/.eden/clients/git",
"state": "RUNNING"
},
"/data/users/johndoe/mercurial": {
"configured": true,
"data_dir": "/home/johndoe/.eden/clients/mercurial",
"state": "RUNNING"
},
"/data/users/johndoe/www": {
"configured": true,
"data_dir": "/home/johndoe/.eden/clients/www",
"state": "NOT_RUNNING"
}
}
""",
json_out.getvalue(),
)
def cure_what_ails_you(
instance: EdenInstance,
dry_run: bool,
mount_table: mtab.MountTable,
fs_util: filesystem.FsUtil,
process_finder: process_finder.ProcessFinder,
out: Optional[ui.Output] = None,
) -> int:
if out is None:
out = ui.get_output()
if not dry_run:
fixer = ProblemFixer(out)
else:
fixer = DryRunFixer(out)
check_working_directory(fixer)
# check OS type, kernel version etc.
check_os.run_operating_system_checks(fixer, instance, out)
# check multiple edenfs running with some rogue stale PIDs
check_rogue_edenfs.check_many_edenfs_are_running(fixer, process_finder)
status = instance.check_health()
if status.status == fb_status.ALIVE:
run_normal_checks(fixer, instance, out, mount_table, fs_util)
elif status.status == fb_status.STARTING:
fixer.add_problem(EdenfsStarting())
elif status.status == fb_status.STOPPING:
fixer.add_problem(EdenfsStopping())
elif status.status == fb_status.DEAD:
run_edenfs_not_healthy_checks(
fixer, instance, out, status, mount_table, fs_util
)
if fixer.num_problems == 0:
out.writeln("Eden is not in use.")
return 0
else:
fixer.add_problem(EdenfsUnexpectedStatus(status))
if fixer.num_problems == 0:
out.writeln("No issues detected.", fg=out.GREEN)
return 0
def problem_count(num: int) -> str:
if num == 1:
return "1 problem"
return f"{num} problems"
if dry_run:
out.writeln(
f"Discovered {problem_count(fixer.num_problems)} during --dry-run",
fg=out.YELLOW,
)
return 1
if fixer.num_fixed_problems:
out.writeln(
f"Successfully fixed {problem_count(fixer.num_fixed_problems)}.",
fg=out.YELLOW,
)
if fixer.num_failed_fixes:
out.writeln(
f"Failed to fix {problem_count(fixer.num_failed_fixes)}.", fg=out.RED
)
if fixer.num_manual_fixes:
if fixer.num_manual_fixes == 1:
msg = f"1 issue requires manual attention."
else:
msg = f"{fixer.num_manual_fixes} issues require manual attention."
out.writeln(msg, fg=out.YELLOW)
if fixer.num_fixed_problems == fixer.num_problems:
return 0
out.write(
"Ask in the Eden Users group if you need help fixing issues with Eden:\n"
"https://fb.facebook.com/groups/eden.users/\n"
)
return 1
def _check_edenfs_health(self) -> HealthStatus:
instance = EdenInstance(str(self.eden_dir), etc_eden_dir=None, home_dir=None)
return instance.check_health()
