def test_path_detector():
ROOTS = ('/var/lib/mesos1/slaves', '/var/lib/mesos2/slaves')
FAKE_ROOT = '/var/blah/blah'
FAKE_CHECKPOINT_DIR = 'ckpt'
path1, path2 = (
ExecutorDetector.path(Match(ROOTS[0], 'slave001', 'framework1', 'executor1', 'latest')),
ExecutorDetector.path(Match(ROOTS[1], 'slave002', 'framework2', 'executor2', 'latest')),
)
with mock.patch('glob.glob', return_value=(path1, path2)) as glob:
mpd = MesosPathDetector(root=FAKE_ROOT, sandbox_path=FAKE_CHECKPOINT_DIR)
paths = list(mpd.get_paths())
assert len(paths) == 2
assert os.path.join(path1, FAKE_CHECKPOINT_DIR) in paths
assert os.path.join(path2, FAKE_CHECKPOINT_DIR) in paths
expected_glob_pattern = os.path.join(*ExecutorDetector.PATTERN) % {
'root': FAKE_ROOT,
'slave_id': '*',
'framework_id': '*',
'executor_id': '*',
'run': '*'
}
glob.assert_called_once_with(expected_glob_pattern)
def test_path_detector():
ROOTS = ('/var/lib/mesos1/slaves', '/var/lib/mesos2/slaves')
FAKE_ROOT = '/var/blah/blah'
FAKE_CHECKPOINT_DIR = 'ckpt'
path1, path2 = (
ExecutorDetector.path(
Match(ROOTS[0], 'slave001', 'framework1', 'executor1', 'latest')),
ExecutorDetector.path(
Match(ROOTS[1], 'slave002', 'framework2', 'executor2', 'latest')),
)
with mock.patch('glob.glob', return_value=(path1, path2)) as glob:
mpd = MesosPathDetector(root=FAKE_ROOT,
sandbox_path=FAKE_CHECKPOINT_DIR)
paths = list(mpd.get_paths())
assert len(paths) == 2
assert os.path.join(path1, FAKE_CHECKPOINT_DIR) in paths
assert os.path.join(path2, FAKE_CHECKPOINT_DIR) in paths
expected_glob_pattern = os.path.join(*ExecutorDetector.PATTERN) % {
'root': FAKE_ROOT,
'slave_id': '*',
'framework_id': '*',
'executor_id': '*',
'run': '*'
}
glob.assert_called_once_with(expected_glob_pattern)
def test_integration():
SLAVES = ('slave001', 'slave123')
FRAMEWORKS = ('framework1', 'framework2')
EXECUTORS = ('executor_a', 'executor_b')
RUNS = ('001', '002', 'latest')
with temporary_dir() as td:
all_groups = set()
for slave, framework, executor, run in itertools.product(
SLAVES, FRAMEWORKS, EXECUTORS, RUNS):
match = Match(td, slave, framework, executor, run)
safe_mkdir(ExecutorDetector.path(match))
all_groups.add(tuple(sorted(match.groups().items())))
for match in ExecutorDetector(td):
assert tuple(sorted(match.groups().items())) in all_groups
def test_find_root():
BAD_PATHS = (
os.path.sep,
'.',
os.path.sep * 10,
'/root/slaves',
'/root/slaves/S/frameworks/F/executors//runs/R',
'root/slaves/S/frameworks/F/executors//runs/R',
)
GOOD_PATHS = (ExecutorDetector.path(DEFAULT_MATCH),
os.path.join(ExecutorDetector.path(DEFAULT_MATCH), 'some',
'other', 'path'))
for cwd in BAD_PATHS:
assert ExecutorDetector.find_root(cwd) is None
for cwd in GOOD_PATHS:
assert ExecutorDetector.find_root(cwd) == 'abcd'
def test_find_root():
BAD_PATHS = (
os.path.sep,
".",
os.path.sep * 10,
"/root/slaves",
"/root/slaves/S/frameworks/F/executors//runs/R",
"root/slaves/S/frameworks/F/executors//runs/R",
)
GOOD_PATHS = (
ExecutorDetector.path(DEFAULT_MATCH),
os.path.join(ExecutorDetector.path(DEFAULT_MATCH), "some", "other", "path"),
)
for cwd in BAD_PATHS:
assert ExecutorDetector.find_root(cwd) is None
for cwd in GOOD_PATHS:
assert ExecutorDetector.find_root(cwd) == "abcd"
def test_path_detector():
ROOTS = ('/var/lib/mesos1/slaves', '/var/lib/mesos2/slaves')
FAKE_ROOT = '/var/blah/blah'
FAKE_CHECKPOINT_DIR = 'ckpt'
path1_symlink, path1, path2 = (
ExecutorDetector.path(Match(ROOTS[0], 'slave001', 'framework1', 'executor1', 'latest')),
ExecutorDetector.path(Match(ROOTS[0], 'slave001', 'framework1', 'executor1', 'asdf-ghjk')),
ExecutorDetector.path(Match(ROOTS[1], 'slave002', 'framework2', 'executor2', 'latest')),
)
with mock.patch('glob.glob', return_value=(path1_symlink, path1, path2)) as glob:
with mock.patch('os.path.realpath', side_effect=(path1, path1, path2)) as realpath:
with mock.patch('os.path.exists', side_effect=(True, True, False)) as exists:
mpd = MesosPathDetector(root=FAKE_ROOT, sandbox_path=FAKE_CHECKPOINT_DIR)
paths = list(mpd.get_paths())
assert len(paths) == 1
assert paths == [os.path.join(path1, FAKE_CHECKPOINT_DIR)]
expected_glob_pattern = os.path.join(*ExecutorDetector.PATTERN) % {
'root': FAKE_ROOT,
'slave_id': '*',
'framework_id': '*',
'executor_id': '*',
'run': '*'
}
assert glob.mock_calls == [mock.call(expected_glob_pattern)]
assert realpath.mock_calls == [
mock.call(os.path.join(path1_symlink)),
mock.call(os.path.join(path1)),
mock.call(os.path.join(path2)),
]
assert exists.mock_calls == [
mock.call(os.path.join(path1, FAKE_CHECKPOINT_DIR)),
mock.call(os.path.join(path1, FAKE_CHECKPOINT_DIR)),
mock.call(os.path.join(path2, FAKE_CHECKPOINT_DIR)),
]
def test_integration():
SLAVES = ("slave001", "slave123")
FRAMEWORKS = ("framework1", "framework2")
EXECUTORS = ("executor_a", "executor_b")
RUNS = ("001", "002", "latest")
with temporary_dir() as td:
all_groups = set()
for slave, framework, executor, run in itertools.product(SLAVES, FRAMEWORKS, EXECUTORS, RUNS):
match = Match(td, slave, framework, executor, run)
safe_mkdir(ExecutorDetector.path(match))
all_groups.add(tuple(sorted(match.groups().items())))
for match in ExecutorDetector(td):
assert tuple(sorted(match.groups().items())) in all_groups
def test_bad_match():
assert ExecutorDetector.match("herpderp") is None
def test_match_inverse():
assert ExecutorDetector.match(ExecutorDetector.path(DEFAULT_MATCH)).groups() == (DEFAULT_MATCH.groups())
def test_bad_match():
assert ExecutorDetector.match('herpderp') is None
def test_match_inverse():
assert ExecutorDetector.match(
ExecutorDetector.path(DEFAULT_MATCH)).groups() == (
DEFAULT_MATCH.groups())