def test_path_detector(): ROOTS = ('/var/lib/mesos1/slaves', '/var/lib/mesos2/slaves') FAKE_ROOT = '/var/blah/blah' FAKE_CHECKPOINT_DIR = 'ckpt' path1, path2 = ( ExecutorDetector.path(Match(ROOTS[0], 'slave001', 'framework1', 'executor1', 'latest')), ExecutorDetector.path(Match(ROOTS[1], 'slave002', 'framework2', 'executor2', 'latest')), ) with mock.patch('glob.glob', return_value=(path1, path2)) as glob: mpd = MesosPathDetector(root=FAKE_ROOT, sandbox_path=FAKE_CHECKPOINT_DIR) paths = list(mpd.get_paths()) assert len(paths) == 2 assert os.path.join(path1, FAKE_CHECKPOINT_DIR) in paths assert os.path.join(path2, FAKE_CHECKPOINT_DIR) in paths expected_glob_pattern = os.path.join(*ExecutorDetector.PATTERN) % { 'root': FAKE_ROOT, 'slave_id': '*', 'framework_id': '*', 'executor_id': '*', 'run': '*' } glob.assert_called_once_with(expected_glob_pattern)
def test_path_detector(): ROOTS = ('/var/lib/mesos1/slaves', '/var/lib/mesos2/slaves') FAKE_ROOT = '/var/blah/blah' FAKE_CHECKPOINT_DIR = 'ckpt' path1, path2 = ( ExecutorDetector.path( Match(ROOTS[0], 'slave001', 'framework1', 'executor1', 'latest')), ExecutorDetector.path( Match(ROOTS[1], 'slave002', 'framework2', 'executor2', 'latest')), ) with mock.patch('glob.glob', return_value=(path1, path2)) as glob: mpd = MesosPathDetector(root=FAKE_ROOT, sandbox_path=FAKE_CHECKPOINT_DIR) paths = list(mpd.get_paths()) assert len(paths) == 2 assert os.path.join(path1, FAKE_CHECKPOINT_DIR) in paths assert os.path.join(path2, FAKE_CHECKPOINT_DIR) in paths expected_glob_pattern = os.path.join(*ExecutorDetector.PATTERN) % { 'root': FAKE_ROOT, 'slave_id': '*', 'framework_id': '*', 'executor_id': '*', 'run': '*' } glob.assert_called_once_with(expected_glob_pattern)
def test_integration(): SLAVES = ('slave001', 'slave123') FRAMEWORKS = ('framework1', 'framework2') EXECUTORS = ('executor_a', 'executor_b') RUNS = ('001', '002', 'latest') with temporary_dir() as td: all_groups = set() for slave, framework, executor, run in itertools.product( SLAVES, FRAMEWORKS, EXECUTORS, RUNS): match = Match(td, slave, framework, executor, run) safe_mkdir(ExecutorDetector.path(match)) all_groups.add(tuple(sorted(match.groups().items()))) for match in ExecutorDetector(td): assert tuple(sorted(match.groups().items())) in all_groups
def test_find_root(): BAD_PATHS = ( os.path.sep, '.', os.path.sep * 10, '/root/slaves', '/root/slaves/S/frameworks/F/executors//runs/R', 'root/slaves/S/frameworks/F/executors//runs/R', ) GOOD_PATHS = (ExecutorDetector.path(DEFAULT_MATCH), os.path.join(ExecutorDetector.path(DEFAULT_MATCH), 'some', 'other', 'path')) for cwd in BAD_PATHS: assert ExecutorDetector.find_root(cwd) is None for cwd in GOOD_PATHS: assert ExecutorDetector.find_root(cwd) == 'abcd'
def test_find_root(): BAD_PATHS = ( os.path.sep, ".", os.path.sep * 10, "/root/slaves", "/root/slaves/S/frameworks/F/executors//runs/R", "root/slaves/S/frameworks/F/executors//runs/R", ) GOOD_PATHS = ( ExecutorDetector.path(DEFAULT_MATCH), os.path.join(ExecutorDetector.path(DEFAULT_MATCH), "some", "other", "path"), ) for cwd in BAD_PATHS: assert ExecutorDetector.find_root(cwd) is None for cwd in GOOD_PATHS: assert ExecutorDetector.find_root(cwd) == "abcd"
def test_path_detector(): ROOTS = ('/var/lib/mesos1/slaves', '/var/lib/mesos2/slaves') FAKE_ROOT = '/var/blah/blah' FAKE_CHECKPOINT_DIR = 'ckpt' path1_symlink, path1, path2 = ( ExecutorDetector.path(Match(ROOTS[0], 'slave001', 'framework1', 'executor1', 'latest')), ExecutorDetector.path(Match(ROOTS[0], 'slave001', 'framework1', 'executor1', 'asdf-ghjk')), ExecutorDetector.path(Match(ROOTS[1], 'slave002', 'framework2', 'executor2', 'latest')), ) with mock.patch('glob.glob', return_value=(path1_symlink, path1, path2)) as glob: with mock.patch('os.path.realpath', side_effect=(path1, path1, path2)) as realpath: with mock.patch('os.path.exists', side_effect=(True, True, False)) as exists: mpd = MesosPathDetector(root=FAKE_ROOT, sandbox_path=FAKE_CHECKPOINT_DIR) paths = list(mpd.get_paths()) assert len(paths) == 1 assert paths == [os.path.join(path1, FAKE_CHECKPOINT_DIR)] expected_glob_pattern = os.path.join(*ExecutorDetector.PATTERN) % { 'root': FAKE_ROOT, 'slave_id': '*', 'framework_id': '*', 'executor_id': '*', 'run': '*' } assert glob.mock_calls == [mock.call(expected_glob_pattern)] assert realpath.mock_calls == [ mock.call(os.path.join(path1_symlink)), mock.call(os.path.join(path1)), mock.call(os.path.join(path2)), ] assert exists.mock_calls == [ mock.call(os.path.join(path1, FAKE_CHECKPOINT_DIR)), mock.call(os.path.join(path1, FAKE_CHECKPOINT_DIR)), mock.call(os.path.join(path2, FAKE_CHECKPOINT_DIR)), ]
def test_integration(): SLAVES = ("slave001", "slave123") FRAMEWORKS = ("framework1", "framework2") EXECUTORS = ("executor_a", "executor_b") RUNS = ("001", "002", "latest") with temporary_dir() as td: all_groups = set() for slave, framework, executor, run in itertools.product(SLAVES, FRAMEWORKS, EXECUTORS, RUNS): match = Match(td, slave, framework, executor, run) safe_mkdir(ExecutorDetector.path(match)) all_groups.add(tuple(sorted(match.groups().items()))) for match in ExecutorDetector(td): assert tuple(sorted(match.groups().items())) in all_groups
def test_bad_match(): assert ExecutorDetector.match("herpderp") is None
def test_match_inverse(): assert ExecutorDetector.match(ExecutorDetector.path(DEFAULT_MATCH)).groups() == (DEFAULT_MATCH.groups())
def test_bad_match(): assert ExecutorDetector.match('herpderp') is None
def test_match_inverse(): assert ExecutorDetector.match( ExecutorDetector.path(DEFAULT_MATCH)).groups() == ( DEFAULT_MATCH.groups())