def test_path_detector():
ROOTS = ('/var/lib/mesos1/slaves', '/var/lib/mesos2/slaves')
FAKE_ROOT = '/var/blah/blah'
FAKE_CHECKPOINT_DIR = 'ckpt'
path1, path2 = (
ExecutorDetector.path(
Match(ROOTS[0], 'slave001', 'framework1', 'executor1', 'latest')),
ExecutorDetector.path(
Match(ROOTS[1], 'slave002', 'framework2', 'executor2', 'latest')),
)
with mock.patch('glob.glob', return_value=(path1, path2)) as glob:
mpd = MesosPathDetector(root=FAKE_ROOT,
sandbox_path=FAKE_CHECKPOINT_DIR)
paths = list(mpd.get_paths())
assert len(paths) == 2
assert os.path.join(path1, FAKE_CHECKPOINT_DIR) in paths
assert os.path.join(path2, FAKE_CHECKPOINT_DIR) in paths
expected_glob_pattern = os.path.join(*ExecutorDetector.PATTERN) % {
'root': FAKE_ROOT,
'slave_id': '*',
'framework_id': '*',
'executor_id': '*',
'run': '*'
}
glob.assert_called_once_with(expected_glob_pattern)
def test_path_detector():
ROOTS = ('/var/lib/mesos1/slaves', '/var/lib/mesos2/slaves')
FAKE_ROOT = '/var/blah/blah'
FAKE_CHECKPOINT_DIR = 'ckpt'
path1, path2 = (
ExecutorDetector.path(Match(ROOTS[0], 'slave001', 'framework1', 'executor1', 'latest')),
ExecutorDetector.path(Match(ROOTS[1], 'slave002', 'framework2', 'executor2', 'latest')),
)
with mock.patch('glob.glob', return_value=(path1, path2)) as glob:
mpd = MesosPathDetector(root=FAKE_ROOT, sandbox_path=FAKE_CHECKPOINT_DIR)
paths = list(mpd.get_paths())
assert len(paths) == 2
assert os.path.join(path1, FAKE_CHECKPOINT_DIR) in paths
assert os.path.join(path2, FAKE_CHECKPOINT_DIR) in paths
expected_glob_pattern = os.path.join(*ExecutorDetector.PATTERN) % {
'root': FAKE_ROOT,
'slave_id': '*',
'framework_id': '*',
'executor_id': '*',
'run': '*'
}
glob.assert_called_once_with(expected_glob_pattern)
def initialize(options):
path_detector = ChainedPathDetector(
FixedPathDetector(options.root),
MesosPathDetector(options.mesos_root),
)
polling_interval = Amount(options.polling_interval_secs, Time.SECONDS)
return TaskObserver(path_detector, interval=polling_interval)
def initialize(options):
path_detector = MesosPathDetector(options.mesos_root)
disk_collector_settings = DiskCollectorSettings(
options.agent_api_url, options.executor_id_json_path,
options.disk_usage_json_path,
Amount(options.task_disk_collection_interval_secs, Time.SECONDS))
return TaskObserver(
path_detector,
Amount(options.polling_interval_secs, Time.SECONDS),
Amount(options.task_process_collection_interval_secs, Time.SECONDS),
enable_mesos_disk_collector=options.enable_mesos_disk_collector,
disk_collector_settings=disk_collector_settings)
def test_path_detector():
ROOTS = ('/var/lib/mesos1/slaves', '/var/lib/mesos2/slaves')
FAKE_ROOT = '/var/blah/blah'
FAKE_CHECKPOINT_DIR = 'ckpt'
path1_symlink, path1, path2 = (
ExecutorDetector.path(Match(ROOTS[0], 'slave001', 'framework1', 'executor1', 'latest')),
ExecutorDetector.path(Match(ROOTS[0], 'slave001', 'framework1', 'executor1', 'asdf-ghjk')),
ExecutorDetector.path(Match(ROOTS[1], 'slave002', 'framework2', 'executor2', 'latest')),
)
with mock.patch('glob.glob', return_value=(path1_symlink, path1, path2)) as glob:
with mock.patch('os.path.realpath', side_effect=(path1, path1, path2)) as realpath:
with mock.patch('os.path.exists', side_effect=(True, True, False)) as exists:
mpd = MesosPathDetector(root=FAKE_ROOT, sandbox_path=FAKE_CHECKPOINT_DIR)
paths = list(mpd.get_paths())
assert len(paths) == 1
assert paths == [os.path.join(path1, FAKE_CHECKPOINT_DIR)]
expected_glob_pattern = os.path.join(*ExecutorDetector.PATTERN) % {
'root': FAKE_ROOT,
'slave_id': '*',
'framework_id': '*',
'executor_id': '*',
'run': '*'
}
assert glob.mock_calls == [mock.call(expected_glob_pattern)]
assert realpath.mock_calls == [
mock.call(os.path.join(path1_symlink)),
mock.call(os.path.join(path1)),
mock.call(os.path.join(path2)),
]
assert exists.mock_calls == [
mock.call(os.path.join(path1, FAKE_CHECKPOINT_DIR)),
mock.call(os.path.join(path1, FAKE_CHECKPOINT_DIR)),
mock.call(os.path.join(path2, FAKE_CHECKPOINT_DIR)),
]
def main(_, options):
path_detector = ChainedPathDetector(
FixedPathDetector(options.root),
MesosPathDetector(options.mesos_root),
)
observer = TaskObserver(path_detector)
observer.start()
root_server = configure_server(observer)
thread = ExceptionalThread(
target=lambda: root_server.run('0.0.0.0', options.port, 'cherrypy'))
thread.daemon = True
thread.start()
sleep_forever()
def initialize(options):
path_detector = MesosPathDetector(options.mesos_root)
return TaskObserver(
path_detector, Amount(options.polling_interval_secs, Time.SECONDS),
Amount(options.task_process_collection_interval_secs, Time.SECONDS),
Amount(options.task_disk_collection_interval_secs, Time.SECONDS))
def register_mesos_root(_, __, value, ___):
register_path_detector(MesosPathDetector(root=value))
from apache.aurora.executor.common.path_detector import MesosPathDetector
from apache.thermos.cli.common import register_path_detector
from apache.thermos.cli.main import register_commands, register_options
register_commands(app)
register_options(app)
def register_mesos_root(_, __, value, ___):
register_path_detector(MesosPathDetector(root=value))
app.add_option(
'--mesos-root',
dest='mesos_root',
type='string',
action='callback',
default=MesosPathDetector.DEFAULT_MESOS_ROOT,
callback=register_mesos_root,
help=
'The mesos root directory to search for Thermos executor sandboxes [default: %default]'
)
# register a default mesos root, since the callback will not be called if no --mesos-root specified.
register_path_detector(MesosPathDetector())
LogOptions.disable_disk_logging()
LogOptions.set_stderr_log_level('google:INFO')
app.main()