def test_chained_path_detector(): root1 = '/var/lib/derp1' root2 = '/var/lib/derp2' fpd1 = FixedPathDetector(path=root1) fpd2 = FixedPathDetector(path=root2) cpd = ChainedPathDetector(fpd1, fpd2) assert set(cpd.get_paths()) == set([root1, root2])
def run_gc(self, root, task_id, retain=False):
"""Run the garbage collection process against the given task_id in the given checkpoint root"""
class FakeTaskKiller(object):
def __init__(self, task_id, checkpoint_root):
pass
def kill(self):
pass
def lose(self):
pass
class FakeTaskGarbageCollector(object):
def __init__(self, root, task_id):
pass
def erase_logs(self):
pass
def erase_metadata(self):
pass
class FastThermosGCExecutor(ThermosGCExecutor):
POLL_WAIT = Amount(1, Time.MILLISECONDS)
detector = functools.partial(FakeExecutorDetector, task_id) if retain else FakeExecutorDetector
executor = FastThermosGCExecutor(
path_detector=FixedPathDetector(root),
task_killer=FakeTaskKiller,
executor_detector=detector,
task_garbage_collector=FakeTaskGarbageCollector,
clock=ThreadedClock(time.time()))
return executor.garbage_collect()
def initialize(options):
path_detector = ChainedPathDetector(
FixedPathDetector(options.root),
MesosPathDetector(options.mesos_root),
)
polling_interval = Amount(options.polling_interval_secs, Time.SECONDS)
return TaskObserver(path_detector, interval=polling_interval)
def __init__(self, checkpoint_root, active_executors=[]):
self._active_executors = active_executors
self._kills = set()
self._losses = set()
self._gcs = set()
ThermosGCExecutor.__init__(
self,
FixedPathDetector(checkpoint_root),
clock=ThreadedClock(time.time()),
executor_detector=lambda: list)
def main(_, opts):
path_detector = FixedPathDetector(opts.root)
task_observer = TaskObserver(path_detector)
task_observer.start()
server = configure_server(task_observer)
thread = ExceptionalThread(
target=lambda: server.run('0.0.0.0', opts.port, 'cherrypy'))
thread.daemon = True
thread.start()
sleep_forever()
def register_options(app):
from apache.thermos.common.constants import DEFAULT_CHECKPOINT_ROOT
clear_path_detectors()
register_path_detector(FixedPathDetector(DEFAULT_CHECKPOINT_ROOT))
app.add_option('--root',
dest='root',
metavar='PATH',
type='string',
default=DEFAULT_CHECKPOINT_ROOT,
action='callback',
callback=register_root,
help="the thermos config root")
def test_fixed_path_detector(): # Default is TaskPath default fpd = FixedPathDetector() assert fpd.get_paths() == [DEFAULT_CHECKPOINT_ROOT] # Non-default root = '/var/lib/derp' fpd = FixedPathDetector(path=root) assert fpd.get_paths() == [root]
def main(_, options):
path_detector = ChainedPathDetector(
FixedPathDetector(options.root),
MesosPathDetector(options.mesos_root),
)
observer = TaskObserver(path_detector)
observer.start()
root_server = configure_server(observer)
thread = ExceptionalThread(
target=lambda: root_server.run('0.0.0.0', options.port, 'cherrypy'))
thread.daemon = True
thread.start()
sleep_forever()
def make_observer(cls):
return TaskObserver(FixedPathDetector(MOCK_BASE_PATH))
def register_root(_, __, value, parser):
parser.values.root = value
register_path_detector(FixedPathDetector(value))
def test_chained_path_detector_constructor():
with pytest.raises(TypeError):
ChainedPathDetector(1, 2, 3)
with pytest.raises(TypeError):
ChainedPathDetector(FixedPathDetector(), 'hello')
def test_fixed_path_detector_constructor():
with pytest.raises(TypeError):
FixedPathDetector(path=234)