def test_mesos_disk_collector_when_connection_error(self):
settings = DiskCollectorSettings(
http_api_url=self.agent_api_url,
executor_id_json_path=self.executor_id_json_path,
disk_usage_json_path=self.disk_usage_json_path,
disk_collection_timeout=self.collection_timeout,
disk_collection_interval=self.collection_interval)
def exception_callback(request, uri, headers):
raise ConnectionError
httpretty.register_uri(method=httpretty.GET,
uri="http://localhost:5051/containers",
body=exception_callback)
collector = MesosDiskCollector(self.sandbox_path, settings)
def wait():
collector.sample()
if collector._thread is not None:
collector._thread.event.wait()
self.assertEquals(collector.value, 0)
wait()
self.assertEquals(collector.value, LOOK_UP_ERROR_VALUE)
self.assertEquals(httpretty.last_request().method, "GET")
self.assertEquals(httpretty.last_request().path, "/containers")
def test_mesos_disk_collector_when_unauthorized(self):
settings = DiskCollectorSettings(
http_api_url=self.agent_api_url,
executor_id_json_path=self.executor_id_json_path,
disk_usage_json_path=self.disk_usage_json_path,
disk_collection_timeout=self.collection_timeout,
disk_collection_interval=self.collection_interval)
json_body = json.dumps([{
"executor_id": "thermos-some-task-id",
"statistics": {
"disk_used_bytes": 100
}
}])
httpretty.register_uri(method=httpretty.GET,
uri="http://localhost:5051/containers",
body=json_body,
status=401)
collector = MesosDiskCollector(self.sandbox_path, settings)
def wait():
collector.sample()
if collector._thread is not None:
collector._thread.event.wait()
self.assertEquals(collector.value, 0)
wait()
self.assertEquals(collector.value, LOOK_UP_ERROR_VALUE)
self.assertEquals(httpretty.last_request().method, "GET")
self.assertEquals(httpretty.last_request().path, "/containers")
def test_mesos_disk_collector_unexpected_response_format(self):
settings = DiskCollectorSettings(
http_api_url=self.agent_api_url,
executor_id_json_path=self.executor_id_json_path,
disk_usage_json_path=self.disk_usage_json_path,
disk_collection_timeout=self.collection_timeout,
disk_collection_interval=self.collection_interval)
json_body = json.dumps({"status": "bad_request"})
httpretty.register_uri(method=httpretty.GET,
uri="http://localhost:5051/containers",
body=json_body,
content_type='application/json')
collector = MesosDiskCollector(self.sandbox_path, settings)
def wait():
collector.sample()
if collector._thread is not None:
collector._thread.event.wait()
assert collector.value == 0
wait()
assert collector.value == LOOK_UP_ERROR_VALUE
self.assertEquals(httpretty.last_request().method, "GET")
self.assertEquals(httpretty.last_request().path, "/containers")
def initialize(options):
path_detector = MesosPathDetector(options.mesos_root)
disk_collector_settings = DiskCollectorSettings(
options.agent_api_url, options.executor_id_json_path,
options.disk_usage_json_path,
Amount(options.task_disk_collection_interval_secs, Time.SECONDS))
return TaskObserver(
path_detector,
Amount(options.polling_interval_secs, Time.SECONDS),
Amount(options.task_process_collection_interval_secs, Time.SECONDS),
enable_mesos_disk_collector=options.enable_mesos_disk_collector,
disk_collector_settings=disk_collector_settings)
def test_mesos_disk_collector(self):
settings = DiskCollectorSettings(
http_api_url=self.agent_api_url,
executor_id_json_path=self.executor_id_json_path,
disk_usage_json_path=self.disk_usage_json_path,
disk_collection_timeout=self.collection_timeout,
disk_collection_interval=self.collection_interval)
first_json_body = json.dumps([{
"executor_id": "thermos-some-task-id",
"statistics": {
"disk_used_bytes": 100
}
}])
second_json_body = json.dumps([{
"executor_id": "thermos-some-task-id",
"statistics": {
"disk_used_bytes": 200
}
}])
httpretty.register_uri(
method=httpretty.GET,
uri="http://localhost:5051/containers",
responses=[
httpretty.Response(body=first_json_body,
content_type='application/json'),
httpretty.Response(body=second_json_body,
content_type='application/json')
])
collector = MesosDiskCollector(self.sandbox_path, settings)
def wait():
collector.sample()
if collector._thread is not None:
collector._thread.event.wait()
assert collector.value == 0
wait()
assert collector.value == 100
wait()
assert collector.value == 200
print(dir(httpretty.last_request()))
self.assertEquals(httpretty.last_request().method, "GET")
self.assertEquals(httpretty.last_request().path, "/containers")
def __init__(self,
path_detector,
interval=POLLING_INTERVAL,
task_process_collection_interval=TaskResourceMonitor.
PROCESS_COLLECTION_INTERVAL,
enable_mesos_disk_collector=False,
disk_collector_settings=DiskCollectorSettings()):
self._detector = ObserverTaskDetector(path_detector, self.__on_active,
self.__on_finished,
self.__on_removed)
self._interval = interval
self._task_process_collection_interval = task_process_collection_interval
self._enable_mesos_disk_collector = enable_mesos_disk_collector
self._disk_collector_settings = disk_collector_settings
self._active_tasks = {} # task_id => ActiveObservedTask
self._finished_tasks = {} # task_id => FinishedObservedTask
self._stop_event = threading.Event()
ExceptionalThread.__init__(self)
Lockable.__init__(self)
self.daemon = True
def test_mesos_disk_collector_bad_disk_usage_selector(self):
settings = DiskCollectorSettings(
http_api_url=self.agent_api_url,
executor_id_json_path=self.executor_id_json_path,
disk_usage_json_path="bad_path",
disk_collection_timeout=self.collection_timeout,
disk_collection_interval=self.collection_interval)
json_body = json.dumps([{
"executor_id": "thermos-some-task-id",
"statistics": {
"disk_used_bytes": 100
}
}])
httpretty.register_uri(method=httpretty.GET,
uri="http://localhost:5051/containers",
responses=[
httpretty.Response(
body=json_body,
content_type='application/json')
])
sandbox_path = "/var/lib/path/to/thermos-some-task-id"
collector = MesosDiskCollector(sandbox_path, settings)
def wait():
collector.sample()
if collector._thread is not None:
collector._thread.event.wait()
self.assertEquals(collector.value, 0)
wait()
self.assertEquals(collector.value, LOOK_UP_ERROR_VALUE)
self.assertEquals(httpretty.last_request().method, "GET")
self.assertEquals(httpretty.last_request().path, "/containers")
def test_mesos_disk_collector_timeout(self):
settings = DiskCollectorSettings(
http_api_url=self.agent_api_url,
executor_id_json_path=self.executor_id_json_path,
disk_usage_json_path=self.disk_usage_json_path,
disk_collection_timeout=Amount(10, Time.MILLISECONDS),
disk_collection_interval=self.collection_interval)
def callback(request, uri, headers):
json_body = json.dumps([{
"executor_id": "thermos-some-task-id",
"statistics": {
"disk_used_bytes": 100
}
}])
sleep(5)
return (200, headers, json_body)
httpretty.register_uri(method=httpretty.GET,
uri="http://localhost:5051/containers",
body=callback)
collector = MesosDiskCollector(self.sandbox_path, settings)
def wait():
collector.sample()
if collector._thread is not None:
collector._thread.event.wait()
self.assertEquals(collector.value, 0)
wait()
self.assertEquals(collector.value, LOOK_UP_ERROR_VALUE)
self.assertEquals(httpretty.last_request().method, "GET")
self.assertEquals(httpretty.last_request().path, "/containers")