def _run_gc(self, task, retain_tasks, retain_start):
"""
Reconcile the set of tasks to retain (provided by the scheduler) with the current state of
executors on this system. Garbage collect tasks/executors as appropriate.
Not re-entrant! Previous executions must complete (and clear self._task_id) before this can be
invoked.
Potentially blocking (e.g. on I/O) in self.garbage_collect()
Args:
task: TaskInfo provided by the slave
retain_tasks: mapping of task_id => ScheduleStatus, describing what the scheduler thinks is
running on this system
retain_start: the time at which the retain_tasks message is effective -- this means that
tasks started after the retain_tasks message is effective are skipped
until future GC runs.
"""
task_id = task.task_id.value
if self._task_id is not None:
raise RuntimeError('_run_gc() called [task_id=%s], but already running [task_id=%s]'
% (task_id, self._task_id))
self._task_id = task_id
self.log('Launching garbage collection [task_id=%s]' % task_id)
self._start_time = retain_start
local_gc, remote_gc, _ = self.reconcile_states(self._driver, retain_tasks)
deleted_tasks = set(retain_tasks).intersection(self.garbage_collect(local_gc)) | remote_gc
if deleted_tasks:
self._driver.sendFrameworkMessage(thrift_serialize(
SchedulerMessage(deletedTasks=DeletedTasks(taskIds=deleted_tasks))))
self.send_update(
self._driver, task.task_id.value, mesos_pb2.TASK_FINISHED, 'Garbage collection finished.')
self.log('Garbage collection complete [task_id=%s]' % task_id)
self._task_id = self._start_time = None
def _run_gc(self, task, retain_tasks, retain_start):
"""
Reconcile the set of tasks to retain (provided by the scheduler) with the current state of
executors on this system. Garbage collect tasks/executors as appropriate.
Not re-entrant! Previous executions must complete (and clear self._task_id) before this can be
invoked.
Potentially blocking (e.g. on I/O) in self.garbage_collect()
Args:
task: TaskInfo provided by the slave
retain_tasks: mapping of task_id => ScheduleStatus, describing what the scheduler thinks is
running on this system
retain_start: the time at which the retain_tasks message is effective -- this means that
tasks started after the retain_tasks message is effective are skipped
until future GC runs.
"""
task_id = task.task_id.value
if self._task_id is not None:
raise RuntimeError('_run_gc() called [task_id=%s], but already running [task_id=%s]'
% (task_id, self._task_id))
self._task_id = task_id
self.log('Launching garbage collection [task_id=%s]' % task_id)
self._start_time = retain_start
local_gc, remote_gc, _ = self.reconcile_states(self._driver, retain_tasks)
deleted_tasks = set(retain_tasks).intersection(self.garbage_collect(local_gc)) | remote_gc
if deleted_tasks:
self._driver.sendFrameworkMessage(thrift_serialize(
SchedulerMessage(deletedTasks=DeletedTasks(taskIds=deleted_tasks))))
self.send_update(
self._driver, task.task_id.value, mesos_pb.TASK_FINISHED, 'Garbage collection finished.')
self.log('Garbage collection complete [task_id=%s]' % task_id)
self._task_id = self._start_time = None
def serialize_art(art, task_id=TASK_ID): td = mesos.TaskInfo() td.slave_id.value = 'ignore_me' td.task_id.value = task_id td.data = thrift_serialize(art) return td
def serialize_art(art, task_id=TASK_ID):
td = mesos_pb2.TaskInfo()
td.slave_id.value = 'ignore_me'
td.task_id.value = task_id
td.data = thrift_serialize(art)
return td
def serialize(thrift_object, protocol_cls=TBinaryProtocol) -> bytes:
return thrift_serialize(thrift_object, TProtocolFactory(protocol_cls))
def thrift_json_serialize(thrift_object) -> bytes:
return thrift_serialize(
thrift_object,
protocol_factory=TSimpleJSONProtocolFactory(),
)
