def _run_gc(self, task, retain_tasks, retain_start): """ Reconcile the set of tasks to retain (provided by the scheduler) with the current state of executors on this system. Garbage collect tasks/executors as appropriate. Not re-entrant! Previous executions must complete (and clear self._task_id) before this can be invoked. Potentially blocking (e.g. on I/O) in self.garbage_collect() Args: task: TaskInfo provided by the slave retain_tasks: mapping of task_id => ScheduleStatus, describing what the scheduler thinks is running on this system retain_start: the time at which the retain_tasks message is effective -- this means that tasks started after the retain_tasks message is effective are skipped until future GC runs. """ task_id = task.task_id.value if self._task_id is not None: raise RuntimeError('_run_gc() called [task_id=%s], but already running [task_id=%s]' % (task_id, self._task_id)) self._task_id = task_id self.log('Launching garbage collection [task_id=%s]' % task_id) self._start_time = retain_start local_gc, remote_gc, _ = self.reconcile_states(self._driver, retain_tasks) deleted_tasks = set(retain_tasks).intersection(self.garbage_collect(local_gc)) | remote_gc if deleted_tasks: self._driver.sendFrameworkMessage(thrift_serialize( SchedulerMessage(deletedTasks=DeletedTasks(taskIds=deleted_tasks)))) self.send_update( self._driver, task.task_id.value, mesos_pb2.TASK_FINISHED, 'Garbage collection finished.') self.log('Garbage collection complete [task_id=%s]' % task_id) self._task_id = self._start_time = None
def _run_gc(self, task, retain_tasks, retain_start): """ Reconcile the set of tasks to retain (provided by the scheduler) with the current state of executors on this system. Garbage collect tasks/executors as appropriate. Not re-entrant! Previous executions must complete (and clear self._task_id) before this can be invoked. Potentially blocking (e.g. on I/O) in self.garbage_collect() Args: task: TaskInfo provided by the slave retain_tasks: mapping of task_id => ScheduleStatus, describing what the scheduler thinks is running on this system retain_start: the time at which the retain_tasks message is effective -- this means that tasks started after the retain_tasks message is effective are skipped until future GC runs. """ task_id = task.task_id.value if self._task_id is not None: raise RuntimeError('_run_gc() called [task_id=%s], but already running [task_id=%s]' % (task_id, self._task_id)) self._task_id = task_id self.log('Launching garbage collection [task_id=%s]' % task_id) self._start_time = retain_start local_gc, remote_gc, _ = self.reconcile_states(self._driver, retain_tasks) deleted_tasks = set(retain_tasks).intersection(self.garbage_collect(local_gc)) | remote_gc if deleted_tasks: self._driver.sendFrameworkMessage(thrift_serialize( SchedulerMessage(deletedTasks=DeletedTasks(taskIds=deleted_tasks)))) self.send_update( self._driver, task.task_id.value, mesos_pb.TASK_FINISHED, 'Garbage collection finished.') self.log('Garbage collection complete [task_id=%s]' % task_id) self._task_id = self._start_time = None
def serialize_art(art, task_id=TASK_ID): td = mesos.TaskInfo() td.slave_id.value = 'ignore_me' td.task_id.value = task_id td.data = thrift_serialize(art) return td
def serialize_art(art, task_id=TASK_ID): td = mesos_pb2.TaskInfo() td.slave_id.value = 'ignore_me' td.task_id.value = task_id td.data = thrift_serialize(art) return td
def serialize(thrift_object, protocol_cls=TBinaryProtocol) -> bytes: return thrift_serialize(thrift_object, TProtocolFactory(protocol_cls))
def thrift_json_serialize(thrift_object) -> bytes: return thrift_serialize( thrift_object, protocol_factory=TSimpleJSONProtocolFactory(), )