Python KubeResourceVersion.checkpoint_resource_version Examples

Example #1

File: kubernetes_executor.py Project: xzhang-comprehend/airflow

    def sync(self):
        if self.running:
            self.log.debug('self.running: %s', self.running)
        if self.queued_tasks:
            self.log.debug('self.queued: %s', self.queued_tasks)
        self.kube_scheduler.sync()

        last_resource_version = None
        while not self.result_queue.empty():
            results = self.result_queue.get()
            key, state, pod_id, resource_version = results
            last_resource_version = resource_version
            self.log.info('Changing state of %s to %s', results, state)
            try:
                self._change_state(key, state, pod_id)
            except Exception as e:
                self.log.exception(
                    'Exception: %s when attempting ' +
                    'to change state of %s to %s, re-queueing.', e, results,
                    state)
                self.result_queue.put(results)

        KubeResourceVersion.checkpoint_resource_version(last_resource_version)

        for i in range(
                min((self.kube_config.worker_pods_creation_batch_size,
                     self.task_queue.qsize()))):
            task = self.task_queue.get()

            try:
                self.kube_scheduler.run_next(task)
            except ApiException:
                self.log.exception('ApiException when attempting ' +
                                   'to run task, re-queueing.')
                self.task_queue.put(task)

Example #2

File: kubernetes_executor.py Project: wooga/airflow

    def sync(self):
        if self.running:
            self.log.debug('self.running: %s', self.running)
        if self.queued_tasks:
            self.log.debug('self.queued: %s', self.queued_tasks)
        self.kube_scheduler.sync()

        last_resource_version = None
        while not self.result_queue.empty():
            results = self.result_queue.get()
            key, state, pod_id, resource_version = results
            last_resource_version = resource_version
            self.log.info('Changing state of %s to %s', results, state)
            self._change_state(key, state, pod_id)

        KubeResourceVersion.checkpoint_resource_version(last_resource_version)

        for i in range(min((self.kube_config.worker_pods_creation_batch_size, self.task_queue.qsize()))):
            task = self.task_queue.get()

            try:
                self.kube_scheduler.run_next(task)
            except ApiException:
                self.log.exception('ApiException when attempting ' +
                                   'to run task, re-queueing.')
                self.task_queue.put(task)

Example #3

File: test_kubernetes.py Project: strackr/composer-airflow

 def test_checkpoint_resource_version(self):
     session = settings.Session()
     KubeResourceVersion.checkpoint_resource_version('7', session)
     self.assertEqual(
         KubeResourceVersion.get_current_resource_version(session), '7')

Example #4

File: test_kubernetes.py Project: Fokko/incubator-airflow

 def test_checkpoint_resource_version(self):
     session = settings.Session()
     KubeResourceVersion.checkpoint_resource_version('7', session)
     self.assertEqual(KubeResourceVersion.get_current_resource_version(session), '7')