def sync(self):
if self.running:
self.log.debug('self.running: %s', self.running)
if self.queued_tasks:
self.log.debug('self.queued: %s', self.queued_tasks)
self.kube_scheduler.sync()
last_resource_version = None
while not self.result_queue.empty():
results = self.result_queue.get()
key, state, pod_id, resource_version = results
last_resource_version = resource_version
self.log.info('Changing state of %s to %s', results, state)
try:
self._change_state(key, state, pod_id)
except Exception as e:
self.log.exception(
'Exception: %s when attempting ' +
'to change state of %s to %s, re-queueing.', e, results,
state)
self.result_queue.put(results)
KubeResourceVersion.checkpoint_resource_version(last_resource_version)
for i in range(
min((self.kube_config.worker_pods_creation_batch_size,
self.task_queue.qsize()))):
task = self.task_queue.get()
try:
self.kube_scheduler.run_next(task)
except ApiException:
self.log.exception('ApiException when attempting ' +
'to run task, re-queueing.')
self.task_queue.put(task)
def sync(self):
if self.running:
self.log.debug('self.running: %s', self.running)
if self.queued_tasks:
self.log.debug('self.queued: %s', self.queued_tasks)
self.kube_scheduler.sync()
last_resource_version = None
while not self.result_queue.empty():
results = self.result_queue.get()
key, state, pod_id, resource_version = results
last_resource_version = resource_version
self.log.info('Changing state of %s to %s', results, state)
self._change_state(key, state, pod_id)
KubeResourceVersion.checkpoint_resource_version(last_resource_version)
if not self.task_queue.empty():
task = self.task_queue.get()
try:
self.kube_scheduler.run_next(task)
except ApiException:
self.log.exception('ApiException when attempting ' +
'to run task, re-queueing.')
self.task_queue.put(task)
def sync(self):
if self.running:
self.log.debug('self.running: %s', self.running)
if self.queued_tasks:
self.log.debug('self.queued: %s', self.queued_tasks)
self.kube_scheduler.sync()
last_resource_version = None
while not self.result_queue.empty():
results = self.result_queue.get()
key, state, pod_id, resource_version = results
last_resource_version = resource_version
self.log.info('Changing state of %s to %s', results, state)
try:
self._change_state(key, state, pod_id)
except Exception as e:
self.log.exception('Exception: %s when attempting ' +
'to change state of %s to %s, re-queueing.', e, results, state)
self.result_queue.put(results)
KubeResourceVersion.checkpoint_resource_version(last_resource_version)
for i in range(min((self.kube_config.worker_pods_creation_batch_size, self.task_queue.qsize()))):
task = self.task_queue.get()
try:
self.kube_scheduler.run_next(task)
except ApiException:
self.log.exception('ApiException when attempting ' +
'to run task, re-queueing.')
self.task_queue.put(task)
def sync(self) -> None:
"""Synchronize task state."""
if self.running:
self.log.debug('self.running: %s', self.running)
if self.queued_tasks:
self.log.debug('self.queued: %s', self.queued_tasks)
if not self.worker_uuid:
raise AirflowException(NOT_STARTED_MESSAGE)
if not self.kube_scheduler:
raise AirflowException(NOT_STARTED_MESSAGE)
if not self.kube_config:
raise AirflowException(NOT_STARTED_MESSAGE)
if not self.result_queue:
raise AirflowException(NOT_STARTED_MESSAGE)
if not self.task_queue:
raise AirflowException(NOT_STARTED_MESSAGE)
self.kube_scheduler.sync()
last_resource_version = None
while True: # pylint: disable=too-many-nested-blocks
try:
results = self.result_queue.get_nowait()
try:
key, state, pod_id, namespace, resource_version = results
last_resource_version = resource_version
self.log.info('Changing state of %s to %s', results, state)
try:
self._change_state(key, state, pod_id, namespace)
except Exception as e: # pylint: disable=broad-except
self.log.exception(
"Exception: %s when attempting to change state of %s to %s, re-queueing.",
e, results, state
)
self.result_queue.put(results)
finally:
self.result_queue.task_done()
except Empty:
break
KubeResourceVersion.checkpoint_resource_version(last_resource_version)
# pylint: disable=too-many-nested-blocks
for _ in range(self.kube_config.worker_pods_creation_batch_size):
try:
task = self.task_queue.get_nowait()
try:
self.kube_scheduler.run_next(task)
except ApiException as e:
if e.reason == "BadRequest":
self.log.error("Request was invalid. Failing task")
key, _, _ = task
self.change_state(key, State.FAILED, e)
else:
self.log.warning('ApiException when attempting to run task, re-queueing. '
'Message: %s', json.loads(e.body)['message'])
self.task_queue.put(task)
finally:
self.task_queue.task_done()
except Empty:
break
def sync(self):
if self.running:
self.log.debug('self.running: %s', self.running)
if self.queued_tasks:
self.log.debug('self.queued: %s', self.queued_tasks)
self.kube_scheduler.sync()
last_resource_version = None
while not self.result_queue.empty():
results = self.result_queue.get()
key, state, pod_id, resource_version = results
last_resource_version = resource_version
self.log.info('Changing state of %s to %s', results, state)
self._change_state(key, state, pod_id)
KubeResourceVersion.checkpoint_resource_version(last_resource_version)
if not self.task_queue.empty():
task = self.task_queue.get()
try:
self.kube_scheduler.run_next(task)
except ApiException:
self.log.exception('ApiException when attempting ' +
'to run task, re-queueing.')
self.task_queue.put(task)
def sync(self):
"""Synchronize task state."""
if self.running:
self.log.debug('self.running: %s', self.running)
if self.queued_tasks:
self.log.debug('self.queued: %s', self.queued_tasks)
self.kube_scheduler.sync()
last_resource_version = None
while True:
try:
results = self.result_queue.get_nowait()
try:
key, state, pod_id, namespace, resource_version = results
last_resource_version = resource_version
self.log.info('Changing state of %s to %s', results, state)
try:
self._change_state(key, state, pod_id, namespace)
except Exception as e:
self.log.exception(
'Exception: %s when attempting ' +
'to change state of %s to %s, re-queueing.', e,
results, state)
self.result_queue.put(results)
finally:
self.result_queue.task_done()
except Empty:
break
KubeResourceVersion.checkpoint_resource_version(last_resource_version)
for _ in range(self.kube_config.worker_pods_creation_batch_size):
try:
task = self.task_queue.get_nowait()
try:
self.kube_scheduler.run_next(task)
except ApiException as e:
self.log.warning(
'ApiException when attempting to run task, re-queueing. '
'Message: %s',
json.loads(e.body)['message'])
self.task_queue.put(task)
except HTTPError as e:
self.log.warning(
'HTTPError when attempting to run task, re-queueing. '
'Exception: %s', str(e))
self.task_queue.put(task)
finally:
self.task_queue.task_done()
except Empty:
break
def sync(self) -> None:
"""Synchronize task state."""
if self.running:
self.log.debug('self.running: %s', self.running)
if self.queued_tasks:
self.log.debug('self.queued: %s', self.queued_tasks)
assert self.kube_scheduler, NOT_STARTED_MESSAGE
assert self.kube_config, NOT_STARTED_MESSAGE
assert self.result_queue, NOT_STARTED_MESSAGE
assert self.task_queue, NOT_STARTED_MESSAGE
self.kube_scheduler.sync()
last_resource_version = None
while True: # pylint: disable=too-many-nested-blocks
try:
results = self.result_queue.get_nowait()
try:
key, state, pod_id, resource_version = results
last_resource_version = resource_version
self.log.info('Changing state of %s to %s', results, state)
try:
self._change_state(key, state, pod_id)
except Exception as e: # pylint: disable=broad-except
self.log.exception(
'Exception: %s when attempting ' +
'to change state of %s to %s, re-queueing.', e,
results, state)
self.result_queue.put(results)
finally:
self.result_queue.task_done()
except Empty:
break
KubeResourceVersion.checkpoint_resource_version(last_resource_version)
# pylint: disable=too-many-nested-blocks
for _ in range(self.kube_config.worker_pods_creation_batch_size):
try:
task = self.task_queue.get_nowait()
try:
self.kube_scheduler.run_next(task)
except ApiException as e:
self.log.warning(
'ApiException when attempting to run task, re-queueing. '
'Message: %s' % json.loads(e.body)['message'])
self.task_queue.put(task)
finally:
self.task_queue.task_done()
except Empty:
break
def sync(self):
self.log.info("self.running: {}".format(self.running))
self.log.info("self.queued: {}".format(self.queued_tasks))
self.kube_scheduler.sync()
last_resource_version = None
while not self.result_queue.empty():
results = self.result_queue.get()
key, state, pod_id, resource_version = results
last_resource_version = resource_version
self.log.info("Changing state of {} to {}".format(results, state))
self._change_state(key, state, pod_id)
KubeResourceVersion.checkpoint_resource_version(
last_resource_version, session=self._session)
if not self.task_queue.empty():
key, command, kube_executor_config = self.task_queue.get()
self.kube_scheduler.run_next((key, command, kube_executor_config))
def sync(self):
if self.running:
self.log.debug('self.running: %s', self.running)
if self.queued_tasks:
self.log.debug('self.queued: %s', self.queued_tasks)
self.kube_scheduler.sync()
last_resource_version = None
while not self.result_queue.empty():
results = self.result_queue.get()
key, state, pod_id, resource_version = results
last_resource_version = resource_version
self.log.info('Changing state of %s to %s', results, state)
self._change_state(key, state, pod_id)
KubeResourceVersion.checkpoint_resource_version(last_resource_version)
if not self.task_queue.empty():
key, command, kube_executor_config = self.task_queue.get()
self.kube_scheduler.run_next((key, command, kube_executor_config))
def sync(self):
if self.running:
self.log.info('self.running: %s', self.running)
if self.queued_tasks:
self.log.info('self.queued: %s', self.queued_tasks)
self.kube_scheduler.sync()
last_resource_version = None
while not self.result_queue.empty():
results = self.result_queue.get()
key, state, pod_id, resource_version = results
last_resource_version = resource_version
self.log.info('Changing state of %s to %s', results, state)
self._change_state(key, state, pod_id)
KubeResourceVersion.checkpoint_resource_version(
last_resource_version, session=self._session)
if not self.task_queue.empty():
key, command, kube_executor_config = self.task_queue.get()
self.kube_scheduler.run_next((key, command, kube_executor_config))
def test_checkpoint_resource_version(self):
session = settings.Session()
KubeResourceVersion.checkpoint_resource_version('7', session)
self.assertEqual(
KubeResourceVersion.get_current_resource_version(session), '7')
def test_checkpoint_resource_version(self):
session = settings.Session()
KubeResourceVersion.checkpoint_resource_version('7', session)
self.assertEqual(KubeResourceVersion.get_current_resource_version(session), '7')
