def watch_app_job_pods(self, cluster):
last_seen_version = None
label_selector = "kae-type in (app, job)"
while True:
try:
if last_seen_version is not None:
watcher = KubeApi.instance().watch_pods(cluster_name=cluster, label_selector=label_selector, resource_version=last_seen_version)
else:
watcher = KubeApi.instance().watch_pods(cluster_name=cluster, label_selector=label_selector)
for event in watcher:
obj = event['object']
labels = obj.metadata.labels or {}
last_seen_version = obj.metadata.resource_version
if 'kae-app-name' in labels:
appname = labels['kae-app-name']
channel = make_app_watcher_channel_name(cluster, appname)
data = {
'object': obj.to_dict(),
'action': event['type'],
}
rds.publish(message=json.dumps(data, cls=VersatileEncoder), channel=channel)
elif 'kae-job-name' in labels:
if event['type'] == 'DELETED':
continue
jobname = labels['kae-job-name']
handle_job_pod_event.delay(jobname, event['raw_object'])
except ProtocolError:
logger.warn('skip this error... because kubernetes disconnect client after default 10m...')
except Exception as e:
# logger.error("---------watch error ------------------")
logger.exception("watch pods workers error")
def on_failure(self, exc, task_id, args, kwargs, einfo):
channel_name = TASK_PUBSUB_CHANNEL.format(task_id=task_id)
failure_msg = {'error': str(exc), 'args': args, 'kwargs': kwargs}
rds.publish(channel_name,
json.dumps(failure_msg, cls=VersatileEncoder))
rds.publish(channel_name, TASK_PUBSUB_EOF.format(task_id=task_id))
msg = 'Console task {}:\nargs\n```\n{}\n```\nkwargs:\n```\n{}\n```\nerror message:\n```\n{}\n```'.format(
self.name, args, kwargs, str(exc))
im_sendmsg(IM_WEBHOOK_CHANNEL, msg)
def on_success(self, retval, task_id, args, kwargs):
channel_name = TASK_PUBSUB_CHANNEL.format(task_id=task_id)
rds.publish(channel_name, TASK_PUBSUB_EOF.format(task_id=task_id))
def stream_output(self, data, task_id=None):
channel_name = TASK_PUBSUB_CHANNEL.format(
task_id=task_id or self.request.id)
rds.publish(channel_name, json.dumps(data, cls=VersatileEncoder))