async def watch_pods():
v1 = client.CoreV1Api()
async with watch.Watch().stream(v1.list_pod_for_all_namespaces) as stream:
async for event in stream:
evt, obj = event['type'], event['object']
print("{} pod {} in NS {}".format(evt, obj.metadata.name,
obj.metadata.namespace))
async def watch_pod_ready_events(self,
namespace: str) -> t.AsyncIterator[str]:
while True:
async for event in watch.Watch().stream(
self.v1_client.list_namespaced_pod,
namespace=namespace,
):
if event['type'] == LISTENING_EVENT_TYPE:
if self.is_pod_ready(event['object']):
pod_name = event['object'].metadata.name
logger.debug('pod is ready',
extra=l_ctx(namespace=namespace,
pod=pod_name))
yield pod_name
elif event['type'] in SKIP_EVENT_TYPES:
pass
elif event['type'] == ERROR_EVENT_TYPE and event['object'].get(
'code') == 410:
# clients must handle the case by recognizing the status code 410 Gone,
# clearing their local cache, performing a list operation,
# and starting the watch from the resourceVersion returned by that new list operation
# (see https://kubernetes.io/docs/reference/using-api/api-concepts/#efficient-detection-of-changes)
# TODO: handle resource_version after https://github.com/tomplus/kubernetes_asyncio/issues/77
break
else:
logger.warning('unhandled event', extra=l_ctx(event=event))
async def monitor(crds, namespace):
log.info(
f"Monitoring charts.k8s.openttd.org in namespace '{namespace}' for changes ..."
)
# Prepare what function we want to call with which parameters
func = crds.list_namespaced_custom_object
args = ["k8s.openttd.org", "v1", namespace, "charts"]
# Start by listing all entries, and emit an 'ADDED' for each existing
# entry. This allows us to get in a known-good-state, and monitor all
# changes after.
initial_list = await func(*args)
for item in initial_list['items']:
_emit_event({"type": "ADDED", "object": item})
# The list has the resource_version we should use as starting point of
# our watch().
resource_version = initial_list['metadata']['resourceVersion']
my_watch = watch.Watch()
# XXX - kubernetes-asyncio has not sync'd with upstream yet.
# See https://github.com/kubernetes-client/python-base/commit/2d69e89dab7134186cbcdaf82381ab6295c6c394
# and https://github.com/tomplus/kubernetes_asyncio/issues/77
# If this gets fixed, the next line can be removed.
my_watch.resource_version = resource_version
async with my_watch.stream(func,
*args,
resource_version=resource_version,
_request_timeout=30) as stream:
async for event in stream:
_emit_event(event)
log.error(f"Monitoring in namespace '{namespace}' stopped unexpectedly")
async def _watch(self):
DBSession = self.connector.DBSession
k8s_config.load_incluster_config()
async with k8s_client.ApiClient() as api:
v1 = k8s_client.CoreV1Api(api)
with open(os.path.join(self.config_dir, 'runner.namespace')) as fp:
namespace = fp.read().strip()
# Find existing run pods
pods = await v1.list_namespaced_pod(
namespace=namespace,
label_selector='app=run',
)
PROM_RUNS.set(0)
for pod in pods.items:
run_id = int(pod.metadata.labels['run'], 10)
logger.info("Found run pod for %d", run_id)
PROM_RUNS.inc()
await self._check_pod(api, run_id, pod)
# Watch changes
watch = k8s_watch.Watch()
f, kwargs = v1.list_namespaced_pod, dict(
namespace=namespace,
label_selector='app=run',
)
while True:
try:
async for event in watch.stream(f, **kwargs):
await self._handle_watch_event(api, DBSession, event)
except k8s_client.ApiException as e:
if e.status != 410:
raise
async def wait_for_pod(self, username: str) -> None:
"""Wait for the pod for a user to complete.
Parameters
----------
username : `str`
Username of user whose pod to wait for.
Raises
------
moneypenny.exceptions.PodNotFound
The user's pod is not there at all.
moneypenny.exceptions.OperationFailed
The pod failed.
moneypenny.exceptions.K8sApiException
Some other Kubernetes API failure.
"""
pod_name = _name_object(username, "pod")
args = (self.v1.list_namespaced_pod, self.namespace)
kwargs = {"field_selector": f"metadata.name={pod_name}"}
try:
async with watch.Watch().stream(*args, **kwargs) as stream:
async for event in stream:
status = event["object"].status
msg = f"New status of {pod_name}: {status.phase}"
self.logger.debug(msg)
if self._is_pod_finished(pod_name, status):
return
except ApiException as e:
if e.status == 404:
raise PodNotFound(f"Pod {pod_name} not found")
msg = "Error checking on {pod_name} pod completion"
self.logger.exception(msg)
raise K8sApiException(e)
async def simple_watch_clusters():
global CLUSTERS
this_clusters = {}
v1 = client.CustomObjectsApi()
async with watch.Watch().stream(v1.list_cluster_custom_object,
"management.cattle.io",
"v3",
"clusters",
timeout_seconds=10) as stream:
async for event in stream:
evt, obj = event['type'], event['object']
if obj['metadata']['name'] == 'local':
continue
cluster_id = obj['metadata']['name']
try:
credentials = {
'apiEndpoint': obj['status']['apiEndpoint'],
'caCert': obj['status']['caCert'],
'serviceAccountToken':
obj['status']['serviceAccountToken'],
}
this_clusters[cluster_id] = credentials
CLUSTERS[cluster_id] = credentials
except Exception as e:
print(f"Wait cluster {cluster_id}")
CLUSTERS = this_clusters
async def k8s_events_internal(self):
chart_namespace_prefix = await self.middleware.call(
'chart.release.get_chart_namespace_prefix')
async with api_client() as (api, context):
watch_obj = watch.Watch()
start_time = datetime.now(tz=tzutc())
async with watch_obj.stream(
context['core_api'].list_event_for_all_namespaces
) as stream:
async for event in stream:
event_obj = event['object']
check_time = event_obj.event_time or event_obj.last_timestamp or event_obj.first_timestamp
if not check_time or start_time > check_time or event[
'type'] != 'ADDED' or (
event_obj.involved_object.uid != NODE_NAME
and not event_obj.metadata.namespace.
startswith(chart_namespace_prefix)):
continue
self.middleware.send_event(
'kubernetes.events',
'ADDED',
uid=event_obj.involved_object.uid,
fields=event_obj.to_dict())
async def watch_namespaces():
async with client.ApiClient() as api:
v1 = client.CoreV1Api(api)
async with watch.Watch().stream(v1.list_namespace) as stream:
async for event in stream:
etype, obj = event['type'], event['object']
print("{} namespace {}".format(etype, obj.metadata.name))
async def clusterroles():
v1rbac = client.RbacAuthorizationV1Api()
async with watch.Watch().stream(v1rbac.list_cluster_role) as stream:
async for event in stream:
print("Event: %s %s %s" % (event['type'], event['object'].kind,
event['object'].metadata.name))
await slack(eventtype=event['type'],
eventkind=event['object'].kind,
eventname=event['object'].metadata.name,
eventns=event['object'].metadata.namespace)
async def persistenvolumes():
v1 = client.CoreV1Api()
async with watch.Watch().stream(v1.list_persistent_volume) as stream:
async for event in stream:
print("Event: %s %s %s" % (event['type'], event['object'].kind,
event['object'].metadata.name))
await slack(eventtype=event['type'],
eventkind=event['object'].kind,
eventname=event['object'].metadata.name,
eventns=event['object'].metadata.namespace)
async def networkpolicies():
v1ext = client.ExtensionsV1beta1Api()
async with watch.Watch().stream(
v1ext.list_network_policy_for_all_namespaces) as stream:
async for event in stream:
print("Event: %s %s %s" % (event['type'], event['object'].kind,
event['object'].metadata.name))
await slack(eventtype=event['type'],
eventkind=event['object'].kind,
eventname=event['object'].metadata.name,
eventns=event['object'].metadata.namespace)
async def jobs():
v1batch = client.BatchV1Api()
async with watch.Watch().stream(
v1batch.list_job_for_all_namespaces) as stream:
async for event in stream:
print("Event: %s %s %s" % (event['type'], event['object'].kind,
event['object'].metadata.name))
await slack(eventtype=event['type'],
eventkind=event['object'].kind,
eventname=event['object'].metadata.name,
eventns=event['object'].metadata.namespace)
async def statefulsets():
v1apps = client.AppsV1Api()
async with watch.Watch().stream(
v1apps.list_stateful_set_for_all_namespaces) as stream:
async for event in stream:
print("Event: %s %s %s" % (event['type'], event['object'].kind,
event['object'].metadata.name))
await slack(eventtype=event['type'],
eventkind=event['object'].kind,
eventname=event['object'].metadata.name,
eventns=event['object'].metadata.namespace)
async def configmaps():
v1 = client.CoreV1Api()
async with watch.Watch().stream(
v1.list_namespaced_config_map('default')) as stream:
async for event in stream:
print("Event: %s %s %s" % (event['type'], event['object'].kind,
event['object'].metadata.name))
await slack(eventtype=event['type'],
eventkind=event['object'].kind,
eventname=event['object'].metadata.name,
eventns=event['object'].metadata.namespace)
async def serviceaccounts():
v1 = client.CoreV1Api()
async with watch.Watch().stream(
v1.list_service_account_for_all_namespaces) as stream:
async for event in stream:
print("Event: %s %s %s" % (event['type'], event['object'].kind,
event['object'].metadata.name))
await slack(eventtype=event['type'],
eventkind=event['object'].kind,
eventname=event['object'].metadata.name,
eventns=event['object'].metadata.namespace)
async def pods():
v1 = client.CoreV1Api()
async with watch.Watch().stream(v1.list_pod_for_all_namespaces) as stream:
async for event in stream:
print("Event: %s %s %s %s" % (event['type'], event['object'].kind,
event['object'].metadata.name,
event['object'].spec.node_name))
await slack(eventtype=event['type'],
eventkind=event['object'].kind,
eventname=event['object'].metadata.name,
eventns=event['object'].metadata.namespace,
eventnodename=event['object'].spec.node_name)
async def monitor(crds):
print(
"Watching for changes in instances.k8s.truebrain.nl for namespace 'default'"
)
stream = watch.Watch().stream(crds.list_namespaced_custom_object,
"k8s.truebrain.nl",
"v1",
"default",
"instances",
_request_timeout=30)
async for event in stream:
del event["raw_object"]
print(json.dumps(event, indent=4))
async def podsecuritypolicies():
v1policy = client.PolicyV1beta1Api()
async with watch.Watch().stream(
v1policy.list_pod_security_policy) as stream:
async for event in stream:
print("Event: %s %s %s %s" % (event['type'], event['object'].kind,
event['object'].metadata.name,
event['object'].spec.node_name))
await slack(eventtype=event['type'],
eventkind=event['object'].kind,
eventname=event['object'].metadata.name,
eventns=event['object'].metadata.namespace,
eventnodename=event['object'].spec.node_name)
async def poddisruptionbudgets():
v1policy = client.PolicyV1beta1Api()
async with watch.Watch().stream(
v1policy.list_pod_disruption_budget_for_all_namespaces) as stream:
async for event in stream:
print("Event: %s %s %s %s" % (event['type'], event['object'].kind,
event['object'].metadata.name,
event['object'].spec.node_name))
await slack(eventtype=event['type'],
eventkind=event['object'].kind,
eventname=event['object'].metadata.name,
eventns=event['object'].metadata.namespace,
eventnodename=event['object'].spec.node_name)
async def simple_watch_nodepools():
global NODEPOOLS
this_nodepools = {}
v1 = client.CustomObjectsApi()
async with watch.Watch().stream(v1.list_cluster_custom_object,
"management.cattle.io",
"v3",
"nodepools",
timeout_seconds=10) as stream:
async for event in stream:
evt, obj = event['type'], event['object']
nodepool_id = f"{obj['metadata']['namespace']}:{obj['metadata']['name']}"
hostnamePrefix = re.sub(r'([-_.])$', '',
obj['spec']['hostnamePrefix'])
this_nodepools[nodepool_id] = hostnamePrefix
NODEPOOLS[nodepool_id] = hostnamePrefix
NODEPOOLS = this_nodepools
async def main():
# Configs can be set in Configuration class directly or using helper
# utility. If no argument provided, the config will be loaded from
# default location.
config.load_kube_config()
v1 = client.CoreV1Api()
count = 10
w = watch.Watch()
async for event in w.stream(v1.list_namespace, timeout_seconds=10):
print("Event: %s %s" % (event['type'], event['object'].metadata.name))
count -= 1
if not count:
w.stop()
print("Ended.")
async def main():
# Configs can be set in Configuration class directly or using helper
# utility. If no argument provided, the config will be loaded from
# default location.
await config.load_kube_config()
v1 = client.CoreV1Api()
count = 10
w = watch.Watch()
async for event in w.stream(v1.list_namespace, timeout_seconds=10):
print("Event: {} {}".format(event['type'], event['object'].metadata.name))
count -= 1
if not count:
w.stop()
print("Ended.")
# An explicit close is necessary to stop the stream
# or use async context manager like in example4.py
w.close()
async def _watch_pods(cls):
logger.info('Watching for all Pod changes...')
v1 = client.CoreV1Api()
w = watch.Watch()
async for event in w.stream(v1.list_pod_for_all_namespaces):
et = event['type'].lower()
if et != 'modified':
continue
# Pod might have been modified.
s = event['object'].status.container_statuses
if s and s[0].state.running:
pod_name = event['raw_object']['metadata']['labels'].get('app')
if pod_name is None:
# Not one of the pods controlled by Asyncy apps.
continue
namespace = event['raw_object']['metadata']['namespace']
logger.info(f'Potentially re-subscribing {pod_name} '
f'for {namespace}...')
pods = cls.subscriptions.get(namespace, {})
sub_ids = pods.get(pod_name, [])
sub_ids_copy = sub_ids.copy() # Use this for iteration.
for sub_id in sub_ids_copy:
try:
await Subscriptions.resubscribe(sub_id,
s[0].container_id)
except NotFoundException: # Stale subscription.
async with cls.sub_lock:
sub_ids.remove(sub_id)
except BaseException as e:
logger.error(f'Exception in resubscribe for '
f'pod_name={pod_name}; sub_id={sub_id}!',
exc_info=e)
logger.debug(f'Stopped watching for Pod changes! Will restart...')
async def simple_watch_nodes(preemptible=False):
global NODEPOOLS
v1 = client.CustomObjectsApi()
async with watch.Watch().stream(v1.list_cluster_custom_object,
"management.cattle.io",
"v3",
"nodes",
timeout_seconds=10) as stream:
async for event in stream:
evt, obj = event['type'], event['object']
if obj['spec']['nodePoolName'] and obj['spec'][
'nodePoolName'] in NODEPOOLS and evt in [
"ADDED", "MODIFIED"
]:
try:
await set_label(obj,
NODEPOOLS[obj['spec']['nodePoolName']],
preemptible)
except Exception as e:
# can't wait
print(f"Wait pool {obj['spec']['nodePoolName']}")
def _get_watcher(self, namespace):
lister = self._get_lister(namespace)
return functools.partial(watch.Watch().stream, lister)
async def watch_cronjobs(queue):
v1 = client.BatchV1beta1Api()
async for event in watch.Watch().stream(
v1.list_cron_job_for_all_namespaces):
await queue.put(event)
async def watch_statefulsets(queue):
v1 = client.AppsV1Api()
async for event in watch.Watch().stream(
v1.list_stateful_set_for_all_namespaces):
await queue.put(event)
async def watch_deployments(queue):
v1 = client.AppsV1Api()
async for event in watch.Watch().stream(
v1.list_deployment_for_all_namespaces):
await queue.put(event)
async def watch_pods():
v1 = client.CoreV1Api()
async for event in watch.Watch().stream(v1.list_pod_for_all_namespaces):
evt, obj = event['type'], event['object']
print(f"{evt} pod {obj.metadata.name} in NS {obj.metadata.namespace}")
async def watch_namespaces():
v1 = client.CoreV1Api()
async for event in watch.Watch().stream(v1.list_namespace):
etype, obj = event['type'], event['object']
print("{} namespace {}".format(etype, obj.metadata.name))
