def log_generator(self, container, timeout=300):
pod = self._get_pod(container)
self.taclib_log.info(f"Waiting for job {container.metadata.name} to run...")
self._wait_for_status(container, "Running")
w = Watch()
self.taclib_log.info(f"Start watching {pod.metadata.name} logs")
while True:
try:
for e in w.stream(
self._c.read_namespaced_pod_log,
name=pod.metadata.name,
namespace=self.namespace,
follow=True,
_request_timeout=config["request_log_timeout"].get(int),
):
yield e.encode()
except ReadTimeoutError:
self.taclib_log.info("Failed to read pod log - timeout error")
job = self._get_job(container.metadata.name)
status = self._get_pod(job).status.phase
if status == "Running":
self.taclib_log.info(
"Pod is still running after failing to fetch logs"
)
self.taclib_log.info("Retrying to fetch pod logs")
continue
break
self.taclib_log.info(f"Stop watching {pod.metadata.name} logs")
def watch_operator(self,
deployment_id: Optional[str] = None,
experiment_id: Optional[str] = None):
GROUP = "argoproj.io"
VERSION = "v1alpha1"
PLURAL = "workflows"
load_kube_config()
api = client.CustomObjectsApi()
while True:
run_id = get_latest_run_id(experiment_id or deployment_id)
if not run_id:
yield "operator not running"
time.sleep(5)
else:
resource_version = list_resource_version(
group=GROUP,
version=VERSION,
namespace=KF_PIPELINES_NAMESPACE,
plural=PLURAL,
)
w = Watch()
stream = w.stream(
api.list_namespaced_custom_object,
group=GROUP,
version=VERSION,
namespace=KF_PIPELINES_NAMESPACE,
plural=PLURAL,
resource_version=resource_version,
label_selector=f"pipeline/runid={run_id}",
pretty="true",
)
for streamline in stream:
yield f"Event: {streamline['type']} {streamline['object']['metadata']['name']}"
def watch_logs(kube_client, pod_info):
"""Stream a single pod's logs
pod logs are streamed directly to sys.stderr,
so that pytest capture can deal with it.
Blocking, should be run in a thread.
Called for each new pod from watch_kubernetes
"""
watch = Watch()
while True:
try:
for event in watch.stream(
func=kube_client.read_namespaced_pod_log,
namespace=pod_info.namespace,
name=pod_info.name,
):
print(f"[{pod_info.name}]: {event}")
except ApiException as e:
if e.status == 400:
# 400 can occur if the container is not yet ready
# wait and retry
time.sleep(1)
continue
elif e.status == 404:
# pod is gone, we are done
return
else:
# unexpeced error
print(f"Error watching logs for {pod_info.name}: {e}", file=sys.stderr)
raise
else:
break
def watch_kubernetes(kube_client, kube_ns):
"""Stream kubernetes events to stdout
so that pytest io capturing can include k8s events and logs
All events are streamed to stdout
When a new pod is started, spawn an additional thread to watch its logs
"""
log_threads = {}
watch = Watch()
for event in watch.stream(
func=kube_client.list_namespaced_event,
namespace=kube_ns,
):
resource = event['object']
obj = resource.involved_object
print(f"k8s event ({event['type']} {obj.kind}/{obj.name}): {resource.message}")
# new pod appeared, start streaming its logs
if (
obj.kind == "Pod"
and event["type"] == "ADDED"
and obj.name not in log_threads
):
log_threads[obj.name] = t = Thread(
target=watch_logs, args=(kube_client, obj), daemon=True
)
t.start()
def cluster_cr_stream() -> Iterator:
w = Watch()
return w.stream(custom_objects_api().list_namespaced_custom_object,
namespace=RAY_NAMESPACE,
group="cluster.ray.io",
version="v1",
plural="rayclusters")
def namespaced_cr_stream(namespace) -> Iterator:
w = Watch()
return w.stream(custom_objects_api().list_namespaced_custom_object,
namespace=namespace,
group=RAY_API_GROUP,
version=RAY_API_VERSION,
plural=RAYCLUSTER_PLURAL)
def start(self):
# Update all deployments to status 'Stopped'.
for deployment in models.Deployment.query.all():
deployment.status = 'Stopped'
deployment.on_status_changed()
db.session.commit()
# Create a watcher for kube events. Sync deployment status on each event.
watch = Watch()
for event in watch.stream(Apps.API.list_namespaced_deployment,
'default',
watch=True):
print('[DEBUG]', event['type'], event['object'].metadata.name,
event['object'].status.available_replicas)
deployment = models.Deployment.query.get_by_ref(
event['object'].metadata.name)
if event['type'] == 'DELETED':
deployment.status = 'Stopped'
elif not event['object'].status.available_replicas:
deployment.status = 'Creating'
elif event['type'] == 'MODIFIED':
deployment.status = 'Created'
else:
deployment.status = 'Running'
db.session.commit()
deployment.on_status_changed()
def cluster_scoped_cr_stream() -> Iterator:
w = Watch()
return w.stream(
custom_objects_api().list_cluster_custom_object,
group=RAY_API_GROUP,
version=RAY_API_VERSION,
plural=RAYCLUSTER_PLURAL)
def log_stream(self, pod, container):
"""
Generates log stream of given pod's container.
Whenever the event source is called, there's a new thread for each pod that listen for new logs and
there's a thread that watches for new pods being created. But there's a limitation within the log generation.
When the client disconnects from the event source, the allocated threads aren't deallocated, not releasing the memory and process used.
Parameters
----------
pod: str
container: str
Yields
------
str
"""
load_kube_config()
v1 = client.CoreV1Api()
w = Watch()
pod_name = pod.metadata.name
namespace = pod.metadata.namespace
container_name = container.name
try:
for streamline in w.stream(
v1.read_namespaced_pod_log,
name=pod_name,
namespace=namespace,
container=container_name,
pretty="true",
tail_lines=0,
timestamps=True,
):
self.queue.put_nowait(streamline)
except RuntimeError as e:
logging.exception(e)
return
except asyncio.CancelledError as e:
logging.exception(e)
return
except ApiException as e:
"""
Expected behavior when trying to connect to a container that isn't ready yet.
"""
logging.exception(e)
except CancelledError as e:
"""
Expected behavior when trying to cancel task
"""
logging.exception(e)
return
def wait_pod_event(self, name, cond):
w = Watch()
for event in w.stream(self.core_api.list_namespaced_pod,
self.namespace,
timeout_seconds=120):
object = event['object']
etype = event['type']
if object.metadata.name != name: continue
if cond(etype, object):
w.stop()
def tail_one(
api_v1: client.CoreV1Api, pod_name: str, namespace: str, tail_lines: int
) -> NoReturn:
watch = Watch()
color_name = io.capture(f"[dim white]{pod_name}[/]", end="")
for line in watch.stream(
api_v1.read_namespaced_pod_log,
pod_name,
namespace,
tail_lines=tail_lines,
):
print(f"{color_name} {line}")
def _thread_tail(
queue: Queue,
api_v1: client.CoreV1Api,
pod_name: str,
pad_width: int,
namespace: str,
) -> NoReturn:
watch = Watch()
padded_name = ("{:<" + str(pad_width) + "}").format(pod_name)
left_col = io.capture(f"[dim white]{padded_name}[/]", end="")
for line in watch.stream(
api_v1.read_namespaced_pod_log, pod_name, namespace, tail_lines=0
):
queue.put(left_col + line)
def wait_for_startup(v1, pods):
pending = set(pods.keys())
watch = Watch()
for event in watch.stream(v1.list_namespaced_pod, namespace=NAMESPACE):
pod = event["object"]
name = pod.metadata.name
if name in pending and pod.status.phase == "Running":
pods[name].started_at = time.monotonic()
logger.trace("Pod {} started in {:.3f} [s]", name,
pods[name].started_at - pods[name].created_at)
pending.remove(name)
if not pending:
return
def watch_job(self, job_name, namespace):
"""
监控job是否完成 bool
"""
watcher = Watch()
try:
for event in watcher.stream(self.batch.list_namespaced_job,
namespace=namespace,
label_selector=f'job-name={job_name}'):
succeed = event['object'].status.succeeded
active = event['object'].status.active
if succeed == 1 and active == None:
watcher.stop()
return True
except Exception as e:
print(e)
return False
def wait_for_deployment_rescale(v1, name, target_replicas):
watch = Watch()
for event in watch.stream(v1.list_namespaced_deployment,
namespace=NAMESPACE):
deployment = event["object"]
if deployment.metadata.name != name:
continue
ready_replicas = deployment.status.ready_replicas
if ready_replicas is None:
ready_replicas = 0
logger.trace("Deployment {} has {} replicas", name, ready_replicas)
if ready_replicas == target_replicas:
return
def wait_for_cleanup(v1, pods):
pending = set(pods)
watch = Watch()
for event in watch.stream(v1.list_namespaced_pod, namespace=NAMESPACE):
type = event["type"]
pod = event["object"]
name = pod.metadata.name
if name in pending and type == "DELETED":
pods[name].exited_at = time.monotonic()
logger.trace("Pod {} exited in {:.3f} [s]", name,
pods[name].exited_at - pods[name].deleted_at)
pending.remove(name)
if not pending:
return
def execute(self, context):
# initialize config
try:
config = kube_config.load_incluster_config()
except:
config = kube_config.load_kube_config()
# create an instance of the API class
api_instance = CustomObjectsApi(ApiClient(config))
# params to create custom object
params = [self.group, self.version, self.namespace, self.plural]
crd_created = self.create_custom_definition(api_instance, *params)
if crd_created:
w = Watch()
for event in w.stream(api_instance.list_namespaced_custom_object, *params, timeout_seconds=self.timeout):
job_name = event.get('object', {}).get('metadata', {}).get('name')
job_state = event.get('object', {}).get('status', {}).get('applicationState', {}).get('state')
if job_name == self.job_name and job_state == "COMPLETED":
break
def watch_deployment_pods(self, deployment_id):
load_kube_config()
v1 = client.CoreV1Api()
w = Watch()
try:
for pod in w.stream(
v1.list_namespaced_pod,
namespace=KF_PIPELINES_NAMESPACE,
label_selector=f"seldon-deployment-id={deployment_id}",
):
if pod["type"] == "ADDED":
pod = pod["object"]
for container in pod.spec.containers:
if container.name not in EXCLUDE_CONTAINERS:
self.loop.run_in_executor(self.pool, self.log_stream, pod, container)
except CancelledError:
"""
Expected behavior when trying to cancel task
"""
w.stop()
return
def watch_workflow_pods(self, experiment_id: str):
# Bug conhecido:
# Um pod que foi encontrado pelo worker de pods pode não ser encontrado pelo worker de logs no caso de experimentos
load_kube_config()
v1 = client.CoreV1Api()
w = Watch()
try:
for pod in w.stream(v1.list_namespaced_pod,
namespace=KF_PIPELINES_NAMESPACE,
label_selector=f"experiment-id={experiment_id}"):
if pod["type"] == "ADDED":
pod = pod["object"]
for container in pod.spec.containers:
if container.name not in EXCLUDE_CONTAINERS and "name" in pod.metadata.annotations:
self.loop.run_in_executor(self.pool, self.log_stream, pod, container)
except CancelledError:
"""
Expected behavior when trying to cancel task
"""
w.stop()
return
def streamEvents(self) -> None:
"""
Watches for changes to the mongo objects in Kubernetes and processes any changes immediately.
"""
event_watcher = Watch()
# start watching from the latest version that we have
if self.cluster_versions:
event_watcher.resource_version = max(
self.cluster_versions.values())
for event in event_watcher.stream(
self.kubernetes_service.listMongoObjects,
_request_timeout=self.STREAM_REQUEST_TIMEOUT):
logging.info("Received event %s", event)
if event["type"] in ("ADDED", "MODIFIED"):
cluster_object = self._parseConfiguration(event["object"])
if cluster_object:
self.checkCluster(cluster_object)
else:
logging.warning(
"Could not validate cluster object, stopping event watcher."
)
event_watcher.stop = True
elif event["type"] in ("DELETED", ):
self.collectGarbage()
else:
logging.warning(
"Could not parse event, stopping event watcher.")
event_watcher.stop = True
# Change the resource version manually because of a bug fixed in a later version of the K8s client:
# https://github.com/kubernetes-client/python-base/pull/64
if isinstance(event.get('object'),
dict) and 'resourceVersion' in event['object'].get(
'metadata', {}):
event_watcher.resource_version = event['object']['metadata'][
'resourceVersion']
def tail_module_log(
layer: "Layer",
module_name: str,
since_seconds: Optional[int] = None,
earliest_pod_start_time: Optional[datetime.datetime] = None,
start_color_idx: int = 15, # White Color
) -> None:
current_pods_monitored: Set[str] = set()
load_opta_kube_config()
v1 = CoreV1Api()
watch = Watch()
count = 0
"""Using the UTC Time stamp as the Kubernetes uses the UTC Timestamps."""
for event in watch.stream(
v1.list_namespaced_pod,
namespace=layer.name,
label_selector=
f"app.kubernetes.io/instance={layer.name}-{module_name}",
):
pod: V1Pod = event["object"]
color_idx = count % (256 - start_color_idx) + start_color_idx
if color_idx in REDS:
count += 1
color_idx = count % (256 - start_color_idx) + start_color_idx
if (earliest_pod_start_time is not None
and pod.metadata.creation_timestamp < earliest_pod_start_time):
continue
if pod.metadata.name not in current_pods_monitored:
current_pods_monitored.add(pod.metadata.name)
new_thread = Thread(
target=tail_pod_log,
args=(layer.name, pod, color_idx, since_seconds),
daemon=True,
)
new_thread.start()
count += 1
def tail_pod_log(namespace: str, pod: V1Pod, color_idx: int,
seconds: Optional[int]) -> None:
v1 = CoreV1Api()
watch = Watch()
print(
f"{fg(color_idx)}Showing the logs for server {pod.metadata.name} of your service{attr(0)}"
)
retry_count = 0
while True:
try:
for logline in watch.stream(
v1.read_namespaced_pod_log,
name=pod.metadata.name,
namespace=namespace,
container="k8s-service",
since_seconds=seconds,
):
print(f"{fg(color_idx)}{pod.metadata.name} {logline}{attr(0)}")
except Exception as e:
if type(e) == ApiException:
if e.status == 404: # type: ignore
print(
f"{fg(color_idx)}Server {pod.metadata.name} has been terminated{attr(0)}"
)
return
if retry_count < 15:
print(
f"{fg(color_idx)}Couldn't get logs, waiting a bit and retrying{attr(0)}"
)
time.sleep(retry_count)
retry_count += 1
else:
logger.error(
f"Got the following error while trying to fetch the logs for pod {pod.metadata.name} in namespace {namespace}: {e}"
)
return
def delete_all(self):
# We must pass a new default API client to avoid urllib conn pool warnings
start_time = datetime.now()
print("Deleting items")
for uid in self.resources["pods"]:
print(" - Pod %s:%s" % uid)
try:
res = self.core_api.delete_namespaced_pod(
namespace=uid[0], name=uid[1], body=V1DeleteOptions())
except:
print(" (issue cleaning up, ignored)")
for uid in self.resources["services"]:
print(" - Service %s:%s" % uid)
try:
res = self.core_api.delete_namespaced_service(namespace=uid[0],
name=uid[1])
except:
print(" (issue cleaning up, ignored)")
for uid in self.resources["ingress"]:
print(" - Ingress %s:%s" % uid)
try:
res = self.beta1_api.delete_namespaced_ingress(
namespace=uid[0], name=uid[1], body=V1DeleteOptions())
except:
print(" (issue cleaning up, ignored)")
self.resources["ingress"] = {}
for uid in self.resources["config_maps"]:
print(" - ConfigMap %s:%s" % uid)
try:
res = self.core_api.delete_namespaced_config_map(
namespace=uid[0], name=uid[1], body=V1DeleteOptions())
except:
print(" (issue cleaning up, ignored)")
self.resources["config_maps"] = {}
for uid in self.resources["role_bindings"]:
print(" - RoleBinding %s:%s" % uid)
try:
res = self.rbac_api.delete_namespaced_role_binding(
namespace=uid[0], name=uid[1], body=V1DeleteOptions())
except:
print(" (issue cleaning up, ignored)")
self.resources["role_bindings"] = {}
for uid in self.resources["roles"]:
print(" - Role %s:%s" % uid)
try:
res = self.rbac_api.delete_namespaced_role(
namespace=uid[0], name=uid[1], body=V1DeleteOptions())
except:
print(" (issue cleaning up, ignored)")
self.resources["roles"] = {}
for uid in self.resources["service_accounts"]:
print(" - ServiceAccount %s:%s" % uid)
try:
res = self.rbac_api.delete_namespaced_service_account(
namespace=uid[0], name=uid[1], body=V1DeleteOptions())
except:
print(" (issue cleaning up, ignored)")
self.resources["service_accounts"] = {}
# Not checking for possibly deleted pods, pods take a while to
# delete and they will not be listed anymore
print("Waiting for pod and service deletion")
#print("Waiting for pods to be deleted: %s" % ', '.join(["%s:%s" % uid for uid in self.resources["pods"]]))
while self.resources["pods"]:
current_pods = [(i.metadata.namespace, i.metadata.name)
for i in self.core_api.list_namespaced_pod(
self.namespace).items]
#print("Current pods: %s" % ', '.join(["%s:%s" % uid for uid in current_pods]))
deleted_pods = [
uid for uid in self.resources["pods"]
if uid not in current_pods
]
#print("Deleted pods: %s" % ', '.join(["%s:%s" % uid for uid in deleted_pods]))
for uid in deleted_pods:
print(" - Pod %s:%s*" % uid)
del self.resources["pods"][uid]
if not self.resources["pods"]: break
#print("Remaining: %s" % ', '.join(["%s:%s" % uid for uid in self.resources["pods"]]))
w = Watch()
for event in w.stream(self.core_api.list_namespaced_pod,
self.namespace,
timeout_seconds=30):
object = event['object']
etype = event['type']
uid = (object.metadata.namespace, object.metadata.name)
if etype == "DELETED" and uid in self.resources["pods"]:
print(" - Pod %s:%s" % uid)
del self.resources["pods"][uid]
if not self.resources["pods"]: w.stop()
#print("Done deleting pods")
#print("Waiting for services to be deleted: %s" % ', '.join(["%s:%s" % uid for uid in self.resources["services"]]))
while self.resources["services"]:
current_services = [(i.metadata.namespace, i.metadata.name)
for i in self.core_api.list_namespaced_service(
self.namespace).items]
#print("Current services: %s" % ', '.join(["%s:%s" % uid for uid in current_services]))
deleted_services = [
uid for uid in self.resources["services"]
if uid not in current_services
]
#print("Deleted services: %s" % ', '.join(["%s:%s" % uid for uid in deleted_services]))
for uid in deleted_services:
print(" - Service %s:%s*" % uid)
del self.resources["services"][uid]
if not self.resources["services"]: break
# There is a short gap here that could trigger a race condition
# but there seems to be no "query and keep watching" API that could
# prevent that.
#print("Remaining: %s" % ', '.join(["%s:%s" % uid for uid in self.resources["services"]]))
w = Watch()
for event in w.stream(self.core_api.list_namespaced_service,
self.namespace,
timeout_seconds=30):
object = event['object']
etype = event['type']
uid = (object.metadata.namespace, object.metadata.name)
if etype == "DELETED" and uid in self.resources["services"]:
print(" - Service %s:%s" % uid)
del self.resources["services"][uid]
if not self.resources["services"]: w.stop()
#print("Done deleting services")
all_deleted_time = datetime.now()
print("All items deleted (deletion took %s)" %
str(all_deleted_time - start_time))
def monitor_pods(self):
# Wrap watch in outer loop, it might get interrupted before we
# are finished looking
printed_all_up = False
start_time = datetime.now()
while self.resources["pods"]:
try:
w = Watch()
for event in w.stream(self.core_api.list_namespaced_pod,
self.namespace):
object = event['object']
etype = event['type']
uid = (object.metadata.namespace, object.metadata.name)
if uid in self.resources["pods"]:
if etype == "MODIFIED":
#print("************************************\n%s %s\n%s" \
# % (etype, object.metadata.name, object))
ready = 0
total = len(object.spec.containers)
pod_name_ip = "n/a"
status = object.status.phase
if object.status.reason is not None:
status = object.status.reason
if object.spec.node_name and object.spec.node_name != "":
pod_name_ip = object.spec.node_name
if object.status.pod_ip and object.status.pod_ip != "":
pod_name_ip += "/" + object.status.pod_ip
initializing = False
# On Kubernetes 1.5, get init container status out of the annotation manually
if not object.status.init_container_statuses \
and object.metadata.annotations \
and "pod.alpha.kubernetes.io/init-container-statuses" in object.metadata.annotations:
jp = json.loads(object.metadata.annotations[
"pod.alpha.kubernetes.io/init-containers"])
js = json.loads(object.metadata.annotations[
"pod.alpha.kubernetes.io/init-container-statuses"]
)
a = ApiClient()
object.spec.init_containers = \
a._ApiClient__deserialize(jp, "list[V1Container]")
object.status.init_container_statuses = \
a._ApiClient__deserialize(js, "list[V1ContainerStatus]")
if object.status.init_container_statuses is not None:
for i, cs in enumerate(
object.status.init_container_statuses):
if cs.state.terminated and cs.state.terminated.exit_code == 0:
continue
elif cs.state.terminated:
if len(cs.state.terminated.reason
) == 0:
if cs.state.terminated.signal != 0:
status = "Init:Signal:%d" % cs.state.terminated.signal
else:
status = "Init:ExitCode:%d" % cs.state.terminated.exit_code
else:
status = "Init:" + cs.state.terminated.reason
initializing = True
elif cs.state.waiting and len(cs.state.waiting.reason) > 0 \
and cs.state.waiting.reason != "PodInitializing":
status = "Init:" + cs.state.waiting.reason
initializing = True
else:
status = "Init:%d/%d" % (
i, len(
object.spec.init_containers))
initializing = True
break
if not initializing and object.status.container_statuses is not None:
for cs in object.status.container_statuses:
if cs.ready: ready += 1
if cs.state.waiting and cs.state.waiting.reason != "":
status = cs.state.waiting.reason
elif cs.state.terminated and cs.state.terminated.reason != "":
status = cs.state.terminated.reason
elif cs.state.terminated and cs.state.terminated.reason == "":
if cs.state.terminated.signal != 0:
status = "Signal:%d" % cs.state.terminated.signal
else:
statis = "ExitCode:%d" % cs.state.terminated.exit_code
print(" - %-24s %-18s %d/%d %s" \
% (object.metadata.name, status, ready, total, pod_name_ip))
self.resources["pods"][uid][
"phase"] = object.status.phase
self.resources["pods"][uid]["status"] = status
self.resources["pods"][uid]["ready"] = ready
self.resources["pods"][uid]["total"] = total
if ((object.status.phase == "Succeeded"
or object.status.phase == "Failed")
and object.metadata.deletion_timestamp
== None):
if object.status.phase == "Failed":
return False
#print("Pod %s/%s is finished" % (object.metadata.namespace, object.metadata.name))
#self.delete_all()
if object.status.container_statuses is not None:
for c in filter(
lambda c: c.state.terminated,
object.status.container_statuses):
# If any container failed, assume overall failure
if c.state.terminated.exit_code != 0:
print(
"Container '%s' of pod '%s:%s' failed"
% (c.name, uid[0], uid[1]))
return False
# If a sufficient container completed, assume overall completion
elif c.name in self.resources["pods"][uid][
"sufficient_containers"]:
print(
"Container '%s' of pod '%s:%s' succeeded, finishing"
% (c.name, uid[0], uid[1]))
return True
if etype == "DELETED":
print("Pod %s/%s has been deleted" %
(object.metadata.namespace,
object.metadata.name))
del self.resources["pods"][uid]
if not self.resources["pods"]:
w.stop()
print("Done watching events")
if not printed_all_up:
all_up = True
for k, p in self.resources["pods"].items():
if p["status"] != "Running":
all_up = False
if p["ready"] != p["total"]:
all_up = False
if all_up:
printed_all_up = True
all_up_time = datetime.now()
print("All pods up and running (setup took %s)" %
str(all_up_time - start_time))
except Exception as e:
if str(e) != "TERM":
print("Exception while monitoring pods")
print(traceback.format_exc())
return False
return True
def tail_namespace_events(
layer: "Layer",
earliest_event_start_time: Optional[datetime.datetime] = None,
color_idx: int = 15, # White Color
) -> None:
load_opta_kube_config()
v1 = EventsV1Api()
watch = Watch()
print(f"{fg(color_idx)}Showing events for namespace {layer.name}{attr(0)}")
retry_count = 0
old_events: List[EventsV1Event] = v1.list_namespaced_event(
namespace=layer.name).items
# Filter by time
if earliest_event_start_time is not None:
# Redefine so mypy doesn't complain about earliest_event_start_time being Optional during lambda call
filter_start_time = earliest_event_start_time
old_events = list(
filter(
lambda x: _event_last_observed(x) > filter_start_time,
old_events,
))
# Sort by timestamp
old_events = sorted(old_events, key=lambda x: _event_last_observed(x))
event: EventsV1Event
for event in old_events:
if do_not_show_event(event):
continue
earliest_event_start_time = _event_last_observed(event)
print(
f"{fg(color_idx)}{earliest_event_start_time} Namespace {layer.name} event: {event.note}{attr(0)}"
)
deleted_pods = set()
while True:
try:
for stream_obj in watch.stream(
v1.list_namespaced_event,
namespace=layer.name,
):
event = stream_obj["object"]
event_time = _event_last_observed(event)
if (earliest_event_start_time is None
or event_time > earliest_event_start_time):
if "Deleted pod:" in event.note:
deleted_pods.add(event.note.split(" ")[-1])
involved_object: Optional[
V1ObjectReference] = event.regarding
if (involved_object is not None
and involved_object.kind == "Pod"
and involved_object.name in deleted_pods):
continue
if do_not_show_event(event):
continue
print(
f"{fg(color_idx)}{event_time} Namespace {layer.name} event: {event.note}{attr(0)}"
)
except ApiException as e:
if retry_count < 5:
print(
f"{fg(color_idx)}Couldn't get logs, waiting a bit and retrying{attr(0)}"
)
time.sleep(1 << retry_count)
retry_count += 1
else:
logger.error(
f"{fg(color_idx)}Got the following error while trying to fetch the events in namespace {layer.name}: {e}"
)
return
except Exception as e:
# print(sys.exc_info()[2])
logger.error(
f"{fg(color_idx)}Got the following error while trying to fetch the events in namespace {layer.name}: {e}{attr(0)}"
)
logger.debug("Event watch exception", exc_info=True)
return
