def _watch_and_update(self):
self.log.info("Starting pod watcher...")
cur_delay = 0.1
while not self.stopped:
start = time.monotonic()
watch = kubernetes.watch.Watch()
try:
resource_version = self._list_and_update()
if not self.first_load_future.done():
# signal that we've loaded our initial data
self.first_load_future.set_result(None)
kwargs = {
"namespace": self.namespace,
"label_selector": self.label_selector,
"resource_version": resource_version,
"_request_timeout": self.request_timeout,
"timeout_seconds": self.timeout_seconds,
}
for ev in watch.stream(self.kube_client.list_namespaced_pod,
**kwargs):
cur_delay = 0.1
pod = ev["object"]
if ev["type"] == "DELETED":
self.pods.pop(pod.metadata.name, None)
else:
self.pods[pod.metadata.name] = pod
if self.stopped:
# Check in inner loop to provide faster shutdown
break
watch_duration = time.monotonic() - start
if watch_duration >= self.restart_seconds:
self.log.debug(
"Restarting pod watcher after %.1f seconds",
watch_duration)
break
except ReadTimeoutError:
# network read time out, just continue and restart the watch
# this could be due to a network problem or just low activity
self.log.warning("Read timeout watching pods, reconnecting")
continue
except Exception as exc:
if cur_delay < 30:
cur_delay = cur_delay * 2
self.log.error(
"Error when watching pods, retrying in %.1f seconds...",
cur_delay,
exc_info=exc,
)
time.sleep(cur_delay)
continue
else:
# no events on watch, reconnect
self.log.debug("Pod watcher timeout, restarting")
finally:
watch.stop()
self.log.debug("Pod watcher stopped")
delete_lb_co(obj)
elif operation == "ERROR":
# this is usually due to a bad resource version pointer
LB_FGTS_RESOURCES_VERSION = crds.list_cluster_custom_object(DOMAIN, "v1", "lb-fgts")['metadata'][
'resourceVersion']
else:
# should not arrive here
raise ValueError
if metadata:
# Covers ERROR case with empty metadata
print("Handling %s on %s" % (operation, metadata['name']))
LB_FGTS_RESOURCES_VERSION = metadata['resourceVersion']
count -= 1
if not count:
watch.stop()
print("end processing LB FGTs events %s" % LB_FGTS_RESOURCES_VERSION)
# Watch and react to change on fortigates CRD (= changes to FGT config)
stream = watch.Watch().stream(crds.list_cluster_custom_object, "fortinet.com", "v1", "fortigates",
resource_version=FGTS_RESOURCE_VERSION, timeout_seconds=timeout)
count = 15
for event in stream:
operation = event['type']
obj = event["object"]
metadata = obj.get("metadata")
# we base the controller to fgt custom object ownership based on the name
# must ignore custom object with different name
if metadata:
if metadata['name'] == fgt_co['metadata']['name']: