async def _watch(self): DBSession = self.connector.DBSession k8s_config.load_incluster_config() async with k8s_client.ApiClient() as api: v1 = k8s_client.CoreV1Api(api) with open(os.path.join(self.config_dir, 'runner.namespace')) as fp: namespace = fp.read().strip() # Find existing run pods pods = await v1.list_namespaced_pod( namespace=namespace, label_selector='app=run', ) PROM_RUNS.set(0) for pod in pods.items: run_id = int(pod.metadata.labels['run'], 10) logger.info("Found run pod for %d", run_id) PROM_RUNS.inc() await self._check_pod(api, run_id, pod) # Watch changes watch = k8s_watch.Watch() f, kwargs = v1.list_namespaced_pod, dict( namespace=namespace, label_selector='app=run', ) while True: try: async for event in watch.stream(f, **kwargs): await self._handle_watch_event(api, DBSession, event) except k8s_client.ApiException as e: if e.status != 410: raise
async def log_stream_websocket(ws): config.load_incluster_config() k8s_client = client.CoreV1Api() name = ws.query_params['name'] namespace = ws.query_params['namespace'] await ws.accept() resp = await k8s_client.read_namespaced_pod_log( name, namespace, tail_lines=TAIL_LINES_DEFAULT, follow=True, _preload_content=False ) while True: try: line = await resp.content.readline() except asyncio.TimeoutError as e: logging.error( f""" Async timeout server side, will recover from client side {e} """) break if not line: break await ws.send_text(line.decode('utf-8')) await ws.close()
async def list_pods(self, check_cond): v1 = client.CoreV1Api() ret = await v1.list_pod_for_all_namespaces() pods = [] tab = Table() tab.add('NAMESPACE', 'PODS', 'VERSION', 'STATUS', 'CONDITIONS', 'NODE') for pod in ret.items: if self.filter.check(pod.metadata): if pod.status.conditions: conds = [ cond.type for cond in pod.status.conditions if 'T' in cond.status ] else: conds = [pod.status.phase] if pod.metadata.deletion_timestamp is not None: conds.append('Terminating') if check_cond(conds): conds_txt = '+'.join(sorted(conds)) vers = [ container.image.split(':')[-1] if ':' in container.image else 'latest' for container in pod.spec.containers ] vers_txt = ';'.join(vers) tab.add(pod.metadata.namespace, pod.metadata.name, vers_txt, pod.status.phase, conds_txt, pod.spec.node_name) pods.append(pod) tab.print() return pods
async def api_client(context=None, api_client_kwargs=None): await config.load_kube_config(config_file=KUBECONFIG_FILE) context = context or {} context['core_api'] = True api_client_kwargs = api_client_kwargs or {} api_client_kwargs.setdefault('request_timeout', 50) api_cl = ApiClient(**api_client_kwargs) user_context = { 'core_api': client.CoreV1Api(api_cl), 'apps_api': client.AppsV1Api(api_cl), 'storage_api': client.StorageV1Api(api_cl), 'batch_api': client.BatchV1Api(api_cl), 'cronjob_batch_api': client.BatchV1beta1Api(api_cl), 'custom_object_api': client.CustomObjectsApi(api_cl), 'extensions_api': client.ApiextensionsV1Api(api_cl), } try: for k in filter(lambda k: context[k], context): if k == 'node': user_context[k] = await get_node(user_context['core_api']) yield api_cl, user_context finally: await api_cl.close()
async def main(): parser = argparse.ArgumentParser() parser.add_argument('timeout_seconds', type=int) parser.add_argument('namespace', type=str) subparsers = parser.add_subparsers(dest='kind') pod_parser = subparsers.add_parser('Pod') pod_parser.add_argument('name', type=str) service_parser = subparsers.add_parser('Service') service_parser.add_argument('name', type=str) service_parser.add_argument('--port', '-p', type=int, default=80) args = parser.parse_args() if args.kind == 'Pod': if 'USE_KUBE_CONFIG' in os.environ: await config.load_kube_config() else: config.load_incluster_config() v1 = client.CoreV1Api() t = wait_for_pod_complete(v1, args.namespace, args.name) else: assert args.kind == 'Service' t = wait_for_service_alive(args.namespace, args.name, args.port) await asyncio.gather(timeout(args.timeout_seconds), t)
async def _get_deployment_logs(namespace, name, tail_lines=TAIL_LINES_DEFAULT): """Gather pod names via K8s label selector""" pods = [] config.load_incluster_config() k8s_client = client.CoreV1Api() try: api_response = await k8s_client.list_namespaced_pod( namespace, label_selector='release={}'.format(name)) for api_items in api_response.items: pods.append(api_items.metadata.name) except ApiException as e: logging.error( f"Exception when calling CoreV1Api->list_namespaced_pod: {e}") # Iterate over list of pods and concatenate logs logs = "" try: for pod in pods: logs += pod + "\n" logs += await k8s_client.read_namespaced_pod_log( pod, namespace, tail_lines=tail_lines) except ApiException as e: logging.error( f"Exception when calling CoreV1Api->read_namespaced_pod_log: {e}") return logs
async def main(): args = parse_args() loader = await config.load_kube_config() api = ApiClient() v1_api = client.CoreV1Api(api) ret = await v1_api.list_namespaced_pod(args.namespace) cmd = [] for pod in ret.items: if pod.metadata.name.startswith(args.pod): for container in pod.spec.containers: cmd.append( print_pod_log(v1_api, pod.metadata.name, args.namespace, container.name, args.lines, args.follow)) if cmd == []: print('No matching PODs !') return if args.follow: # autorefresh gcp token cmd.append(config.refresh_token(loader)) await asyncio.wait(cmd) await api.close()
async def watch_pods(): v1 = client.CoreV1Api() async with watch.Watch().stream(v1.list_pod_for_all_namespaces) as stream: async for event in stream: evt, obj = event['type'], event['object'] print("{} pod {} in NS {}".format(evt, obj.metadata.name, obj.metadata.namespace))
async def watch_namespaces(): async with client.ApiClient() as api: v1 = client.CoreV1Api(api) async with watch.Watch().stream(v1.list_namespace) as stream: async for event in stream: etype, obj = event['type'], event['object'] print("{} namespace {}".format(etype, obj.metadata.name))
async def setup(self, app): await super().setup(app) try: # Not a coroutine for some reason config.load_incluster_config() except config.ConfigException: await config.load_kube_config() self.api_client = client.ApiClient() self.core_client = client.CoreV1Api(api_client=self.api_client) self.custom_client = client.CustomObjectsApi( api_client=self.api_client) self.cluster_waiters = defaultdict(Flag) self.clusters = {} self.username_to_clusters = defaultdict(dict) self.queue = WorkQueue() self.informer = Informer( parent=self, name="cluster", client=self.custom_client, method="list_cluster_custom_object", method_kwargs=dict( group="gateway.dask.org", version=self.crd_version, plural="daskclusters", label_selector=self.label_selector, ), on_update=self.on_cluster_event, on_delete=self.on_cluster_event, ) await self.informer.start() self.sync_task = asyncio.ensure_future(self.sync_clusters_loop())
async def k8s() -> K8sAsync: configuration = client.Configuration() await config.load_kube_config(client_configuration=configuration) api_client = client.ApiClient(configuration) return K8sAsync(core_client=client.CoreApi(api_client), v1_client=client.CoreV1Api(api_client))
async def log_job(request, ws, job, pod_id, namespace, container): job_uuid = job.uuid.hex if job_uuid in request.app.job_logs_ws_managers: ws_manager = request.app.job_logs_ws_managers[job_uuid] else: ws_manager = SocketManager() request.app.job_logs_ws_managers[job_uuid] = ws_manager ws_manager.add_socket(ws) # Stream phase changes status = None while status != JobLifeCycle.RUNNING and not JobLifeCycle.is_done(status): job.refresh_from_db() if status != job.last_status: status = job.last_status await notify_ws(ws=ws, message=get_status_message(status)) if should_disconnect(ws=ws, ws_manager=ws_manager): return await asyncio.sleep(SOCKET_SLEEP) if JobLifeCycle.is_done(status): await notify_ws(ws=ws, message=get_status_message(status)) return config.load_incluster_config() k8s_api = client.CoreV1Api() await log_job_pod(k8s_api=k8s_api, ws=ws, ws_manager=ws_manager, pod_id=pod_id, container=container, namespace=namespace)
def list_obj_mapping(self): obj_map = {} v1 = client.CoreV1Api() for method in dir(v1): rer = re.search(r'^list_(.*)_for_all_namespaces$', method) if rer is not None and method != 'list_event_for_all_namespaces': obj_map[rer.group(1)] = getattr(v1, method) return obj_map
async def on_startup(app): if 'BATCH_USE_KUBE_CONFIG' in os.environ: await config.load_kube_config() else: config.load_incluster_config() app['k8s_client'] = client.CoreV1Api() app['dbpool'] = await create_database_pool()
async def init(cls): logger.info('Init!') if Config.IN_CLUSTER: config.load_incluster_config() else: await config.load_kube_config() cls.v1 = client.CoreV1Api() t = threading.Thread(target=cls.init_watch_all_pods) t.setDaemon(True) t.start()
async def persistenvolumes(): v1 = client.CoreV1Api() async with watch.Watch().stream(v1.list_persistent_volume) as stream: async for event in stream: print("Event: %s %s %s" % (event['type'], event['object'].kind, event['object'].metadata.name)) await slack(eventtype=event['type'], eventkind=event['object'].kind, eventname=event['object'].metadata.name, eventns=event['object'].metadata.namespace)
async def _get_all_namespaces(): """Get all namespaces""" config.load_incluster_config() k8s_client = client.CoreV1Api() namespaces = [] ret = await k8s_client.list_namespace(watch=False) for i in ret.items: namespaces.append(i.metadata.name) return namespaces
async def serviceaccounts(): v1 = client.CoreV1Api() async with watch.Watch().stream( v1.list_service_account_for_all_namespaces) as stream: async for event in stream: print("Event: %s %s %s" % (event['type'], event['object'].kind, event['object'].metadata.name)) await slack(eventtype=event['type'], eventkind=event['object'].kind, eventname=event['object'].metadata.name, eventns=event['object'].metadata.namespace)
async def configmaps(): v1 = client.CoreV1Api() async with watch.Watch().stream( v1.list_namespaced_config_map('default')) as stream: async for event in stream: print("Event: %s %s %s" % (event['type'], event['object'].kind, event['object'].metadata.name)) await slack(eventtype=event['type'], eventkind=event['object'].kind, eventname=event['object'].metadata.name, eventns=event['object'].metadata.namespace)
async def main(): # Configs can be set in Configuration class directly or using helper # utility. If no argument provided, the config will be loaded from # default location. await config.load_kube_config() v1 = client.CoreV1Api() print("Listing pods with their IPs:") ret = await v1.list_pod_for_all_namespaces() for i in ret.items: print(i.status.pod_ip, i.metadata.namespace, i.metadata.name)
async def pods(): v1 = client.CoreV1Api() async with watch.Watch().stream(v1.list_pod_for_all_namespaces) as stream: async for event in stream: print("Event: %s %s %s %s" % (event['type'], event['object'].kind, event['object'].metadata.name, event['object'].spec.node_name)) await slack(eventtype=event['type'], eventkind=event['object'].kind, eventname=event['object'].metadata.name, eventns=event['object'].metadata.namespace, eventnodename=event['object'].spec.node_name)
async def main(): # Configs can be set in Configuration class directly or using helper # utility. If no argument provided, the config will be loaded from # default location. config.load_kube_config() v1 = client.CoreV1Api() print("Try to find a pod with busybox (name busybox*) ...") ret = await v1.list_pod_for_all_namespaces() for i in ret.items: if i.metadata.name.startswith('busybox'): pod = i.metadata.name namespace = i.metadata.namespace print('Buxy box', pod, 'namespace', namespace) break else: print('Busybox not found !') return v1_ws = client.CoreV1Api(api_client=WsApiClient()) exec_command = [ '/bin/sh', '-c', 'echo This message goes to stderr >&2; echo This message goes to stdout' ] resp = v1_ws.connect_get_namespaced_pod_exec(pod, namespace, command=exec_command, stderr=True, stdin=False, stdout=True, tty=False) ret = await resp print("Response: ", ret)
async def _describe_pod(pod, namespace): """Describes pod""" config.load_incluster_config() k8s_client = client.CoreV1Api() try: ret = await k8s_client.read_namespaced_pod( pod, namespace, pretty='true') except ApiException as e: logging.error( f"Exception when calling CoreV1Api->read_namespaced_pod: {e}") return ret
async def _get_pod_logs(pod, namespace, tail_lines=TAIL_LINES_DEFAULT): """Read pod logs""" config.load_incluster_config() k8s_client = client.CoreV1Api() try: ret = await k8s_client.read_namespaced_pod_log( pod, namespace, tail_lines=tail_lines) except ApiException as e: logging.error( f"Exception when calling CoreV1Api->read_namespaced_pod: {e}") return ret
async def do_images(self): v1 = client.CoreV1Api() ret = await v1.list_pod_for_all_namespaces() tab = Table() tab.add('IMAGE', 'NAMESACE', 'POD-STATUS') for pod in ret.items: if self.filter.check(pod.metadata): for container in pod.spec.containers: tab.add(container.image, pod.metadata.namespace, pod.status.phase) tab.print(distinct=True)
async def do_delete_broken(self): v1 = client.CoreV1Api() to_delete = await self.list_pods(lambda conds: 'Ready' not in conds) confirm = input('Do you want to delete these pods [YES/NO]?') if confirm == 'YES': for i, pod in enumerate(to_delete): pod_namespace = pod.metadata.namespace pod_name = pod.metadata.name print('{}/{} {}/{} deleting...'.format(i + 1, len(to_delete), pod_namespace, pod_name)) await v1.delete_namespaced_pod(pod_name, pod_namespace, client.V1DeleteOptions())
async def main(): logging.basicConfig(format="%(asctime)s %(message)s", level=logging.INFO) try: config.load_incluster_config() logging.debug('Acquired credentials from service account') except: await config.load_kube_config() logging.debug('Acquired credentials from kubeconfig') v1 = client.CoreV1Api() while True: await label_newest_node(v1, namespace, user_node_selector, attractor_label) await asyncio.sleep(10)
async def main(): # Configs can be set in Configuration class directly or using helper # utility. If no argument provided, the config will be loaded from # default location. await config.load_kube_config() # use the context manager to close http sessions automatically async with ApiClient() as api: v1 = client.CoreV1Api(api) print("Listing pods with their IPs:") ret = await v1.list_pod_for_all_namespaces() for i in ret.items: print(i.status.pod_ip, i.metadata.namespace, i.metadata.name)
async def get(cls) -> CoreV1Api: if cls.core_api: return cls.core_api else: try: try: await config.load_kube_config() except FileNotFoundError: config.load_incluster_config() cls.core_api = client.CoreV1Api(client.ApiClient()) return cls.core_api except Exception: logger.exception(f'Failed to initialize {cls.__name__}') raise
async def setup(self, k8s_config=None): if not k8s_config: if self.in_cluster: config.load_incluster_config() else: await config.load_kube_config() self.api_client = client.api_client.ApiClient() else: self.api_client = client.api_client.ApiClient(configuration=k8s_config) self.k8s_api = client.CoreV1Api(self.api_client) self.k8s_batch_api = client.BatchV1Api(self.api_client) self.k8s_beta_api = client.ExtensionsV1beta1Api(self.api_client) self.k8s_custom_object_api = client.CustomObjectsApi(self.api_client) self.k8s_version_api = client.VersionApi(self.api_client)