async def _watch(self):
DBSession = self.connector.DBSession
k8s_config.load_incluster_config()
async with k8s_client.ApiClient() as api:
v1 = k8s_client.CoreV1Api(api)
with open(os.path.join(self.config_dir, 'runner.namespace')) as fp:
namespace = fp.read().strip()
# Find existing run pods
pods = await v1.list_namespaced_pod(
namespace=namespace,
label_selector='app=run',
)
PROM_RUNS.set(0)
for pod in pods.items:
run_id = int(pod.metadata.labels['run'], 10)
logger.info("Found run pod for %d", run_id)
PROM_RUNS.inc()
await self._check_pod(api, run_id, pod)
# Watch changes
watch = k8s_watch.Watch()
f, kwargs = v1.list_namespaced_pod, dict(
namespace=namespace,
label_selector='app=run',
)
while True:
try:
async for event in watch.stream(f, **kwargs):
await self._handle_watch_event(api, DBSession, event)
except k8s_client.ApiException as e:
if e.status != 410:
raise
async def log_stream_websocket(ws):
config.load_incluster_config()
k8s_client = client.CoreV1Api()
name = ws.query_params['name']
namespace = ws.query_params['namespace']
await ws.accept()
resp = await k8s_client.read_namespaced_pod_log(
name,
namespace,
tail_lines=TAIL_LINES_DEFAULT,
follow=True, _preload_content=False
)
while True:
try:
line = await resp.content.readline()
except asyncio.TimeoutError as e:
logging.error(
f"""
Async timeout server side, will recover from client side {e}
""")
break
if not line:
break
await ws.send_text(line.decode('utf-8'))
await ws.close()
async def list_pods(self, check_cond):
v1 = client.CoreV1Api()
ret = await v1.list_pod_for_all_namespaces()
pods = []
tab = Table()
tab.add('NAMESPACE', 'PODS', 'VERSION', 'STATUS', 'CONDITIONS', 'NODE')
for pod in ret.items:
if self.filter.check(pod.metadata):
if pod.status.conditions:
conds = [
cond.type for cond in pod.status.conditions
if 'T' in cond.status
]
else:
conds = [pod.status.phase]
if pod.metadata.deletion_timestamp is not None:
conds.append('Terminating')
if check_cond(conds):
conds_txt = '+'.join(sorted(conds))
vers = [
container.image.split(':')[-1]
if ':' in container.image else 'latest'
for container in pod.spec.containers
]
vers_txt = ';'.join(vers)
tab.add(pod.metadata.namespace, pod.metadata.name,
vers_txt, pod.status.phase, conds_txt,
pod.spec.node_name)
pods.append(pod)
tab.print()
return pods
async def api_client(context=None, api_client_kwargs=None):
await config.load_kube_config(config_file=KUBECONFIG_FILE)
context = context or {}
context['core_api'] = True
api_client_kwargs = api_client_kwargs or {}
api_client_kwargs.setdefault('request_timeout', 50)
api_cl = ApiClient(**api_client_kwargs)
user_context = {
'core_api': client.CoreV1Api(api_cl),
'apps_api': client.AppsV1Api(api_cl),
'storage_api': client.StorageV1Api(api_cl),
'batch_api': client.BatchV1Api(api_cl),
'cronjob_batch_api': client.BatchV1beta1Api(api_cl),
'custom_object_api': client.CustomObjectsApi(api_cl),
'extensions_api': client.ApiextensionsV1Api(api_cl),
}
try:
for k in filter(lambda k: context[k], context):
if k == 'node':
user_context[k] = await get_node(user_context['core_api'])
yield api_cl, user_context
finally:
await api_cl.close()
async def main():
parser = argparse.ArgumentParser()
parser.add_argument('timeout_seconds', type=int)
parser.add_argument('namespace', type=str)
subparsers = parser.add_subparsers(dest='kind')
pod_parser = subparsers.add_parser('Pod')
pod_parser.add_argument('name', type=str)
service_parser = subparsers.add_parser('Service')
service_parser.add_argument('name', type=str)
service_parser.add_argument('--port', '-p', type=int, default=80)
args = parser.parse_args()
if args.kind == 'Pod':
if 'USE_KUBE_CONFIG' in os.environ:
await config.load_kube_config()
else:
config.load_incluster_config()
v1 = client.CoreV1Api()
t = wait_for_pod_complete(v1, args.namespace, args.name)
else:
assert args.kind == 'Service'
t = wait_for_service_alive(args.namespace, args.name, args.port)
await asyncio.gather(timeout(args.timeout_seconds), t)
async def _get_deployment_logs(namespace, name, tail_lines=TAIL_LINES_DEFAULT):
"""Gather pod names via K8s label selector"""
pods = []
config.load_incluster_config()
k8s_client = client.CoreV1Api()
try:
api_response = await k8s_client.list_namespaced_pod(
namespace, label_selector='release={}'.format(name))
for api_items in api_response.items:
pods.append(api_items.metadata.name)
except ApiException as e:
logging.error(
f"Exception when calling CoreV1Api->list_namespaced_pod: {e}")
# Iterate over list of pods and concatenate logs
logs = ""
try:
for pod in pods:
logs += pod + "\n"
logs += await k8s_client.read_namespaced_pod_log(
pod, namespace, tail_lines=tail_lines)
except ApiException as e:
logging.error(
f"Exception when calling CoreV1Api->read_namespaced_pod_log: {e}")
return logs
async def main():
args = parse_args()
loader = await config.load_kube_config()
api = ApiClient()
v1_api = client.CoreV1Api(api)
ret = await v1_api.list_namespaced_pod(args.namespace)
cmd = []
for pod in ret.items:
if pod.metadata.name.startswith(args.pod):
for container in pod.spec.containers:
cmd.append(
print_pod_log(v1_api, pod.metadata.name, args.namespace,
container.name, args.lines, args.follow))
if cmd == []:
print('No matching PODs !')
return
if args.follow:
# autorefresh gcp token
cmd.append(config.refresh_token(loader))
await asyncio.wait(cmd)
await api.close()
async def watch_pods():
v1 = client.CoreV1Api()
async with watch.Watch().stream(v1.list_pod_for_all_namespaces) as stream:
async for event in stream:
evt, obj = event['type'], event['object']
print("{} pod {} in NS {}".format(evt, obj.metadata.name,
obj.metadata.namespace))
async def watch_namespaces():
async with client.ApiClient() as api:
v1 = client.CoreV1Api(api)
async with watch.Watch().stream(v1.list_namespace) as stream:
async for event in stream:
etype, obj = event['type'], event['object']
print("{} namespace {}".format(etype, obj.metadata.name))
async def setup(self, app):
await super().setup(app)
try:
# Not a coroutine for some reason
config.load_incluster_config()
except config.ConfigException:
await config.load_kube_config()
self.api_client = client.ApiClient()
self.core_client = client.CoreV1Api(api_client=self.api_client)
self.custom_client = client.CustomObjectsApi(
api_client=self.api_client)
self.cluster_waiters = defaultdict(Flag)
self.clusters = {}
self.username_to_clusters = defaultdict(dict)
self.queue = WorkQueue()
self.informer = Informer(
parent=self,
name="cluster",
client=self.custom_client,
method="list_cluster_custom_object",
method_kwargs=dict(
group="gateway.dask.org",
version=self.crd_version,
plural="daskclusters",
label_selector=self.label_selector,
),
on_update=self.on_cluster_event,
on_delete=self.on_cluster_event,
)
await self.informer.start()
self.sync_task = asyncio.ensure_future(self.sync_clusters_loop())
async def k8s() -> K8sAsync:
configuration = client.Configuration()
await config.load_kube_config(client_configuration=configuration)
api_client = client.ApiClient(configuration)
return K8sAsync(core_client=client.CoreApi(api_client),
v1_client=client.CoreV1Api(api_client))
async def log_job(request, ws, job, pod_id, namespace, container):
job_uuid = job.uuid.hex
if job_uuid in request.app.job_logs_ws_managers:
ws_manager = request.app.job_logs_ws_managers[job_uuid]
else:
ws_manager = SocketManager()
request.app.job_logs_ws_managers[job_uuid] = ws_manager
ws_manager.add_socket(ws)
# Stream phase changes
status = None
while status != JobLifeCycle.RUNNING and not JobLifeCycle.is_done(status):
job.refresh_from_db()
if status != job.last_status:
status = job.last_status
await notify_ws(ws=ws, message=get_status_message(status))
if should_disconnect(ws=ws, ws_manager=ws_manager):
return
await asyncio.sleep(SOCKET_SLEEP)
if JobLifeCycle.is_done(status):
await notify_ws(ws=ws, message=get_status_message(status))
return
config.load_incluster_config()
k8s_api = client.CoreV1Api()
await log_job_pod(k8s_api=k8s_api,
ws=ws,
ws_manager=ws_manager,
pod_id=pod_id,
container=container,
namespace=namespace)
def list_obj_mapping(self):
obj_map = {}
v1 = client.CoreV1Api()
for method in dir(v1):
rer = re.search(r'^list_(.*)_for_all_namespaces$', method)
if rer is not None and method != 'list_event_for_all_namespaces':
obj_map[rer.group(1)] = getattr(v1, method)
return obj_map
async def on_startup(app):
if 'BATCH_USE_KUBE_CONFIG' in os.environ:
await config.load_kube_config()
else:
config.load_incluster_config()
app['k8s_client'] = client.CoreV1Api()
app['dbpool'] = await create_database_pool()
async def init(cls):
logger.info('Init!')
if Config.IN_CLUSTER:
config.load_incluster_config()
else:
await config.load_kube_config()
cls.v1 = client.CoreV1Api()
t = threading.Thread(target=cls.init_watch_all_pods)
t.setDaemon(True)
t.start()
async def persistenvolumes():
v1 = client.CoreV1Api()
async with watch.Watch().stream(v1.list_persistent_volume) as stream:
async for event in stream:
print("Event: %s %s %s" % (event['type'], event['object'].kind,
event['object'].metadata.name))
await slack(eventtype=event['type'],
eventkind=event['object'].kind,
eventname=event['object'].metadata.name,
eventns=event['object'].metadata.namespace)
async def _get_all_namespaces():
"""Get all namespaces"""
config.load_incluster_config()
k8s_client = client.CoreV1Api()
namespaces = []
ret = await k8s_client.list_namespace(watch=False)
for i in ret.items:
namespaces.append(i.metadata.name)
return namespaces
async def serviceaccounts():
v1 = client.CoreV1Api()
async with watch.Watch().stream(
v1.list_service_account_for_all_namespaces) as stream:
async for event in stream:
print("Event: %s %s %s" % (event['type'], event['object'].kind,
event['object'].metadata.name))
await slack(eventtype=event['type'],
eventkind=event['object'].kind,
eventname=event['object'].metadata.name,
eventns=event['object'].metadata.namespace)
async def configmaps():
v1 = client.CoreV1Api()
async with watch.Watch().stream(
v1.list_namespaced_config_map('default')) as stream:
async for event in stream:
print("Event: %s %s %s" % (event['type'], event['object'].kind,
event['object'].metadata.name))
await slack(eventtype=event['type'],
eventkind=event['object'].kind,
eventname=event['object'].metadata.name,
eventns=event['object'].metadata.namespace)
async def main():
# Configs can be set in Configuration class directly or using helper
# utility. If no argument provided, the config will be loaded from
# default location.
await config.load_kube_config()
v1 = client.CoreV1Api()
print("Listing pods with their IPs:")
ret = await v1.list_pod_for_all_namespaces()
for i in ret.items:
print(i.status.pod_ip, i.metadata.namespace, i.metadata.name)
async def pods():
v1 = client.CoreV1Api()
async with watch.Watch().stream(v1.list_pod_for_all_namespaces) as stream:
async for event in stream:
print("Event: %s %s %s %s" % (event['type'], event['object'].kind,
event['object'].metadata.name,
event['object'].spec.node_name))
await slack(eventtype=event['type'],
eventkind=event['object'].kind,
eventname=event['object'].metadata.name,
eventns=event['object'].metadata.namespace,
eventnodename=event['object'].spec.node_name)
async def main():
# Configs can be set in Configuration class directly or using helper
# utility. If no argument provided, the config will be loaded from
# default location.
config.load_kube_config()
v1 = client.CoreV1Api()
print("Try to find a pod with busybox (name busybox*) ...")
ret = await v1.list_pod_for_all_namespaces()
for i in ret.items:
if i.metadata.name.startswith('busybox'):
pod = i.metadata.name
namespace = i.metadata.namespace
print('Buxy box', pod, 'namespace', namespace)
break
else:
print('Busybox not found !')
return
v1_ws = client.CoreV1Api(api_client=WsApiClient())
exec_command = [
'/bin/sh', '-c',
'echo This message goes to stderr >&2; echo This message goes to stdout'
]
resp = v1_ws.connect_get_namespaced_pod_exec(pod,
namespace,
command=exec_command,
stderr=True,
stdin=False,
stdout=True,
tty=False)
ret = await resp
print("Response: ", ret)
async def _describe_pod(pod, namespace):
"""Describes pod"""
config.load_incluster_config()
k8s_client = client.CoreV1Api()
try:
ret = await k8s_client.read_namespaced_pod(
pod, namespace, pretty='true')
except ApiException as e:
logging.error(
f"Exception when calling CoreV1Api->read_namespaced_pod: {e}")
return ret
async def _get_pod_logs(pod, namespace, tail_lines=TAIL_LINES_DEFAULT):
"""Read pod logs"""
config.load_incluster_config()
k8s_client = client.CoreV1Api()
try:
ret = await k8s_client.read_namespaced_pod_log(
pod, namespace, tail_lines=tail_lines)
except ApiException as e:
logging.error(
f"Exception when calling CoreV1Api->read_namespaced_pod: {e}")
return ret
async def do_images(self):
v1 = client.CoreV1Api()
ret = await v1.list_pod_for_all_namespaces()
tab = Table()
tab.add('IMAGE', 'NAMESACE', 'POD-STATUS')
for pod in ret.items:
if self.filter.check(pod.metadata):
for container in pod.spec.containers:
tab.add(container.image, pod.metadata.namespace,
pod.status.phase)
tab.print(distinct=True)
async def do_delete_broken(self):
v1 = client.CoreV1Api()
to_delete = await self.list_pods(lambda conds: 'Ready' not in conds)
confirm = input('Do you want to delete these pods [YES/NO]?')
if confirm == 'YES':
for i, pod in enumerate(to_delete):
pod_namespace = pod.metadata.namespace
pod_name = pod.metadata.name
print('{}/{} {}/{} deleting...'.format(i + 1, len(to_delete),
pod_namespace,
pod_name))
await v1.delete_namespaced_pod(pod_name, pod_namespace,
client.V1DeleteOptions())
async def main():
logging.basicConfig(format="%(asctime)s %(message)s", level=logging.INFO)
try:
config.load_incluster_config()
logging.debug('Acquired credentials from service account')
except:
await config.load_kube_config()
logging.debug('Acquired credentials from kubeconfig')
v1 = client.CoreV1Api()
while True:
await label_newest_node(v1, namespace, user_node_selector,
attractor_label)
await asyncio.sleep(10)
async def main():
# Configs can be set in Configuration class directly or using helper
# utility. If no argument provided, the config will be loaded from
# default location.
await config.load_kube_config()
# use the context manager to close http sessions automatically
async with ApiClient() as api:
v1 = client.CoreV1Api(api)
print("Listing pods with their IPs:")
ret = await v1.list_pod_for_all_namespaces()
for i in ret.items:
print(i.status.pod_ip, i.metadata.namespace, i.metadata.name)
async def get(cls) -> CoreV1Api:
if cls.core_api:
return cls.core_api
else:
try:
try:
await config.load_kube_config()
except FileNotFoundError:
config.load_incluster_config()
cls.core_api = client.CoreV1Api(client.ApiClient())
return cls.core_api
except Exception:
logger.exception(f'Failed to initialize {cls.__name__}')
raise
async def setup(self, k8s_config=None):
if not k8s_config:
if self.in_cluster:
config.load_incluster_config()
else:
await config.load_kube_config()
self.api_client = client.api_client.ApiClient()
else:
self.api_client = client.api_client.ApiClient(configuration=k8s_config)
self.k8s_api = client.CoreV1Api(self.api_client)
self.k8s_batch_api = client.BatchV1Api(self.api_client)
self.k8s_beta_api = client.ExtensionsV1beta1Api(self.api_client)
self.k8s_custom_object_api = client.CustomObjectsApi(self.api_client)
self.k8s_version_api = client.VersionApi(self.api_client)
