def build_app_status_command():
ctx = context()
appname = ctx.obj['appname']
pod_cmd = [
'get',
'pod',
'-owide',
# add this sort so that abnormal pods appear on top
'--sort-by={.status.phase}',
'-lapp.kubernetes.io/name={appname}',
]
ctx.obj['watch_pod_command'] = pod_cmd
if tell_pods_count() > 13:
ctx.obj['too_many_pods'] = True
ctx.obj[
'watch_pod_title'] = f'(digested, only showing weird pods) k {list2cmdline(pod_cmd)}'
else:
ctx.obj['too_many_pods'] = False
ctx.obj['watch_pod_title'] = f'k {list2cmdline(pod_cmd)}'
top_cmd = ['top', 'po', '-l', f'app.kubernetes.io/name={appname}']
ctx.obj['watch_top_command'] = top_cmd
if ctx.obj['too_many_pods']:
ctx.obj['watch_top_title'] = f'(digested) k {list2cmdline(top_cmd)}'
else:
ctx.obj['watch_top_title'] = f'k {list2cmdline(top_cmd)}'
def _build_without_prepare():
obj = context().obj
values = obj['values']
build_clause = values['build']
build_clause['env'] = {'build_env': BUILD_TREASURE_NAME}
del build_clause['prepare']
lain_build(stage=stage, push=False)
def _build():
obj = context().obj
values = obj['values']
build_clause = values['build']
build_clause['script'].append(
f'echo {RANDOM_STRING} >> {BUILD_TREASURE_NAME}')
lain_build(stage=stage)
def build_cluster_status_command():
ctx = context()
pod_cmd = ctx.obj['watch_bad_pod_command'] = [
'get',
'po',
'--all-namespaces',
'-owide',
]
ctx.obj['watch_bad_pod_title'] = f'k {list2cmdline(pod_cmd)}'
def build_cluster_status():
ctx = context()
build_cluster_status_command()
# building pods container
bad_pod_text_control = FormattedTextControl(
text=lambda: CONTENT_VENDERER['pod_text'])
bad_pod_win = Win(content=bad_pod_text_control)
bad_pod_title = ctx.obj['watch_bad_pod_title']
bad_pod_container = HSplit([
Win(
height=1,
content=Title(bad_pod_title),
),
bad_pod_win,
])
# building nodes container
bad_node_text_control = FormattedTextControl(
text=lambda: CONTENT_VENDERER['node_text'])
bad_node_window = Win(content=bad_node_text_control)
bad_node_container = HSplit([
Win(
height=1,
content=Title('bad nodes'),
),
bad_node_window,
])
parts = [bad_pod_container, bad_node_container]
global_urls = ctx.obj.get('global_urls')
if global_urls:
ingress_text_control = FormattedTextControl(
text=lambda: CONTENT_VENDERER['ingress_text'])
ingress_window = Win(content=ingress_text_control,
height=lambda: tell_screen_height(0.4))
ingress_container = HSplit([
Win(height=1, content=Title('bad url requests')),
ingress_window,
])
parts.append(ingress_container)
# building root container
root_container = HSplit(parts)
kb = KeyBindings()
@kb.add('c-c', eager=True)
@kb.add('c-q', eager=True)
def _(event):
event.app.exit()
app = Application(
key_bindings=kb,
layout=Layout(root_container),
full_screen=True,
)
app.create_background_task(refresh_admin_content())
return app
def bad_node_text():
ctx = context()
cmd = ctx.obj['watch_node_command'] = ['get', 'node', '--no-headers']
res = kubectl(*cmd, timeout=2, capture_output=True, check=False)
if rc(res):
return ensure_str(res.stderr)
all_nodes = ensure_str(res.stdout)
bad_nodes = [
line for line in all_nodes.splitlines() if ' Ready ' not in line
]
return '\n'.join(bad_nodes)
def with_extra_values_file():
obj = context().obj
dic = {'labels': {'foo': 'bar'}}
f = NamedTemporaryFile(prefix='values-extra', suffix='.yaml')
yadu(dic, f)
f.seek(0)
obj['extra_values_file'] = f
try:
return tell_helm_options()
finally:
del f
def __init__(self, endpoint=None):
if not endpoint:
cc = tell_cluster_config()
endpoint = cc.get('prometheus')
if not endpoint:
raise click.Abort(
f'prometheus not provided in cluster config: {cc}')
ctx = context(silent=True)
self.query_range = (ctx.obj.get('values', {}).get(
'prometheus_query_range', '7d') if ctx else '7d')
self.query_step = int(int(parse_timespan(self.query_range)) / 1440)
self.endpoint = endpoint
def diff_k8s_secret(self, old, new):
secret_name = old['metadata']['name']
diff = diff_dict(old['data'], new['data'])
if not sum(len(l) for l in diff.values()):
# do not send notification on empty diff
return
ctx = context()
report = self.k8s_secret_diff_template.render(
secret_name=secret_name,
executor=tell_executor(),
cluster=ctx.obj['cluster'],
**diff,
)
return self.send_msg(report)
def pod_text(too_many_pods=None):
ctx = context()
appname = ctx.obj['appname']
if too_many_pods is None:
too_many_pods = ctx.obj['too_many_pods']
res, pods = get_pods(appname=appname,
headers=True,
show_only_bad_pods=too_many_pods)
if rc(res):
return ensure_str(res.stderr)
CONTENT_VENDERER['bad_pods'] = pods
report = '\n'.join(pods)
return report
def top_text(too_many_pods=None):
"""display kubectl top results"""
ctx = context()
cmd = ctx.obj['watch_top_command']
res = kubectl(*cmd, timeout=9, capture_output=True, check=False)
stdout = ensure_str(res.stdout)
if too_many_pods is None:
too_many_pods = ctx.obj['too_many_pods']
if stdout and too_many_pods:
report = kubectl_top_digest(stdout)
else:
report = stdout or ensure_str(res.stderr)
return report
def tell_webhook_client(hook_url=None):
ctx = context()
obj = ctx.obj
config = obj.get('values', {}).get('webhook', {})
hook_url = hook_url or config.get('url')
if not hook_url:
return
clusters_to_notify = config.pop('clusters', None) or set()
cluster = obj['cluster']
if clusters_to_notify and cluster not in clusters_to_notify:
return
pr = urlparse(hook_url)
if pr.netloc == 'open.feishu.cn':
return FeishuWebhook(hook_url, **config)
if pr.netloc == 'hooks.slack.com':
return SlackIncomingWebhook(hook_url, **config)
raise NotImplementedError(f'webhook not implemented for {hook_url}')
def _prepare():
obj = context().obj
values = obj['values']
build_clause = values['build']
build_clause['prepare']['env'] = {
'prepare_env': BUILD_TREASURE_NAME,
'escape_test': 'space test & newline \n test',
}
build_clause['prepare']['keep'].extend([
'foo/thing.txt',
'bar',
])
build_clause['prepare']['script'].extend([
f'echo {RANDOM_STRING} > {BUILD_TREASURE_NAME}',
'mkdir foo bar',
'touch foo/thing.txt bar/thing.txt',
])
lain_build(stage=stage)
def global_ingress_text():
ctx = context()
global_urls = ctx.obj['global_urls']
if not global_urls:
return ''
rl = []
results = []
def tidy_report(re):
if not re.request:
return ''
report = {'url': re.request.url}
if isinstance(re, requests.Response):
code = re.status_code
if code >= 502 or (code == 404 and re.text.strip()
== DEFAULT_BACKEND_RESPONSE):
report.update({
'status': re.status_code,
'text': re.text,
})
elif isinstance(re, requests.exceptions.RequestException):
report.update({
'status': re.__class__.__name__,
'text': str(re),
})
else:
raise ValueError(f'cannot process this request result: {re}')
return report
simple = ctx.obj.get('simple')
with ThreadPoolExecutor(max_workers=len(global_urls)) as executor:
for url in global_urls:
rl.append(executor.submit(test_url, url))
for future in as_completed(rl):
single_report = tidy_report(future.result())
if not single_report or not single_report.get('status'):
continue
if simple or len(results) < tell_screen_height(0.4):
results.append(single_report)
render_ctx = {'results': sorted(results, key=itemgetter('url'))}
res = ingress_text_template.render(**render_ctx)
return res
def ingress_text():
ctx = context()
urls = ctx.obj.get('urls')
if not urls:
return ''
rl = []
results = []
def tidy_report(re):
if not re.request:
return ''
report = {'url': re.request.url}
if isinstance(re, requests.Response):
report.update({
'status': re.status_code,
'text': re.text,
})
elif isinstance(re, requests.exceptions.RequestException):
report.update({
'status': re.__class__.__name__,
'text': str(re),
})
else:
raise ValueError(f'cannot process this request result: {re}')
return report
# why use ThreadPoolExecutor?
# because we can't use loop.run_until_complete in the main thread
# and why is that?
# because prompt_toolkit application itself runs in a asyncio eventloop
# you can't tell the current eventloop to run something for you if the
# invoker itself lives in that eventloop
# ref: https://bugs.python.org/issue22239
with ThreadPoolExecutor(max_workers=len(urls)) as executor:
for url in urls:
rl.append(executor.submit(test_url, url))
for future in as_completed(rl):
results.append(tidy_report(future.result()))
render_ctx = {'results': sorted(results, key=itemgetter('url'))}
res = ingress_text_template.render(**render_ctx)
return res
def send_deploy_message(self,
stderr=None,
rollback_revision=None,
previous_revision=None):
ctx = context()
obj = ctx.obj
git_revision = obj.get('git_revision')
if git_revision:
res = git(
'log',
'-n',
'1',
'--pretty=format:%s',
git_revision,
check=False,
capture_output=True,
)
if rc(res):
commit_msg = ensure_str(res.stderr)
else:
commit_msg = ensure_str(res.stdout)
else:
commit_msg = 'N/A'
if previous_revision:
cherry = tell_cherry(git_revision=previous_revision,
capture_output=True)
else:
cherry = ''
executor = tell_executor()
text = self.deploy_message_template.render(
executor=executor,
commit_msg=commit_msg,
stderr=stderr,
cherry=cherry,
rollback_revision=rollback_revision,
**ctx.obj,
)
return self.send_msg(text)
def _release():
obj = context().obj
values = obj['values']
values['release'] = {
'env': {
'release_env': BUILD_TREASURE_NAME
},
'dest_base':
'python:latest',
'workdir':
DEFAULT_WORKDIR,
'script': [],
'copy': [
{
'src': '/lain/app/treasure.txt',
'dest': '/lain/app/treasure.txt'
},
{
'src': '/lain/app/treasure.txt',
'dest': '/etc'
},
],
}
lain_build(stage=stage, push=False)
def no_build_and_override_registry():
obj = context().obj
values = obj['values']
del values['build']
pairs = [('registry', RANDOM_STRING)]
return tell_helm_options(pairs)
def get_helm_values():
ctx = context()
helm_values = ctx.obj['values']
return helm_values
def build_app_status():
ctx = context()
build_app_status_command()
# building pods container
pod_text_control = FormattedTextControl(
text=lambda: CONTENT_VENDERER['pod_text'])
pod_win = Win(content=pod_text_control)
pod_title = ctx.obj['watch_pod_title']
pod_container = HSplit([
Win(
height=1,
content=Title(pod_title),
),
pod_win,
])
# building top container
top_text_control = FormattedTextControl(
text=lambda: CONTENT_VENDERER['top_text'])
top_win = Win(content=top_text_control)
top_title = ctx.obj['watch_top_title']
top_container = HSplit([
Win(
height=1,
content=Title(top_title),
),
top_win,
])
# building events container
events_text_control = FormattedTextControl(
text=lambda: CONTENT_VENDERER['event_text'])
events_window = Win(content=events_text_control)
events_container = HSplit([
Win(
height=1,
content=Title('events and messages for pods in weird states'),
),
events_window,
])
parts = [pod_container, top_container, events_container]
# building ingress container
urls = ctx.obj.get('urls')
if urls:
ingress_text_control = FormattedTextControl(
text=lambda: CONTENT_VENDERER['ingress_text'])
ingress_window = Win(content=ingress_text_control,
height=len(urls) + 3)
ingress_container = HSplit([
Win(height=1, content=Title('url requests')),
ingress_window,
])
parts.append(ingress_container)
# building root container
root_container = HSplit(parts)
kb = KeyBindings()
@kb.add('c-c', eager=True)
@kb.add('c-q', eager=True)
def _(event):
event.app.exit()
app = Application(
key_bindings=kb,
layout=Layout(root_container),
full_screen=True,
)
app.create_background_task(refresh_content())
return app
