def check_rbac(actions, app=None, cluster=None, user=None):
"""
check if a user has the permission, cluster is optional argument,
:param actions:
:param app: if set to None, then this function will not check app
:param cluster: if set to None, then this function will not check cluster
:param user:
:return:
"""
if user is None:
user = g.user
roles = get_roles_by_user(user)
logger.debug(f"roles: {roles}, user: {user}")
if not roles:
return False
for role in roles:
# kae admin can do anything
if RBACAction.KAE_ADMIN in role.action_list:
return True
# check cluster
if cluster and role.cluster_list and (cluster
not in role.cluster_list):
continue
# check app
if app is not None:
if len(role.app_names) > 0 and app.name not in role.app_names:
continue
# app admin can do anything on specified app
if RBACAction.ADMIN in role.action_list:
return True
if len(set(actions) - set(role.action_list)) == 0:
return True
return False
def _refresh_thread_func(self, *args, **kwargs):
logger.info("starting sso refresher thread")
while True:
logger.debug("refresh sso cache.")
try:
self.refresh()
except Exception:
logger.exception("error when refresh sso, retry...")
time.sleep(REFRESH_INTERVAL)
def wrapper(*args, **kwargs):
nonlocal obj
try:
return obj(*args, **kwargs)
except KeycloakError:
# retry
logger.debug("got keycloak error, retry.")
self.keycloak_admin = KeycloakAdmin(*self._args, **self._kwargs)
obj = getattr(self.keycloak_admin, item)
return obj(*args, **kwargs)
def heartbeat_sender():
nonlocal need_exit
interval = WS_HEARTBEAT_TIMEOUT - 3
if interval <= 0:
interval = WS_HEARTBEAT_TIMEOUT
try:
while need_exit is False:
time.sleep(interval)
try:
# send a null character to client
logger.debug("send PING")
send_ping(socket)
except WebSocketError as e:
need_exit = True
return
finally:
logger.debug("pod entry heartbeat greenlet exit")
def list_app(self, start=0, limit=500):
from console.models.rbac import RBACAction, get_roles_by_user
from console.models.app import App
roles = get_roles_by_user(self)
seen_app_names = set()
apps = []
logger.debug(f"role list(user: {self.username}) {roles}")
for role in roles:
if RBACAction.KAE_ADMIN in role.action_list:
apps = App.get_all()
break
if RBACAction.ADMIN in role.action_list or RBACAction.GET in role.action_list:
# remove duplicate
for app in role.app_list:
if app.name not in seen_app_names:
apps.append(app)
seen_app_names.add(app.name)
# sort
apps = sorted(apps, key=lambda app: app.name)
return apps[start:start + limit]
def get_clusters_by_user(user):
if user is None:
user = g.user
all_clusters = get_cluster_names()
roles = get_roles_by_user(user)
logger.debug(f"roles: {roles}, user: {user}")
if not roles:
return []
role_clusters = set()
for role in roles:
# kae admin can do anything
if RBACAction.KAE_ADMIN in role.action_list:
return all_clusters
# check cluster
role_clusters.update(role.cluster_list)
if len(role_clusters) == len(all_clusters):
return list(role_clusters)
return list(role_clusters)
def get_current_user(require_token=False, scopes_required=None):
user = None
# logger.debug(f"request headers: {request.headers}")
auth = request.authorization
# token authentication
token = None
if 'Authorization' in request.headers and request.headers[
'Authorization'].startswith('Bearer '):
token = request.headers['Authorization'].split(None, 1)[1].strip()
if 'access_token' in request.form:
token = request.form['access_token']
elif 'access_token' in request.args:
token = request.args['access_token']
if token is not None:
validity = oidc.validate_token(token, scopes_required)
logger.debug(f"validity: {validity}, token: {token}")
if (validity is True) or (not require_token):
user = User(g.oidc_token_info)
elif auth is not None:
# Basic authentication
# we use keycloak admin user and password to do authentication,
# and pass a real user through form, argument, or http header
# this is mainly used in feishu bot, because sso's admin account can't get token of individual user
username, password = auth.username, auth.password
if username != KEYCLOAK_ADMIN_USER or password != KEYCLOAK_ADMIN_PASSWD:
logger.debug("invalid user/password")
return None
if 'real_user' in request.form:
real_user = request.form['real_user']
elif 'real_user' in request.args:
real_user = request.args['real_user']
else:
real_user = request.headers.get("X-Real-User")
if real_user is not None:
user = User.get_by_username(real_user)
else:
# try to get current user from session
username = session.get('current_username', None)
logger.debug(f"user from session: {username}")
if username is not None:
user = User.get_by_username(username)
if not user:
session.pop('current_username')
if user:
session['current_username'] = user.username
return user
def celery_task_stream_response(celery_task_ids,
timeout=0,
exit_when_timeout=True):
if isinstance(celery_task_ids, str):
celery_task_ids = celery_task_ids,
task_progress_channels = [
TASK_PUBSUB_CHANNEL.format(task_id=id_) for id_ in celery_task_ids
]
pubsub = rds.pubsub()
pubsub.subscribe(task_progress_channels)
try:
while pubsub.subscribed:
resp = pubsub.get_message(timeout=timeout)
if resp is None:
if exit_when_timeout:
logger.warn("pubsub timeout {}".format(celery_task_ids))
return None
continue
raw_content = resp['data']
# omit the initial message where item['data'] is 1L
if not isinstance(raw_content, (bytes, str)):
continue
content = raw_content
if isinstance(content, bytes):
content = content.decode('utf-8')
logger.debug('Got pubsub message: %s', content)
# task will publish TASK_PUBSUB_EOF at success or failure
if content.startswith('CELERY_TASK_DONE'):
finished_task_id = content[content.find(':') + 1:]
finished_task_channel = TASK_PUBSUB_CHANNEL.format(
task_id=finished_task_id)
logger.debug(
'Task %s finished, break celery_task_stream_response',
finished_task_id)
pubsub.unsubscribe(finished_task_channel)
else:
yield content
finally:
logger.debug("celery stream response exit ************")
pubsub.unsubscribe()
pubsub.close()
def resp_sender():
nonlocal need_exit
try:
while sh.is_open() and need_exit is False:
sh.update(timeout=1)
if sh.peek_stdout():
msg = sh.read_stdout()
logger.debug("STDOUT: %s" % msg)
socket.send(msg)
if sh.peek_stderr():
msg = sh.read_stderr()
logger.debug("STDERR: %s" % msg)
socket.send(msg)
except ProtocolError:
logger.warn('kubernetes disconnect client after default 10m...')
except WebSocketError as e:
logger.warn('client socket is closed')
except Exception as e:
logger.warn("unknown exception: {}".format(str(e)))
finally:
need_exit = True
logger.debug("exec output sender greenlet exit")
def enter_pod(socket, appname):
payload = None
while True:
message = socket.receive()
if message is None:
return
try:
payload = pod_entry_schema.loads(message)
break
except ValidationError as e:
socket.send(json.dumps(e.messages))
except JSONDecodeError as e:
socket.send(json.dumps({'error': str(e)}))
app = App.get_by_name(appname)
if not app:
socket.send(
make_errmsg('app {} not found'.format(appname), jsonize=True))
return
if not g.user.granted_to_app(app):
socket.send(
make_errmsg(
'You\'re not granted to this app, ask administrators for permission',
jsonize=True))
return
args = payload.data
podname = args['podname']
cluster = args['cluster']
namespace = args['namespace']
container = args.get('container', None)
sh = KubeApi.instance().exec_shell(podname,
namespace=namespace,
cluster_name=cluster,
container=container)
need_exit = False
def heartbeat_sender():
nonlocal need_exit
interval = WS_HEARTBEAT_TIMEOUT - 3
if interval <= 0:
interval = WS_HEARTBEAT_TIMEOUT
try:
while need_exit is False:
time.sleep(interval)
try:
# send a null character to client
logger.debug("send PING")
send_ping(socket)
except WebSocketError as e:
need_exit = True
return
finally:
logger.debug("pod entry heartbeat greenlet exit")
def resp_sender():
nonlocal need_exit
try:
while sh.is_open() and need_exit is False:
sh.update(timeout=1)
if sh.peek_stdout():
msg = sh.read_stdout()
logger.debug("STDOUT: %s" % msg)
socket.send(msg)
if sh.peek_stderr():
msg = sh.read_stderr()
logger.debug("STDERR: %s" % msg)
socket.send(msg)
except ProtocolError:
logger.warn('kubernetes disconnect client after default 10m...')
except WebSocketError as e:
logger.warn('client socket is closed')
except Exception as e:
logger.warn("unknown exception: {}".format(str(e)))
finally:
need_exit = True
logger.debug("exec output sender greenlet exit")
gevent.spawn(resp_sender)
gevent.spawn(heartbeat_sender)
# to avoid lost mysql connection exception
db.session.remove()
try:
while need_exit is False:
# get command from client
message = socket.receive()
if message is None:
logger.info("client socket closed")
break
sh.write_stdin(message)
continue
finally:
need_exit = True
logger.debug("pod entry greenlet exit")
def build_app(socket, appname):
"""Build an image for the specified release.
---
definitions:
BuildArgs:
type: object
properties:
tag:
type: object
parameters:
- name: appname
in: path
type: string
required: true
- name: build_args
in: body
required: true
schema:
$ref: '#/definitions/BuildArgs'
responses:
200:
description: multiple stream messages
schema:
$ref: '#/definitions/StreamMessage'
400:
description: Error information
schema:
$ref: '#/definitions/Error'
examples:
error: "xxx"
"""
payload = None
total_msg = []
client_closed = False
phase = ""
def handle_msg(ss):
nonlocal phase
try:
m = json.loads(ss)
except:
return False
if m['success'] is False:
total_msg.append(m['error'])
return False
if phase != m['phase']:
phase = m['phase']
total_msg.append("***** PHASE {}".format(m['phase']))
raw_data = m.get('raw_data', None)
if raw_data is None:
raw_data = {}
if raw_data.get('error', None):
total_msg.append((str(raw_data)))
return False
if phase.lower() == "pushing":
if len(raw_data) == 1 and 'status' in raw_data:
total_msg.append(raw_data['status'])
elif 'id' in raw_data and 'status' in raw_data:
# TODO: make the output like docker push
total_msg.append("{}:{}".format(raw_data['id'],
raw_data['status']))
elif 'digest' in raw_data:
total_msg.append("{}: digest: {} size: {}".format(
raw_data.get('status'), raw_data['digest'],
raw_data.get('size')))
else:
total_msg.append(str(m))
else:
total_msg.append(m['msg'])
return True
while True:
message = socket.receive()
if message is None:
return
try:
payload = build_args_schema.loads(message)
break
except ValidationError as e:
socket.send(json.dumps(e.messages))
except JSONDecodeError as e:
socket.send(json.dumps({'error': str(e)}))
args = payload.data
tag = args["tag"]
block = args['block']
app = App.get_by_name(appname)
if not app:
socket.send(
make_errmsg('app {} not found'.format(appname), jsonize=True))
return
if not g.user.granted_to_app(app):
socket.send(
make_errmsg(
'You\'re not granted to this app, ask administrators for permission',
jsonize=True))
return
release = app.get_release_by_tag(tag)
if not release:
socket.send(
make_errmsg('release {} not found.'.format(tag), jsonize=True))
return
if release.build_status:
socket.send(make_msg("Finished", msg="already built", jsonize=True))
return
def heartbeat_sender():
nonlocal client_closed
interval = WS_HEARTBEAT_TIMEOUT - 3
if interval <= 0:
interval = WS_HEARTBEAT_TIMEOUT
while client_closed is False:
try:
time.sleep(interval)
send_ping(socket)
except WebSocketError as e:
client_closed = True
return
gevent.spawn(heartbeat_sender)
app_redis_key = make_app_redis_key(appname)
# don't allow multiple build tasks for single app
lock_name = "__app_lock_{}_build_aaa".format(appname)
lck = redis_lock.Lock(rds, lock_name, expire=30, auto_renewal=True)
with gevent.Timeout(APP_BUILD_TIMEOUT, False):
if lck.acquire(blocking=block):
async_result = build_image.delay(appname, tag)
rds.hset(app_redis_key, "build-task-id", async_result.task_id)
db.session.remove()
try:
for m in celery_task_stream_response(async_result.task_id,
900):
# after 10 minutes, we still can't get output message, so we exit the build task
if m is None:
async_result.revoke(terminate=True)
socket.send(
make_errmsg(
"doesn't receive any messages in last 15 minutes, build task for app {} seems to be stuck"
.format(appname),
jsonize=True))
break
try:
if client_closed is False:
socket.send(m)
except WebSocketError as e:
client_closed = True
logger.warn(
"Can't send build msg to client: {}".format(
str(e)))
if handle_msg(m) is False:
break
except gevent.Timeout:
async_result.revoke(terminate=True)
logger.debug("********* build gevent timeout")
socket.send(
make_errmsg("timeout when build app {}".format(appname),
jsonize=True))
except Exception as e:
async_result.revoke(terminate=True)
socket.send(
make_errmsg("error when build app {}: {}".format(
appname, str(e)),
jsonize=True))
finally:
lck.release()
rds.hdel(app_redis_key, "build-task-id")
logger.debug("************ terminate task")
# after build exit, we send an email to the user
if phase.lower() != "finished":
subject = "KAE: Failed to build {}:{}".format(appname, tag)
bearychat_msg = "KAE: Failed to build **{}:{}**".format(
appname, tag)
text_title = '<h2 style="color: #ff6161;"> Build Failed </h2>'
build_result_text = '<strong style="color:#ff6161;"> build terminates prematurely.</strong>'
else:
release.update_build_status(True)
subject = 'KAE: build {}:{} successfully'.format(
appname, tag)
bearychat_msg = 'KAE: build **{}:{}** successfully'.format(
appname, tag)
text_title = '<h2 style="color: #00d600;"> Build Success </h2>'
build_result_text = '<strong style="color:#00d600; font-weight: 600">Build %s %s done.</strong>' % (
appname, tag)
email_text_tpl = '''<div>
<div>{}</div>
<div style="background:#000; padding: 15px; color: #c4c4c4;">
<pre>{}</pre>
</div>
</div>'''
email_text = email_text_tpl.format(
text_title,
html.escape("\n".join(total_msg)) + '\n' +
build_result_text)
email_list = [u.email for u in app.users]
send_email(email_list, subject, email_text)
bearychat_sendmsg(BEARYCHAT_CHANNEL, bearychat_msg)
else:
socket.send(
make_msg(
"Unknown",
msg=
"there seems exist another build task, try to fetch output",
jsonize=True))
build_task_id = rds.hget(app_redis_key, "build-task-id")
if not build_task_id:
socket.send(
make_errmsg("can't get build task id", jsonize=True))
return
if isinstance(build_task_id, bytes):
build_task_id = build_task_id.decode('utf8')
for m in celery_task_stream_response(build_task_id, 900):
# after 10 minutes, we still can't get output message, so we exit the build task
try:
if m is None:
socket.send(
make_errmsg(
"doesn't receive any messages in last 15 minutes, build task for app {} seems to be stuck"
.format(appname),
jsonize=True))
break
if handle_msg(m) is False:
break
if client_closed is False:
socket.send(m)
except WebSocketError as e:
client_closed = True
break