def test_app(test_db):
App.get_or_create(default_appname, git=default_git, apptype="web")
app = App.get_by_name(default_appname)
assert app.name == default_appname
app.delete()
app = App.get_by_name(default_appname)
assert app is None
def test_app(test_db):
u = User.get_by_username(FAKE_USER['username'])
assert u is not None
App.get_or_create(default_appname,
git=default_git,
apptype="web",
subscribers=[u])
app = App.get_by_name(default_appname)
assert app.name == default_appname
assert len(app.subscriber_list) == 1
assert app.subscriber_list[0].username == FAKE_USER['username']
app.delete()
app = App.get_by_name(default_appname)
assert app is None
def test_role(test_db):
username = "******"
group_id = "abcd-1234"
App.get_or_create(default_appname, git=default_git, apptype="web")
app = App.get_by_name(default_appname)
assert app.name == default_appname
rolename = f"app-{default_appname}-reader"
role = Role.create(rolename, [app], [RBACAction.GET])
assert Role.get_by_name("134haha") is None
assert Role.get(role.id + 100000) is None
role = Role.get(role.id)
assert role.name == rolename
UserRoleBinding.create(username, role)
GroupRoleBinding.create(group_id, role)
roles = UserRoleBinding.get_roles_by_name(username)
assert len(roles) == 1
assert roles[0].name == rolename
roles = GroupRoleBinding.get_roles_by_id(group_id)
assert len(roles) == 1
assert roles[0].name == rolename
role.delete()
roles = UserRoleBinding.get_roles_by_name(username)
assert len(roles) == 0
roles = GroupRoleBinding.get_roles_by_id(group_id)
assert len(roles) == 0
def prepare(appname, username, group_id):
App.get_or_create(appname, git=default_git, apptype="web")
app = App.get_by_name(appname)
assert app.name == appname
app_reader_name, app_writer_name, app_admin_name = f"app-{app.name}-reader", f"app-{app.name}-writer", f"app-{app.name}-admin"
app_reader = Role.create(app_reader_name, [app], [RBACAction.GET])
app_writer = Role.create(app_writer_name, [app], [RBACAction.GET])
app_admin = Role.create(app_admin_name, [app], [RBACAction.GET])
UserRoleBinding.create(username, app_admin)
GroupRoleBinding.create(group_id, app_reader)
def create_role(args):
"""
create a role
"""
apps = []
for appname in args["apps"]:
app = App.get_by_name(appname)
apps.append(app)
actions = [str2action(act_txt) for act_txt in args.get("actions", [])]
r = Role.create(
name=args['name'],
apps=apps,
actions=actions,
clusters=args.get('clusters', []),
)
for username in args.get("users", []):
UserRoleBinding.create(username, r)
for groupname in args.get("groups", []):
grp = Group.get_by_name(groupname)
GroupRoleBinding.create(grp['id'], r)
return DEFAULT_RETURN_VALUE
def test_delete_app(test_db):
username = "******"
group_id = "abcd-1234"
reader_name = f"app-{default_appname}-reader"
writer_name = f"app-{default_appname}-reader"
admin_name = f"app-{default_appname}-admin"
prepare(default_appname, username, group_id)
app = App.get_by_name(default_appname)
assert app is not None
assert app.roles.count() == 3
assert Role.get_by_name(reader_name) is not None
assert Role.get_by_name(writer_name) is not None
assert Role.get_by_name(admin_name) is not None
delete_roles_relate_to_app(app)
assert Role.get_by_name(reader_name) is None
assert Role.get_by_name(writer_name) is None
assert Role.get_by_name(admin_name) is None
assert app.roles.count() == 0
app.delete()
def enter_pod(socket, appname):
payload = None
while True:
message = socket.receive()
if message is None:
return
try:
payload = pod_entry_schema.loads(message)
break
except ValidationError as e:
socket.send(json.dumps(e.messages))
except JSONDecodeError as e:
socket.send(json.dumps({'error': str(e)}))
app = App.get_by_name(appname)
if not app:
socket.send(
make_errmsg('app {} not found'.format(appname), jsonize=True))
return
if not g.user.granted_to_app(app):
socket.send(
make_errmsg(
'You\'re not granted to this app, ask administrators for permission',
jsonize=True))
return
args = payload.data
podname = args['podname']
cluster = args['cluster']
namespace = args['namespace']
container = args.get('container', None)
sh = KubeApi.instance().exec_shell(podname,
namespace=namespace,
cluster_name=cluster,
container=container)
need_exit = False
def heartbeat_sender():
nonlocal need_exit
interval = WS_HEARTBEAT_TIMEOUT - 3
if interval <= 0:
interval = WS_HEARTBEAT_TIMEOUT
try:
while need_exit is False:
time.sleep(interval)
try:
# send a null character to client
logger.debug("send PING")
send_ping(socket)
except WebSocketError as e:
need_exit = True
return
finally:
logger.debug("pod entry heartbeat greenlet exit")
def resp_sender():
nonlocal need_exit
try:
while sh.is_open() and need_exit is False:
sh.update(timeout=1)
if sh.peek_stdout():
msg = sh.read_stdout()
logger.debug("STDOUT: %s" % msg)
socket.send(msg)
if sh.peek_stderr():
msg = sh.read_stderr()
logger.debug("STDERR: %s" % msg)
socket.send(msg)
except ProtocolError:
logger.warn('kubernetes disconnect client after default 10m...')
except WebSocketError as e:
logger.warn('client socket is closed')
except Exception as e:
logger.warn("unknown exception: {}".format(str(e)))
finally:
need_exit = True
logger.debug("exec output sender greenlet exit")
gevent.spawn(resp_sender)
gevent.spawn(heartbeat_sender)
# to avoid lost mysql connection exception
db.session.remove()
try:
while need_exit is False:
# get command from client
message = socket.receive()
if message is None:
logger.info("client socket closed")
break
sh.write_stdin(message)
continue
finally:
need_exit = True
logger.debug("pod entry greenlet exit")
def build_app(socket, appname):
"""Build an image for the specified release.
---
definitions:
BuildArgs:
type: object
properties:
tag:
type: object
parameters:
- name: appname
in: path
type: string
required: true
- name: build_args
in: body
required: true
schema:
$ref: '#/definitions/BuildArgs'
responses:
200:
description: multiple stream messages
schema:
$ref: '#/definitions/StreamMessage'
400:
description: Error information
schema:
$ref: '#/definitions/Error'
examples:
error: "xxx"
"""
payload = None
total_msg = []
client_closed = False
phase = ""
def handle_msg(ss):
nonlocal phase
try:
m = json.loads(ss)
except:
return False
if m['success'] is False:
total_msg.append(m['error'])
return False
if phase != m['phase']:
phase = m['phase']
total_msg.append("***** PHASE {}".format(m['phase']))
raw_data = m.get('raw_data', None)
if raw_data is None:
raw_data = {}
if raw_data.get('error', None):
total_msg.append((str(raw_data)))
return False
if phase.lower() == "pushing":
if len(raw_data) == 1 and 'status' in raw_data:
total_msg.append(raw_data['status'])
elif 'id' in raw_data and 'status' in raw_data:
# TODO: make the output like docker push
total_msg.append("{}:{}".format(raw_data['id'],
raw_data['status']))
elif 'digest' in raw_data:
total_msg.append("{}: digest: {} size: {}".format(
raw_data.get('status'), raw_data['digest'],
raw_data.get('size')))
else:
total_msg.append(str(m))
else:
total_msg.append(m['msg'])
return True
while True:
message = socket.receive()
if message is None:
return
try:
payload = build_args_schema.loads(message)
break
except ValidationError as e:
socket.send(json.dumps(e.messages))
except JSONDecodeError as e:
socket.send(json.dumps({'error': str(e)}))
args = payload.data
tag = args["tag"]
block = args['block']
app = App.get_by_name(appname)
if not app:
socket.send(
make_errmsg('app {} not found'.format(appname), jsonize=True))
return
if not g.user.granted_to_app(app):
socket.send(
make_errmsg(
'You\'re not granted to this app, ask administrators for permission',
jsonize=True))
return
release = app.get_release_by_tag(tag)
if not release:
socket.send(
make_errmsg('release {} not found.'.format(tag), jsonize=True))
return
if release.build_status:
socket.send(make_msg("Finished", msg="already built", jsonize=True))
return
def heartbeat_sender():
nonlocal client_closed
interval = WS_HEARTBEAT_TIMEOUT - 3
if interval <= 0:
interval = WS_HEARTBEAT_TIMEOUT
while client_closed is False:
try:
time.sleep(interval)
send_ping(socket)
except WebSocketError as e:
client_closed = True
return
gevent.spawn(heartbeat_sender)
app_redis_key = make_app_redis_key(appname)
# don't allow multiple build tasks for single app
lock_name = "__app_lock_{}_build_aaa".format(appname)
lck = redis_lock.Lock(rds, lock_name, expire=30, auto_renewal=True)
with gevent.Timeout(APP_BUILD_TIMEOUT, False):
if lck.acquire(blocking=block):
async_result = build_image.delay(appname, tag)
rds.hset(app_redis_key, "build-task-id", async_result.task_id)
db.session.remove()
try:
for m in celery_task_stream_response(async_result.task_id,
900):
# after 10 minutes, we still can't get output message, so we exit the build task
if m is None:
async_result.revoke(terminate=True)
socket.send(
make_errmsg(
"doesn't receive any messages in last 15 minutes, build task for app {} seems to be stuck"
.format(appname),
jsonize=True))
break
try:
if client_closed is False:
socket.send(m)
except WebSocketError as e:
client_closed = True
logger.warn(
"Can't send build msg to client: {}".format(
str(e)))
if handle_msg(m) is False:
break
except gevent.Timeout:
async_result.revoke(terminate=True)
logger.debug("********* build gevent timeout")
socket.send(
make_errmsg("timeout when build app {}".format(appname),
jsonize=True))
except Exception as e:
async_result.revoke(terminate=True)
socket.send(
make_errmsg("error when build app {}: {}".format(
appname, str(e)),
jsonize=True))
finally:
lck.release()
rds.hdel(app_redis_key, "build-task-id")
logger.debug("************ terminate task")
# after build exit, we send an email to the user
if phase.lower() != "finished":
subject = "KAE: Failed to build {}:{}".format(appname, tag)
bearychat_msg = "KAE: Failed to build **{}:{}**".format(
appname, tag)
text_title = '<h2 style="color: #ff6161;"> Build Failed </h2>'
build_result_text = '<strong style="color:#ff6161;"> build terminates prematurely.</strong>'
else:
release.update_build_status(True)
subject = 'KAE: build {}:{} successfully'.format(
appname, tag)
bearychat_msg = 'KAE: build **{}:{}** successfully'.format(
appname, tag)
text_title = '<h2 style="color: #00d600;"> Build Success </h2>'
build_result_text = '<strong style="color:#00d600; font-weight: 600">Build %s %s done.</strong>' % (
appname, tag)
email_text_tpl = '''<div>
<div>{}</div>
<div style="background:#000; padding: 15px; color: #c4c4c4;">
<pre>{}</pre>
</div>
</div>'''
email_text = email_text_tpl.format(
text_title,
html.escape("\n".join(total_msg)) + '\n' +
build_result_text)
email_list = [u.email for u in app.users]
send_email(email_list, subject, email_text)
bearychat_sendmsg(BEARYCHAT_CHANNEL, bearychat_msg)
else:
socket.send(
make_msg(
"Unknown",
msg=
"there seems exist another build task, try to fetch output",
jsonize=True))
build_task_id = rds.hget(app_redis_key, "build-task-id")
if not build_task_id:
socket.send(
make_errmsg("can't get build task id", jsonize=True))
return
if isinstance(build_task_id, bytes):
build_task_id = build_task_id.decode('utf8')
for m in celery_task_stream_response(build_task_id, 900):
# after 10 minutes, we still can't get output message, so we exit the build task
try:
if m is None:
socket.send(
make_errmsg(
"doesn't receive any messages in last 15 minutes, build task for app {} seems to be stuck"
.format(appname),
jsonize=True))
break
if handle_msg(m) is False:
break
if client_closed is False:
socket.send(m)
except WebSocketError as e:
client_closed = True
break
def get_app_pods_events(socket, appname):
payload = None
socket_active_ts = time.time()
while True:
message = socket.receive()
if message is None:
return
try:
payload = cluster_canary_schema.loads(message)
break
except ValidationError as e:
socket.send(json.dumps(e.messages))
except JSONDecodeError as e:
socket.send(json.dumps({'error': str(e)}))
args = payload.data
cluster = args['cluster']
canary = args['canary']
name = "{}-canary".format(appname) if canary else appname
channel = make_app_watcher_channel_name(cluster, name)
ns = DEFAULT_APP_NS
app = App.get_by_name(appname)
if not app:
socket.send(
make_errmsg('app {} not found'.format(appname), jsonize=True))
return
if not g.user.granted_to_app(app):
socket.send(
make_errmsg(
'You\'re not granted to this app, ask administrators for permission',
jsonize=True))
return
# since this request may pend long time, so we remove the db session
# otherwise we may get error like `sqlalchemy.exc.TimeoutError: QueuePool limit of size 50 overflow 10 reached, connection timed out`
with session_removed():
pod_list = KubeApi.instance().get_app_pods(name,
cluster_name=cluster,
namespace=ns)
pods = pod_list.to_dict()
for item in pods['items']:
data = {
'object': item,
'action': "ADDED",
}
socket.send(json.dumps(data, cls=VersatileEncoder))
pubsub = rds.pubsub()
pubsub.subscribe(channel)
need_exit = False
def check_client_socket():
nonlocal need_exit
while need_exit is False:
if socket.receive() is None:
need_exit = True
break
def heartbeat_sender():
nonlocal need_exit, socket_active_ts
interval = WS_HEARTBEAT_TIMEOUT - 3
if interval <= 0:
interval = WS_HEARTBEAT_TIMEOUT
while need_exit is False:
now = time.time()
if now - socket_active_ts <= (interval - 1):
time.sleep(interval - (now - socket_active_ts))
else:
try:
send_ping(socket)
socket_active_ts = time.time()
except WebSocketError as e:
need_exit = True
return
gevent.spawn(check_client_socket)
gevent.spawn(heartbeat_sender)
try:
while need_exit is False:
resp = pubsub.get_message(timeout=30)
if resp is None:
continue
if resp['type'] == 'message':
raw_content = resp['data']
# omit the initial message where resp['data'] is 1L
if not isinstance(raw_content, (bytes, str)):
continue
content = raw_content
if isinstance(content, bytes):
content = content.decode('utf-8')
socket.send(content)
socket_active_ts = time.time()
finally:
# need close the connection created by PUB/SUB,
# otherwise it will cause too many redis connections
pubsub.unsubscribe()
pubsub.close()
need_exit = True
logger.info("ws connection closed")