def test_db(request, app):
def check_service_host(uri):
"""只能在本地或者容器里跑测试"""
u = urlparse(uri)
return u.hostname in (
'localhost', '127.0.0.1') or 'hub.ricebook.net__ci__' in u.hostname
if not (check_service_host(app.config['SQLALCHEMY_DATABASE_URI'])
and check_service_host(app.config['REDIS_URL'])):
raise Exception('Need to run test on localhost or in container')
db.create_all()
app = App.get_or_create(default_appname, git=default_git)
app.add_env_set(default_env_name, default_env)
Release.create(app, default_sha, make_specs_text())
Combo.create(default_appname,
default_combo_name,
'web',
default_podname,
networks=[default_network_name],
cpu_quota=default_cpu_quota,
memory=default_memory,
count=1,
envname=default_env_name)
def teardown():
db.session.remove()
db.drop_all()
rds.flushdb()
request.addfinalizer(teardown)
def fake_release(port=None, erection_timeout=120):
'''
start new container using current specs might cause port conflict if
using host network mode, must modify port before renew container
'''
sha = fake_sha(40)
if port:
entrypoints = {
'web': {
'cmd': 'python -m http.server --bind 0.0.0.0 {}'.format(port),
'ports': [str(port)],
'healthcheck': {
'http_url': '/{}'.format(artifact_filename),
'http_port': int(port),
'http_code': 200,
},
}
}
else:
entrypoints = default_entrypoints
app = App.get_or_create(default_appname)
r = Release.create(app, sha, make_specs_text(entrypoints=entrypoints,
erection_timeout=erection_timeout))
r.update_image(test_app_image)
return sha
def trigger_tackle_routine(self):
"""
gather all apps that has tackle rule defined, and check each rule to
decide what strategy to apply (async)
should only run within celery worker
"""
apps = App.get_apps_with_tackle_rule()
for app in apps:
tackle_single_app.delay(app.name)
def deal_with_agent_etcd_change(self, key, deploy_info):
container_id = deploy_info['ID']
healthy = deploy_info['Healthy']
appname = deploy_info['Name']
app = App.get_by_name(appname)
container = Container.get_by_container_id(container_id)
if not container:
return
previous_deploy_info = container.deploy_info
container.update_deploy_info(deploy_info)
# TODO: use new ELB lib
# 只要是健康, 无论如何也做一次 ELB 更新, 一方面是反正不贵,
# 另一方面如果之前哪里出错了没更新成功, 下一次更新还有可能修好
if healthy:
logger.info('ELB: ADD [%s, %s, %s, %s, %s]', container.appname, container.podname, container.entrypoint_name, container_id, container.publish)
update_elb_for_containers(container)
else:
logger.info('ELB: REMOVE [%s, %s, %s, %s, %s]', container.appname, container.podname, container.entrypoint_name, container_id, container.publish)
update_elb_for_containers(container, UpdateELBAction.REMOVE)
# 处理完了 ELB, 再根据前后状态决定要发什么报警信息
subscribers = app.subscribers
msg = ''
# TODO: acquire exit-code
# TODO: handle cronjob containers
previous_healthy = previous_deploy_info.get('Healthy')
if healthy:
# 健康万岁
if previous_healthy is None:
# 容器第一次健康, 说明刚刚初始化好, 就不需要报警了, mark 一下就好
container.mark_initialized()
elif previous_healthy is False:
# 容器病好了, 要汇报好消息, 但是如果是第一次病好,
# 那说明只是初始化成功, 这种情况就没必要报警了,
# 每个容器都会经历一次, 只需要 mark 一下就好
if container.initialized:
msg = 'Container resurge: {}'.format(container)
else:
container.mark_initialized()
else:
# 之前也健康, 那就不用管了
pass
else:
# 生病了
if previous_healthy is None:
# 说明刚刚初始化好, 这时候不健康也是正常的, 可以忽略
pass
elif previous_healthy is False:
# 之前就不健康, 那说明已经发过报警了, 就不要骚扰用户了
pass
else:
# 之前是健康的, 现在病了, 当然要报警
msg = 'Container sick: {}'.format(container)
notbot_sendmsg(subscribers, msg)
def _get_app(appname):
app = App.get_by_name(appname)
if not app:
abort(404, 'App not found: {}'.format(appname))
if not g.user.granted_to_app(app):
abort(
403,
'You\'re not granted to this app, ask administrators for permission'
)
return app
def tackle_single_app(appname):
app = App.get_by_name(appname)
rule = app.tackle_rule
app_status_assembler = app.app_status_assembler
# check container status
for rule in rule.get('container_tackle_rule', []):
for c in app_status_assembler.container_status:
dangers = c.eval_expressions(rule['situations'])
if dangers:
method = container_tackle_strategy_lib[rule['strategy']]
logger.warn('%s container %s in DANGER: %s, tackle strategy %s', appname, c, dangers, method)
method(c, dangers, **rule.get('kwargs', {}))
def renew_container(self, old_container_id, sha=None, user_id=None):
"""renew one container to a certain version, if no version provided, just
renew this container"""
old_container = Container.get_by_container_id(old_container_id)
appname = old_container.appname
app = App.get_by_name(appname)
if sha:
release = app.get_release(sha)
else:
sha = old_container.sha
release = old_container.release
task_id = self.request.id
# if erection_timeout == 0, there'll be no smooth renewal, but remove the
# old container first, and then start new container
if not release.specs.erection_timeout:
remove_container_message = remove_container(old_container_id, task_id=task_id)[0]
if not remove_container_message.success:
raise ActionError('Remove old container {} failed'.format(old_container))
create_container_message = create_container(old_container.zone,
user_id,
appname,
sha,
old_container.combo_name,
task_id=task_id)[0]
if not create_container_message.success:
raise ActionError('Create new container failed: {}'.format(create_container_message.error))
else:
create_container_message = create_container(old_container.zone,
user_id,
appname,
sha,
old_container.combo_name,
task_id=task_id)[0]
if not create_container_message.success:
raise ActionError('Create new container failed: {}'.format(create_container_message.error))
container_id = create_container_message.id
container = Container.get_by_container_id(container_id)
if not container.wait_for_erection():
remove_container_message = remove_container(container_id, task_id=task_id)[0]
raise ActionError('New container {} did\'t became healthy, remove result: {}'.format(container, remove_container_message))
remove_container_message = remove_container(old_container_id, task_id=task_id)[0]
if not remove_container_message.success:
raise ActionError('New container {}, but remove old container {} failed'.format(container, old_container_id))
# reason see the end of create_container definition
if not task_id:
return [remove_container_message, create_container_message]
def test_permissions(test_db, client):
FAKE_USER['privileged'] = 0
user = User.create(**FAKE_USER)
res = client.get(url_for('app.list_app'))
assert res.json == []
res = client.get(url_for('app.get_app', appname=default_appname))
assert res.status_code == 403
app = App.get_by_name(default_appname)
app.grant_user(user)
res = client.get(url_for('app.list_app'))
assert len(res.json) == 1
res = client.get(url_for('app.get_app', appname=default_appname))
assert res.status_code == 200
def register_release(args):
"""Register a release of the specified app
:<json string appname: required
:<json string sha: required, must be length 40
:<json string git: required, the repo address using git protocol, e.g. :code:`[email protected]:projecteru2/citadel.git`
:<json string specs_text: required, the yaml specs for this app
:<json string branch: optional git branch
:<json string git_tag: optional git tag
:<json string commit_message: optional commit message
:<json string author: optional author
"""
appname = args['appname']
git = args['git']
sha = args['sha']
specs_text = args['specs_text']
branch = args.get('branch')
git_tag = args.get('git_tag')
commit_message = args.get('commit_message')
author = args.get('author')
app = App.get_or_create(appname, git)
if not app:
abort(400,
'Error during create an app (%s, %s, %s)' % (appname, git, sha))
try:
release = Release.create(app,
sha,
specs_text,
branch=branch,
git_tag=git_tag,
author=author,
commit_message=commit_message)
except (IntegrityError, ValidationError) as e:
abort(400, str(e))
if release.raw:
release.update_image(release.specs.base)
return release
def list_app(self):
from citadel.models.app import AppUserRelation, App
if self.privileged:
return App.get_all()
rs = AppUserRelation.query.filter_by(user_id=self.id)
return [App.get_by_name(r.appname) for r in rs]
def renew(socket):
"""Create a new container to substitute the old one, this API can be used
to upgrade a app to a specified version, or simply re-create a container
using the same combo.
Things can go wrong at any step, the example response
was the output generated by ``renew("1aa8a638a153b393ee423c0a8c158757b13ab74591ade036b6e73ac33a4bdeac", "3641aca")``
which failed because the newly created container didn't become healthy
within the given time limit.
:<json list container_ids: required, a list of container_id
:<json string sha: required, minimum length is 7
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Type: application/json
{
"podname": "eru",
"nodename": "c1-eru-2.ricebook.link",
"id": "2f123f1abcdfc8208b298c89e10bcd8f48f9fdb25c9eb7874ea5cc7199825e6e",
"name": "test-app_web_rvrhPg",
"error": "",
"success": true,
"cpu": {"0": 20},
"quota": 0.2,
"memory": 134217728,
"publish": {"bridge": "172.17.0.5:6789"},
"hook": "hook output",
"__class__": "CreateContainerMessage"
}
{
"id": "2f123f1abcdfc8208b298c89e10bcd8f48f9fdb25c9eb7874ea5cc7199825e6e",
"success": true,
"message": "hook output",
"__class__": "RemoveContainerMessage"
}
{
"error": "New container <Container test-zone:test-app:3641aca:web:2f123f1 did't became healthy, remove result: id: 2f123f1abcdfc8208b298c89e10bcd8f48f9fdb25c9eb7874ea5cc7199825e6e success: true",
"args": ["1aa8a638a153b393ee423c0a8c158757b13ab74591ade036b6e73ac33a4bdeac", "3641aca"],
"kwargs": {"user_id": null}
}
"""
payload = None
while True:
message = socket.receive()
try:
payload = renew_schema.loads(message)
break
except ValidationError as e:
socket.send(json.dumps(e.messages))
except JSONDecodeError as e:
socket.send(json.dumps({'error': str(e)}))
args = payload.data
containers = [
Container.get_by_container_id(id_) for id_ in args['container_ids']
]
appnames = {c.appname for c in containers}
sha = args['sha']
if len(appnames) > 1 and sha:
socket.send(
json.dumps({
'error':
'cannot provide sha when renewing containers of multiple apps: {}'
.format(appnames)
}))
socket.close()
appname = appnames.pop()
app = App.get_by_name(appname)
if not app:
socket.send(json.dumps({'error': 'app {} not found'.format(appname)}))
socket.close()
task_ids = []
for c in containers:
async_result = renew_container.delay(c.container_id,
sha,
user_id=session.get('user_id'))
task_ids.append(async_result.task_id)
for m in celery_task_stream_response(task_ids):
logger.debug(m)
socket.send(m)
def deploy(socket):
"""Create containers for the specified release
:<json string appname: required
:<json string zone: required
:<json string sha: required, minimum length is 7
:<json string combo_name: required, specify the combo to use, you can
update combo value using this API, so all parameters in the
:http:post:`/api/app/(appname)/combo` are supported
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Type: application/json
{
"podname": "eru",
"nodename": "c1-eru-2.ricebook.link",
"id": "9c91d06cb165e829e8e0ad5d5b5484c47d4596af04122489e4ead677113cccb4",
"name": "test-app_web_kMqYFQ",
"error": "",
"success": true,
"cpu": {"0": 20},
"quota": 0.2,
"memory": 134217728,
"publish": {"bridge": "172.17.0.5:6789"},
"hook": "I am the hook output",
"__class__": "CreateContainerMessage"
}
"""
payload = None
while True:
message = socket.receive()
try:
payload = deploy_schema.loads(message)
break
except ValidationError as e:
socket.send(json.dumps(e.messages))
except JSONDecodeError as e:
socket.send(json.dumps({'error': str(e)}))
args = payload.data
appname = args['appname']
app = App.get_by_name(appname)
if not app:
socket.send(json.dumps({'error': 'app {} not found'.format(appname)}))
socket.close()
combo_name = args['combo_name']
combo = app.get_combo(combo_name)
if not combo:
socket.send(
json.dumps({
'error':
'combo {} for app {} not found'.format(combo_name, app)
}))
socket.close()
combo.update(**{k: v for k, v in args.items() if hasattr(combo, k)})
async_result = create_container.delay(user_id=session.get('user_id'),
**args)
for m in celery_task_stream_response(async_result.task_id):
logger.debug(m)
socket.send(m)