def task_log(task_id):
ws = request.environ['wsgi.websocket']
task = Task.get(task_id)
if not task:
ws.close()
_log.info('Task %s not found, close websocket' % task_id)
return 'websocket closed'
notifier = TaskNotifier(task)
try:
pub = rds.pubsub()
pub.subscribe(task.publish_key)
for line in notifier.get_store_logs():
ws.send(line)
if task.finished:
return ''
for line in pub.listen():
if line['data'] == consts.PUB_END_MESSAGE:
break
if line['type'] != 'message':
continue
ws.send(line['data'])
except geventwebsocket.WebSocketError, e:
_log.exception(e)
def build_docker_image(task_id, base, file_path):
task = Task.get(task_id)
if not task:
_log.error('Task (id=%s) not found, quit', task_id)
return
_log.info('Task<id=%s>: Start on host %s', task_id, task.host.ip)
notifier = TaskNotifier(task)
app = task.app
host = task.host
version = task.version
try:
repo, tag = base.split(':', 1)
repo = repo if repo.startswith('eru/') else 'eru/' + repo.strip('/')
_log.info('Task<id=%s>: Pull base image (base=%s)', task_id, base)
notifier.store_and_broadcast(dockerjob.pull_image(host, repo, tag))
_log.info('Task<id=%s>: Build image (base=%s)', task_id, base)
notifier.store_and_broadcast(
dockerjob.build_image(host, version, base, file_path))
_log.info('Task<id=%s>: Push image (base=%s)', task_id, base)
last_line = notifier.store_and_broadcast(
dockerjob.push_image(host, version))
dockerjob.remove_image(version, host)
except Exception, e:
task.finish(consts.TASK_FAILED)
task.reason = str(e.message)
notifier.pub_fail()
_log.error('Task<id=%s>, exception', task_id)
_log.exception(e)
def remove_containers(task_id, cids, rmi=False):
task = Task.get(task_id)
if not task:
_log.error('Task (id=%s) not found, quit', task_id)
return
_log.info('Task<id=%s>: Start on host %s', task_id, task.host.ip)
notifier = TaskNotifier(task)
containers = Container.get_multi(cids)
if not containers:
_log.error('Task (id=%s) no container found, quit')
return
host = containers[0].host
for c in containers:
c.in_removal = 1
container_ids = [c.container_id for c in containers if c]
try:
set_flag_for_agent(container_ids)
for c in containers:
remove_container_backends(c)
_log.info('Task<id=%s>: Container (cid=%s) backends removed',
task_id, c.short_id)
appnames = {c.appname for c in containers}
publish_to_service_discovery(*appnames)
time.sleep(3)
dockerjob.remove_host_containers(containers, host)
_log.info('Task<id=%s>: Containers (cids=%s) removed', task_id, cids)
if rmi:
try:
dockerjob.remove_image(task.version, host)
except Exception as e:
_log.error('Task<id=%s>, fail to remove image', task_id, e)
except Exception as e:
task.finish(consts.TASK_FAILED)
task.reason = str(e.message)
notifier.pub_fail()
_log.error('Task<id=%s> exception', task_id)
_log.exception(e)
else:
for c in containers:
c.delete()
task.finish(consts.TASK_SUCCESS)
task.reason = 'ok'
notifier.pub_success()
remove_container_for_agent(host, container_ids)
remove_flag_for_agent(container_ids)
_log.info('Task<id=%s>: Done', task_id)
def create_containers(task_id,
ncontainer,
nshare,
cores,
network_ids,
spec_ips=None):
"""
执行task_id的任务. 部署ncontainer个容器, 占用*_core_ids这些核, 绑定到networks这些子网
"""
_log.info('Task<id=%s>: Started', task_id)
task = Task.get(task_id)
if not task:
_log.error('Task (id=%s) not found, quit', task_id)
return
if spec_ips is None:
spec_ips = []
need_network = bool(network_ids)
networks = [ipam.get_pool(n) for n in network_ids]
notifier = TaskNotifier(task)
host = task.host
version = task.version
entrypoint = task.props['entrypoint']
env = task.props['env']
ports = task.props['ports']
args = task.props['args']
# use raw
image = task.props['image']
callback_url = task.props['callback_url']
cpu_shares = int(float(nshare) / host.pod.core_share *
1024) if nshare else 1024
cids = []
backends = []
entry = version.appconfig.entrypoints[entrypoint]
for fcores, pcores in _iter_cores(cores, ncontainer):
cores_for_one_container = {'full': fcores, 'part': pcores}
# 在宿主机上创建容器
try:
cid, cname = dockerjob.create_one_container(
host,
version,
entrypoint,
env,
fcores + pcores,
ports=ports,
args=args,
cpu_shares=cpu_shares,
image=image,
need_network=need_network)
except Exception as e:
# 写给celery日志看
_log.exception(e)
host.release_cores(cores_for_one_container, nshare)
continue
# 容器记录下来
c = Container.create(cid, host, version, cname, entrypoint,
cores_for_one_container, env, nshare,
callback_url)
# 为容器创建网络栈
# 同时把各种信息都记录下来
# 如果失败, 清除掉所有记录和宿主机上的容器
# 循环下一次尝试
cidrs = [n.netspace for n in networks]
if not ipam.allocate_ips(cidrs, cid, spec_ips):
_clean_failed_containers(cid)
continue
notifier.notify_agent(c)
add_container_for_agent(host, c)
add_container_backends(c)
cids.append(cid)
backends.extend(c.get_backends())
c.callback_report(status='start')
health_check = entry.get('health_check', '')
if health_check and backends:
urls = [b + health_check for b in backends]
if not wait_health_check(urls):
# TODO 这里要么回滚要么报警
_log.info('Task<id=%s>: Done, but something went error', task_id)
return
publish_to_service_discovery(version.name)
task.finish(consts.TASK_SUCCESS)
task.reason = 'ok'
task.container_ids = cids
notifier.pub_success()
_log.info('Task<id=%s>: Done', task_id)
