def task_log(task_id): ws = request.environ['wsgi.websocket'] task = Task.get(task_id) if not task: ws.close() _log.info('Task %s not found, close websocket' % task_id) return 'websocket closed' notifier = TaskNotifier(task) try: pub = rds.pubsub() pub.subscribe(task.publish_key) for line in notifier.get_store_logs(): ws.send(line) if task.finished: return '' for line in pub.listen(): if line['data'] == consts.PUB_END_MESSAGE: break if line['type'] != 'message': continue ws.send(line['data']) except geventwebsocket.WebSocketError, e: _log.exception(e)
def build_docker_image(task_id, base, file_path): task = Task.get(task_id) if not task: _log.error('Task (id=%s) not found, quit', task_id) return _log.info('Task<id=%s>: Start on host %s', task_id, task.host.ip) notifier = TaskNotifier(task) app = task.app host = task.host version = task.version try: repo, tag = base.split(':', 1) repo = repo if repo.startswith('eru/') else 'eru/' + repo.strip('/') _log.info('Task<id=%s>: Pull base image (base=%s)', task_id, base) notifier.store_and_broadcast(dockerjob.pull_image(host, repo, tag)) _log.info('Task<id=%s>: Build image (base=%s)', task_id, base) notifier.store_and_broadcast( dockerjob.build_image(host, version, base, file_path)) _log.info('Task<id=%s>: Push image (base=%s)', task_id, base) last_line = notifier.store_and_broadcast( dockerjob.push_image(host, version)) dockerjob.remove_image(version, host) except Exception, e: task.finish(consts.TASK_FAILED) task.reason = str(e.message) notifier.pub_fail() _log.error('Task<id=%s>, exception', task_id) _log.exception(e)
def remove_containers(task_id, cids, rmi=False): task = Task.get(task_id) if not task: _log.error('Task (id=%s) not found, quit', task_id) return _log.info('Task<id=%s>: Start on host %s', task_id, task.host.ip) notifier = TaskNotifier(task) containers = Container.get_multi(cids) if not containers: _log.error('Task (id=%s) no container found, quit') return host = containers[0].host for c in containers: c.in_removal = 1 container_ids = [c.container_id for c in containers if c] try: set_flag_for_agent(container_ids) for c in containers: remove_container_backends(c) _log.info('Task<id=%s>: Container (cid=%s) backends removed', task_id, c.short_id) appnames = {c.appname for c in containers} publish_to_service_discovery(*appnames) time.sleep(3) dockerjob.remove_host_containers(containers, host) _log.info('Task<id=%s>: Containers (cids=%s) removed', task_id, cids) if rmi: try: dockerjob.remove_image(task.version, host) except Exception as e: _log.error('Task<id=%s>, fail to remove image', task_id, e) except Exception as e: task.finish(consts.TASK_FAILED) task.reason = str(e.message) notifier.pub_fail() _log.error('Task<id=%s> exception', task_id) _log.exception(e) else: for c in containers: c.delete() task.finish(consts.TASK_SUCCESS) task.reason = 'ok' notifier.pub_success() remove_container_for_agent(host, container_ids) remove_flag_for_agent(container_ids) _log.info('Task<id=%s>: Done', task_id)
def create_containers(task_id, ncontainer, nshare, cores, network_ids, spec_ips=None): """ 执行task_id的任务. 部署ncontainer个容器, 占用*_core_ids这些核, 绑定到networks这些子网 """ _log.info('Task<id=%s>: Started', task_id) task = Task.get(task_id) if not task: _log.error('Task (id=%s) not found, quit', task_id) return if spec_ips is None: spec_ips = [] need_network = bool(network_ids) networks = [ipam.get_pool(n) for n in network_ids] notifier = TaskNotifier(task) host = task.host version = task.version entrypoint = task.props['entrypoint'] env = task.props['env'] ports = task.props['ports'] args = task.props['args'] # use raw image = task.props['image'] callback_url = task.props['callback_url'] cpu_shares = int(float(nshare) / host.pod.core_share * 1024) if nshare else 1024 cids = [] backends = [] entry = version.appconfig.entrypoints[entrypoint] for fcores, pcores in _iter_cores(cores, ncontainer): cores_for_one_container = {'full': fcores, 'part': pcores} # 在宿主机上创建容器 try: cid, cname = dockerjob.create_one_container( host, version, entrypoint, env, fcores + pcores, ports=ports, args=args, cpu_shares=cpu_shares, image=image, need_network=need_network) except Exception as e: # 写给celery日志看 _log.exception(e) host.release_cores(cores_for_one_container, nshare) continue # 容器记录下来 c = Container.create(cid, host, version, cname, entrypoint, cores_for_one_container, env, nshare, callback_url) # 为容器创建网络栈 # 同时把各种信息都记录下来 # 如果失败, 清除掉所有记录和宿主机上的容器 # 循环下一次尝试 cidrs = [n.netspace for n in networks] if not ipam.allocate_ips(cidrs, cid, spec_ips): _clean_failed_containers(cid) continue notifier.notify_agent(c) add_container_for_agent(host, c) add_container_backends(c) cids.append(cid) backends.extend(c.get_backends()) c.callback_report(status='start') health_check = entry.get('health_check', '') if health_check and backends: urls = [b + health_check for b in backends] if not wait_health_check(urls): # TODO 这里要么回滚要么报警 _log.info('Task<id=%s>: Done, but something went error', task_id) return publish_to_service_discovery(version.name) task.finish(consts.TASK_SUCCESS) task.reason = 'ok' task.container_ids = cids notifier.pub_success() _log.info('Task<id=%s>: Done', task_id)