def rm_containers(): cids = request.get_json()['cids'] if not all(len(cid) >= 7 for cid in cids): abort(400, 'must given at least 7 chars for container_id') version_dict = {} for cid in cids: container = Container.get_by_container_id(cid) if not container: continue version_dict.setdefault((container.version, container.host), []).append(container) ts, watch_keys = [], [] for (version, host), containers in version_dict.iteritems(): cids = [c.id for c in containers] task = Task.create(consts.TASK_REMOVE, version, host, {'container_ids': cids}) all_host_cids = [c.id for c in Container.get_multi_by_host(host) if c and c.version_id == version.id] need_to_delete_image = set(cids) == set(all_host_cids) remove_containers.apply_async( args=(task.id, cids, need_to_delete_image), task_id='task:%d' % task.id ) ts.append(task.id) watch_keys.append(task.result_key) return {'r': 0, 'msg': 'ok', 'tasks': ts, 'watch_keys': watch_keys}
def _create_task(type_, version, host, ncontainer, cores, nshare, networks, spec_ips, entrypoint, env, image=""): network_ids = [n.id for n in networks] task_props = { "ncontainer": ncontainer, "entrypoint": entrypoint, "env": env, "full_cores": [c.label for c in cores.get("full", [])], "part_cores": [c.label for c in cores.get("part", [])], "nshare": nshare, "networks": network_ids, "image": image, } task = Task.create(type_, version, host, task_props) if not task: return None try: create_containers_with_macvlan.apply_async( args=(task.id, ncontainer, nshare, cores, network_ids, spec_ips), task_id="task:%d" % task.id ) except Exception as e: logger.exception(e) host.release_cores(cores) return task
def rm_containers(): cids = request.get_json()['cids'] if not all(len(cid) >= 7 for cid in cids): abort(400, 'must given at least 7 chars for container_id') version_dict = {} for cid in cids: container = Container.get_by_container_id(cid) if not container: continue version_dict.setdefault((container.version, container.host), []).append(container) task_ids, watch_keys = [], [] for (version, host), containers in version_dict.iteritems(): cids = [c.id for c in containers] task = Task.create(TASK_REMOVE, version, host, {'container_ids': cids}) all_host_cids = [c.id for c in Container.get_multi_by_host(host) if c and c.version_id == version.id] need_to_delete_image = set(cids) == set(all_host_cids) remove_containers.apply_async( args=(task.id, cids, need_to_delete_image), task_id='task:%d' % task.id ) task_ids.append(task.id) watch_keys.append(task.result_key) return {'tasks': task_ids, 'watch_keys': watch_keys}
def build_image_v2(): # form post appname = request.form.get('appname', default='') version = request.form.get('version', default='') base = request.form.get('base', default='') if not base: abort(400, 'base image must be set') _, version = _get_app_and_version(appname, version) if ':' not in base: base = base + ':latest' host = Host.get_random_public_host() if not host: abort(406, 'no host is available') # if no artifacts.zip is set # ignore and just do the cloning and building file_path = None if 'artifacts.zip' in request.files: f = request.files['artifacts.zip'] file_path = os.path.join(tempfile.mkdtemp(), secure_filename(f.filename)) f.save(file_path) task = Task.create(TASK_BUILD, version, host, {'base': base}) build_docker_image.apply_async( args=(task.id, base, file_path), task_id='task:%d' % task.id ) return {'task': task.id, 'watch_key': task.result_key}
def _create_task(type_, version, host, ncontainer, cores, nshare, networks, spec_ips, entrypoint, env, image=''): network_ids = [n.id for n in networks] task_props = { 'ncontainer': ncontainer, 'entrypoint': entrypoint, 'env': env, 'full_cores': [c.label for c in cores.get('full', [])], 'part_cores': [c.label for c in cores.get('part', [])], 'nshare': nshare, 'networks': network_ids, 'image': image, } task = Task.create(type_, version, host, task_props) if not task: return None try: create_containers_with_macvlan.apply_async( args=(task.id, ncontainer, nshare, cores, network_ids, spec_ips), task_id='task:%d' % task.id ) except Exception as e: logger.exception(e) host.release_cores(cores) return task
def build_image(group_name, pod_name, appname): data = request.get_json() group, pod, application, version = validate_instance(group_name, pod_name, appname, data["version"]) # TODO # 这个group可以用这个pod不? # 这个group可以build这个version不? base = data["base"] host = pod.get_random_host() task = Task.create(consts.TASK_BUILD, version, host, {"base": base}) build_docker_image.apply_async(args=(task.id, base), task_id="task:%d" % task.id) return {"r": 0, "msg": "ok", "task": task.id, "watch_key": task.result_key}
def offline_version(group_name, pod_name, appname): data = request.get_json() group, pod, application, version = validate_instance(group_name, pod_name, appname, data["version"]) d = {} ts, keys = [], [] for container in version.containers.all(): d.setdefault(container.host, []).append(container) for host, containers in d.iteritems(): cids = [c.id for c in containers] task_props = {"container_ids": cids} task = Task.create(consts.TASK_REMOVE, version, host, task_props) remove_containers.apply_async(args=(task.id, cids, True), task_id="task:%d" % task.id) ts.append(task.id) keys.append(task.result_key) return {"r": 0, "msg": "ok", "tasks": ts, "watch_keys": keys}
def build_image(group_name, pod_name, appname): data = request.get_json() group, pod, _, version = validate_instance(group_name, pod_name, appname, data['version']) base = data['base'] if ':' not in base: base = base + ':latest' host = Host.get_random_public_host() or pod.get_random_host() task = Task.create(consts.TASK_BUILD, version, host, {'base': base}) build_docker_image.apply_async( args=(task.id, base), task_id='task:%d' % task.id ) return {'r': 0, 'msg': 'ok', 'task': task.id, 'watch_key': task.result_key}
def build_image(group_name, pod_name, appname): data = request.get_json() group, pod, application, version = validate_instance(group_name, pod_name, appname, data['version']) # TODO # 这个group可以用这个pod不? # 这个group可以build这个version不? base = data['base'] host = pod.get_random_host() task = Task.create(consts.TASK_BUILD, version, host, {'base': base}) build_docker_image.apply_async( args=(task.id, base), task_id='task:%d' % task.id ) return {'r': 0, 'msg': 'ok', 'task': task.id, 'watch_key': task.result_key}
def rm_containers(): cids = request.get_json()["cids"] version_dict = {} ts, watch_keys = [], [] for cid in cids: container = Container.get_by_container_id(cid) if not container: continue version_dict.setdefault((container.version, container.host), []).append(container) for (version, host), containers in version_dict.iteritems(): cids = [c.id for c in containers] task_props = {"container_ids": cids} task = Task.create(consts.TASK_REMOVE, version, host, task_props) remove_containers.apply_async(args=(task.id, cids, False), task_id="task:%d" % task.id) ts.append(task.id) watch_keys.append(task.result_key) return {"r": 0, "msg": "ok", "tasks": ts, "watch_keys": watch_keys}
def _create_task(version, host, ncontainer, cores, nshare, networks, ports, args, spec_ips, route, entrypoint, env, image=''): network_ids = [n.id for n in networks] # host 模式不允许绑定 vlan entry = version.appconfig['entrypoints'][entrypoint] if entry.get('network_mode') == 'host': network_ids = [] task_props = { 'ncontainer': ncontainer, 'entrypoint': entrypoint, 'env': env, 'full_cores': [c.label for c in cores.get('full', [])], 'part_cores': [c.label for c in cores.get('part', [])], 'ports': ports, 'args': args, 'nshare': nshare, 'networks': network_ids, 'image': image, 'route': route, } task = Task.create(consts.TASK_CREATE, version, host, task_props) if not task: return None if cores: try: create_containers_with_macvlan.apply_async( args=(task.id, ncontainer, nshare, cores, network_ids, spec_ips), task_id='task:%d' % task.id ) except Exception as e: logger.exception(e) host.release_cores(cores) else: try: create_containers_with_macvlan_public.apply_async( args=(task.id, ncontainer, nshare, network_ids, spec_ips), task_id='task:%d' % task.id ) except Exception as e: logger.exception(e) return task
def build_image(): data = request.get_json() _, version = _get_app_and_version(**data) base = data['base'] if ':' not in base: base = base + ':latest' host = Host.get_random_public_host() if not host: abort(406, 'no host is available') task = Task.create(TASK_BUILD, version, host, {'base': base}) build_docker_image.apply_async( args=(task.id, base, None), task_id='task:%d' % task.id ) return {'task': task.id, 'watch_key': task.result_key}
def rm_containers(group_name, pod_name, appname): data = request.get_json() group, pod, application, version = validate_instance(group_name, pod_name, appname, data['version']) host = Host.get_by_name(data['host']) # 直接拿前ncontainer个吧, 反正他们都等价的 containers = host.get_containers_by_version(version)[:int(data['ncontainer'])] try: cids = [c.id for c in containers] task_props = {'container_ids': cids} task = Task.create(code.TASK_REMOVE, version, host, task_props) remove_containers.apply_async( args=(task.id, cids, False), task_id='task:%d' % task.id ) return {'r': 0, 'msg': 'ok', 'task': task.id, 'watch_key': task.result_key} except Exception, e: logger.exception(e) return {'r': 1, 'msg': str(e), 'task': None, 'watch_key': None}
def offline_version(group_name, pod_name, appname): data = request.get_json() group, pod, application, version = validate_instance(group_name, pod_name, appname, data['version']) d = {} ts, keys = [], [] for container in version.containers.all(): d.setdefault(container.host, []).append(container) for host, containers in d.iteritems(): cids = [c.id for c in containers] task_props = {'container_ids': cids} task = Task.create(consts.TASK_REMOVE, version, host, task_props) remove_containers.apply_async( args=(task.id, cids, True), task_id='task:%d' % task.id ) ts.append(task.id) keys.append(task.result_key) return {'r': 0, 'msg': 'ok', 'tasks': ts, 'watch_keys': keys}
def offline_version(): data = request.get_json() pod, _, version = _get_instances(**data) d = {} for container in version.containers.all(): d.setdefault(container.host, []).append(container) task_ids, watch_keys = [], [] for host, containers in d.iteritems(): cids = [c.id for c in containers] task = Task.create(TASK_REMOVE, version, host, {'container_ids': cids}) remove_containers.apply_async( args=(task.id, cids, True), task_id='task:%d' % task.id ) task_ids.append(task.id) watch_keys.append(task.result_key) return {'tasks': task_ids, 'watch_keys': watch_keys}
def migrate_container(container_id, need_to_remove=True): container = Container.get_by_container_id(container_id) if not container: _log.error('container %s is not found, ignore migration', container_id) return ncore, nshare = container.host.pod.get_core_allocation(container.ncore) host_cores = average_schedule(container.host.pod, 1, ncore, nshare, None) if not host_cores: _log.error('not enough cores to migrate') return cids = [container.id] spec_ips = cidrs = container.get_ips() (host, container_count), cores = next(host_cores.iteritems()) props = { 'ncontainer': 1, 'entrypoint': container.entrypoint, 'env': container.env, 'full_cores': [c.label for c in cores.get('full', [])], 'part_cores': [c.label for c in cores.get('part', [])], 'ports': None, 'args': None, 'nshare': nshare, 'networks': cidrs, 'image': None, 'route': '', 'callback_url': container.callback_url, 'container_ids': cids, } task = Task.create(consts.TASK_MIGRATE, container.version, host, props) if not task: _log.error('create migrate task error') return _log.info('start migration...') if need_to_remove: remove_containers.apply(args=(task.id, cids, False), task_id='task:%s' % task.id) create_containers.apply(args=(task.id, 1, nshare, cores, cidrs, spec_ips), task_id='task:%s' % task.id) _log.info('migration done')
def rm_containers(): cids = request.get_json()['cids'] version_dict = {} ts, watch_keys = [], [] for cid in cids: container = Container.get_by_container_id(cid) if not container: continue version_dict.setdefault((container.version, container.host), []).append(container) for (version, host), containers in version_dict.iteritems(): cids = [c.id for c in containers] task_props = {'container_ids': cids} task = Task.create(consts.TASK_REMOVE, version, host, task_props) remove_containers.apply_async( args=(task.id, cids, False), task_id='task:%d' % task.id ) ts.append(task.id) watch_keys.append(task.result_key) return {'r': 0, 'msg': 'ok', 'tasks': ts, 'watch_keys': watch_keys}
def clean_app(app_name): app = App.get_by_name(app_name) if not app: print 'app %s not found' % app_name return containers = app.list_containers(limit=None) version_dict = {} for c in containers: if not c: continue version_dict.setdefault((c.version, c.host), []).append(c) for (version, host), cs in version_dict.iteritems(): cids = [c.id for c in cs] task_props = {'container_ids': cids} task = Task.create(TASK_REMOVE, version, host, task_props) remove_containers.apply_async(args=(task.id, cids, False), task_id='task:%s' % task.id) print task print 'done, waiting...'
def _create_task(type_, version, host, ncontainer, cores, networks, entrypoint, env): try: core_ids = [c.id for c in cores] network_ids = [n.id for n in networks] task_props = { 'ncontainer': ncontainer, 'entrypoint': entrypoint, 'env': env, 'cores': core_ids, 'networks': network_ids, } task = Task.create(type_, version, host, task_props) create_containers_with_macvlan.apply_async( args=(task.id, ncontainer, core_ids, network_ids), task_id='task:%d' % task.id ) return task except Exception, e: logger.exception(e) host.release_cores(cores)
def migrate_container(container_id, need_to_remove=True): container = Container.get_by_container_id(container_id) if not container: _log.error('container %s is not found, ignore migration', container_id) return ncore, nshare= container.host.pod.get_core_allocation(container.ncore) host_cores = average_schedule(container.host.pod, 1, ncore, nshare, None) if not host_cores: _log.error('not enough cores to migrate') return cids = [container.id] spec_ips = cidrs = container.get_ips() (host, container_count), cores = next(host_cores.iteritems()) props = { 'ncontainer': 1, 'entrypoint': container.entrypoint, 'env': container.env, 'full_cores': [c.label for c in cores.get('full', [])], 'part_cores': [c.label for c in cores.get('part', [])], 'ports': None, 'args': None, 'nshare': nshare, 'networks': cidrs, 'image': None, 'route': '', 'callback_url': container.callback_url, 'container_ids': cids, } task = Task.create(consts.TASK_MIGRATE, container.version, host, props) if not task: _log.error('create migrate task error') return _log.info('start migration...') if need_to_remove: remove_containers.apply(args=(task.id, cids, False), task_id='task:%s' % task.id) create_containers.apply(args=(task.id, 1, nshare, cores, cidrs, spec_ips), task_id='task:%s' % task.id) _log.info('migration done')
def clean_app(app_name): app = App.get_by_name(app_name) if not app: print 'app %s not found' % app_name return containers = app.list_containers(limit=None) version_dict = {} for c in containers: if not c: continue version_dict.setdefault((c.version, c.host), []).append(c) for (version, host), cs in version_dict.iteritems(): cids = [c.id for c in cs] task_props = {'container_ids': cids} task = Task.create(TASK_REMOVE, version, host, task_props) remove_containers.apply_async( args=(task.id, cids, False), task_id='task:%s' % task.id ) print task print 'done, waiting...'
def fillup_data(): app, _ = create_app_with_celery() with app.app_context(): db.drop_all() db.create_all() group = Group.create('test-group', 'test-group') pod = Pod.create('test-pod', 'test-pod') pod.assigned_to_group(group) r = requests.get('http://192.168.59.103:2375/info').json() host = Host.create(pod, '192.168.59.103:2375', r['Name'], r['ID'], r['NCPU'], r['MemTotal']) host.assigned_to_group(group) app = App.get_or_create('nbetest', 'http://git.hunantv.com/platform/nbetest.git', 'token') app.add_version('96cbf8c68ed214f105d9f79fa4f22f0e80e75cf3') app.assigned_to_group(group) version = app.get_version('96cbf8') host_cores = group.get_free_cores(pod, 2, 2) cores = [] for (host, cn), coress in host_cores.iteritems(): print host, cn, coress cores = coress print cores ports = host.get_free_ports(2) print ports props = { 'entrypoint': 'web', 'ncontainer': 2, 'env': 'PROD', 'cores': [c.id for c in cores], 'ports': [p.id for p in ports], } task = Task.create(1, version, host, props) print task.props host.occupy_cores(cores) host.occupy_ports(ports) print group.get_free_cores(pod, 1, 1)
def _create_task(version, host, ncontainer, cores, nshare, networks, ports, args, spec_ips, entrypoint, env, image='', callback_url=''): # host 模式不允许绑定 vlan entry = version.appconfig['entrypoints'][entrypoint] if entry.get('network_mode') == 'host': cidrs = [] else: cidrs = [n.cidr for n in networks] task_props = { 'ncontainer': ncontainer, 'entrypoint': entrypoint, 'env': env, 'full_cores': [c.label for c in cores.get('full', [])], 'part_cores': [c.label for c in cores.get('part', [])], 'ports': ports, 'args': args, 'nshare': nshare, 'networks': cidrs, 'image': image, 'route': entry.get('network_route', ''), 'callback_url': callback_url, } task = Task.create(TASK_CREATE, version, host, task_props) if not task: return None try: create_containers.apply_async( args=(task.id, ncontainer, nshare, cores, cidrs, spec_ips), task_id='task:%d' % task.id ) except Exception as e: _log.exception(e) host.release_cores(cores) return task
def rm_containers(): cids = request.get_json()['cids'] if not all(len(cid) >= 7 for cid in cids): raise EruAbortException(consts.HTTP_BAD_REQUEST, 'must given at least 7 chars for container_id') version_dict = {} ts, watch_keys = [], [] for cid in cids: container = Container.get_by_container_id(cid) if not container: continue version_dict.setdefault((container.version, container.host), []).append(container) for (version, host), containers in version_dict.iteritems(): cids = [c.id for c in containers] task_props = {'container_ids': cids} task = Task.create(consts.TASK_REMOVE, version, host, task_props) remove_containers.apply_async( args=(task.id, cids, False), task_id='task:%d' % task.id ) ts.append(task.id) watch_keys.append(task.result_key) return {'r': 0, 'msg': 'ok', 'tasks': ts, 'watch_keys': watch_keys}