예제 #1
0
def clean_image():
    for host in Host.query.all():
        try:
            client = get_docker_client(host.addr)
        except:
            print 'can not connect, maybe tls problem.'
            continue

        try:
            client.ping()
        except:
            print 'client not connected, skipped.'
            continue

        for c in client.containers(all=True):
            if need_to_delete_container(c['Image'], c['Names'][0]):
                try:
                    client.remove_container(c['Id'])
                    print 'container %s cleaned.' % c['Names'][0]
                except:
                    print 'container %s still running.' % c['Names'][0]

        for i in client.images():
            try:
                client.remove_image(i['Id'])
                print 'image %s cleaned.' % i['RepoTags'][0]
            except:
                print 'conflict, image %s is still being used.' % i['RepoTags'][0]

        print 'host %s cleaned.' % host.ip
예제 #2
0
파일: sys.py 프로젝트: CMGS/eru-core
def create_host():
    """为了文件, 只好不用json了"""
    addr = request.form.get('addr', default='')
    pod_name = request.form.get('pod_name', default='')
    if not (addr and pod_name):
        abort(400, 'Need addr and pod_name')

    pod = Pod.get_by_name(pod_name)
    if not pod:
        abort(400, 'No pod found')

    # 存证书, 没有就算了
    if all(k in request.files for k in ['ca', 'cert', 'key']):
        try:
            ca, cert, key = request.files['ca'], request.files['cert'], request.files['key']
            save_docker_certs(addr.split(':', 1)[0], ca.read(), cert.read(), key.read())
        finally:
            ca.close()
            cert.close()
            key.close()

    try:
        client = get_docker_client(addr, force_flush=True)
        info = client.info()
    except Exception as e:
        abort(400, 'Docker daemon error on host %s, error: %s' % (addr, e.message))

    if not Host.create(pod, addr, info['Name'], info['ID'], info['NCPU'], info['MemTotal']):
        abort(400, 'Host create error.')
    return 201, {'r':0, 'msg': consts.OK}
예제 #3
0
파일: sys.py 프로젝트: BlueKarl/eru-core
def create_host():
    """为了文件, 只好不用json了"""
    addr = request.form.get('addr', type=str, default='')
    pod_name = request.form.get('pod_name', type=str, default='')
    if not (addr and pod_name):
        raise EruAbortException(consts.HTTP_BAD_REQUEST, 'need addr and pod_name')

    pod = Pod.get_by_name(pod_name)
    if not pod:
        raise EruAbortException(consts.HTTP_BAD_REQUEST, 'No pod found')

    # 存证书, 没有就算了
    try:
        ca, cert, key = request.files['ca'], request.files['cert'], request.files['key']
        save_docker_certs(addr.split(':', 1)[0], ca.read(), cert.read(), key.read())
    finally:
        ca.close()
        cert.close()
        key.close()

    try:
        client = get_docker_client(addr, force_flush=True)
        info = client.info()
    except Exception:
        raise EruAbortException(consts.HTTP_BAD_REQUEST, 'Docker daemon error on host %s' % addr)

    if not Host.create(pod, addr, info['Name'], info['ID'], info['NCPU'], info['MemTotal']):
        raise EruAbortException(consts.HTTP_BAD_REQUEST)
    return consts.HTTP_CREATED, {'r':0, 'msg': consts.OK}
예제 #4
0
파일: dockerjob.py 프로젝트: CMGS/eru-core
def remove_container_by_cid(cids, host):
    client = get_docker_client(host.addr)
    for cid in cids:
        try:
            __stop_container(client, cid)
            client.remove_container(cid)
        except docker.errors.APIError as e:
            if 'no such id' in str(e).lower():
                logger.info('%s not found, just delete it' % cid)
                continue
            raise
예제 #5
0
파일: dockerjob.py 프로젝트: CMGS/eru-core
def remove_host_containers(containers, host):
    """删除这个host上的这些容器"""
    client = get_docker_client(host.addr)
    for c in containers:
        try:
            __stop_container(client, c.container_id)
            client.remove_container(c.container_id)
        except docker.errors.APIError as e:
            if 'no such id' in str(e).lower():
                logger.info('%s not found, just delete it' % c.container_id)
                continue
            raise
예제 #6
0
파일: dockerjob.py 프로젝트: CMGS/eru-core
def build_image(host, version, base):
    """
    用 host 机器, 以 base 为基础镜像, 为 version 构建
    一个稍后可以运行的镜像.
    """
    client = get_docker_client(host.addr)
    appname = version.app.name
    repo = '{0}/{1}'.format(config.DOCKER_REGISTRY, appname)
    rev = version.short_sha
    tag = '{0}:{1}'.format(repo, rev)

    with build_image_environment(version, base, rev) as build_path:
        return client.build(path=build_path, rm=True, forcerm=True, tag=tag)
예제 #7
0
파일: dockerjob.py 프로젝트: CMGS/eru-core
def remove_image(version, host):
    """在host上删除掉version的镜像"""
    client = get_docker_client(host.addr)
    appconfig = version.appconfig
    appname = appconfig.appname
    image = '{0}/{1}:{2}'.format(config.DOCKER_REGISTRY, appname, version.short_sha)
    try:
        client.remove_image(image)
    except docker.errors.APIError as e:
        if 'no such image' in str(e).lower():
            logger.info('%s not found, just delete it' % image)
        else:
            raise
예제 #8
0
def create_host():
    data = request.get_json()
    addr = data['addr']

    pod = Pod.get_by_name(data['pod_name'])
    if not pod:
        raise EruAbortException(consts.HTTP_BAD_REQUEST, 'No pod found')

    try:
        client = get_docker_client(addr)
        info = client.info()
    except Exception:
        raise EruAbortException(consts.HTTP_BAD_REQUEST, 'Docker daemon error on host %s' % addr)

    if not Host.create(pod, addr, info['Name'], info['ID'], info['NCPU'], info['MemTotal']):
        raise EruAbortException(consts.HTTP_BAD_REQUEST)
    return consts.HTTP_CREATED, {'r':0, 'msg': consts.OK}
예제 #9
0
파일: websockets.py 프로젝트: CMGS/eru-core
def container_log(cid):
    stderr = request.args.get('stderr', type=int, default=0)
    stdout = request.args.get('stdout', type=int, default=0)
    tail = request.args.get('tail', type=int, default=10)

    # docker client's argument
    if tail == 0:
        tail = 'all'

    ws = request.environ['wsgi.websocket']
    container = Container.get_by_container_id(cid)
    if not container:
        ws.close()
        logger.info('Container %s not found, close websocket' % cid)
        return 'websocket closed'
    try:
        client = get_docker_client(container.host.addr)
        for line in client.logs(cid, stream=True, stderr=bool(stderr), stdout=bool(stdout), tail=tail):
            ws.send(line)
    except geventwebsocket.WebSocketError, e:
        logger.exception(e)
예제 #10
0
파일: dockerjob.py 프로젝트: CMGS/eru-core
def create_one_container(host, version, entrypoint, env='prod',
        cores=None, ports=None, args=None, cpu_shares=1024, image='', need_network=False):
    # raw方式有些设定不同
    is_raw = bool(image)

    if cores is None:
        cores = []
    if ports is None:
        ports = []
    if args is None:
        args = []

    client = get_docker_client(host.addr)
    local_images = {r['RepoTags'][0] for r in client.images()}

    appconfig = version.appconfig
    appname = appconfig.appname
    entry = appconfig.entrypoints[entrypoint]
    envconfig = version.get_resource_config(env)
    # replace $port1...
    cmd = replace_ports(entry['cmd'], ports)
    # add extend arguments
    cmd = cmd + ' '.join([''] + args)
    if not is_raw:
        network = 'network' if need_network else 'nonetwork'
        cmd = '/usr/local/bin/launcher %s ' % network + cmd

    network_mode = entry.get('network_mode', config.DOCKER_NETWORK_MODE)
    mem_limit = entry.get('mem_limit', 0)
    restart_policy = {'MaximumRetryCount': 3, 'Name': entry.get('restart', 'no')} # could be no/always/on-failure

    # raw 模式下可以选择暴露端口
    def get_ports(expose):
        inport, hostport = expose.split(':')
        return int(inport), int(hostport)

    exposes = [get_ports(expose) for expose in entry.get('exposes', [])]
    exposed_ports = None
    port_bindings = None
    if is_raw and exposes:
        exposed_ports = [p for p, _ in exposes]
        port_bindings = dict(exposes)

    if not image:
        image = '{0}/{1}:{2}'.format(config.DOCKER_REGISTRY, appname, version.short_sha)

    if image not in local_images:
        repo, tag = image.split(':', 1)
        for line in client.pull(repo, tag, stream=True,
                insecure_registry=config.DOCKER_REGISTRY_INSECURE):
            print line

    env_dict = {
        'APP_NAME': appname,
        'ERU_RUNENV': env.upper(),
        'ERU_POD': host.pod.name,
        'ERU_HOST': host.name,
    }
    env_dict.update(envconfig.to_env_dict())

    volumes = ['/writable-proc/sys']
    volumes.extend(appconfig.get('volumes', []))

    binds = {'/proc/sys': {'bind': '/writable-proc/sys', 'ro': False}}
    binds.update(appconfig.get('binds', {}))

    if config.ERU_CONTAINER_PERMDIR:
        permdir = config.ERU_CONTAINER_PERMDIR % appname
        env_dict['ERU_PERMDIR'] = permdir
        volumes.append(permdir)
        binds[config.ERU_HOST_PERMDIR % appname] =  {'bind': permdir, 'ro': False}

    # container name: {appname}_{entrypoint}_{ident_id}
    container_name = '_'.join([appname, entrypoint, gen_salt(6)])
    # cpuset: '0,1,2,3'
    cpuset = ','.join([c.label for c in cores])
    # host_config, include log_config
    host_config = create_host_config(
        binds=binds,
        network_mode=network_mode,
        log_config=LogConfig(type=config.DOCKER_LOG_DRIVER),
        ulimits=[Ulimit(name='nofile', soft=65535, hard=65535)],
        restart_policy=restart_policy,
        mem_limit=mem_limit,
        port_bindings=port_bindings,
    )
    container = client.create_container(
        image=image,
        command=cmd,
        environment=env_dict,
        name=container_name,
        cpuset=cpuset,
        working_dir=None if is_raw else '/%s' % appname,
        network_disabled=config.DOCKER_NETWORK_DISABLED,
        volumes=volumes,
        host_config=host_config,
        cpu_shares=cpu_shares,
        ports=exposed_ports,
    )
    container_id = container['Id']

    client.start(container=container_id)
    return container_id, container_name
예제 #11
0
파일: dockerjob.py 프로젝트: CMGS/eru-core
def pull_image(host, repo, tag):
    client = get_docker_client(host.addr)
    return client.pull(repo, tag=tag, stream=True, insecure_registry=config.DOCKER_REGISTRY_INSECURE)
예제 #12
0
파일: dockerjob.py 프로젝트: CMGS/eru-core
def push_image(host, version):
    client = get_docker_client(host.addr)
    appname = version.app.name
    repo = '{0}/{1}'.format(config.DOCKER_REGISTRY, appname)
    rev = version.short_sha
    return client.push(repo, tag=rev, stream=True, insecure_registry=config.DOCKER_REGISTRY_INSECURE)
예제 #13
0
파일: dockerjob.py 프로젝트: CMGS/eru-core
def stop_containers(containers, host):
    """停止这个host上的这些容器"""
    client = get_docker_client(host.addr)
    for c in containers:
        __stop_container(client, c.container_id)
예제 #14
0
파일: dockerjob.py 프로젝트: CMGS/eru-core
def start_containers(containers, host):
    """启动这个host上的这些容器"""
    client = get_docker_client(host.addr)
    for c in containers:
        client.start(c.container_id)
예제 #15
0
파일: dockerjob.py 프로젝트: CMGS/eru-core
def execute_container(host, container_id, cmd, stream=False):
    """在容器里跑一个命令, stream的话返回一个generator"""
    client = get_docker_client(host.addr)
    exec_id = client.exec_create(container_id, cmd, stream=stream)
    return client.exec_start(exec_id)
예제 #16
0
def create_one_container(host, version, entrypoint, env='prod',
        cores=None, ports=None, cpu_shares=1024, image=''):
    if cores is None:
        cores = []
    if ports is None:
        ports = []

    client = get_docker_client(host.addr)
    local_images = {r['RepoTags'][0] for r in client.images()}

    appconfig = version.appconfig
    appname = appconfig.appname
    entry = appconfig.entrypoints[entrypoint]
    envconfig = version.get_resource_config(env)
    cmd = replace_ports(entry['cmd'], ports)

    network_mode = entry.get('network_mode', config.DOCKER_NETWORK_MODE)

    if not image:
        image = '{0}/{1}:{2}'.format(config.DOCKER_REGISTRY, appname, version.short_sha)

    if image not in local_images:
        repo, tag = image.split(':', 1)
        for line in client.pull(repo, tag, stream=True,
                insecure_registry=config.DOCKER_REGISTRY_INSECURE):
            print line

    env_dict = {
        'APP_NAME': appname,
        'ERU_RUNENV': env.upper(),
        'ERU_POD': host.pod.name,
        'ERU_HOST': host.name,
    }
    env_dict.update(envconfig.to_env_dict())

    volumes = ['/writable-proc/sys']
    volumes.extend(appconfig.get('volumes', []))

    binds = {'/proc/sys': {'bind': '/writable-proc/sys', 'ro': False}}
    binds.update(appconfig.get('binds', {}))

    if config.ERU_CONTAINER_PERMDIR:
        permdir = config.ERU_CONTAINER_PERMDIR % appname
        env_dict['ERU_PERMDIR'] = permdir
        volumes.append(permdir)
        binds[config.ERU_HOST_PERMDIR % appname] =  {'bind': permdir, 'ro': False}

    # container name: {appname}_{entrypoint}_{ident_id}
    container_name = '_'.join([appname, entrypoint, random_string(6)])
    # cpuset: '0,1,2,3'
    cpuset = ','.join([c.label for c in cores])
    # host_config, include log_config
    host_config = create_host_config(
        binds=binds,
        network_mode=network_mode,
        log_config=LogConfig(type=config.DOCKER_LOG_DRIVER),
        ulimits=[Ulimit(name='nofile', soft=65535, hard=65535)],
    )
    container = client.create_container(
        image=image,
        command=cmd,
        environment=env_dict,
        entrypoint=None if image else 'launch',
        name=container_name,
        cpuset=cpuset,
        working_dir=None if image else '/%s' % appname,
        network_disabled=config.DOCKER_NETWORK_DISABLED,
        volumes=volumes,
        host_config=host_config,
        cpu_shares=cpu_shares,
    )
    container_id = container['Id']

    client.start(container=container_id)
    return container_id, container_name