Exemple #1
0
def create_nginx_config(session, docker):
    nginx_config = nginx_config_manager.Conf()
    container = {}
    for row in docker.containers():
        container[row["Id"]] = row
    for dbcontainer in session.query(DbContainer).order_by(DbContainer.created).group_by(DbContainer.image).all():
        if not dbcontainer.container_id in container:
            click.secho("Container %r not running. (Image %r)" % (dbcontainer.container_id, dbcontainer.image),
                        fg="red")
            # TODO: start container
            continue
        inspect = docker.inspect_container(dbcontainer.container_id)
        config = json.loads(dbcontainer.config)
        settings = {
            "CONTAINER_IP": inspect["NetworkSettings"]["IPAddress"],
            "DEPLOY_GIT_VERSION": config.get("git_version")
        }

        ngxa = config.get("nginx")
        for ngx in ngxa:
            nginx_server = nginx_config_manager.Server()
            for keys in ngx.get("keys"):
                for k, v in keys.items():
                    nginx_server.add(nginx_config_manager.Key(k, v.format(**settings)))

            for l, keys_array in ngx.get("locations").items():
                nginx_location = nginx_config_manager.Location(l)
                for keys in keys_array:
                    for k, v in keys.items():
                        nginx_location.add(nginx_config_manager.Key(k, v.format(**settings)))
                nginx_server.add(nginx_location)
            nginx_config.add(nginx_config_manager.Comment("Server %r" % dbcontainer.image))
            nginx_config.add(nginx_server)

    return nginx_config_manager.dumps(nginx_config)
def recreate_volume_container(docker, cname):
    print 'recreate_volume_container,',
    if cname in ' '.join([y.encode('utf-8').replace('/', '') for x in 
        docker.containers(all=True) for y in x['Names'] ]):
        print '"%s" container exist' % cname
        remove_container(docker, cname)
    else:
        print '"%s" container not exist' % cname
    create_container(docker, cname)
def recreate_volume_container(docker, cname):
    print 'recreate_volume_container,',
    if cname in ' '.join([
            y.encode('utf-8').replace('/', '')
            for x in docker.containers(all=True) for y in x['Names']
    ]):
        print '"%s" container exist' % cname
        remove_container(docker, cname)
    else:
        print '"%s" container not exist' % cname
    create_container(docker, cname)
Exemple #4
0
def main():

    containers1 = []

    epoll = select.epoll()

    start = 0

    buffy = {}

    try:
        with open('/srv/events/since') as f:
            docker.Container.since = int(f.read().rstrip())
        print('since', docker.Container.since)
    except FileNotFoundError:
        pass

    with riemann_client.client.QueuedClient(riemann_client.transport.TCPTransport('localhost', 5555)) as client:

        while 1:

            # tight loops are bad mmkay
            time.sleep(0.05)

            if time.time() - start >= 1.0:
                start = time.time()

                containers2 = docker.containers()

                a = [x for x in containers1 if x not in containers2]
                b = [x for x in containers2 if x not in containers1]

                for container in a:
                    print('remove', container)

                    container.logs_stop(epoll)

                    if container.logs_fd is not None:
                        if buffy.get(container.logs_fd):
                            print(container, 'logs', repr(buffy[container.logs_fd]))
                        try:
                            del buffy[container.logs_fd]
                        except KeyError:
                            pass

                    container.stats_stop(epoll)

                    if container.stats_fd is not None:
                        if buffy.get(container.stats_fd):
                            print(container, 'stats', repr(buffy[container.stats_fd]))
                        try:
                            del buffy[container.stats_fd]
                        except KeyError:
                            pass

                    containers1.remove(container)

                for container in b:

                    try:
                        info = container.inspect()
                    except docker.HTTPError:
                        continue

                    container._info = info

                    if info['Config']['Tty']:
                        continue

                    print('append', container)

                    try:
                        container.logs_start(epoll)
                    except docker.HTTPError as exc:
                        print(container, exc)
                        continue

                    try:
                        container.stats_start(epoll)
                    except docker.HTTPError as exc:
                        print(container, exc)
                        continue

                    containers1.append(container)

                for container in containers1:
                    container.logs_check(epoll)
                    container.stats_check(epoll)

                #
                docker.Container.since = int(time.time()) - 10
                with open('/srv/events/since', 'w') as f:
                    print(docker.Container.since, file=f)
                #print('since', docker.Container.since)

            #
            for fd, event in epoll.poll(0):

                container = None
                try:
                    container = [x for x in containers1 if x.logs_fd and x.logs_fd == fd][0]
                except IndexError:
                    pass
                try:
                    container = [x for x in containers1 if x.stats_fd and x.stats_fd == fd][0]
                except IndexError:
                    pass

                assert container is not None

                data = os.read(fd, 8192)

                # XXX
                if data == b'0\r\n\r\n':
                    #print(container, data)
                    continue

                data = buffy.get(fd, b'') + data

                # XXX
                if not re.match(rb'[0-9a-f]+\r\n', data, re.I):
                    m = re.search(rb'\r\n[0-9a-f]+\r\n', data, re.I)
                    if m:
                        print(container, 'skipping', repr(data[:m.start() + 2]))
                        data = data[m.start() + 2:]
                    else:
                        print(container, 'unusable', repr(data))
                        try:
                            del buffy[container.logs_fd]
                        except KeyError:
                            pass
                        continue

                while 1:
                    data, line = docker.parse(data)
                    if not line:
                        break

                    if container is not None:
                        if fd == container.logs_fd:
                            handle_log(client, container, line)
                        if fd == container.stats_fd:
                            handle_stat(client, container, line)

                    if not data:
                        break

                buffy[fd] = data
Exemple #5
0
def main():
    riemann_host = os.getenv('RIEMANN_HOST', 'localhost')
    riemann_port = int(os.getenv('RIEMANN_PORT', '5555'))

    containers1 = []

    epoll = select.epoll()

    start = 0

    buffy = {}

    try:
        with open('/srv/events/since') as f:
            docker.Container.since = int(f.read().rstrip())
        print('since', docker.Container.since)
    except FileNotFoundError:
        pass
    except ValueError:
        pass

    with riemann_client.client.QueuedClient(riemann_client.transport.TCPTransport(riemann_host, riemann_port)) as client:

        while 1:

            # tight loops are bad mmkay
            time.sleep(0.05)

            if time.time() - start >= 1.0:
                start = time.time()

                containers2 = docker.containers()

                a = [x for x in containers1 if x not in containers2]
                b = [x for x in containers2 if x not in containers1]

                for container in a:
                    print('remove', container)

                    container.logs_stop(epoll)

                    if container.logs_fd is not None:
                        if buffy.get(container.logs_fd):
                            print(container, 'logs', 'remaining', summarise(repr(buffy[container.logs_fd])))
                        try:
                            del buffy[container.logs_fd]
                        except KeyError:
                            pass

                    container.stats_stop(epoll)

                    if container.stats_fd is not None:
                        if buffy.get(container.stats_fd):
                            print(container, 'stats', 'remaining', summarise(repr(buffy[container.stats_fd])))
                        try:
                            del buffy[container.stats_fd]
                        except KeyError:
                            pass

                    containers1.remove(container)

                for container in b:

                    try:
                        info = container.inspect()
                    except docker.HTTPError:
                        continue

                    container._info = info

                    if info['Config']['Tty']:
                        continue

                    print('append', container)

                    try:
                        container.logs_start(epoll)
                    except docker.HTTPError as exc:
                        print(container, exc)
                        continue

                    try:
                        container.stats_start(epoll)
                    except docker.HTTPError as exc:
                        print(container, exc)
                        continue

                    containers1.append(container)

                for container in containers1:
                    container.logs_check(epoll)
                    container.stats_check(epoll)

                #
                docker.Container.since = int(time.time()) - 10
                with open('/srv/events/since', 'w') as f:
                    print(docker.Container.since, file=f)
                #print('since', docker.Container.since)

            #
            for fd, event in epoll.poll(0):

                container = None
                try:
                    container = [x for x in containers1 if x.logs_fd and x.logs_fd == fd][0]
                except IndexError:
                    pass
                try:
                    container = [x for x in containers1 if x.stats_fd and x.stats_fd == fd][0]
                except IndexError:
                    pass

                assert container is not None

                data = os.read(fd, 8192)

                data = buffy.get(fd, b'') + data

                while 1:
                    data, line = docker.parse(data)
                    if not line:
                        break

                    if container is not None:
                        if fd == container.logs_fd:
                            handle_log(client, container, line)
                        if fd == container.stats_fd:
                            handle_stat(client, container, line)

                    if not data:
                        break

                buffy[fd] = data
Exemple #6
0
def gitreceive(obj, repository, basename, revision, nginx):
    docker = get_docker_client()

    with open(os.path.join(repository, "Deploy.yaml")) as f:
        config = yaml.load(f)

    config["git_version"] = revision

    tag = "%s/%s" % (obj.get("build_org"), basename)
    session = obj.get("db")

    # find old running container
    echo("Looking for running containers")
    running_containers = []
    for check in docker.containers():
        if check.get("Image") == "%s:latest" % tag:
            click.echo("Found %s (%s)" % (check.get("Id"), check.get("Status")))
            running_containers.append(check.get("Id"))
    if not len(running_containers):
        click.echo("non found.")

    # build docker
    echo('Building Dockerfile')
    buildout = docker.build(path=repository, tag=tag, quiet=False, nocache=True, rm=True, stream=True)

    for l in buildout:
        row = json.loads(l)
        if row.get("stream"):
            click.echo("   " + row["stream"].strip())
            del row["stream"]
        if len(row):
            click.echo("%r" % row)

    # create container
    echo("Creating Docker Container")

    environment = config.get("environment", {})
    environment["DEPLOY_GIT_VERSION"] = config["git_version"]

    container = docker.create_container(tag,
                                        detach=True,
                                        ports=config.get("ports"),
                                        environment=environment,
                                        name=config.get("name"))
    click.echo("container %r" % container)
    container_id = container.get("Id")
    dbcontainer = DbContainer(container_id=container_id, image=tag, config=json.dumps(config))
    session.add(dbcontainer)
    session.commit()

    echo("Configuring Stuff")
    info = docker.inspect_image(tag)
    binds = {}
    volumes = info.get("ContainerConfig").get("Volumes")
    for vol in volumes:
        vpath = os.path.join(os.getcwd(), "VOLUMES", basename, vol[1:])
        os.makedirs(vpath, exist_ok=True)
        binds[vpath] = {
            'bind': vol,
            'ro': False
        }
    click.echo("Binds %r" % binds)

    # stopping old containers
    if len(running_containers):
        echo("Stopping old containers")
        for cont in running_containers:
            dbcontainer = session.query(DbContainer).get(cont)
            if dbcontainer:
                session.delete(dbcontainer)
            docker.stop(cont)

    # start container
    echo("Starting Container")

    start = docker.start(container_id, publish_all_ports=False, binds=binds)
    click.echo(start)

    if nginx:
        echo("writing nginx config file and reloading nginx")
        nginx.write(create_nginx_config(session, docker))
        subprocess.call(["sudo", "-n", "/usr/sbin/nginx", "-s", "reload"])