def create_nginx_config(session, docker):
nginx_config = nginx_config_manager.Conf()
container = {}
for row in docker.containers():
container[row["Id"]] = row
for dbcontainer in session.query(DbContainer).order_by(DbContainer.created).group_by(DbContainer.image).all():
if not dbcontainer.container_id in container:
click.secho("Container %r not running. (Image %r)" % (dbcontainer.container_id, dbcontainer.image),
fg="red")
# TODO: start container
continue
inspect = docker.inspect_container(dbcontainer.container_id)
config = json.loads(dbcontainer.config)
settings = {
"CONTAINER_IP": inspect["NetworkSettings"]["IPAddress"],
"DEPLOY_GIT_VERSION": config.get("git_version")
}
ngxa = config.get("nginx")
for ngx in ngxa:
nginx_server = nginx_config_manager.Server()
for keys in ngx.get("keys"):
for k, v in keys.items():
nginx_server.add(nginx_config_manager.Key(k, v.format(**settings)))
for l, keys_array in ngx.get("locations").items():
nginx_location = nginx_config_manager.Location(l)
for keys in keys_array:
for k, v in keys.items():
nginx_location.add(nginx_config_manager.Key(k, v.format(**settings)))
nginx_server.add(nginx_location)
nginx_config.add(nginx_config_manager.Comment("Server %r" % dbcontainer.image))
nginx_config.add(nginx_server)
return nginx_config_manager.dumps(nginx_config)
def recreate_volume_container(docker, cname):
print 'recreate_volume_container,',
if cname in ' '.join([y.encode('utf-8').replace('/', '') for x in
docker.containers(all=True) for y in x['Names'] ]):
print '"%s" container exist' % cname
remove_container(docker, cname)
else:
print '"%s" container not exist' % cname
create_container(docker, cname)
def recreate_volume_container(docker, cname):
print 'recreate_volume_container,',
if cname in ' '.join([
y.encode('utf-8').replace('/', '')
for x in docker.containers(all=True) for y in x['Names']
]):
print '"%s" container exist' % cname
remove_container(docker, cname)
else:
print '"%s" container not exist' % cname
create_container(docker, cname)
def main():
containers1 = []
epoll = select.epoll()
start = 0
buffy = {}
try:
with open('/srv/events/since') as f:
docker.Container.since = int(f.read().rstrip())
print('since', docker.Container.since)
except FileNotFoundError:
pass
with riemann_client.client.QueuedClient(riemann_client.transport.TCPTransport('localhost', 5555)) as client:
while 1:
# tight loops are bad mmkay
time.sleep(0.05)
if time.time() - start >= 1.0:
start = time.time()
containers2 = docker.containers()
a = [x for x in containers1 if x not in containers2]
b = [x for x in containers2 if x not in containers1]
for container in a:
print('remove', container)
container.logs_stop(epoll)
if container.logs_fd is not None:
if buffy.get(container.logs_fd):
print(container, 'logs', repr(buffy[container.logs_fd]))
try:
del buffy[container.logs_fd]
except KeyError:
pass
container.stats_stop(epoll)
if container.stats_fd is not None:
if buffy.get(container.stats_fd):
print(container, 'stats', repr(buffy[container.stats_fd]))
try:
del buffy[container.stats_fd]
except KeyError:
pass
containers1.remove(container)
for container in b:
try:
info = container.inspect()
except docker.HTTPError:
continue
container._info = info
if info['Config']['Tty']:
continue
print('append', container)
try:
container.logs_start(epoll)
except docker.HTTPError as exc:
print(container, exc)
continue
try:
container.stats_start(epoll)
except docker.HTTPError as exc:
print(container, exc)
continue
containers1.append(container)
for container in containers1:
container.logs_check(epoll)
container.stats_check(epoll)
#
docker.Container.since = int(time.time()) - 10
with open('/srv/events/since', 'w') as f:
print(docker.Container.since, file=f)
#print('since', docker.Container.since)
#
for fd, event in epoll.poll(0):
container = None
try:
container = [x for x in containers1 if x.logs_fd and x.logs_fd == fd][0]
except IndexError:
pass
try:
container = [x for x in containers1 if x.stats_fd and x.stats_fd == fd][0]
except IndexError:
pass
assert container is not None
data = os.read(fd, 8192)
# XXX
if data == b'0\r\n\r\n':
#print(container, data)
continue
data = buffy.get(fd, b'') + data
# XXX
if not re.match(rb'[0-9a-f]+\r\n', data, re.I):
m = re.search(rb'\r\n[0-9a-f]+\r\n', data, re.I)
if m:
print(container, 'skipping', repr(data[:m.start() + 2]))
data = data[m.start() + 2:]
else:
print(container, 'unusable', repr(data))
try:
del buffy[container.logs_fd]
except KeyError:
pass
continue
while 1:
data, line = docker.parse(data)
if not line:
break
if container is not None:
if fd == container.logs_fd:
handle_log(client, container, line)
if fd == container.stats_fd:
handle_stat(client, container, line)
if not data:
break
buffy[fd] = data
def main():
riemann_host = os.getenv('RIEMANN_HOST', 'localhost')
riemann_port = int(os.getenv('RIEMANN_PORT', '5555'))
containers1 = []
epoll = select.epoll()
start = 0
buffy = {}
try:
with open('/srv/events/since') as f:
docker.Container.since = int(f.read().rstrip())
print('since', docker.Container.since)
except FileNotFoundError:
pass
except ValueError:
pass
with riemann_client.client.QueuedClient(riemann_client.transport.TCPTransport(riemann_host, riemann_port)) as client:
while 1:
# tight loops are bad mmkay
time.sleep(0.05)
if time.time() - start >= 1.0:
start = time.time()
containers2 = docker.containers()
a = [x for x in containers1 if x not in containers2]
b = [x for x in containers2 if x not in containers1]
for container in a:
print('remove', container)
container.logs_stop(epoll)
if container.logs_fd is not None:
if buffy.get(container.logs_fd):
print(container, 'logs', 'remaining', summarise(repr(buffy[container.logs_fd])))
try:
del buffy[container.logs_fd]
except KeyError:
pass
container.stats_stop(epoll)
if container.stats_fd is not None:
if buffy.get(container.stats_fd):
print(container, 'stats', 'remaining', summarise(repr(buffy[container.stats_fd])))
try:
del buffy[container.stats_fd]
except KeyError:
pass
containers1.remove(container)
for container in b:
try:
info = container.inspect()
except docker.HTTPError:
continue
container._info = info
if info['Config']['Tty']:
continue
print('append', container)
try:
container.logs_start(epoll)
except docker.HTTPError as exc:
print(container, exc)
continue
try:
container.stats_start(epoll)
except docker.HTTPError as exc:
print(container, exc)
continue
containers1.append(container)
for container in containers1:
container.logs_check(epoll)
container.stats_check(epoll)
#
docker.Container.since = int(time.time()) - 10
with open('/srv/events/since', 'w') as f:
print(docker.Container.since, file=f)
#print('since', docker.Container.since)
#
for fd, event in epoll.poll(0):
container = None
try:
container = [x for x in containers1 if x.logs_fd and x.logs_fd == fd][0]
except IndexError:
pass
try:
container = [x for x in containers1 if x.stats_fd and x.stats_fd == fd][0]
except IndexError:
pass
assert container is not None
data = os.read(fd, 8192)
data = buffy.get(fd, b'') + data
while 1:
data, line = docker.parse(data)
if not line:
break
if container is not None:
if fd == container.logs_fd:
handle_log(client, container, line)
if fd == container.stats_fd:
handle_stat(client, container, line)
if not data:
break
buffy[fd] = data
def gitreceive(obj, repository, basename, revision, nginx):
docker = get_docker_client()
with open(os.path.join(repository, "Deploy.yaml")) as f:
config = yaml.load(f)
config["git_version"] = revision
tag = "%s/%s" % (obj.get("build_org"), basename)
session = obj.get("db")
# find old running container
echo("Looking for running containers")
running_containers = []
for check in docker.containers():
if check.get("Image") == "%s:latest" % tag:
click.echo("Found %s (%s)" % (check.get("Id"), check.get("Status")))
running_containers.append(check.get("Id"))
if not len(running_containers):
click.echo("non found.")
# build docker
echo('Building Dockerfile')
buildout = docker.build(path=repository, tag=tag, quiet=False, nocache=True, rm=True, stream=True)
for l in buildout:
row = json.loads(l)
if row.get("stream"):
click.echo(" " + row["stream"].strip())
del row["stream"]
if len(row):
click.echo("%r" % row)
# create container
echo("Creating Docker Container")
environment = config.get("environment", {})
environment["DEPLOY_GIT_VERSION"] = config["git_version"]
container = docker.create_container(tag,
detach=True,
ports=config.get("ports"),
environment=environment,
name=config.get("name"))
click.echo("container %r" % container)
container_id = container.get("Id")
dbcontainer = DbContainer(container_id=container_id, image=tag, config=json.dumps(config))
session.add(dbcontainer)
session.commit()
echo("Configuring Stuff")
info = docker.inspect_image(tag)
binds = {}
volumes = info.get("ContainerConfig").get("Volumes")
for vol in volumes:
vpath = os.path.join(os.getcwd(), "VOLUMES", basename, vol[1:])
os.makedirs(vpath, exist_ok=True)
binds[vpath] = {
'bind': vol,
'ro': False
}
click.echo("Binds %r" % binds)
# stopping old containers
if len(running_containers):
echo("Stopping old containers")
for cont in running_containers:
dbcontainer = session.query(DbContainer).get(cont)
if dbcontainer:
session.delete(dbcontainer)
docker.stop(cont)
# start container
echo("Starting Container")
start = docker.start(container_id, publish_all_ports=False, binds=binds)
click.echo(start)
if nginx:
echo("writing nginx config file and reloading nginx")
nginx.write(create_nginx_config(session, docker))
subprocess.call(["sudo", "-n", "/usr/sbin/nginx", "-s", "reload"])