class DockerEventManager(Thread):
def __init__(self, manager, url=None):
super(DockerEventManager, self).__init__()
self.manager = manager
self.url = url
self.daemon = True
self.client = Client(self.url)
def run(self):
for payload in self.client.events():
event = loads(payload)
status = event.pop("status")
docker_event = DOCKER_EVENTS.get(status)
if docker_event is not None:
self.manager.fire(docker_event(**event), "docker")
else:
print(
"WARNING: Unknown Docker Event <{0:s}({1:s})>".format(
status, repr(event)
),
file=sys.stderr
)
def stop(self):
self.client.close()
class DockerUtils:
def __init__(self, socketPath):
self._cli = Client(base_url=socketPath, version='auto')
def getStats(self, containerName):
stats_obj = self._cli.stats(containerName, True, True)
return stats_obj
def getInfo(self, containerId):
info_obj = self._cli.inspect_container(container=containerId)
return info_obj
def getEvents(self, filters):
events_obj = self._cli.events(filters= filters, decode=True)
return events_obj
def getLogs(self, containerName, streamLog):
log_string = self._cli.logs(container = containerName, stream = streamLog)
return log_string
def getRegexpFromLogs(self, log_string, regularExpression, group):
match = re.search(r""+regularExpression, log_string, re.MULTILINE)
if match:
return match.group(group)
else:
return None
def main():
# register the exit signals
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
args = parse_args()
global hosts_path
hosts_path = args.file
docker = Client(base_url='unix://%s'%args.socket)
events = docker.events(decode=True)
#get running containers
for c in docker.containers(quiet=True,all=False):
container_id = c["Id"]
container = get_container_data(docker, container_id)
hosts[container_id] = container
update_hosts_file()
#listen for events to keep the hosts file updated
for e in events:
status = e["status"];
if status =="start":
container_id = e["id"]
container = get_container_data(docker, container_id)
hosts[container_id] = container
update_hosts_file()
if status=="stop" or status=="die" or status=="destroy":
container_id = e["id"]
if container_id in hosts:
hosts.pop(container_id)
update_hosts_file()
def main():
cli = Client(base_url="unix://var/run/docker.sock")
hosts = {"static": OrderedDict(), "dynamic": OrderedDict()}
for container in cli.containers():
data = container["Image"].split("/")
if len(data) != 2:
continue
host, name = data
if host == "tellendil":
if name in ["static", "dynamic"]:
hosts[name][container["Id"]] = container["NetworkSettings"]["Networks"]["bridge"]["IPAddress"]
update_conf(safe_get(hosts, "static"), safe_get(hosts, "dynamic"))
for event in cli.events(decode=True):
data = event.get("from", "").split("/")
if len(data) != 2:
continue
host, name = data
if event["Action"] == "die" and host == "tellendil":
if name in ["static", "dynamic"]:
hosts[name].pop(event["id"])
update_conf(safe_get(hosts, "static"), safe_get(hosts, "dynamic"))
elif event["Action"] == "start" and host == "tellendil":
if name in ["static", "dynamic"]:
hosts[name][event["id"]] = cli.inspect_container(event["id"])["NetworkSettings"]["Networks"]["bridge"]["IPAddress"]
update_conf(safe_get(hosts, "static"), safe_get(hosts, "dynamic"))
class DockerUtils:
def __init__(self, socketPath):
self._cli = Client(base_url=socketPath, version='auto')
def getStats(self, containerName):
stats_obj = self._cli.stats(containerName, False, False)
return stats_obj
def getInfo(self, containerId):
info_obj = self._cli.inspect_container(container=containerId)
return info_obj
def getEvents(self, filters):
events_obj = self._cli.events(filters=filters, decode=True)
return events_obj
def getLogs(self, containerName, streamLog):
log_string = self._cli.logs(container=containerName, stream=streamLog)
return log_string
def getRegexpFromLogs(self, log_string, regularExpression, group):
match = re.search(r"" + regularExpression, log_string, re.MULTILINE)
if match:
return match.group(group)
else:
return None
class DockerEventManager(Thread):
def __init__(self, manager, url=None):
super(DockerEventManager, self).__init__()
self.manager = manager
self.url = url
self.daemon = True
self.client = Client(self.url)
def run(self):
for payload in self.client.events():
event = loads(payload)
status = event.pop("status")
docker_event = DOCKER_EVENTS.get(status)
if docker_event is not None:
self.manager.fire(docker_event(**event), "docker")
else:
print("WARNING: Unknown Docker Event <{0:s}({1:s})>".format(
status, repr(event)),
file=sys.stderr)
def stop(self):
self.client.close()
def sync(docker_client: docker.Client, zk: KazooClient, node_path: str,
advertise_name: str):
events = docker_client.events()
for event_str in events:
if end:
break
event = json.loads(event_str.decode('utf-8'))
if 'Type' in event and event['Type'] == 'container' and 'id' in event:
container_id = event['id']
container_zk_path = "{}/{}".format(node_path, container_id)
if 'Action' in event:
if event['Action'] == 'destroy':
update_data(zk, container_zk_path, {}, advertise_name)
else:
try:
container = docker_client.inspect_container(
container_id)
update_data(zk, container_zk_path, container,
advertise_name)
except docker.errors.APIError:
pass
except requests.exceptions.HTTPError:
pass
def main():
cli = Client(base_url='unix://var/run/docker.sock')
for event in cli.events(decode=True):
eventStatus = get_status(event)
if eventStatus in ['create', 'die']:
newImgHash = getNewContImageHash(cli, event)
print(newImgHash)
extendPCR(newImgHash)
def loop(self):
if not Client: raise ImportError('could not import docker client')
docker = Client(base_url=self._docker_url)
logger.info("waiting for events ...")
for event in docker.events(decode=True):
# it looks like this {"status":"die","id":"123","from":"foobar/eggs:latest","time":1434047926}
if event['status'] in self._watch_set:
docker_inspects=[docker.inspect_container(cn['Id']) for cn in docker.containers()]
self.reconfig(docker_inspects)
def main():
cli = Client(base_url='unix://var/run/docker.sock')
with daemon.DaemonContext():
for event in cli.events(decode=True):
eventStatus = get_status(event)
if eventStatus in ['create', 'die']:
newImgHash = getNewContImageHash(cli, event)
extendPCR(newImgHash)
performAction(eventStatus)
def ddns():
client = Client(base_url='unix://var/run/docker.sock')
for event in client.events():
event = json.loads(event)
if event['status'] == 'start':
container = client.inspect_container(event['id'])
name = container['Name']
ip = container['NetworkSettings']['IPAddress']
update(name[1:], ip)
def loop(self):
if not Client: raise ImportError('could not import docker client')
docker = Client(base_url=self._docker_url)
logger.info("waiting for events ...")
for event in docker.events(decode=True):
# it looks like this {"status":"die","id":"123","from":"foobar/eggs:latest","time":1434047926}
if event['status'] in self._watch_set:
docker_inspects = [
docker.inspect_container(cn['Id'])
for cn in docker.containers()
]
self.reconfig(docker_inspects)
class DockerManager:
def __init__(self):
self.cli = Client(base_url='unix://var/run/docker.sock')
self.events = self.cli.events()
def run(self):
while True:
message = self.events.next()
try:
self.handler(json.loads(message))
except Exception, e:
traceback.print_exc()
def event_stream(images=['redis1', ]):
"""
Read events from docker's event stream and call the container monitor
"""
client = Client(base_url='unix://var/run/docker.sock')
logging.info('Listening for docker events...')
try:
for event in client.events():
event = json.loads(event)
container_monitor(event)
except KeyboardInterrupt:
sys.exit(0)
def listener():
'''
Listen to docker events, and invoke event_handler if a
container was started.
'''
log.info('Listening for Docker events')
cli = Client(version='auto')
for event in cli.events():
event = json.loads(event.decode('utf-8'))
if event.get('status') == 'start':
log.info('Event - Container starting: %s', event)
try:
event_handler(event)
except Exception as event_exception:
notifier(False, str(event_exception))
continue
def event_stream(images=[
'redis1',
]):
"""
Read events from docker's event stream and call the container monitor
"""
client = Client(base_url='unix://var/run/docker.sock')
logging.info('Listening for docker events...')
try:
for event in client.events():
event = json.loads(event)
container_monitor(event)
except KeyboardInterrupt:
sys.exit(0)
class MonitorThread(Thread):
def __init__(self, app, sock, dockerEvents):
super(MonitorThread, self).__init__()
self.app = app
self.sock = sock
self.dockerEvents = dockerEvents
self.cli = Client(base_url=self.sock)
def run(self):
# Listen for Docker events
for event in self.cli.events():
event = json.loads(event.decode('utf-8'))
if event.get("status") in self.dockerEvents:
self.app.updateProxy()
def stop(self):
self.cli.close()
def listen():
logging.info("Eveli is alive!")
cli = Client(base_url='unix:///var/run/docker.sock')
# Just in case, run the networks-fixer at startup as well
ensure_networks(cli)
# Could use decorators to register the event handlers when we get more of them
event_handlers = [handler_nginx_reload, handler_compose_networks]
for event in cli.events():
event = json.loads(event.decode('utf-8'))
for handler in event_handlers:
try:
handler(cli, event)
except:
logging.exception("Event handler %s failed", handler.__name__)
def event_producer(docker_url, queue, monitor, events):
try:
client = Client(base_url=docker_url, version="auto")
if events:
filters = dict(event=events)
else:
filters = None
for raw_event in client.events(decode=True, filters=filters):
log.debug("Received event %s", raw_event)
event = DotAccessDict(time=raw_event['time'],
container_id=raw_event['id'],
status=raw_event['status'],
image=raw_event.get('from'),
details={})
if raw_event['status'] != 'destroy':
try:
raw = client.inspect_container(raw_event['id'])
except Exception as e:
log.error("can't get container details for %s: %s",
raw_event['id'], e)
raw = {}
event['details'] = raw
if event['details']:
event['name'] = event['details']['Name'].replace('/', '', 1)
else:
event['name'] = event['container_id']
event = add_dot_access(event)
queue.put(("ev", event))
except Exception as e:
log.error("Error contacting docker daemon: %s", e, exc_info=True)
finally:
monitor.set()
def event_producer(docker_url, queue, monitor, events):
try:
client = Client(base_url=docker_url, version="auto")
if events:
filters = dict(event=events)
else:
filters = None
for raw_event in client.events(decode=True, filters=filters):
log.debug("Received event %s", raw_event)
event = DotAccessDict(
time=raw_event['time'],
container_id=raw_event['id'],
status=raw_event['status'],
image=raw_event.get('from'),
details={})
if raw_event['status'] != 'destroy':
try:
raw = client.inspect_container(raw_event['id'])
except Exception as e:
log.error("can't get container details for %s: %s", raw_event['id'], e)
raw = {}
event['details'] = raw
if event['details']:
event['name'] = event['details']['Name'].replace('/', '', 1)
else:
event['name'] = event['container_id']
event = add_dot_access(event)
queue.put(("ev", event))
except Exception as e:
log.error("Error contacting docker daemon: %s", e, exc_info=True)
finally:
monitor.set()
def main():
"""
Main method.
This method holds what you want to execute when
the script is run on command line.
"""
args = get_arguments()
setup_logging(args)
docker_server = Client(**(kwargs_from_env()))
LOGGER.debug(
('Setting up a bind server with '
'ip {} for domain {} and zone {}').format(args.server, args.domain,
args.zone))
bind = BindServer(args.server, args.domain, args.key, args.zone)
if args.register_running:
LOGGER.info('Registering existing containers')
containers = docker_server.containers()
for configuration in containers:
container = Container(configuration)
LOGGER.debug(
('Registering container with '
'hostname {} and id {}').format(container.hostname,
container.id))
bind.register(container)
for event in docker_server.events():
data = json.loads((event.decode('utf-8')))
status = data.get('status', False)
if status in ('start', 'destroy', 'die'):
LOGGER.debug('Got status {}'.format(status))
details = docker_server.inspect_container(data.get('id'))
container = Container(details)
if status == 'start':
LOGGER.info('Trying to add {}'.format(container.hostname))
bind.register(container)
elif status in ('destroy', 'die'):
LOGGER.info('Trying to delete {}'.format(container.hostname))
bind.delete(container)
def main():
PORT = int(os.environ['HTTP_PORT'])
Handler = http.server.SimpleHTTPRequestHandler
httpd = socketserver.TCPServer(("", PORT), Handler)
print("serving at port", PORT)
server = Thread(target=httpd.serve_forever)
server.start()
monitors = {}
cli = Client(base_url='unix://var/run/docker.sock')
events = cli.events(since=0,decode=True)
for e in events:
if 'id' not in e:
print('WARNIG: id not in e',e)
continue
cid = e['id']
status = e['status']
if status in ['start']:
print('Starting Monitor on',cid)
if cid in monitors:
print('WARNIG: Unexpected cid')
continue
stop = Event()
mon = Monitor(cid,stop)
monitors[cid] = (mon,stop)
mon.start()
elif status in ['die']:
print('Stoping Monitor on',cid)
if cid not in monitors:
print('WARNIG: Unexpected cid')
continue
(mon,stop) = monitors[cid]
stop.set()
mon.join(1)
print('Stopped')
del monitors[cid]
elif status in ['create','destroy','kill','stop','commit','delete']:
print(status)
pass
else:
print('WARNIG: Unexpected status:',e)
pass
def main():
cli = Client(base_url="unix://var/run/docker.sock")
hosts = {"static": OrderedDict(), "dynamic": OrderedDict()}
for container in cli.containers():
data = container["Image"].split("/")
if len(data) != 2:
continue
host, name = data
if host == "tellendil":
if name in ["static", "dynamic"]:
hosts[name][container["Id"]] = container["NetworkSettings"][
"Networks"]["bridge"]["IPAddress"]
update_conf(safe_get(hosts, "static"), safe_get(hosts, "dynamic"))
for event in cli.events(decode=True):
data = event.get("from", "").split("/")
if len(data) != 2:
continue
host, name = data
if event["Action"] == "die" and host == "tellendil":
if name in ["static", "dynamic"]:
hosts[name].pop(event["id"])
update_conf(safe_get(hosts, "static"),
safe_get(hosts, "dynamic"))
elif event["Action"] == "start" and host == "tellendil":
if name in ["static", "dynamic"]:
hosts[name][event["id"]] = cli.inspect_container(
event["id"]
)["NetworkSettings"]["Networks"]["bridge"]["IPAddress"]
update_conf(safe_get(hosts, "static"),
safe_get(hosts, "dynamic"))
def dockerClient():
global client, events
client = Client(base_url='unix://var/run/docker.sock', version='auto')
events = client.events(decode=True)
return binds
# --- Run loop
# Connect to docker via docker.sock or TCP
if os.environ.get('DOCKER_HOST') is not None:
kwargs = kwargs_from_env()
kwargs['tls'].assert_hostname = False
cli = Client(**kwargs)
else:
cli = Client(base_url="unix://var/run/docker.sock")
# Blocking event loop
try:
for event in cli.events(decode=True):
# On tag events check for running containers running this image
if event['Action'] == "tag":
imageId = event['Actor']['ID']
for container in cli.containers():
name = event['Actor']['Attributes']['name'].replace(":latest", "")
nameShort = event['Actor']['Attributes']['name']
container_data = cli.inspect_container(container["Id"])
# On container found remove the container and reload it preserving env, ports, and mappings
if container_data["Config"]["Image"] == nameShort or container_data["Config"]["Image"] == name:
print("reloading %s\n" % event['Actor']['Attributes']['name'])
cli.remove_container(container["Id"], force=True)
result = cli.create_container(
import os
import json
import sys
import logging
import etcd
from docker import Client
cli = Client(base_url='unix:///var/run/docker.sock')
events = cli.events(decode=True)
logging.basicConfig(level=logging.DEBUG)
etcd_hostname = 'etcd'
etcd_client = etcd.Client(host=etcd_hostname)
def get_container(message):
container = message.get('Actor')
container['Attributes']['ID'] = container['ID']
return container['Attributes']
def get_envvar(container, to_find):
container_details = cli.inspect_container(container['ID'])
env = container_details['Config']['Env']
for envvar in env:
if envvar.startswith(to_find + '='):
return envvar.split('=')[1]
return None
def get_container_hostname(container):
vhosts = dict()
for container in c.containers():
inspect = c.inspect_container(container['Id'])
for env in inspect['Config']['Env']:
if '=' in env:
item, value = env.split('=', 2)
if item == 'VIRTUAL_HOST':
if ':' in value:
vhost, port = value.split(':', 2)
else:
vhost = value
if 'Ports' in container and len(container) > 0 and 'PublicPort' in container['Ports'][0]:
port = container['Ports'][0]['PublicPort']
else:
port = None
if port:
if vhost not in vhosts: vhosts[vhost] = []
vhosts[vhost].append(port)
return vhosts
if __name__ == "__main__":
c = Client(base_url=DOCKER_SOCKET)
for e in c.events():
event = json.loads(e)
if event['status'] in [ 'start', 'stop', 'die' ]:
data['vhosts'] = get_vhosts()
try:
requests.post(API_URL, json = data)
except Exception, e:
print e.message
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
filename=(args.log_file if args.log_file != '-' else None))
if args.zone is None:
args.zone = args.domain
logging.info("Starting with arguments %s", args)
c = Client(**(kwargs_from_env()))
resolver = Resolver()
resolver.nameservers = [args.server]
if args.catchup:
logging.info("Registering existing containers")
containers = c.containers()
for container in containers:
register_container(container["Id"])
events = c.events(decode=True)
for event in events:
logging.debug("Event message: %s", repr(event))
if event['Action'] == "start":
register_container(event['id'])
elif event['Action'] == "die":
remove_container(event['id'])
# 2014-11-28T15:32:04.000000000+01:00 a3d66b00acc9adbdbdbc91cc664d2d94b6a07cc4295c5cf54fcc595e2aa92a43: (from mongo:latest) restart
# 2015-03-05T08:36:14.000000000+01:00 eb75c1a5ad836d008b0fd66bf6b1ea353510175e8caa619e59d9851029b1ceca: (from ggtools/zabbix-server:latest) exec_start: ifconfig eth0
def dock_events(api):
cli = Client(base_url='tcp://'+api['docker-ip'] +':2375')
return cli.events()
class Agent(object):
"""
Collects stats from all containers on a single Docker host, appending
container name and id fields and publishing to redis
params:
- docker_host(str): full base_url of a Docker host to connect to.
(e.g. 'tcp://127.0.0.1:4243')
- redis_host(str): redis host to connect to. default 127.0.0.1
- redis_port(int): port to connect to redis host on. default 6379
"""
def __init__(self,docker_host,redis_host='127.0.0.1',redis_port=6379):
self.docker = Client(base_url=docker_host)
self.source = self.docker.info()['Name']
self.ncpu = self.docker.info()['NCPU']
self.redis = StrictRedis(host=redis_host,port=redis_port,db=0)
self.children = []
self.stopped = False
log.info('Connected to Docker API at url %s' % docker_host)
output('starting collector on source %s' % self.source)
self.start()
def start(self):
signal.signal(signal.SIGINT, self._sig_handler)
#start a collector for all existing containers
for cid in [ c['Id'] for c in self.docker.containers() ]:
self._add_collector(cid)
#start event listener
self._event_listener()
def _sig_handler(self, signal, frame):
self.stopped = True
sys.exit(0)
def _event_listener(self):
"""
Listen for docker events and dynamically add or remove
stat collectors based on start and die events
"""
output('started event listener')
for event in self.docker.events():
event = json.loads(event.decode('utf-8'))
if event['status'] == 'start':
self._add_collector(event['id'])
if event['status'] == 'die':
self._remove_collector(event['id'])
def _collector(self,cid,cname):
"""
Collector instance collects stats via Docker API streaming web socket,
appending container name and source, and publishing to redis
params:
- cid(str): ID of container to collect stats from
- cname(str): Name of container
"""
sleep(5) # sleep to allow container to fully start
output('started collector for container %s' % cid)
stats = self.docker.stats(cid, decode=True)
for stat in stats:
#append additional information to the returned stat
stat['container_name'] = cname
stat['container_id'] = cid
stat['source'] = self.source
stat['ncpu'] = self.ncpu
self.redis.publish('statsquid', msgpack.packb(stat))
if self.stopped:
break
#####
# collector methods
#####
def _add_collector(self,cid):
log.debug('creating collector for container %s' % cid)
cname = self.docker.inspect_container(cid)['Name'].strip('/')
p = Process(target=self._collector,name=cid,args=(cid,cname))
p.start()
self.children.append(p)
def _remove_collector(self,cid):
c = self._get_collector(cid)
c.terminate()
while c.is_alive():
sleep(.2)
output('collector stopped for container %s' % cid)
self.children = [ c for c in self.children if c.name != cid ]
def _get_collector(self,cid):
return [ p for p in self.children if p.name == cid ][0]
class ECSIDMapAgent():
def __init__(self, server_endpoint, log_level):
self.id_map = {}
self.new_id_map = {}
self.server_endpoint = server_endpoint
self.logger = self._setup_logger(log_level)
self.backoff_time = 2
self.current_backoff_time = self.backoff_time
self.current_retry = 0
self.max_retries = 2
self.hostname = gethostname()
self.instance_ip = self.get_instance_metadata('local-ipv4') # set these at object constr. as they don't change
self.instance_id = self.get_instance_metadata('instance-id')
self.instance_type = self.get_instance_metadata('instance-type')
self.instance_az = self.get_instance_metadata('placement/availability-zone')
self.docker_client = Client(base_url='unix://var/run/docker.sock', version='1.21')
@staticmethod
def _setup_logger(log_level):
logger = logging.getLogger('ecs_id_mapper_agent')
logger.setLevel(log_level.upper())
logger.propagate = False
stderr_logs = logging.StreamHandler()
stderr_logs.setLevel(getattr(logging, log_level))
stderr_logs.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
logger.addHandler(stderr_logs)
return logger
def _retry(self):
self.logger.debug('backoff: {} retry: {} max_retry {}'.format(self.current_backoff_time,
self.current_retry, self.max_retries))
if self.current_retry >= self.max_retries:
self.logger.info('Max _retry reached. Aborting')
self.current_retry = 0
self.current_backoff_time = self.backoff_time
return False
else:
self.current_retry += 1
self.logger.info('Sleeping for {} seconds'.format(str(self.current_backoff_time)))
time.sleep(self.current_backoff_time)
self.current_backoff_time **= 2
return True
def _http_connect(self, url, timeout=1):
self.logger.debug('Making connection to: {}'.format(url))
while True:
try:
r = requests.get(url, timeout=timeout)
return r
except requests.exceptions.ConnectionError:
self.logger.error('Connection error accessing URL {}'.format(str(url)))
if not self._retry():
return None
except requests.exceptions.Timeout:
self.logger.error(
'Connection timeout accessing URL {}. Current timeout value {}'.format(url, str(timeout)))
if not self._retry():
return None
def get_instance_metadata(self, path):
self.logger.info('Checking instance metadata for {}'.format(path))
metadata = self._http_connect('http://169.254.169.254/latest/meta-data/{}'.format(path))
if metadata:
return metadata.text
else:
return ""
@staticmethod
def get_container_ports(container_id):
try:
cmd = ["/usr/bin/docker", "port", container_id[:12]]
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output, errors = p.communicate()
if errors or len(output) < 1:
return "0", "0"
cport, hport = output.split("-")
cport = cport.split('/')[0]
hport = hport.split(':')[1].strip()
return cport, hport
except (OSError, ValueError):
return "0", "0"
def get_ecs_agent_tasks(self):
self.logger.info('Requesting data from ECS agent')
ecs_agent_tasks_response = self._http_connect('http://127.0.0.1:51678/v1/tasks')
ecs_agent_metadata_response = self._http_connect('http://127.0.0.1:51678/v1/metadata')
if ecs_agent_tasks_response and ecs_agent_metadata_response:
ecs_agent_tasks = ecs_agent_tasks_response.json()
ecs_agent_metadata = ecs_agent_metadata_response.json()
else:
ecs_agent_tasks = None
ecs_agent_metadata = None
return False
id_map = {}
cluster_name = ecs_agent_metadata['Cluster']
ecs_agent_version = ecs_agent_metadata['Version']
for task in ecs_agent_tasks['Tasks']:
task_id = str(task['Arn'].split(":")[-1][5:])
desired_status = str(task['DesiredStatus'])
known_status = str(task['KnownStatus'])
task_name = str(task['Family'])
task_version = str(task['Version'])
for container in task['Containers']:
docker_id = str(container['DockerId'])
if desired_status == "RUNNING":
container_port, instance_port = self.get_container_ports(docker_id)
else:
container_port, instance_port = "0", "0"
container_name = str(container['Name'])
pkey = hashlib.sha256()
pkey.update(docker_id)
pkey.update(task_id)
pkey.update(desired_status)
id_map[pkey.hexdigest()] = {'container_id': docker_id,
'container_name': container_name,
'container_port': container_port,
'task_id': task_id,
'task_name': task_name,
'task_version': task_version,
'instance_port': instance_port,
'instance_ip': self.instance_ip,
'instance_id': self.instance_id,
'instance_type': self.instance_type,
'instance_az': self.instance_az,
'desired_status': desired_status,
'known_status': known_status,
'host_name': self.hostname,
'cluster_name': cluster_name,
'ecs_agent_version': ecs_agent_version,
'sample_time': time.time()}
# Update internal state
self.new_id_map = copy.deepcopy(id_map)
def report_event(self, event_id, action):
for id in event_id:
self.logger.info('Reporting new container event. Action {}. Event id: {}'.format(action, id))
headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
while True:
try:
r = requests.post(path.join(self.server_endpoint, 'report/event'),
headers=headers,
data=json.dumps({'event_id': id, 'event': action, 'timestamp': time.time()}))
self.logger.debug("HTTP response: " + str(r.status_code))
break
except requests.exceptions.ConnectionError:
self.logger.info('Unable to connect to server endpoint. Sleeping for {} seconds'.format(
str(self.current_backoff_time)))
if not self._retry():
break
def report_map(self):
self.logger.info('Reporting current id map')
headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
while True:
try:
r = requests.post(path.join(self.server_endpoint, 'report/map'),
headers=headers, data=json.dumps(self.id_map))
self.logger.debug("HTTP response: " + str(r.status_code))
break
except requests.exceptions.ConnectionError:
self.logger.info('Unable to connect to server endpoint. Sleeping for {} seconds'.format(
str(self.current_backoff_time)))
if not self._retry():
break
def compare_hash(self):
self.logger.info('Comparing known state to current state')
containers_added = set(self.new_id_map.keys()) - set(self.id_map.keys())
containers_removed = set(self.id_map.keys()) - set(self.new_id_map.keys())
if len(containers_added) > 0:
self.logger.info('Containers added {}'.format(containers_added))
self.report_event(containers_added, 'added')
if len(containers_removed) > 0:
self.logger.info('Containers removed {}'.format(containers_removed))
self.report_event(containers_removed, 'removed')
if len(containers_removed) > 0 or len(containers_added) > 0:
self.id_map = copy.deepcopy(self.new_id_map)
self.report_map()
else:
self.logger.info('No container actions to report')
def run(self):
"""
Blocking method to run agent
:return:
"""
self.logger.info('Starting agent')
for event in self.docker_client.events(decode=True):
self.logger.debug(str(event))
if event['status'] == 'start' or event['status'] == 'die':
agent.get_ecs_agent_tasks()
agent.compare_hash()
class Watcher(object):
"""
The watcher that will be observing the docker events
"""
def __init__(self):
"""
the watcher will be created with a docker-py client
and a set of filters for event watching
:return:
"""
self.client = Client()
self.filters = {
'event': ['start', 'restart', 'die', 'stop'],
'type': 'container'
}
self.host = Host('docker')
def process_event(self, event):
"""
the processing of the event, the event will be a json encoded raw string
representation of the event
:param event:
:return:
"""
self.host.reset()
self.host_entries()
self.host.save()
self.create_host_file()
def create_host_file(self):
"""
:return:
"""
f = open('/hosts', 'w+')
f.write(self.base + "\n" + str(self.host))
f.close()
def start(self):
"""
starts the observer and the event stream for processing
:return:
"""
with open('/hosts.d/base', 'r+') as fl:
self.base = fl.read()
self.host_entries()
self.host.save()
self.create_host_file()
for event in self.client.events(filters=self.filters):
self.process_event(event)
def process_container_host(self, container):
"""
:param container:
:return:
"""
details = self.client.inspect_container(container['Id'])
hostname = details['Config']['Hostname']
if details['Config']['Domainname'] != "":
hostname += '.'+details['Config']['Domainname']
names = [name.split('/').pop() for name in container['Names']] + [hostname]
for ip in map(lambda (k, v): v['IPAddress'], details['NetworkSettings']['Networks'].iteritems()):
for name in names:
self.host.add(ip, name)
def main(arguments):
logging.basicConfig(level=logging.INFO)
# retrieve configuration file with default values
config_parser = argparse.ArgumentParser(add_help=False)
config_parser.add_argument('-c', '--config', nargs='?')
(config_arg, arguments) = config_parser.parse_known_args(arguments)
defaults = dict(DEFAULTS)
if config_arg.config is not None:
if os.path.isfile(config_arg.config):
try:
configp = ConfigParser.SafeConfigParser()
configp.read(config_arg.config)
defaults.update(dict(configp.items('docker-listen')))
except Exception:
logging.exception('File %s can not be read', config_arg.config)
else:
logging.warn('File %s can not be read', config_arg.config)
# fix boolean value
if defaults['systemctl_enabled'] in ('True', 'yes', '1'):
defaults['systemctl_enabled'] = True
defaults['sighup_enabled'] = False
elif defaults['sighup_enabled'] in ('True', 'yes', '1'):
defaults['systemctl_enabled'] = False
defaults['sighup_enabled'] = True
else:
defaults['systemctl_enabled'] = False
defaults['sighup_enabled'] = False
# clean process name
defaults['sighup_process_name'] = defaults['sighup_process_name'].replace('\'', '')
defaults['systemctl_service_name'] = defaults['systemctl_service_name'].replace('\'', '')
# retrieve configuration ; configuration file provides defaults values
parser = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('-c', '--config', nargs='?', metavar='FILE', help='load configuration from .ini config file (section docker-listen)')
parser.add_argument('--docker-url', nargs='?', metavar='URL', help='docker socket path (unix://var/run/docker.sock) or docker url')
parser.add_argument('--systemctl-enabled', nargs='?', type=bool, choices=('yes', 'no'), help='systemctl is enable ?')
parser.add_argument('--systemctl-service-name', metavar='NAME', nargs='?', help='name of the service to restart')
parser.add_argument('--sighup-enabled', nargs='?', type=bool, choices=('yes', 'no'), help='sighup process on events ?')
parser.add_argument('--sighup-process-name', metavar='NAME', nargs='?', help='name of the process to sighup (with killall)')
parser.add_argument('--hosts-dir', nargs='?', metavar='DIR_PATH', help='directory where hosts files are stored ; all files in this directory will be deleted')
parser.add_argument('--log-level', nargs='?', choices=('DEBUG', 'INFO', 'WARN', 'ERROR'))
logging.debug('Using defaults %s', pprint.pformat(defaults))
parser.set_defaults(**defaults)
configuration = parser.parse_args(arguments)
# set logging level and start working
logging.getLogger('').setLevel(configuration.log_level)
logging.info('Current configuration : %s', pprint.pformat(configuration))
# check configuration
if not os.path.isdir(configuration.hosts_dir):
logging.error('hosts_dir \'%s\' is not a directory. Stopping.', configuration.hosts_dir)
return 2
try:
client = Client(base_url=configuration.docker_url)
client.ping()
except Exception:
logging.exception('Error communicating with docker socket %s. Stopping.', configuration.docker_url)
return 2
logging.info('Docker-listen started')
events = client.events(decode=True)
os.umask(0000)
clean_all(configuration)
init_all(configuration, client)
try:
for event in events:
logging.debug(event)
if event['Action'] == 'connect':
handle_start(configuration, client, event)
elif event['Action'] == 'disconnect':
handle_stop(configuration, client, event)
except Exception:
logging.exception('Error processing docker events. Stopping.')
return 2
return 0
if __name__ == '__main__':
docker = Client()
hostname = getenv('HOSTNAME', None)
if hostname:
if '.' not in hostname:
hostname = "%s%s" % (hostname, DOMAIN_SUFFIX)
print "Adding %s" % hostname
hosts = Hosts(HOSTS_PATH)
my_ip = commands.getoutput("ip -4 -f inet -o addr show eth0 | awk '{print $4}' | cut -d/ -f1")
hosts.set_one(hostname, my_ip)
hosts.write(HOSTS_PATH)
print "Go to http://%s/" % hostname
for event in docker.events():
event = json.loads(event)
if 'status' not in event:
continue
if event['status'] == 'start':
hostname = get_hostname(event['id'])
if hostname is None:
continue
print "Adding %s" % hostname
hosts = Hosts(HOSTS_PATH)
hosts.set_one(hostname, get_ip(event['id']))
hosts.write(HOSTS_PATH)
elif event['status'] == 'die':
hostname = get_hostname(event['id'])
print "Removing %s" % hostname
hosts = Hosts(HOSTS_PATH)
class DockerSpy:
"""
# Usage:
docker = dockerspy(container_api_url='unix://var/run/docker.sock')
# print docker information
print(docker.describe_containers)
print(docker.env(container='chargeback-sync'))
print(docker.virtual_port(container='chargeback-sync'))
print(docker.ip_address(container='chargeback-sync'))
# listen to events from docker API, similar to docker events
docker.events()
# print all containers information as a dictionary structure
docker.containers
"""
def __init__(self,
container=None,
container_api_url='unix://var/run/docker.sock'):
"""
:rtype: object
:type dockerspy: object
"""
self._client = Client(base_url=container_api_url)
self.containers = {}
if container:
self.containers[container] = self._client.inspect_container(
container=container)
self.tagged_containers = {}
def get_containers(self):
"""
:rtype : list
:return: containers
"""
try:
# list live containers names using list comprehension
containers = [
container['Names'][0].replace('/', '')
for container in self._client.containers()
]
return containers
except Exception:
raise
@staticmethod
def _into_dict(kv_config):
env_dict = {}
for item in kv_config:
k, v = item.split('=', 1)
env_dict[k] = v
return env_dict
@property
def describe_containers(self):
# update self.containers
for name in self.get_containers():
self.containers[name] = self._client.inspect_container(name)
self.containers[name]['Config']['Env'] = self._into_dict(
kv_config=self.containers[name]['Config']['Env'])
return self.containers
def events(self):
"""
"""
try:
return self._client.events(decode=True)
except KeyboardInterrupt:
pass
def env(self, container=None):
"""
:rtype: dict
:return: container's environment variable
"""
"""
# internal method to transform k=v environment variable into dict
def _into_dict(kv_config):
env_dict = {}
for item in kv_config:
k, v = item.split('=', 1)
env_dict[k] = v
return env_dict
"""
# If self.cotnainers exists or grater then 0 return Config.Env
if self.containers:
return self.containers[container]['Config']['Env']
else:
return self.describe_containers[container]['Config']['Env']
def virtual_port(self, container):
# internal method to transform k=v environment variable into dict
"""
def _into_dict(kv_config):
env_dict = {}
for item in kv_config:
k, v = item.split('=', 1)
env_dict[k] = v
return env_dict
"""
# return config env of a container in dictionary format
if self.containers:
return self.env(container)['VIRTUAL_PORT']
else:
return self.describe_containers[container]['Config']['Env'][
'VIRTUAL_PORT']
def ip_address(self, container):
if self.containers:
return self.containers[container]['NetworkSettings']['IPAddress']
else:
return self.describe_containers[container]['NetworkSettings'][
'IPAddress']
def memberof(self, service_name):
pass
def dockerClient():
global client, events
client = Client(base_url='unix://var/run/docker.sock', version='auto')
events = client.events(decode=True)
class DockerSpy:
"""
# Usage:
docker = dockerspy(container_api_url='unix://var/run/docker.sock')
# print docker information
print(docker.describe_containers)
print(docker.env(container='chargeback-sync'))
print(docker.virtual_port(container='chargeback-sync'))
print(docker.ip_address(container='chargeback-sync'))
# listen to events from docker API, similar to docker events
docker.events()
# print all containers information as a dictionary structure
docker.containers
"""
def __init__(self, container=None, container_api_url='unix://var/run/docker.sock'):
"""
:rtype: object
:type dockerspy: object
"""
self._client = Client(base_url=container_api_url)
self.containers = {}
if container:
self.containers[container] = self._client.inspect_container(container=container)
self.tagged_containers = {}
def get_containers(self):
"""
:rtype : list
:return: containers
"""
try:
# list live containers names using list comprehension
containers = [container['Names'][0].replace('/', '') for container in self._client.containers()]
return containers
except Exception:
raise
@staticmethod
def _into_dict(kv_config):
env_dict = {}
for item in kv_config:
k, v = item.split('=', 1)
env_dict[k] = v
return env_dict
@property
def describe_containers(self):
# update self.containers
for name in self.get_containers():
self.containers[name] = self._client.inspect_container(name)
self.containers[name]['Config']['Env'] = self._into_dict(kv_config=self.containers[name]['Config']['Env'])
return self.containers
def events(self):
"""
"""
try:
return self._client.events(decode=True)
except KeyboardInterrupt:
pass
def env(self, container=None):
"""
:rtype: dict
:return: container's environment variable
"""
"""
# internal method to transform k=v environment variable into dict
def _into_dict(kv_config):
env_dict = {}
for item in kv_config:
k, v = item.split('=', 1)
env_dict[k] = v
return env_dict
"""
# If self.cotnainers exists or grater then 0 return Config.Env
if self.containers:
return self.containers[container]['Config']['Env']
else:
return self.describe_containers[container]['Config']['Env']
def virtual_port(self, container):
# internal method to transform k=v environment variable into dict
"""
def _into_dict(kv_config):
env_dict = {}
for item in kv_config:
k, v = item.split('=', 1)
env_dict[k] = v
return env_dict
"""
# return config env of a container in dictionary format
if self.containers:
return self.env(container)['VIRTUAL_PORT']
else:
return self.describe_containers[container]['Config']['Env']['VIRTUAL_PORT']
def ip_address(self, container):
if self.containers:
return self.containers[container]['NetworkSettings']['IPAddress']
else:
return self.describe_containers[container]['NetworkSettings']['IPAddress']
def memberof(self, service_name):
pass
class Collector:
def __init__(self):
self.logger = logging.getLogger("Monitor")
self.INFLUXDB = {}
self.INFLUXDB['HOST'] = os.getenv("INFLUXDB_HOST","localhost")
self.INFLUXDB['PORT'] = os.getenv("INFLUXDB_PORT","8086")
self.INFLUXDB['USERNAME'] = os.getenv("INFLUXDB_USERNAME","root")
self.INFLUXDB['PASSWORD'] = os.getenv("INFLUXDB_PASSWORD","root")
self.INFLUXDB['DBNAME'] = os.getenv("INFLUXDB_DBNAME","autoscaling")
self.INFLUXDB['SERIES'] = os.getenv("INFLUXDB_SERIES","monitoring")
self.COLLECT_TIME_INTERVAL = int(os.getenv("COLLECT_TIME_INTERVAL","3"))
self.BATH_TIME_INTERVAL = int(os.getenv("BATH_TIME_INTERVAL","10"))
self.influxdb_client = InfluxDBClient(
host=self.INFLUXDB['HOST'],
port=self.INFLUXDB['PORT'],
username=self.INFLUXDB['USERNAME'],
password=self.INFLUXDB['PASSWORD'],
database=self.INFLUXDB['DBNAME']
)
self.docker_client = Client(base_url='unix://var/run/docker.sock')
self.data_bath = []
self.current_data = 0
def collecting(self, container_id):
mesos_task_id = ""
app_name = ""
container_envs = self.docker_client.inspect_container(container_id)['Config']['Env']
for env in container_envs:
if env.startswith('MESOS_TASK_ID'):
mesos_task_id = env.split('=')[1]
index = mesos_task_id.rfind('.')
app_name = mesos_task_id[:index]
break
while True:
try:
stat = self.docker_client.stats(container_id, decode="utf8", stream=False)
cpu_usage = (stat["cpu_stats"]["cpu_usage"]["total_usage"] - stat["precpu_stats"]["cpu_usage"]["total_usage"])*100/math.pow(10,9)/len(stat["cpu_stats"]["cpu_usage"]["percpu_usage"])
mem_usage = stat["memory_stats"]["usage"] / stat["memory_stats"]["limit"] * 100
current_time = datetime.now().timestamp()
data = [current_time, container_id, app_name, mesos_task_id, cpu_usage, mem_usage]
self.logger.debug("Append: "+str(data))
self.data_bath.append(data)
time.sleep(self.COLLECT_TIME_INTERVAL)
except errors.NotFound as e:
self.logger.info("Container {} has gone away".format(container_id))
break
except Exception as e:
self.logger.error("Error "+str(e))
break
def send_data(self):
while True:
try:
time.sleep(self.BATH_TIME_INTERVAL)
if self.data_bath:
data = dict()
data['name'] = self.INFLUXDB["SERIES"]
data['columns'] = ['time', 'container_id', 'app_uuid', 'mesos_task_id', "cpu_usage", "mem_usage"]
data['points'] = self.data_bath
self.data_bath = []
self.logger.info("Send data ...")
self.logger.debug(str(data))
self.influxdb_client.write_points([data])
except Exception as e:
self.logger.error("Error "+str(e))
def run(self):
with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:
executor.submit(self.send_data)
containers = self.docker_client.containers()
collectors = {}
for container in containers:
collectors[container["Id"]] = executor.submit(self.collecting, container["Id"])
events = self.docker_client.events(decode=True)
for event in events:
if event["status"] == "start":
self.logger.info("Start collector for: "+event["id"])
collectors[event["id"]] = executor.submit(self.collecting, event["id"])
elif event["status"] == "die":
try:
self.logger.info("Cancel collector for: "+event["id"])
collectors[event["id"]].cancel()
except Exception as e:
self.logger.debug("Exception when Cancel collector: {} : {}".format(str(e), event["id"]))
def dock_events(api):
cli = Client(base_url='tcp://' + api['docker-ip'] + ':2375')
return cli.events()
class DockerSwarmInterface:
def __init__(self, swarm_url, swarm_tls_ca_cert, swarm_tls_cert,
swarm_tls_key, swarm_allow_insecure, app_info, netscaler):
tls_config = False
if not swarm_allow_insecure:
if swarm_url.startswith("tcp"):
swarm_url = swarm_url.replace("tcp", "https")
logger.info("Using swarm url %s" % swarm_url)
tls_config = tls.TLSConfig(client_cert=(swarm_tls_cert,
swarm_tls_key),
verify=swarm_tls_ca_cert,
assert_hostname=False)
self.client = Client(base_url=swarm_url, tls=tls_config)
self.app_info = app_info
self.netskaler = netscaler
self.lock = threading.Lock()
def get_backends_for_app(self, app_label):
logger.info("Getting backends for app label %s" % app_label)
containers = self.client.containers(filters={'status': 'running',
'label': [app_label]})
portConfigs = [n['Ports'] for n in containers]
"""
[[{u'Type': u'tcp', u'PrivatePort': 443},
{u'IP': u'0.0.0.0', u'Type': u'tcp', u'PublicPort': 807, u'PrivatePort': 80}],
[{u'IP': u'0.0.0.0', u'Type': u'tcp', u'PublicPort': 806, u'PrivatePort': 80},
{u'Type': u'tcp', u'PrivatePort': 443}]]
"""
result = []
for ports in portConfigs:
for port in ports:
if port.get('PublicPort'):
# TODO: handle the case where more than one port is exposed
result.append((port['IP'], port['PublicPort']))
return result
def configure_ns_for_app(self, app_key, appname):
self.lock.acquire()
try:
app_label = app_key + "=" + appname
backends = self.get_backends_for_app(app_label)
# backends = map(lambda y: ("192.168.99.100", y[1]), backends)
# TODO: remove above for actual swarm. With plain docker machine,
# host IP is "0.0.0.0" -- that cannot be load balanced. Docker
# swarm supplies correct host IP.
logger.debug("Backends are %s" % str(backends))
self.netskaler.configure_app(appname, backends)
finally:
self.lock.release()
def configure_all(self):
app_key = self.app_info['appkey']
appnames = map(lambda x: x['name'], self.app_info['apps'])
logger.info("Configuring for app names: %s" % str(appnames))
for appname in appnames:
self.configure_ns_for_app(app_key, appname)
self.watch_all_apps()
self.wait_for_all()
def watch_app(self, app_key, appname):
app_label = app_key + "=" + appname
events = self.client.events(
filters={"event": ["start", "kill", "die"],
"label": [app_label]})
for e in events:
evj = json.loads(e)
status = evj['status']
c_id = evj['id']
if status in ['start', 'die', 'kill']:
# TODO: BUG in docker swarm events does not actually apply
# filters. Use 'docker ps' to verify the app that is changing
# belongs to this thread
logger.debug("Event status: %s, id: %.12s" %
(evj['status'], evj['id']))
containers = self.client.containers(
all=True,
filters={"label": [app_label]})
container_ids = [c.get('Id') for c in containers
if c.get('Id') == c_id]
if container_ids:
logger.info("Configuring NS for app %s, "
"container id=%.12s" % (appname, c_id))
self.configure_ns_for_app(app_key, appname)
def watch_all_apps(self):
app_key = self.app_info['appkey']
appnames = map(lambda x: x['name'], self.app_info['apps'])
for appname in appnames:
logger.debug("Watching for events for app: %s" % str(appname))
t = threading.Thread(target=self.watch_app,
args=(app_key, appname,))
t.start()
def wait_for_all(self):
main_thread = threading.currentThread()
for t in threading.enumerate():
if t is main_thread:
continue
logging.debug('joining %s', t.getName())
t.join()
import os
import json
import sys
import logging
import etcd
logging.basicConfig(level=logging.DEBUG)
etcd_hostname = 'etcd'
etcd_client = etcd.Client(host=etcd_hostname)
from docker import Client
client = Client(base_url='unix://var/run/docker.sock')
events = client.events(decode=True)
def get_container(message):
container = message['Actor']['Attributes']
id = message['id']
container['Env'] = client.inspect_container(id)['Config']['Env']
container['Id'] = id
print(container)
return container
def get_envvar(container, to_find):
for envvar in container['Env']:
envvar = envvar.split('=')
if envvar[0] == to_find:
return envvar[1]
def main(arguments):
logging.basicConfig(level=logging.INFO)
# retrieve configuration file with default values
config_parser = argparse.ArgumentParser(add_help=False)
config_parser.add_argument('-c', '--config', nargs='?')
(config_arg, arguments) = config_parser.parse_known_args(arguments)
defaults = dict(DEFAULTS)
if config_arg.config is not None:
if os.path.isfile(config_arg.config):
try:
configp = ConfigParser.SafeConfigParser()
configp.read(config_arg.config)
defaults.update(dict(configp.items('docker-listen')))
except Exception:
logging.exception('File %s can not be read', config_arg.config)
else:
logging.warn('File %s can not be read', config_arg.config)
# fix boolean value
if defaults['systemctl_enabled'] in ('True', 'yes', '1'):
defaults['systemctl_enabled'] = True
defaults['sighup_enabled'] = False
elif defaults['sighup_enabled'] in ('True', 'yes', '1'):
defaults['systemctl_enabled'] = False
defaults['sighup_enabled'] = True
else:
defaults['systemctl_enabled'] = False
defaults['sighup_enabled'] = False
# clean process name
defaults['sighup_process_name'] = defaults['sighup_process_name'].replace('\'', '')
defaults['systemctl_service_name'] = defaults['systemctl_service_name'].replace('\'', '')
# retrieve configuration ; configuration file provides defaults values
parser = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('-c', '--config', nargs='?', metavar='FILE', help='load configuration from .ini config file (section docker-listen)')
parser.add_argument('--docker-url', nargs='?', metavar='URL', help='docker socket path (unix://var/run/docker.sock) or docker url')
parser.add_argument('--systemctl-enabled', nargs='?', type=bool, choices=('yes', 'no'), help='systemctl is enable ?')
parser.add_argument('--systemctl-service-name', metavar='NAME', nargs='?', help='name of the service to restart')
parser.add_argument('--sighup-enabled', nargs='?', type=bool, choices=('yes', 'no'), help='sighup process on events ?')
parser.add_argument('--sighup-process-name', metavar='NAME', nargs='?', help='name of the process to sighup (with killall)')
parser.add_argument('--hosts-dir', nargs='?', metavar='DIR_PATH', help='directory where hosts files are stored ; all files in this directory will be deleted')
parser.add_argument('--log-level', nargs='?', choices=('DEBUG', 'INFO', 'WARN', 'ERROR'))
logging.debug('Using defaults %s', pprint.pformat(defaults))
parser.set_defaults(**defaults)
configuration = parser.parse_args(arguments)
# set logging level and start working
logging.getLogger('').setLevel(configuration.log_level)
logging.info('Current configuration : %s', pprint.pformat(configuration))
# check configuration
if not os.path.isdir(configuration.hosts_dir):
logging.error('hosts_dir \'%s\' is not a directory. Stopping.', configuration.hosts_dir)
return 2
try:
client = Client(base_url=configuration.docker_url)
client.ping()
except Exception:
logging.exception('Error communicating with docker socket %s. Stopping.', configuration.docker_url)
return 2
logging.info('Docker-listen started')
events = client.events(decode=True)
os.umask(0000)
clean_all(configuration)
init_all(configuration, client)
try:
for event in events:
logging.debug(event)
if event['Action'] == 'connect':
handle_start(configuration, client, event)
elif event['Action'] == 'disconnect':
handle_stop(configuration, client, event)
except Exception:
logging.exception('Error processing docker events. Stopping.')
return 2
return 0
class Dock2dns():
def __init__(self, socket = None, etcd = None, domain = None, ttl = 0, retry = 5, sleep = 2, debug = None):
self.socket = socket
self.etcd = etcd
self.domain = domain
self.ttl = ttl
self.retry = retry
self.sleep = sleep
self.client = Client(base_url = self.socket)
self.debug = debug
def log(self,message):
try:
sys.stdout.write(message)
sys.stdout.write("\n")
sys.stdout.flush()
except:
pass
def inspect(self, id):
try:
inspect = self.client.inspect_container(id)
return inspect
except:
return None
def req(self, method, uri, data = None):
payload = {}
if data:
payload["value"] = json.dumps(data)
payload["ttl"] = self.ttl
self.log("%s %s %s" % (method,uri,data))
for r in range(self.retry):
try:
r = requests.request(method, uri, data = payload)
break
except:
time.sleep(self.sleep)
pass
def update(self, event):
config = self.inspect(event["id"])
if not config:
return
if config["Config"]["Hostname"] != "":
dnsname = config["Config"]["Hostname"]
else:
dnsname = config["Name"].replace("/","")
uri="%s/v2/keys/skydns/%s/%s" % (self.etcd, "/".join(self.domain.split(".")[::-1]), dnsname )
method = None
data = None
# message = "UPDATE %s %s %s" % (event["status"], dnsname, config["NetworkSettings"]["IPAddress"])
# self.log(message)
if event["status"] == "start":
data = { "Host" : config["NetworkSettings"]["IPAddress"] }
method = "PUT"
elif event["status"] in ("die", "stop"):
data = {}
method = "DELETE"
if method:
treq = threading.Thread(target = self.req, args = (method,uri,data,))
treq.start()
def listen(self):
while True:
e = self.client.events()
for data in e:
eventdata = json.loads(data)
self.update(eventdata)
for rule in iptables_rules:
# FYI: http://stackoverflow.com/questions/11269575/how-to-hide-output-of-subprocess-in-python-2-7
ret = subprocess.call(shlex.split('{0} {1}'.format(iptables_check_command, rule)), stdout=DEVNULL, stderr=subprocess.STDOUT)
if ret != 0:
command = '{0} {1}'.format(iptables_insert_command, rule)
syslog.syslog(syslog.LOG_INFO, 'network_container:{0}'.format(command))
subprocess.call(shlex.split(command))
def on_stop(event):
for rule in iptables_rules:
ret = subprocess.call(shlex.split('{0} {1}'.format(iptables_check_command, rule)), stdout=DEVNULL, stderr=subprocess.STDOUT)
if ret == 0:
command = '{0} {1}'.format(iptables_delete_command, rule)
syslog.syslog(syslog.LOG_INFO, 'network_container:{0}'.format(command))
subprocess.call(shlex.split(command))
event_handlers = {}
event_handlers['start'] = on_start
event_handlers['stop'] = on_stop
# TODO: Use library to handle event loop
# TODO: Use fluentd for logging
gen = client.events(filters={'event': ['start', 'stop'], 'name': 'iptables'})
while True:
# wait till event is invoked.
event = json.loads(next(gen))
syslog.syslog(syslog.LOG_INFO, 'network_container:{0}'.format(event))
handler = event_handlers.get(event['status'])
if handler:
handler(event)
class Collector:
def __init__(self):
self.logger = logging.getLogger("Monitor")
self.INFLUXDB = {}
self.INFLUXDB['HOST'] = os.getenv("INFLUXDB_HOST", "localhost")
self.INFLUXDB['PORT'] = os.getenv("INFLUXDB_PORT", "8086")
self.INFLUXDB['USERNAME'] = os.getenv("INFLUXDB_USERNAME", "root")
self.INFLUXDB['PASSWORD'] = os.getenv("INFLUXDB_PASSWORD", "root")
self.INFLUXDB['DBNAME'] = os.getenv("INFLUXDB_DBNAME", "autoscaling")
self.INFLUXDB['SERIES'] = os.getenv("INFLUXDB_SERIES", "monitoring")
self.COLLECT_TIME_INTERVAL = int(
os.getenv("COLLECT_TIME_INTERVAL", "3"))
self.BATH_TIME_INTERVAL = int(os.getenv("BATH_TIME_INTERVAL", "10"))
self.influxdb_client = InfluxDBClient(
host=self.INFLUXDB['HOST'],
port=self.INFLUXDB['PORT'],
username=self.INFLUXDB['USERNAME'],
password=self.INFLUXDB['PASSWORD'],
database=self.INFLUXDB['DBNAME'])
self.docker_client = Client(base_url='unix://var/run/docker.sock')
self.data_bath = []
self.current_data = 0
def collecting(self, container_id):
mesos_task_id = ""
app_name = ""
container_envs = self.docker_client.inspect_container(
container_id)['Config']['Env']
for env in container_envs:
if env.startswith('MESOS_TASK_ID'):
mesos_task_id = env.split('=')[1]
index = mesos_task_id.rfind('.')
app_name = mesos_task_id[:index]
break
while True:
try:
stat = self.docker_client.stats(container_id,
decode="utf8",
stream=False)
cpu_usage = (
stat["cpu_stats"]["cpu_usage"]["total_usage"] -
stat["precpu_stats"]["cpu_usage"]["total_usage"]
) * 100 / math.pow(10, 9) / len(
stat["cpu_stats"]["cpu_usage"]["percpu_usage"])
mem_usage = stat["memory_stats"]["usage"] / stat[
"memory_stats"]["limit"] * 100
current_time = datetime.now().timestamp()
data = [
current_time, container_id, app_name, mesos_task_id,
cpu_usage, mem_usage
]
self.logger.debug("Append: " + str(data))
self.data_bath.append(data)
time.sleep(self.COLLECT_TIME_INTERVAL)
except errors.NotFound as e:
self.logger.info(
"Container {} has gone away".format(container_id))
break
except Exception as e:
self.logger.error("Error " + str(e))
break
def send_data(self):
while True:
try:
time.sleep(self.BATH_TIME_INTERVAL)
if self.data_bath:
data = dict()
data['name'] = self.INFLUXDB["SERIES"]
data['columns'] = [
'time', 'container_id', 'app_uuid', 'mesos_task_id',
"cpu_usage", "mem_usage"
]
data['points'] = self.data_bath
self.data_bath = []
self.logger.info("Send data ...")
self.logger.debug(str(data))
self.influxdb_client.write_points([data])
except Exception as e:
self.logger.error("Error " + str(e))
def run(self):
with concurrent.futures.ThreadPoolExecutor(
max_workers=100) as executor:
executor.submit(self.send_data)
containers = self.docker_client.containers()
collectors = {}
for container in containers:
collectors[container["Id"]] = executor.submit(
self.collecting, container["Id"])
events = self.docker_client.events(decode=True)
for event in events:
if event["status"] == "start":
self.logger.info("Start collector for: " + event["id"])
collectors[event["id"]] = executor.submit(
self.collecting, event["id"])
elif event["status"] == "die":
try:
self.logger.info("Cancel collector for: " +
event["id"])
collectors[event["id"]].cancel()
except Exception as e:
self.logger.debug(
"Exception when Cancel collector: {} : {}".format(
str(e), event["id"]))
if hostname:
if '.' not in hostname:
hostname = "{}{}".format(hostname, DOMAIN_SUFFIX)
print("Adding {}".format(hostname))
hosts = Hosts(HOSTS_PATH)
my_ip = check_output([
"/bin/sh", "-c",
"ip -4 -f inet -o addr show eth0 | awk '{print $4}' | cut -d/ -f1"
])
my_ip = my_ip.decode().strip()
print("My IP: {}".format(my_ip))
hosts.set_one(hostname, my_ip)
hosts.write(HOSTS_PATH)
print("Go to http://%s/" % hostname)
for event in docker.events():
event = json.loads(event)
if 'status' not in event:
continue
if event['status'] == 'start':
hostname = get_hostname(event['id'])
if hostname is None:
print(
"ERR: Event 'start' received but no hostname found for {}".
format(event['id']))
continue
container_ip = get_ip(event['id'])
if not container_ip:
print("ERR: Could not find IP address for {}".format(hostname))
continue
print("Adding {} as {}".format(hostname, container_ip))
def myjob(url):
def download(url):
return urllib2.urlopen(url).read()
elapsed = yield sleep_task(random.uniform(0.0, 3.0))
#~ sys.stderr.write("[slept_for:%0.2f]" % elapsed)
#~ sys.stderr.write("[start_download:%s]" % url)
#~ html = yield threaded_task(download, url)
#~ sys.stderr.write("[done:%s:%d]" % (url, len(html)))
def basso_continuo():
sys.stderr.write(".")
return True
now = datetime.datetime.now()
since = now #- datetime.timedelta(seconds=10)
live_events = docker_client.events(since=since)
while True:
print '1'
next(live_events)
print '2'
urls = ["http://www.google.com", "http://python.com", "http://www.pygtk.org"]
jobs = [start_job(myjob(url)) for url in urls]
# See how easily can we raise a exception in the job couroutine:
# gobject.timeout_add(1000, lambda: jobs[0].throw(JobStopped))
gobject.timeout_add(100, basso_continuo)
loop = gobject.MainLoop()
loop.run()
class DockerSwarmInterface:
def __init__(self, swarm_url, swarm_tls_ca_cert, swarm_tls_cert,
swarm_tls_key, swarm_allow_insecure, app_info, netscaler):
tls_config = False
if not swarm_allow_insecure:
if swarm_url.startswith("tcp"):
swarm_url = swarm_url.replace("tcp", "https")
logger.info("Using swarm url %s" % swarm_url)
tls_config = tls.TLSConfig(client_cert=(swarm_tls_cert,
swarm_tls_key),
verify=swarm_tls_ca_cert,
assert_hostname=False)
self.client = Client(base_url=swarm_url, tls=tls_config)
self.app_info = app_info
self.netskaler = netscaler
self.lock = threading.Lock()
def get_backends_for_app(self, app_label):
logger.info("Getting backends for app label %s" % app_label)
containers = self.client.containers(filters={
'status': 'running',
'label': [app_label]
})
portConfigs = [n['Ports'] for n in containers]
"""
[[{u'Type': u'tcp', u'PrivatePort': 443},
{u'IP': u'0.0.0.0', u'Type': u'tcp', u'PublicPort': 807, u'PrivatePort': 80}],
[{u'IP': u'0.0.0.0', u'Type': u'tcp', u'PublicPort': 806, u'PrivatePort': 80},
{u'Type': u'tcp', u'PrivatePort': 443}]]
"""
result = []
for ports in portConfigs:
for port in ports:
if port.get('PublicPort'):
# TODO: handle the case where more than one port is exposed
result.append((port['IP'], port['PublicPort']))
return result
def configure_ns_for_app(self, app_key, appname):
self.lock.acquire()
try:
app_label = app_key + "=" + appname
backends = self.get_backends_for_app(app_label)
# backends = map(lambda y: ("192.168.99.100", y[1]), backends)
# TODO: remove above for actual swarm. With plain docker machine,
# host IP is "0.0.0.0" -- that cannot be load balanced. Docker
# swarm supplies correct host IP.
logger.debug("Backends are %s" % str(backends))
self.netskaler.configure_app(appname, backends)
finally:
self.lock.release()
def configure_all(self):
app_key = self.app_info['appkey']
appnames = map(lambda x: x['name'], self.app_info['apps'])
logger.info("Configuring for app names: %s" % str(appnames))
for appname in appnames:
self.configure_ns_for_app(app_key, appname)
self.watch_all_apps()
self.wait_for_all()
def watch_app(self, app_key, appname):
app_label = app_key + "=" + appname
events = self.client.events(filters={
"event": ["start", "kill", "die"],
"label": [app_label]
})
for e in events:
evj = json.loads(e)
status = evj['status']
c_id = evj['id']
if status in ['start', 'die', 'kill']:
# TODO: BUG in docker swarm events does not actually apply
# filters. Use 'docker ps' to verify the app that is changing
# belongs to this thread
logger.debug("Event status: %s, id: %.12s" %
(evj['status'], evj['id']))
containers = self.client.containers(
all=True, filters={"label": [app_label]})
container_ids = [
c.get('Id') for c in containers if c.get('Id') == c_id
]
if container_ids:
logger.info("Configuring NS for app %s, "
"container id=%.12s" % (appname, c_id))
self.configure_ns_for_app(app_key, appname)
def watch_all_apps(self):
app_key = self.app_info['appkey']
appnames = map(lambda x: x['name'], self.app_info['apps'])
for appname in appnames:
logger.debug("Watching for events for app: %s" % str(appname))
t = threading.Thread(target=self.watch_app,
args=(
app_key,
appname,
))
t.start()
def wait_for_all(self):
main_thread = threading.currentThread()
for t in threading.enumerate():
if t is main_thread:
continue
logging.debug('joining %s', t.getName())
t.join()
for e in valid_events:
key, value = e.replace('ACT_', '').split('=')
runner_file = '{}/{}'.format(runner_dir, key)
if os.path.isfile(runner_file):
inspect_file = '{}/{}_{}.json'.format(inspect_dir, docker_id,
status)
with open(inspect_file, 'w') as fp:
fp.write(json.dumps(inspect, indent=2))
try:
# We need to keep a referrence to the process so we can remove the reference
# when they are done. See comment below.
# We also need to spawn the process in the backround, therefor, Popen, not call.
procs.append(
subprocess.Popen([runner_file, value, docker_id, status]))
# Delete the references to completed processes so python will do garbage collection
# to kill old processes. They will be zombies from the time they are done to when
# this code is run. A zombie doesnt use any resources, but its nice to get rid of them.
procs[:] = [proc for proc in procs if proc.poll() is None]
except OSError:
# Probably not executable..
return
for i in c.containers(filters={'status': 'running'}):
handler(docker_id=i['Id'], status='running')
# c.events() returns a blocking generator
for event in c.events(decode=True):
handler(docker_id=event['id'], status=event['status'])
class Agent(object):
def __init__(self, docker_url='unix:///var/run/docker.sock', handler_paths=None, script_paths=None, once=False, watch=False):
self.docker = Client(base_url=docker_url)
exec_prefix = os.path.dirname(sys.argv[0])
script_paths = relative_to_exec_dirs([
'../libexec/docker-glue',
'../../libexec/docker-glue',
'/usr/libexec/docker-glue',
'./handler-scripts',
])
self.script_paths = script_paths
handler_paths = relative_to_exec_dirs(['../etc', '../../etc', '/etc', '.'], suffix='/docker-glue.d')
self.load_handlers(handler_paths)
if once: self.once('none', '')
elif watch: self.loop()
def resolve_script(self, script):
if os.path.isabs(script): return script
for path in self.script_paths:
fn = os.path.realpath(os.path.join(path, script))
if os.path.exists(fn) and os.access(fn, os.X_OK): return fn
return None
def load_handlers(self, paths):
handlers=[]
by_event=defaultdict(list)
done=set()
for path in paths:
if path in done: continue
logger.info("looking for handlers in [%s]", path)
done.add(path)
for ini_file in glob.glob(os.path.join(path, '*.ini')):
logger.info("parsing [%s]",ini_file)
ini = ConfigParser.RawConfigParser()
ini.read(ini_file)
handler=factory(ini.get('handler', 'class'), self, ini_file, ini)
handlers.append(handler)
for event in handler.events:
by_event[event].append(handler)
logger.debug("handlers by event: %r",by_event)
self.handlers = handlers
self.handlers_by_event = by_event
def once(self, event, container_id):
# event can be: create, destroy, die, exec_create, exec_start, export, kill, oom, pause, restart, start, stop, unpause
triggers_none=False
handlers=self.handlers_by_event[event]
if event!='none': handlers+=self.handlers_by_event['all']
logger.debug('got handlers %r', handlers)
for handler in handlers:
logger.debug("passing event=%r container_id=%r to handler=%r", event, container_id, handler)
if not handler.enabled: continue
handler.handle(event, container_id)
triggers_none|=handler.triggers_none
if event!='none' and triggers_none: self.once('none', '')
def loop(self):
logger.info("waiting for docker events")
self.once('none', '')
for event in self.docker.events(decode=True):
# event looks like this {"status":"die","id":"123","from":"foobar/eggs:latest","time":1434047926}
self.once(event['status'], event['id'])