def __init__(self, config, cache):
self.config = config
self.containers = []
self.cache = cache
self.docker = DockerPyClient(config['remote'], config['username'],
config['password'], config['email'])
self.log = logging.getLogger(__name__)
def __init__(self, config, cache):
self.config = config
self.containers = []
self.cache = cache
self.docker = DockerPyClient(config['remote'], config['username'], config['password'], config['email'])
self.log = logging.getLogger(__name__)
class DockerUp(object):
"""
Service for synchronizing locally running Docker containers with an external
configuration file. If available, EC2 user-data is used as the configuration file,
otherwise dockerup looks in /etc/dockerup/dockerup.json by default (override with --config).
This script can be run on-demand or via a cron job.
Sample config file is shown below:
{
"containers": [
{
"type": "docker",
"name": "historical-app",
"image": "barchart/historical-app-alpha",
"portMappings": [
{
"containerPort": "8080",
"hostPort": "8080"
}
]
},
{
"type": "docker",
"name": "logstash-forwarder",
"image": "barchart/logstash-forwarder",
"volumes": [
{
"containerPath": "/var/log/containers",
"hostPath": "/var/log/ext",
"mode": "ro"
}
]
}
]
}
"""
def __init__(self, config, cache):
self.config = config
self.containers = []
self.cache = cache
self.docker = DockerPyClient(config['remote'], config['username'], config['password'], config['email'])
self.log = logging.getLogger(__name__)
def pull_allowed(self, entry):
if 'pull' in self.config and not self.config['pull']:
return False
if not 'update' in entry or not 'pull' in entry['update'] or entry['update']['pull']:
return True
return False
def update(self, entry):
if not 'image' in entry:
self.log.warn('No image defined for container, skipping')
return
current = self.status(entry)
updated = self.updated(entry)
if current['Image'] is None or self.pull_allowed(entry):
updated = self.docker.pull(entry['image']) or updated
if updated or not current['Running']:
if 'links' in entry:
# Has dependency on another container, let's give Docker time to bring
# bring previous container up fully before attempting to launch
time.sleep(5)
if current['Running']:
return self.update_next_window(entry, current)
else:
return self.update_launch()(entry)
return current
def update_next_window(self, entry, status):
if 'update' in entry and 'rolling' in entry['update'] and entry['update']['rolling']:
# TODO use central coordinator service to wait for available update window
log.warn('Rolling updates not yet supported')
return self.update_replace(entry, status)
# Eager update: start new container first (primarily to facilitate self-upgrade
# of the dockerup management container itself)
def is_eager(self, entry):
if 'update' in entry and 'eager' in entry['update'] and entry['update']['eager']:
if 'name' in entry:
log.warn('Skipping eager update due to container name conflict')
return False
if 'portMappings' in entry:
for mapping in entry['portMappings']:
if 'hostPort' in mapping:
log.warn('Skipping eager update due to host port conflict')
return False
return True
return False
def update_replace(self, entry, status):
if self.is_eager(entry):
return self.update_launch(self.update_stop(status))(entry)
return self.update_stop(status, self.update_launch())(entry)
def update_stop(self, status, callback=None):
def actual(entry):
self.log.debug('Stopping old container: %s' % status['Id'])
self.stop(status)
if callback:
return callback(entry)
return status
return actual
def update_launch(self, callback=None):
def actual(entry):
status = self.status(entry)
self.log.info('current status %s' % status)
if status['Image']:
try:
# Stop all dependencies, they will get updated/restarted
self.stop_dependencies(entry)
self.log.debug('Starting new container')
self.run(entry)
if 'signal' in entry:
for target in entry['signal'].keys():
self.docker_signal(target, entry['signal'][target])
status = self.status(entry)
except Exception as e:
self.log.error('Could not run container: %s' % e)
else:
self.log.error('!!!!!!!!!!!!!!!!Image not found: %s' % entry['image'])
if callback:
callback(entry)
return status
return actual
def status(self, entry):
image = self.docker.image(entry['image'])
container = self.docker.container(image['Id']) if image else None
return {
'Id': container['Id'] if container else None,
'Tag': container['Image'] if container else None,
'Image': image['Id'] if image else None,
'Running': container['Running'] if container else False
}
def updated(self, entry):
updated = False
cachefile = '%s/%s.json' % (self.cache, self.__cache_name(entry))
if os.path.exists(cachefile):
with open(cachefile) as local:
if json.dumps(entry) != local.read():
updated = True
else:
updated = True
if updated:
with open(cachefile, 'w') as local:
json.dump(entry, local)
return updated
def __cache_name(self, entry):
image_clean = entry['image'].replace(':', '_').replace('/', '_')
if 'name' in entry:
return '%s-%s' % (image_clean, entry['name'])
return image_clean
def run(self, config):
if 'type' in config and config['type'] != 'docker':
return False
return self.docker.run(config)
def stop(self, status, remove=True):
self.docker.stop(status['Id'], remove)
# Shutdown containers with unrecognized images to avoid resource conflicts
def shutdown_unknown(self, entries=None):
existing = []
catalog = []
def cached_entry(cached):
cachefile = '%s/%s' % (self.cache, cached)
with open(cachefile) as local:
return json.load(local)
if entries:
catalog.extend(entries)
catalog.extend([cached_entry(cf) for cf in os.listdir(self.cache) if cf.endswith('.json')])
for entry in catalog:
status = self.status(entry)
if status['Id']:
existing.append(status['Id'])
self.log.debug('Cleaning up orphaned containers')
# Iterate through running containers and stop them if they don't match a cached config
[self.docker.stop(c['Id']) for c in self.docker.containers() if c['Running'] and not c['Id'] in existing]
# Remove old log files from last run shutdown (gives logstash some time to process final messages)
ids = [c['Id'] for c in self.docker.containers() if c['Running']]
if os.path.exists('/var/log/ext'):
for entry in os.listdir('/var/log/ext'):
if os.path.isdir('/var/log/ext/%s' % entry) and entry not in ids:
self.log.info('Removing old logs for %s' % entry)
shutil.rmtree('/var/log/ext/%s' % entry)
# Shutdown leftover containers from old configurations
def cleanup(self, valid):
self.log.debug('Cleaning up missing configurations')
for entry in os.listdir(self.cache):
if not entry.endswith('.json'):
continue
cachefile = '%s/%s' % (self.cache, entry)
with open(cachefile) as local:
cached = json.load(local)
status = self.status(cached)
if status['Id'] and not status['Id'] in valid:
os.unlink(cachefile)
self.stop(status)
def update_config(self):
config = {}
containers = []
def merge(cfg):
if 'containers' in cfg:
containers.extend(cfg['containers'])
del cfg['containers']
config.update(cfg)
if 'confdir' in self.config:
merge(conf.files_config(self.config['confdir']))
if 'aws' in self.config and self.config['aws']:
merge(conf.aws_config())
self.containers = DependencyResolver(containers).resolve()
self.config.update(config)
def stop_dependencies(self, entry):
if 'name' in entry:
for container in DependencyResolver(self.containers).downstream(entry['name']):
status = self.status(container)
if status['Id']:
self.log.info('Dependent container %s will be restarted to maintain link consistency' % status['Id'])
self.stop(status)
# Run a single sync cycle
def sync(self):
# Update container config
self.update_config()
# Rare occurence, kill containers that have an unknown image tag
# Usually due to manual updates, may be required to avoid port binding conflicts
self.shutdown_unknown(self.containers)
# Process configuration and store running container IDs
running = [self.update(container)['Id'] for container in self.containers]
# Cleanup containers with no config
self.cleanup(running)
# Remove unused containers/images from Docker
self.docker.cleanup()
def start(self):
if 'server' in self.config and self.config['server']:
signal.signal(signal.SIGTERM, self.handle_signal)
# TODO connect to control queue (SQS?) for update broadcasts
while True:
try:
try:
self.sync()
except Exception as e:
self.log.error('Error in sync loop: %s' % e.message)
self.log.debug(traceback.format_exc())
# Separate sleep from sync loop to prevent logspam
time.sleep(self.config['interval'])
except Exception as e:
# Sleep interrupted, just go to next loop
pass
else:
self.sync()
def handle_signal(self, signo, stack):
self.log.info('Received signal %s, shutting down' % signo)
sys.exit(1)
def shutdown(self):
self.log.info('Shutting down')
sys.exit(0)
class DockerUp(object):
"""
Service for synchronizing locally running Docker containers with an external
configuration file. If available, EC2 user-data is used as the configuration file,
otherwise dockerup looks in /etc/dockerup/dockerup.json by default (override with --config).
This script can be run on-demand or via a cron job.
Sample config file is shown below:
{
"containers": [
{
"type": "docker",
"name": "historical-app",
"image": "barchart/historical-app-alpha",
"portMappings": [
{
"containerPort": "8080",
"hostPort": "8080"
}
]
},
{
"type": "docker",
"name": "logstash-forwarder",
"image": "barchart/logstash-forwarder",
"volumes": [
{
"containerPath": "/var/log/containers",
"hostPath": "/var/log/ext",
"mode": "ro"
}
]
}
]
}
"""
def __init__(self, config, cache):
self.config = config
self.containers = []
self.cache = cache
self.docker = DockerPyClient(config['remote'], config['username'],
config['password'], config['email'])
self.log = logging.getLogger(__name__)
def pull_allowed(self, entry):
if 'pull' in self.config and not self.config['pull']:
return False
if not 'update' in entry or not 'pull' in entry['update'] or entry[
'update']['pull']:
return True
return False
def update(self, entry):
if not 'image' in entry:
self.log.warn('No image defined for container, skipping')
return
current = self.status(entry)
updated = self.updated(entry)
if current['Image'] is None or self.pull_allowed(entry):
updated = self.docker.pull(entry['image']) or updated
if updated or not current['Running']:
if 'links' in entry:
# Has dependency on another container, let's give Docker time to bring
# bring previous container up fully before attempting to launch
time.sleep(5)
if current['Running']:
return self.update_next_window(entry, current)
else:
return self.update_launch()(entry)
return current
def update_next_window(self, entry, status):
if 'update' in entry and 'rolling' in entry['update'] and entry[
'update']['rolling']:
# TODO use central coordinator service to wait for available update window
log.warn('Rolling updates not yet supported')
return self.update_replace(entry, status)
# Eager update: start new container first (primarily to facilitate self-upgrade
# of the dockerup management container itself)
def is_eager(self, entry):
if 'update' in entry and 'eager' in entry['update'] and entry[
'update']['eager']:
if 'name' in entry:
log.warn(
'Skipping eager update due to container name conflict')
return False
if 'portMappings' in entry:
for mapping in entry['portMappings']:
if 'hostPort' in mapping:
log.warn(
'Skipping eager update due to host port conflict')
return False
return True
return False
def update_replace(self, entry, status):
if self.is_eager(entry):
return self.update_launch(self.update_stop(status))(entry)
return self.update_stop(status, self.update_launch())(entry)
def update_stop(self, status, callback=None):
def actual(entry):
self.log.debug('Stopping old container: %s' % status['Id'])
self.stop(status)
if callback:
return callback(entry)
return status
return actual
def update_launch(self, callback=None):
def actual(entry):
status = self.status(entry)
if status['Image']:
try:
# Stop all dependencies, they will get updated/restarted
self.stop_dependencies(entry)
self.log.debug('Starting new container')
self.run(entry)
if 'signal' in entry:
for target in entry['signal'].keys():
self.docker_signal(target, entry['signal'][target])
status = self.status(entry)
except Exception as e:
self.log.error('Could not run container: %s' % e)
else:
self.log.error('Image not found: %s' % entry['image'])
if callback:
callback(entry)
return status
return actual
def status(self, entry):
image = self.docker.image(entry['image'])
container = self.docker.container(image['Id']) if image else None
return {
'Id': container['Id'] if container else None,
'Tag': container['Image'] if container else None,
'Image': image['Id'] if image else None,
'Running': container['Running'] if container else False
}
def updated(self, entry):
updated = False
cachefile = '%s/%s.json' % (self.cache, self.__cache_name(entry))
if os.path.exists(cachefile):
with open(cachefile) as local:
if json.dumps(entry) != local.read():
updated = True
else:
updated = True
if updated:
with open(cachefile, 'w') as local:
json.dump(entry, local)
return updated
def __cache_name(self, entry):
image_clean = entry['image'].replace(':', '_').replace('/', '_')
if 'name' in entry:
return '%s-%s' % (image_clean, entry['name'])
return image_clean
def run(self, config):
if 'type' in config and config['type'] != 'docker':
return False
return self.docker.run(config)
def stop(self, status, remove=True):
self.docker.stop(status['Id'], remove)
# Shutdown containers with unrecognized images to avoid resource conflicts
def shutdown_unknown(self, entries=None):
existing = []
catalog = []
def cached_entry(cached):
cachefile = '%s/%s' % (self.cache, cached)
with open(cachefile) as local:
return json.load(local)
if entries:
catalog.extend(entries)
catalog.extend([
cached_entry(cf) for cf in os.listdir(self.cache)
if cf.endswith('.json')
])
for entry in catalog:
status = self.status(entry)
if status['Id']:
existing.append(status['Id'])
self.log.debug('Cleaning up orphaned containers')
# Iterate through running containers and stop them if they don't match a cached config
[
self.docker.stop(c['Id']) for c in self.docker.containers()
if c['Running'] and not c['Id'] in existing
]
# Remove old log files from last run shutdown (gives logstash some time to process final messages)
ids = [c['Id'] for c in self.docker.containers() if c['Running']]
if os.path.exists('/var/log/ext'):
for entry in os.listdir('/var/log/ext'):
if os.path.isdir(
'/var/log/ext/%s' % entry) and entry not in ids:
self.log.info('Removing old logs for %s' % entry)
shutil.rmtree('/var/log/ext/%s' % entry)
# Shutdown leftover containers from old configurations
def cleanup(self, valid):
self.log.debug('Cleaning up missing configurations')
for entry in os.listdir(self.cache):
if not entry.endswith('.json'):
continue
cachefile = '%s/%s' % (self.cache, entry)
with open(cachefile) as local:
cached = json.load(local)
status = self.status(cached)
if status['Id'] and not status['Id'] in valid:
os.unlink(cachefile)
self.stop(status)
def update_config(self):
config = {}
containers = []
def merge(cfg):
if 'containers' in cfg:
containers.extend(cfg['containers'])
del cfg['containers']
config.update(cfg)
if 'confdir' in self.config:
merge(conf.files_config(self.config['confdir']))
if 'aws' in self.config and self.config['aws']:
merge(conf.aws_config())
self.containers = DependencyResolver(containers).resolve()
self.config.update(config)
def stop_dependencies(self, entry):
if 'name' in entry:
for container in DependencyResolver(self.containers).downstream(
entry['name']):
status = self.status(container)
if status['Id']:
self.log.info(
'Dependent container %s will be restarted to maintain link consistency'
% status['Id'])
self.stop(status)
# Run a single sync cycle
def sync(self):
# Update container config
self.update_config()
# Rare occurence, kill containers that have an unknown image tag
# Usually due to manual updates, may be required to avoid port binding conflicts
self.shutdown_unknown(self.containers)
# Process configuration and store running container IDs
running = [
self.update(container)['Id'] for container in self.containers
]
# Cleanup containers with no config
self.cleanup(running)
# Remove unused containers/images from Docker
self.docker.cleanup()
def start(self):
if 'server' in self.config and self.config['server']:
signal.signal(signal.SIGTERM, self.handle_signal)
# TODO connect to control queue (SQS?) for update broadcasts
while True:
try:
try:
self.sync()
except Exception as e:
self.log.error('Error in sync loop: %s' % e.message)
self.log.debug(traceback.format_exc())
# Separate sleep from sync loop to prevent logspam
self.log.info('Config: %s' % self.config['interval'])
time.sleep(float(self.config['interval']))
except Exception as e:
self.log.error('Error in sync loop: %s' % e.message)
pass
else:
self.sync()
def handle_signal(self, signo, stack):
self.log.info('Received signal %s, shutting down' % signo)
sys.exit(1)
def shutdown(self):
self.log.info('Shutting down')
sys.exit(0)
