def test_named_volumes(self):
"""Make sure that named volumes are correctly registered"""
volume_name = "control_unittest_volume{}".format(
random.randint(1, 65535))
self.container_volumes.append(volume_name)
self.image = 'busybox'
self.conf = {
"image": self.image,
"container": {
"name": self.container_name,
"hostname": "busybox",
"volumes": ["{}:/var".format(volume_name)]
}
}
serv = create_service(self.conf, './Controlfile')
container = Container(serv).create(prod=False)
container.start()
self.assertEqual(len(container.inspect['Mounts']), 1)
self.assertEqual(len(container.inspect['Mounts'][0]['Name']),
len(volume_name))
self.assertEqual(container.inspect['Mounts'][0]['Destination'], '/var')
self.assertTrue(container.inspect['Mounts'][0]['Source'].startswith(
'/var/lib/docker/volumes'),
msg="Unexpected mount source: {}".format(
container.inspect['Mounts'][0]['Source']))
def test_anon_volumes(self):
"""
Make sure that anonymous volumes are correctly registered
Currently unimplemented and will fail
"""
self.image = 'busybox'
self.conf = {
"image": self.image,
"container": {
"name": self.container_name,
"hostname": "busybox",
"volumes": ["/var"]
}
}
serv = create_service(self.conf, './Controlfile')
container = Container(serv).create(prod=False)
container.start()
self.assertEqual(len(container.inspect['Mounts']), 1)
self.assertEqual(len(container.inspect['Mounts'][0]['Name']), 65)
self.assertEqual(container.inspect['Mounts'][0]['Destination'], '/var')
self.assertTrue(container.inspect['Mounts'][0]['Source'].startswith(
'/var/lib/docker/volumes'),
msg="Unexpected mount source: {}".format(
container.inspect['Mount'][0]['Source']))
def test_unspecified_service(self):
"""
A Control service does not need to specify a service name if it
specifies a container name
"""
service = {"image": "busybox", "container": {"name": "example"}}
name, ret = normalize_service(
create_service(deepcopy(service), './Controlfile'), {}, {})
self.assertEqual(ret['container']['name'], "example")
self.assertEqual(name, "example")
self.assertIn('hostname', ret['container'])
self.assertEqual(ret['container']['hostname'], "example")
def test_name_suffixes(self):
"""check for suffix changes"""
service = {
"service": "example",
"image": "busybox",
"container": {
"name": "example",
"hostname": "service"
}
}
options = {"name": {"suffix": ".company"}}
name, ret = normalize_service(
create_service(deepcopy(service), './Controlfile'), options, {})
self.assertEqual(ret['container']['name'], "example.company")
self.assertEqual(name, "example")
self.assertEqual(ret['container']['hostname'], "service")
del service['service']
name, ret = normalize_service(
create_service(deepcopy(service), './Controlfile'), options, {})
self.assertEqual(ret['container']['name'], "example.company")
self.assertEqual(name, "example",
"service name is affected by name suffix")
self.assertEqual(ret['container']['hostname'], "service")
def create_service(self, data, service_name, options, variables, ctrlfile):
"""
Determine if data is a Metaservice or Uniservice
Variables only exist to be applied if they have been defined up the
chain of discovered Controlfiles. You don't get to randomly define a
variable somewhere in a web of included Controlfiles and have that
apply everywhere.
"""
self.logger.debug('Received %i variables', len(variables))
while 'controlfile' in data:
ctrlfile = data['controlfile']
# TODO write a test that gets a FileNotFound thrown from here
data = self.read_in_file(ctrlfile)
data['service'] = service_name
services_in_data = 'services' in data
services_is_list = isinstance(data.get('services', None), list)
if services_in_data:
self.logger.debug('metaservice named %s', service_name)
self.logger.debug('services_is_list %s', services_is_list)
self.logger.debug(data)
# Recursive step
if services_in_data and not services_is_list:
self.logger.debug('found Metaservice %s', data['service'])
metaservice = MetaService(data, ctrlfile)
opers = satisfy_nested_options(outer=options,
inner=data.get('options', {}))
nvars = copy.deepcopy(variables)
nvars.update(_substitute_vars(data.get('vars', {}), variables))
nvars.update(os.environ)
for name, serv in data['services'].items():
metaservice.services += self.create_service(
serv, name, opers, nvars, ctrlfile)
self.push_service_into_list(metaservice.service, metaservice)
return metaservice.services
# No more recursing, we have concrete services now
try:
serv = create_service(data, ctrlfile)
except InvalidControlfile as e:
self.logger.warning(e)
return []
variables['SERVICE'] = serv.service
if isinstance(serv, ImageService):
name, service = normalize_service(serv, options, variables)
self.push_service_into_list(name, service)
return [name]
self.push_service_into_list(serv.service, serv)
return [serv.service]
def test_happy_path(self):
"""Make sure that the defaults still work"""
self.conf = {
"image": 'busybox',
"container": {
"name": self.container_name,
"hostname": "happy_path"
}
}
serv = create_service(deepcopy(self.conf), './Controlfile')
container = Container(serv)
self.assertEqual(container.service.expected_timeout, 10)
self.assertEqual(container.service['name'], self.container_name)
self.assertEqual(container.service['hostname'], self.conf['container']['hostname'])
self.assertEqual(container.service.image, self.conf['image'])
def test_dns_search(self):
"""test that dns search makes it into the host config"""
self.image = 'busybox'
self.conf = {
"image": self.image,
"container": {
"name": self.container_name,
"dns_search": [
"example"
]
}
}
serv = create_service(deepcopy(self.conf), './Controlfile')
conf_copy = serv.prepare_container_options(prod=False)
self.assertEqual(
conf_copy['host_config']['DnsSearch'][0],
self.conf["container"]["dns_search"][0])
def test_expected_timeout(self):
"""Test mirroring unspecified values and overriding default timeout"""
self.conf = {
"image": 'busybox',
"expected_timeout": 3,
"container": {
"name": self.container_name
}
}
serv = create_service(deepcopy(self.conf), './Controlfile')
container = Container(serv)
self.assertEqual(container.service.expected_timeout, 3)
self.assertEqual(container.service['name'], self.container_name)
self.assertEqual(
container.service['hostname'],
self.container_name,
msg='Unspecified hostname not being mirrored from container name')
self.assertEqual(container.service.image, self.conf['image'])
def test_env_var_parsing(self):
"""
Need to test that environment variables are always getting parsed
correctly
"""
self.image = 'busybox'
self.conf = {
"image": self.image,
"container": {
"name": self.container_name,
"hostname": "grafana",
"environment": [
"PASSWORD=password",
]
}
}
serv = create_service(deepcopy(self.conf), './Controlfile')
conf_copy = serv.prepare_container_options(prod=False)
self.assertEqual(conf_copy['environment'][0], self.conf['container']['environment'][0])
def test_value_substitution(self):
"""Test name substitution working"""
self.image = 'busybox'
self.conf = {
"image": self.image,
"container": {
"name": "{container}.{{{{COLLECTIVE}}}}".format(container=self.container_name),
"environment": [
"DOMAIN={{COLLECTIVE}}.petrode.com"
],
"volumes": [
"/mnt/log/{{COLLECTIVE}}:/var/log"
],
"dns_search": [
"petrode",
"{{COLLECTIVE}}.petrode"
]
}
}
os.environ['COLLECTIVE'] = 'example'
serv = create_service(self.conf, './Controlfile')
conf_copy = serv.prepare_container_options(prod=False)
self.assertEqual(
conf_copy['name'],
'{container}.{{{{COLLECTIVE}}}}'.format(container=self.container_name))
self.assertEqual(
conf_copy['environment'][0],
'DOMAIN={{COLLECTIVE}}.petrode.com')
self.assertEqual(
conf_copy['host_config']['Binds'][0],
"/mnt/log/{{COLLECTIVE}}:/var/log")
self.assertEqual(
conf_copy['volumes'][0],
"/var/log")
self.assertEqual(
conf_copy['host_config']['DnsSearch'][0],
"petrode")
self.assertEqual(
conf_copy['host_config']['DnsSearch'][1],
"{{COLLECTIVE}}.petrode")
del os.environ['COLLECTIVE']
def test_volume_parsing(self):
"""Make sure that volumes get created correctly"""
self.image = 'busybox'
self.conf = {
"image": self.image,
"container": {
"name": "grafana",
"hostname": "grafana",
"volumes": [
"/var",
"named-user:/usr",
"/mnt/usrbin:/usr/bin",
]
}
}
serv = create_service(deepcopy(self.conf), './Controlfile')
conf_copy = serv.prepare_container_options(prod=False)
self.assertEqual(conf_copy['host_config']['Binds'][0], self.conf['container']['volumes'][1])
self.assertEqual(conf_copy['host_config']['Binds'][1], self.conf['container']['volumes'][2])
self.assertEqual(conf_copy['volumes'][0], '/var')
self.assertEqual(conf_copy['volumes'][1], '/usr')
self.assertEqual(conf_copy['volumes'][2], '/usr/bin')
def test_mounted_volumes(self):
"""Make sure that mounted volumes are correctly registered"""
import tempfile
temp_dir = tempfile.TemporaryDirectory()
self.image = 'busybox'
self.conf = {
"image": self.image,
"container": {
"name": self.container_name,
"hostname": "busybox",
"volumes": ["{}:/var".format(temp_dir.name)]
}
}
serv = create_service(self.conf, './Controlfile')
container = Container(serv).create(prod=False)
container.start()
self.assertEqual(len(container.inspect['Mounts']), 1)
self.assertEqual(container.inspect['Mounts'][0]['Destination'], '/var')
self.assertTrue(
container.inspect['Mounts'][0]['Source'] == temp_dir.name,
msg="Unexpected mount source: {}".format(
container.inspect['Mounts'][0]['Source']))
temp_dir.cleanup()
def __init__(self, controlfile_location, force_user=False):
"""
There's two types of Controlfiles. A multi-service file that
allows some meta-operations on top of the other kind of
Controlfile. The second kind is the single service Controlfile.
Full Controlfiles can reference both kinds files to load in more
options for meta-services.
"""
self.logger = logging.getLogger('control.controlfile.Controlfile')
self.services = {
"required":
MetaService(
{
'service': 'required',
'required': True,
'services': []
}, controlfile_location),
"optional":
MetaService(
{
'service': 'optional',
'required': False,
'services': []
}, controlfile_location)
}
variables = {
"CONTROL_DIR": dn(dn(dn(os.path.abspath(__file__)))),
"CONTROL_PATH": dn(dn(os.path.abspath(__file__))),
"CONTROL_SESSION_UUID": uuid.uuid4(),
"UID": os.getuid(),
"GID": os.getgid(),
"HOSTNAME": socket.gethostname(),
}
git = {}
with subprocess.Popen(['git', 'rev-parse', '--show-toplevel'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE) as p:
p.wait()
if p.returncode == 0:
root_dir, _ = p.communicate()
root_dir = root_dir.decode('utf-8').strip()
git['GIT_ROOT_DIR'] = root_dir
with subprocess.Popen(
['git', 'rev-parse', '--abbrev-ref', 'HEAD'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE) as q:
q.wait()
if q.returncode == 0:
branch, _ = q.communicate()
git['GIT_BRANCH'] = branch.decode('utf-8').strip()
with subprocess.Popen(['git', 'rev-parse', 'HEAD'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE) as q:
q.wait()
if q.returncode == 0:
commit, _ = q.communicate()
git['GIT_COMMIT'] = commit.decode('utf-8').strip()
git['GIT_SHORT_COMMIT'] = git['GIT_COMMIT'][:7]
variables.update(git)
variables.update(os.environ)
data = self.read_in_file(controlfile_location)
if not data:
raise InvalidControlfile(controlfile_location, "empty Controlfile")
# Check if this is a single service Controlfile, if it is, wrap in a
# metaservice.
if 'services' not in data:
# Temporarily create a service to take advantage of service name guessing
serv = create_service(copy.deepcopy(data), controlfile_location)
data = {"services": {serv.service: data}}
# Check if we are running --as-me, if we are, make sure that we start
# the process with the user's UID and GID
if force_user:
if 'options' not in data:
data['options'] = {}
if 'user' not in data['options']:
data['options']['user'] = {}
data['options']['user']['replace'] = "{UID}:{GID}"
self.logger.debug("variables to substitute in: %s", variables)
self.create_service(data, 'all', {}, variables, controlfile_location)
