def test_jinja_render_silent(self):
context = {
"base_distro": "debian",
"base_tag": "jessie",
"maintainer": "some maintainer",
"duck": {
"egg": "needle"
}
}
content = jinja_utils.jinja_render(self.filename,
context,
functions=[utils.address],
ignore_undefined=True)
self.assertEqual(
"debian\njessie\nsome maintainer\nneedle\nneedle\n"
"keystone.ccp.svc.cluster.local\n"
"keystone.ccp.svc.cluster.local", content)
context = {"base_distro": "debian"}
content = jinja_utils.jinja_render(self.filename,
context,
functions=[utils.address],
ignore_undefined=True)
self.assertEqual(
"debian\n\n\n\n\nkeystone.ccp.svc.cluster.local\n"
"keystone.ccp.svc.cluster.local", content)
def create_rendered_dockerfile(dockerfile, tmp_path, config):
src_dir = os.path.dirname(dockerfile['path'])
dest_dir = os.path.join(tmp_path, dockerfile['name'])
os.makedirs(dest_dir)
dockerfilename = os.path.join(dest_dir, 'Dockerfile')
with open(dockerfilename, 'w') as f:
f.write(dockerfile['content'])
for source_name in dockerfile['sources']:
prepare_source(source_name, dockerfile['name'], dest_dir, config)
for render_file in dockerfile['render_files']:
fpath = os.path.join(src_dir, render_file['src'])
opath = os.path.join(dest_dir, render_file['dest'])
content = jinja_utils.jinja_render(fpath, config['render'])
with open(opath, 'wb') as f:
f.write(content.encode('utf-8'))
for filename in os.listdir(src_dir):
if 'Dockerfile' in filename:
continue
full_filename = os.path.join(src_dir, filename)
if os.path.isfile(full_filename):
shutil.copy(full_filename, dest_dir)
elif os.path.isdir(full_filename):
shutil.copytree(full_filename, os.path.join(dest_dir, filename))
return dockerfilename
def get_deploy_components_info(rendering_context=None):
if rendering_context is None:
rendering_context = CONF.configs._dict
components_map = {}
for component_ref in CONF.repositories.repos:
component_name = component_ref['name']
service_dir = os.path.join(CONF.repositories.path, component_name,
'service')
if not os.path.isdir(service_dir):
continue
REPO_NAME_PREFIX = "fuel-ccp-"
if component_name.startswith(REPO_NAME_PREFIX):
component_name = component_name[len(REPO_NAME_PREFIX):]
for service_file in os.listdir(service_dir):
if service_file.endswith('.yaml'):
LOG.debug("Rendering service definition: %s", service_file)
content = jinja_utils.jinja_render(os.path.join(
service_dir, service_file),
rendering_context,
functions=[address])
LOG.debug("Parse service definition: %s", service_file)
service_definition = yaml.load(content)
service_name = service_definition['service']['name']
components_map[service_name] = {
'component_name': component_name,
'service_dir': service_dir,
'service_content': service_definition
}
return components_map
def render_dockerfile(path, name, config):
LOG.debug('%s: Rendering dockerfile', name)
sources = set()
parent = [] # Could've been None if we could use nonlocal
render_files = []
def copy_sources(source_name, cont_dir):
if source_name not in config['sources']:
raise ValueError('No such source: %s' % source_name)
sources.add(source_name)
return 'COPY %s %s' % (source_name, cont_dir)
def image_spec(image_name):
if parent:
raise RuntimeError('You can use image_spec only once in FROM line')
parent.append(image_name)
return images.image_spec(image_name, add_address=CONF.builder.push)
def render(fname):
if fname.endswith('.j2'):
oname = fname[:-3]
else:
oname = fname + '.rendered'
render_files.append({
'src': fname,
'dest': oname
})
return oname
content = jinja_utils.jinja_render(path, config['render'],
[copy_sources, image_spec, render])
return content, sources, render_files, parent[0] if parent else None
def _get_service_files_hash(files, configs):
data = {}
if files:
for filename, f in files.items():
data[filename] = jinja_utils.jinja_render(f["content"],
configs, [utils.address],
ignore_undefined=True)
dump = json.dumps(data, sort_keys=True).encode("utf-8")
return hashlib.sha1(dump).hexdigest()
def _get_service_files_hash(files, configs):
data = {}
if files:
for filename, f in files.items():
data[filename] = jinja_utils.jinja_render(
f["content"], configs, [utils.address],
ignore_undefined=True)
dump = json.dumps(data, sort_keys=True).encode("utf-8")
return hashlib.sha1(dump).hexdigest()
def _get_service_files_hash(service_dir, files, configs):
data = {}
if files:
for filename, f in files.items():
path = os.path.join(service_dir, "files", f["content"])
data[filename] = jinja_utils.jinja_render(
path, configs, ignore_undefined=True)
dump = json.dumps(data, sort_keys=True).encode("utf-8")
return hashlib.sha1(dump).hexdigest()
def get_deploy_components_info(rendering_context=None):
if rendering_context is None:
rendering_context = CONF.configs._dict
components_map = {}
for repo in get_repositories_paths():
service_dir = os.path.join(repo, "service")
if not os.path.isdir(service_dir):
continue
component_name = get_component_name_from_repo_path(repo)
component = {
"name": component_name,
"upgrades": {},
"service_dir": service_dir,
}
upgrade_dir = os.path.join(service_dir, "upgrade")
if os.path.isdir(upgrade_dir):
for upgrade_fname in os.listdir(upgrade_dir):
if not upgrade_fname.endswith('.yaml'):
continue
LOG.debug("Loading upgrade definition: %s", upgrade_fname)
with open(os.path.join(upgrade_dir, upgrade_fname)) as f:
upgrade_def = yaml.load(f)
key = upgrade_fname[:-len('.yaml')]
component['upgrades'][key] = upgrade_def
for service_file in os.listdir(service_dir):
if service_file.endswith('.yaml'):
LOG.debug("Rendering service definition: %s", service_file)
content = jinja_utils.jinja_render(os.path.join(
service_dir, service_file),
rendering_context,
functions=[address])
LOG.debug("Parse service definition: %s", service_file)
service_definition = yaml.load(content)
service_name = service_definition['service']['name']
components_map[service_name] = {
'component': component,
'component_name': component_name,
'service_dir': service_dir,
'service_content': service_definition
}
return components_map
def test_jinja_render_strict(self):
context = {
"base_distro": "debian",
"base_tag": "jessie",
"maintainer": "some maintainer",
"duck": {
"egg": "needle"
}
}
content = jinja_utils.jinja_render(self.filename,
context,
functions=[utils.address])
self.assertEqual(
"debian\njessie\nsome maintainer\nneedle\nneedle\nkeystone.ccp",
content)
context = {"base_distro": "debian"}
self.assertRaises(exceptions.UndefinedError, jinja_utils.jinja_render,
self.filename, context)
def create_rendered_dockerfile(path, name, tmp_path, config):
content = jinja_utils.jinja_render(path, config)
src_dir = os.path.dirname(path)
dest_dir = os.path.join(tmp_path, name)
os.makedirs(dest_dir)
dockerfilename = os.path.join(dest_dir, 'Dockerfile')
with open(dockerfilename, 'w') as f:
f.write(content)
for filename in os.listdir(src_dir):
if 'Dockerfile' in filename:
continue
full_filename = os.path.join(src_dir, filename)
if os.path.isfile(full_filename):
shutil.copy(full_filename, dest_dir)
elif os.path.isdir(full_filename):
shutil.copytree(full_filename, os.path.join(dest_dir, filename))
return dockerfilename
def list_actions():
"""List of available actions.
:returns: list -- list of all available actions
"""
actions = []
for repo in utils.get_repositories_paths():
component_name = utils.get_component_name_from_repo_path(repo)
action_path = os.path.join(repo, "service", "actions")
if not os.path.isdir(action_path):
continue
for filename in os.listdir(action_path):
if filename.endswith(".yaml"):
action_file = os.path.join(action_path, filename)
conf = utils.get_rendering_config()
data = jinja_utils.jinja_render(action_file, conf._dict)
for action_dict in yaml.load(data).get("actions", ()):
actions.append(Action(component=component_name,
component_dir=repo,
**action_dict))
return actions
def list_actions():
"""List of available actions.
:returns: list -- list of all available actions
"""
actions = []
for repo in utils.get_repositories_paths():
component_name = utils.get_component_name_from_repo_path(repo)
action_path = os.path.join(repo, "service", "actions")
if not os.path.isdir(action_path):
continue
for filename in os.listdir(action_path):
if filename.endswith(".yaml"):
action_file = os.path.join(action_path, filename)
conf = utils.get_rendering_config()
data = jinja_utils.jinja_render(action_file, conf._dict)
for action_dict in yaml.load(data).get("actions", ()):
actions.append(
Action(component=component_name,
component_dir=repo,
**action_dict))
return actions
def get_deploy_components_info():
rendering_context = get_rendering_config()
service_definitions_map = get_service_definitions_map()
services_map = {}
custom_services_map = {}
for repo in get_repositories_paths():
service_dir = os.path.join(repo, "service")
if not os.path.isdir(service_dir):
continue
component_name = get_component_name_from_repo_path(repo)
component = {
"name": component_name,
"upgrades": {},
"service_dir": service_dir,
}
upgrade_dir = os.path.join(service_dir, "upgrade")
if os.path.isdir(upgrade_dir):
for upgrade_fname in os.listdir(upgrade_dir):
if not upgrade_fname.endswith('.yaml'):
continue
LOG.debug("Loading upgrade definition: %s", upgrade_fname)
with open(os.path.join(upgrade_dir, upgrade_fname)) as f:
upgrade_def = yaml.load(f)
key = upgrade_fname[:-len('.yaml')]
component['upgrades'][key] = upgrade_def
for service_file in os.listdir(service_dir):
if service_file.endswith('.yaml'):
LOG.debug("Rendering service definition: %s", service_file)
content = jinja_utils.jinja_render(
os.path.join(service_dir, service_file),
rendering_context._dict, functions=[address]
)
LOG.debug("Parse service definition: %s", service_file)
service_definition = yaml.load(content)
service_name = service_definition['service']['name']
services_map[service_name] = {
'component': component,
'component_name': component_name,
'service_dir': service_dir,
'service_content': service_definition
}
for svc in service_definitions_map.get(service_name, ()):
LOG.debug("Rendering service definition: %s for '%s' "
"service", service_file, svc)
context = copy.deepcopy(rendering_context)
context['_current_service'] = svc
extend_with_service_configs(svc, context)
content = jinja_utils.jinja_render(
os.path.join(service_dir, service_file),
context._dict, functions=[address]
)
LOG.debug("Parse service definition: %s for '%s' "
"service", service_file, svc)
service_definition = yaml.load(content)
service_definition['service']['name'] = svc
custom_services_map[svc] = {
'component': component,
'component_name': component_name,
'service_dir': service_dir,
'service_content': service_definition
}
deps_map = get_dependencies_map(services_map)
services_map.update(custom_services_map)
for svc_name, svc in services_map.items():
process_dependencies(svc, deps_map, services_map)
return services_map
def get_deploy_components_info():
rendering_context = get_rendering_config()
service_definitions_map = get_service_definitions_map()
services_map = {}
custom_services_map = {}
for repo in get_repositories_paths():
service_dir = os.path.join(repo, "service")
if not os.path.isdir(service_dir):
continue
component_name = get_component_name_from_repo_path(repo)
component = {
"name": component_name,
"upgrades": {},
"service_dir": service_dir,
}
upgrade_dir = os.path.join(service_dir, "upgrade")
if os.path.isdir(upgrade_dir):
for upgrade_fname in os.listdir(upgrade_dir):
if not upgrade_fname.endswith('.yaml'):
continue
LOG.debug("Loading upgrade definition: %s", upgrade_fname)
with open(os.path.join(upgrade_dir, upgrade_fname)) as f:
upgrade_def = yaml.load(f)
key = upgrade_fname[:-len('.yaml')]
component['upgrades'][key] = upgrade_def
for service_file in os.listdir(service_dir):
if service_file.endswith('.yaml'):
LOG.debug("Rendering service definition: %s", service_file)
content = jinja_utils.jinja_render(os.path.join(
service_dir, service_file),
rendering_context._dict,
functions=[address])
LOG.debug("Parse service definition: %s", service_file)
service_definition = yaml.load(content)
service_name = service_definition['service']['name']
services_map[service_name] = {
'component': component,
'component_name': component_name,
'service_dir': service_dir,
'service_content': service_definition
}
for svc in service_definitions_map.get(service_name, ()):
LOG.debug(
"Rendering service definition: %s for '%s' "
"service", service_file, svc)
context = copy.deepcopy(rendering_context)
context['_current_service'] = svc
extend_with_service_configs(svc, context)
content = jinja_utils.jinja_render(os.path.join(
service_dir, service_file),
context._dict,
functions=[address])
LOG.debug(
"Parse service definition: %s for '%s' "
"service", service_file, svc)
service_definition = yaml.load(content)
service_definition['service']['name'] = svc
custom_services_map[svc] = {
'component': component,
'component_name': component_name,
'service_dir': service_dir,
'service_content': service_definition
}
deps_map = get_dependencies_map(services_map)
services_map.update(custom_services_map)
for svc_name, svc in services_map.items():
process_dependencies(svc, deps_map, services_map)
return services_map
