def _upload_templates(swift_client,
container_name,
tht_root,
roles_file=None,
plan_env_file=None,
networks_file=None):
"""tarball up a given directory and upload it to Swift to be extracted"""
with tempfile.NamedTemporaryFile() as tmp_tarball:
tarball.create_tarball(tht_root, tmp_tarball.name)
tarball.tarball_extract_to_swift_container(swift_client,
tmp_tarball.name,
container_name)
# Optional override of the roles_data.yaml file
if roles_file:
_upload_file(swift_client, container_name,
constants.OVERCLOUD_ROLES_FILE,
utils.rel_or_abs_path(roles_file, tht_root))
# Optional override of the network_data.yaml file
if networks_file:
_upload_file(swift_client, container_name,
constants.OVERCLOUD_NETWORKS_FILE, networks_file)
# Optional override of the plan-environment.yaml file
if plan_env_file:
# TODO(jpalanis): Instead of overriding default file,
# merging the user override plan-environment with default
# plan-environment file will avoid explict merging issues.
_upload_file(swift_client, container_name, constants.PLAN_ENVIRONMENT,
plan_env_file)
def create_plan_from_templates(clients,
name,
tht_root,
roles_file=None,
generate_passwords=True,
plan_env_file=None,
networks_file=None,
validate_stack=True):
workflow_client = clients.workflow_engine
swift_client = clients.tripleoclient.object_store
print("Creating Swift container to store the plan")
result = create_container(workflow_client, container=name)
if result:
# create_container returns 'None' on success and a string with
# the error message when failing.
raise exceptions.PlanCreationError(
"Unable to create plan. {}".format(result))
print("Creating plan from template files in: {}".format(tht_root))
_upload_templates(swift_client, name, tht_root,
utils.rel_or_abs_path(roles_file, tht_root),
plan_env_file, networks_file)
try:
create_deployment_plan(clients,
container=name,
generate_passwords=generate_passwords,
validate_stack=validate_stack)
except exceptions.WorkflowServiceError:
swiftutils.delete_container(swift_client, name)
raise
def create_plan_from_templates(clients,
name,
tht_root,
roles_file=None,
generate_passwords=True,
plan_env_file=None,
networks_file=None,
validate_stack=True,
verbosity_level=0):
swift_client = clients.tripleoclient.object_store
print("Creating Swift container to store the plan")
plan_utils.create_plan_container(swift_client, name)
print("Creating plan from template files in: {}".format(tht_root))
_upload_templates(swift_client, name, tht_root,
utils.rel_or_abs_path(roles_file, tht_root),
plan_env_file, networks_file)
try:
create_deployment_plan(container=name,
generate_passwords=generate_passwords,
plan_env_file=plan_env_file,
validate_stack=validate_stack,
verbosity_level=verbosity_level)
except exceptions.WorkflowServiceError:
swiftutils.delete_container(swift_client, name)
raise
def _validate_env_files_paths():
"""Verify the non-matching templates path vs env files paths"""
tht_path = CONF.get('templates') or constants.TRIPLEO_HEAT_TEMPLATES
roles_file = utils.rel_or_abs_path(
CONF.get('roles_file') or constants.UNDERCLOUD_ROLES_FILE, tht_path)
# get the list of jinja templates normally rendered for UC installations
LOG.debug(_("Using roles file {0} from {1}").format(roles_file, tht_path))
process_templates = os.path.join(tht_path, 'tools/process-templates.py')
python_interpreter = "/usr/bin/python{}".format(sys.version_info[0])
p = _run_live_command([
python_interpreter, process_templates, '--roles-data', roles_file,
'--dry-run'
],
name='process-templates-dry-run',
cwd=tht_path,
wait=False)
# parse the list for the rendered from j2 file names
result = p.communicate()[0]
j2_files_list = []
for line in result.split("\n"):
if ((line.startswith('dry run') or line.startswith('jinja2'))
and line.endswith('.yaml')):
bname = os.path.basename(line.split(' ')[-1])
if line.startswith('dry run'):
j2_files_list.append(bname)
if line.startswith('jinja2'):
j2_files_list.append(bname.replace('.j2', ''))
for env_file in CONF['custom_env_files']:
env_file_abs = os.path.abspath(env_file)
if (os.path.dirname(env_file_abs) != os.path.abspath(tht_path)
and os.path.basename(env_file) in j2_files_list):
msg = _('Heat environment external to the templates dir '
'can not reference j2 processed file %s') % env_file_abs
LOG.error(msg)
raise FailedValidation(msg)
def update_plan_from_templates(clients,
name,
tht_root,
roles_file=None,
generate_passwords=True,
plan_env_file=None,
networks_file=None,
keep_env=False,
validate_stack=True):
swift_client = clients.tripleoclient.object_store
passwords = None
keep_file_contents = {}
roles_file = utils.rel_or_abs_path(roles_file, tht_root)
if keep_env:
# Dict items are (remote_name, local_name). local_name may be
# None in which case we only try to load from Swift (remote).
keep_map = {
constants.PLAN_ENVIRONMENT: plan_env_file,
constants.USER_ENVIRONMENT: None,
constants.OVERCLOUD_ROLES_FILE: roles_file,
constants.OVERCLOUD_NETWORKS_FILE: networks_file,
}
# Also try to fetch any files under 'user-files/'
# dir. local_name is always None for these
keep_map.update(
dict(
map(lambda path: (path, None),
_list_user_files(swift_client, name))))
keep_file_contents = _load_content_or_file(swift_client, name,
keep_map)
else:
passwords = _load_passwords(swift_client, name)
# TODO(dmatthews): Removing the existing plan files should probably be
# a Mistral action.
print("Removing the current plan files")
swiftutils.empty_container(swift_client, name)
# Until we have a well defined plan update workflow in
# tripleo-common we need to manually reset the environments and
# parameter_defaults here. This is to ensure that no environments
# are in the plan environment but not actually in swift.
# See bug: https://bugs.launchpad.net/tripleo/+bug/1623431
#
# Currently this is being done incidentally because we overwrite
# the existing plan-environment.yaml with the skeleton one in THT
# when updating the templates. Once LP#1623431 is resolved we may
# need to special-case plan-environment.yaml to avoid this.
print("Uploading new plan files")
if keep_env:
_upload_templates(swift_client, name, tht_root)
for filename in keep_file_contents:
_upload_file_content(swift_client, name, filename,
keep_file_contents[filename])
else:
_upload_templates(swift_client, name, tht_root, roles_file,
plan_env_file, networks_file)
_update_passwords(swift_client, name, passwords)
update_deployment_plan(clients,
container=name,
generate_passwords=generate_passwords,
source_url=None,
validate_stack=validate_stack)
def test_norm_path_rel(self, mock_exists):
self.assertEqual(
utils.rel_or_abs_path('baz/foobar.yaml', '/bar'),
'/bar/baz/foobar.yaml')
def test_norm_path_abs(self, mock_exists):
self.assertEqual(
utils.rel_or_abs_path('/foobar.yaml', '/tmp'),
'/foobar.yaml')
def _setup_heat_environments(self, parsed_args):
"""Process tripleo heat templates with jinja and deploy into work dir
* Process j2/install additional templates there
* Return the environments list for futher processing as a new base.
The first two items are reserved for the
overcloud-resource-registry-puppet.yaml and passwords files.
"""
self.log.warning(_("** Handling template files **"))
env_files = []
# TODO(aschultz): in overcloud deploy we have a --environments-dir
# we might want to handle something similar for this
# (shardy) alternatively perhaps we should rely on the plan-environment
# environments list instead?
if parsed_args.environment_files:
env_files.extend(parsed_args.environment_files)
# ensure any user provided templates get copied into tht_render
user_environments = self._normalize_user_templates(
parsed_args.templates, self.tht_render, env_files)
# generate jinja templates by its work dir location
self.log.debug(_("Using roles file %s") % self.roles_file)
process_templates = os.path.join(parsed_args.templates,
'tools/process-templates.py')
args = [
'python', process_templates, '--roles-data', self.roles_file,
'--output-dir', self.tht_render
]
if utils.run_command_and_log(self.log, args, cwd=self.tht_render) != 0:
# TODO(aschultz): improve error messaging
msg = _("Problems generating templates.")
self.log.error(msg)
raise exceptions.DeploymentError(msg)
# NOTE(aschultz): the next set of environment files are system included
# so we have to include them at the front of our environment list so a
# user can override anything in them.
# Include any environments from the plan-environment.yaml
plan_env_path = utils.rel_or_abs_path(
self.tht_render, parsed_args.plan_environment_file)
with open(plan_env_path, 'r') as f:
plan_env_data = yaml.safe_load(f)
environments = [
utils.rel_or_abs_path(self.tht_render, e.get('path'))
for e in plan_env_data.get('environments', {})
]
# this will allow the user to overwrite passwords with custom envs
pw_file = self._update_passwords_env(self.output_dir)
environments.append(pw_file)
# use deployed-server because we run os-collect-config locally
deployed_server_env = os.path.join(self.tht_render, 'environments',
'config-download-environment.yaml')
environments.append(deployed_server_env)
# use deployed-server because we run os-collect-config locally
deployed_server_env = os.path.join(
self.tht_render, 'environments',
'deployed-server-noop-ctlplane.yaml')
environments.append(deployed_server_env)
self.log.info(
_("Deploying templates in the directory {0}").format(
os.path.abspath(self.tht_render)))
maps_file = os.path.join(self.tht_render,
'tripleoclient-hosts-portmaps.yaml')
ip_nw = netaddr.IPNetwork(parsed_args.local_ip)
ip = str(ip_nw.ip)
if parsed_args.control_virtual_ip:
c_ip = parsed_args.control_virtual_ip
else:
c_ip = ip
if parsed_args.public_virtual_ip:
p_ip = parsed_args.public_virtual_ip
else:
p_ip = ip
tmp_env = self._generate_hosts_parameters(parsed_args, p_ip)
tmp_env.update(
self._generate_portmap_parameters(
ip,
ip_nw,
c_ip,
p_ip,
stack_name=parsed_args.stack,
role_name=self._get_primary_role_name()))
with open(maps_file, 'w') as env_file:
yaml.safe_dump({'parameter_defaults': tmp_env},
env_file,
default_flow_style=False)
environments.append(maps_file)
# NOTE(aschultz): this doesn't get copied into tht_root but
# we always include the hieradata override stuff last.
if parsed_args.hieradata_override:
environments.append(
self._process_hieradata_overrides(
parsed_args.hieradata_override,
parsed_args.standalone_role))
# Create a persistent drop-in file to indicate the stack
# virtual state changes
stack_vstate_dropin = os.path.join(
self.tht_render, '%s-stack-vstate-dropin.yaml' % parsed_args.stack)
with open(stack_vstate_dropin, 'w') as dropin_file:
yaml.safe_dump(
{'parameter_defaults': {
'StackAction': self.stack_action
}},
dropin_file,
default_flow_style=False)
environments.append(stack_vstate_dropin)
return environments + user_environments
def get_parser(self, prog_name):
parser = super(TripleOImagePrepare, self).get_parser(prog_name)
try:
roles_file = utils.rel_or_abs_path(
constants.OVERCLOUD_ROLES_FILE,
constants.TRIPLEO_HEAT_TEMPLATES)
except exceptions.DeploymentError:
roles_file = None
parser.add_argument(
'--environment-file', '-e', metavar='<file path>',
action='append', dest='environment_files',
help=_('Environment file containing the ContainerImagePrepare '
'parameter which specifies all prepare actions. '
'Also, environment files specifying which services are '
'containerized. Entries will be filtered to only contain '
'images used by containerized services. (Can be specified '
'more than once.)')
)
parser.add_argument(
'--environment-directory', metavar='<HEAT ENVIRONMENT DIRECTORY>',
action='append', dest='environment_directories',
default=[os.path.expanduser(constants.DEFAULT_ENV_DIRECTORY)],
help=_('Environment file directories that are automatically '
'added to the environment. '
'Can be specified more than once. Files in directories are '
'loaded in ascending sort order.')
)
parser.add_argument(
'--roles-file', '-r', dest='roles_file',
default=roles_file,
help=_(
'Roles file, overrides the default %s in the t-h-t templates '
'directory used for deployment. May be an '
'absolute path or the path relative to the templates dir.'
) % constants.OVERCLOUD_ROLES_FILE
)
parser.add_argument(
"--output-env-file",
dest="output_env_file",
metavar='<file path>',
help=_("File to write heat environment file which specifies all "
"image parameters. Any existing file will be overwritten."),
)
parser.add_argument(
'--dry-run',
dest='dry_run',
action='store_true',
default=False,
help=_('Do not perform any pull, modify, or push operations. '
'The environment file will still be populated as if these '
'operations were performed.')
)
parser.add_argument(
"--cleanup",
dest="cleanup",
metavar='<full, partial, none>',
default=image_uploader.CLEANUP_FULL,
help=_("Cleanup behavior for local images left after upload. "
"The default 'full' will attempt to delete all local "
"images. 'partial' will leave images required for "
"deployment on this host. 'none' will do no cleanup.")
)
return parser
def get_parser(self, prog_name):
parser = super(PrepareImageFiles, self).get_parser(prog_name)
try:
roles_file = utils.rel_or_abs_path(
constants.OVERCLOUD_ROLES_FILE,
constants.TRIPLEO_HEAT_TEMPLATES)
except exceptions.DeploymentError:
roles_file = None
defaults = kolla_builder.container_images_prepare_defaults()
parser.add_argument(
"--template-file",
dest="template_file",
default=kolla_builder.DEFAULT_TEMPLATE_FILE,
metavar='<yaml template file>',
help=_("YAML template file which the images config file will be "
"built from.\n"
"Default: %s") % kolla_builder.DEFAULT_TEMPLATE_FILE,
)
parser.add_argument(
"--push-destination",
dest="push_destination",
metavar='<location>',
help=_("Location of image registry to push images to. "
"If specified, a push_destination will be set for every "
"image entry."),
)
parser.add_argument(
"--tag",
dest="tag",
default=defaults['tag'],
metavar='<tag>',
help=_("Override the default tag substitution. "
"If --tag-from-label is specified, "
"start discovery with this tag.\n"
"Default: %s") % defaults['tag'],
)
parser.add_argument(
"--tag-from-label",
dest="tag_from_label",
metavar='<image label>',
help=_("Use the value of the specified label(s) to discover the "
"tag. Labels can be combined in a template format, "
"for example: {version}-{release}"),
)
parser.add_argument(
"--namespace",
dest="namespace",
default=defaults['namespace'],
metavar='<namespace>',
help=_("Override the default namespace substitution.\n"
"Default: %s") % defaults['namespace'],
)
parser.add_argument(
"--prefix",
dest="prefix",
default=defaults['name_prefix'],
metavar='<prefix>',
help=_("Override the default name prefix substitution.\n"
"Default: %s") % defaults['name_prefix'],
)
parser.add_argument(
"--suffix",
dest="suffix",
default=defaults['name_suffix'],
metavar='<suffix>',
help=_("Override the default name suffix substitution.\n"
"Default: %s") % defaults['name_suffix'],
)
parser.add_argument(
'--set',
metavar='<variable=value>',
action='append',
help=_('Set the value of a variable in the template, even if it '
'has no dedicated argument such as "--suffix".')
)
parser.add_argument(
"--exclude",
dest="excludes",
metavar='<regex>',
default=[],
action="append",
help=_("Pattern to match against resulting imagename entries to "
"exclude from the final output. Can be specified multiple "
"times."),
)
parser.add_argument(
"--include",
dest="includes",
metavar='<regex>',
default=[],
action="append",
help=_("Pattern to match against resulting imagename entries to "
"include in final output. Can be specified multiple "
"times, entries not matching any --include will be "
"excluded. --exclude is ignored if --include is used."),
)
parser.add_argument(
"--output-images-file",
dest="output_images_file",
metavar='<file path>',
help=_("File to write resulting image entries to, as well as "
"stdout. Any existing file will be overwritten."),
)
parser.add_argument(
'--environment-file', '-e', metavar='<file path>',
action='append', dest='environment_files',
help=_('Environment files specifying which services are '
'containerized. Entries will be filtered to only contain '
'images used by containerized services. (Can be specified '
'more than once.)')
)
parser.add_argument(
'--environment-directory', metavar='<HEAT ENVIRONMENT DIRECTORY>',
action='append', dest='environment_directories',
default=[os.path.expanduser(constants.DEFAULT_ENV_DIRECTORY)],
help=_('Environment file directories that are automatically '
'added to the update command. Entries will be filtered '
'to only contain images used by containerized services. '
'Can be specified more than once. Files in directories are '
'loaded in ascending sort order.')
)
parser.add_argument(
"--output-env-file",
dest="output_env_file",
metavar='<file path>',
help=_("File to write heat environment file which specifies all "
"image parameters. Any existing file will be overwritten."),
)
parser.add_argument(
'--roles-file', '-r', dest='roles_file',
default=roles_file,
help=_(
'Roles file, overrides the default %s in the t-h-t templates '
'directory used for deployment. May be an '
'absolute path or the path relative to the templates dir.'
) % constants.OVERCLOUD_ROLES_FILE
)
parser.add_argument(
'--modify-role',
dest='modify_role',
help=_('Name of ansible role to run between every image upload '
'pull and push.')
)
parser.add_argument(
'--modify-vars',
dest='modify_vars',
help=_('Ansible variable file containing variables to use when '
'invoking the role --modify-role.')
)
return parser
