def _upload_templates(swift_client, container_name, tht_root, roles_file=None, plan_env_file=None, networks_file=None): """tarball up a given directory and upload it to Swift to be extracted""" with tempfile.NamedTemporaryFile() as tmp_tarball: tarball.create_tarball(tht_root, tmp_tarball.name) tarball.tarball_extract_to_swift_container(swift_client, tmp_tarball.name, container_name) # Optional override of the roles_data.yaml file if roles_file: _upload_file(swift_client, container_name, constants.OVERCLOUD_ROLES_FILE, utils.rel_or_abs_path(roles_file, tht_root)) # Optional override of the network_data.yaml file if networks_file: _upload_file(swift_client, container_name, constants.OVERCLOUD_NETWORKS_FILE, networks_file) # Optional override of the plan-environment.yaml file if plan_env_file: # TODO(jpalanis): Instead of overriding default file, # merging the user override plan-environment with default # plan-environment file will avoid explict merging issues. _upload_file(swift_client, container_name, constants.PLAN_ENVIRONMENT, plan_env_file)
def create_plan_from_templates(clients, name, tht_root, roles_file=None, generate_passwords=True, plan_env_file=None, networks_file=None, validate_stack=True): workflow_client = clients.workflow_engine swift_client = clients.tripleoclient.object_store print("Creating Swift container to store the plan") result = create_container(workflow_client, container=name) if result: # create_container returns 'None' on success and a string with # the error message when failing. raise exceptions.PlanCreationError( "Unable to create plan. {}".format(result)) print("Creating plan from template files in: {}".format(tht_root)) _upload_templates(swift_client, name, tht_root, utils.rel_or_abs_path(roles_file, tht_root), plan_env_file, networks_file) try: create_deployment_plan(clients, container=name, generate_passwords=generate_passwords, validate_stack=validate_stack) except exceptions.WorkflowServiceError: swiftutils.delete_container(swift_client, name) raise
def create_plan_from_templates(clients, name, tht_root, roles_file=None, generate_passwords=True, plan_env_file=None, networks_file=None, validate_stack=True, verbosity_level=0): swift_client = clients.tripleoclient.object_store print("Creating Swift container to store the plan") plan_utils.create_plan_container(swift_client, name) print("Creating plan from template files in: {}".format(tht_root)) _upload_templates(swift_client, name, tht_root, utils.rel_or_abs_path(roles_file, tht_root), plan_env_file, networks_file) try: create_deployment_plan(container=name, generate_passwords=generate_passwords, plan_env_file=plan_env_file, validate_stack=validate_stack, verbosity_level=verbosity_level) except exceptions.WorkflowServiceError: swiftutils.delete_container(swift_client, name) raise
def _validate_env_files_paths(): """Verify the non-matching templates path vs env files paths""" tht_path = CONF.get('templates') or constants.TRIPLEO_HEAT_TEMPLATES roles_file = utils.rel_or_abs_path( CONF.get('roles_file') or constants.UNDERCLOUD_ROLES_FILE, tht_path) # get the list of jinja templates normally rendered for UC installations LOG.debug(_("Using roles file {0} from {1}").format(roles_file, tht_path)) process_templates = os.path.join(tht_path, 'tools/process-templates.py') python_interpreter = "/usr/bin/python{}".format(sys.version_info[0]) p = _run_live_command([ python_interpreter, process_templates, '--roles-data', roles_file, '--dry-run' ], name='process-templates-dry-run', cwd=tht_path, wait=False) # parse the list for the rendered from j2 file names result = p.communicate()[0] j2_files_list = [] for line in result.split("\n"): if ((line.startswith('dry run') or line.startswith('jinja2')) and line.endswith('.yaml')): bname = os.path.basename(line.split(' ')[-1]) if line.startswith('dry run'): j2_files_list.append(bname) if line.startswith('jinja2'): j2_files_list.append(bname.replace('.j2', '')) for env_file in CONF['custom_env_files']: env_file_abs = os.path.abspath(env_file) if (os.path.dirname(env_file_abs) != os.path.abspath(tht_path) and os.path.basename(env_file) in j2_files_list): msg = _('Heat environment external to the templates dir ' 'can not reference j2 processed file %s') % env_file_abs LOG.error(msg) raise FailedValidation(msg)
def update_plan_from_templates(clients, name, tht_root, roles_file=None, generate_passwords=True, plan_env_file=None, networks_file=None, keep_env=False, validate_stack=True): swift_client = clients.tripleoclient.object_store passwords = None keep_file_contents = {} roles_file = utils.rel_or_abs_path(roles_file, tht_root) if keep_env: # Dict items are (remote_name, local_name). local_name may be # None in which case we only try to load from Swift (remote). keep_map = { constants.PLAN_ENVIRONMENT: plan_env_file, constants.USER_ENVIRONMENT: None, constants.OVERCLOUD_ROLES_FILE: roles_file, constants.OVERCLOUD_NETWORKS_FILE: networks_file, } # Also try to fetch any files under 'user-files/' # dir. local_name is always None for these keep_map.update( dict( map(lambda path: (path, None), _list_user_files(swift_client, name)))) keep_file_contents = _load_content_or_file(swift_client, name, keep_map) else: passwords = _load_passwords(swift_client, name) # TODO(dmatthews): Removing the existing plan files should probably be # a Mistral action. print("Removing the current plan files") swiftutils.empty_container(swift_client, name) # Until we have a well defined plan update workflow in # tripleo-common we need to manually reset the environments and # parameter_defaults here. This is to ensure that no environments # are in the plan environment but not actually in swift. # See bug: https://bugs.launchpad.net/tripleo/+bug/1623431 # # Currently this is being done incidentally because we overwrite # the existing plan-environment.yaml with the skeleton one in THT # when updating the templates. Once LP#1623431 is resolved we may # need to special-case plan-environment.yaml to avoid this. print("Uploading new plan files") if keep_env: _upload_templates(swift_client, name, tht_root) for filename in keep_file_contents: _upload_file_content(swift_client, name, filename, keep_file_contents[filename]) else: _upload_templates(swift_client, name, tht_root, roles_file, plan_env_file, networks_file) _update_passwords(swift_client, name, passwords) update_deployment_plan(clients, container=name, generate_passwords=generate_passwords, source_url=None, validate_stack=validate_stack)
def test_norm_path_rel(self, mock_exists): self.assertEqual( utils.rel_or_abs_path('baz/foobar.yaml', '/bar'), '/bar/baz/foobar.yaml')
def test_norm_path_abs(self, mock_exists): self.assertEqual( utils.rel_or_abs_path('/foobar.yaml', '/tmp'), '/foobar.yaml')
def _setup_heat_environments(self, parsed_args): """Process tripleo heat templates with jinja and deploy into work dir * Process j2/install additional templates there * Return the environments list for futher processing as a new base. The first two items are reserved for the overcloud-resource-registry-puppet.yaml and passwords files. """ self.log.warning(_("** Handling template files **")) env_files = [] # TODO(aschultz): in overcloud deploy we have a --environments-dir # we might want to handle something similar for this # (shardy) alternatively perhaps we should rely on the plan-environment # environments list instead? if parsed_args.environment_files: env_files.extend(parsed_args.environment_files) # ensure any user provided templates get copied into tht_render user_environments = self._normalize_user_templates( parsed_args.templates, self.tht_render, env_files) # generate jinja templates by its work dir location self.log.debug(_("Using roles file %s") % self.roles_file) process_templates = os.path.join(parsed_args.templates, 'tools/process-templates.py') args = [ 'python', process_templates, '--roles-data', self.roles_file, '--output-dir', self.tht_render ] if utils.run_command_and_log(self.log, args, cwd=self.tht_render) != 0: # TODO(aschultz): improve error messaging msg = _("Problems generating templates.") self.log.error(msg) raise exceptions.DeploymentError(msg) # NOTE(aschultz): the next set of environment files are system included # so we have to include them at the front of our environment list so a # user can override anything in them. # Include any environments from the plan-environment.yaml plan_env_path = utils.rel_or_abs_path( self.tht_render, parsed_args.plan_environment_file) with open(plan_env_path, 'r') as f: plan_env_data = yaml.safe_load(f) environments = [ utils.rel_or_abs_path(self.tht_render, e.get('path')) for e in plan_env_data.get('environments', {}) ] # this will allow the user to overwrite passwords with custom envs pw_file = self._update_passwords_env(self.output_dir) environments.append(pw_file) # use deployed-server because we run os-collect-config locally deployed_server_env = os.path.join(self.tht_render, 'environments', 'config-download-environment.yaml') environments.append(deployed_server_env) # use deployed-server because we run os-collect-config locally deployed_server_env = os.path.join( self.tht_render, 'environments', 'deployed-server-noop-ctlplane.yaml') environments.append(deployed_server_env) self.log.info( _("Deploying templates in the directory {0}").format( os.path.abspath(self.tht_render))) maps_file = os.path.join(self.tht_render, 'tripleoclient-hosts-portmaps.yaml') ip_nw = netaddr.IPNetwork(parsed_args.local_ip) ip = str(ip_nw.ip) if parsed_args.control_virtual_ip: c_ip = parsed_args.control_virtual_ip else: c_ip = ip if parsed_args.public_virtual_ip: p_ip = parsed_args.public_virtual_ip else: p_ip = ip tmp_env = self._generate_hosts_parameters(parsed_args, p_ip) tmp_env.update( self._generate_portmap_parameters( ip, ip_nw, c_ip, p_ip, stack_name=parsed_args.stack, role_name=self._get_primary_role_name())) with open(maps_file, 'w') as env_file: yaml.safe_dump({'parameter_defaults': tmp_env}, env_file, default_flow_style=False) environments.append(maps_file) # NOTE(aschultz): this doesn't get copied into tht_root but # we always include the hieradata override stuff last. if parsed_args.hieradata_override: environments.append( self._process_hieradata_overrides( parsed_args.hieradata_override, parsed_args.standalone_role)) # Create a persistent drop-in file to indicate the stack # virtual state changes stack_vstate_dropin = os.path.join( self.tht_render, '%s-stack-vstate-dropin.yaml' % parsed_args.stack) with open(stack_vstate_dropin, 'w') as dropin_file: yaml.safe_dump( {'parameter_defaults': { 'StackAction': self.stack_action }}, dropin_file, default_flow_style=False) environments.append(stack_vstate_dropin) return environments + user_environments
def get_parser(self, prog_name): parser = super(TripleOImagePrepare, self).get_parser(prog_name) try: roles_file = utils.rel_or_abs_path( constants.OVERCLOUD_ROLES_FILE, constants.TRIPLEO_HEAT_TEMPLATES) except exceptions.DeploymentError: roles_file = None parser.add_argument( '--environment-file', '-e', metavar='<file path>', action='append', dest='environment_files', help=_('Environment file containing the ContainerImagePrepare ' 'parameter which specifies all prepare actions. ' 'Also, environment files specifying which services are ' 'containerized. Entries will be filtered to only contain ' 'images used by containerized services. (Can be specified ' 'more than once.)') ) parser.add_argument( '--environment-directory', metavar='<HEAT ENVIRONMENT DIRECTORY>', action='append', dest='environment_directories', default=[os.path.expanduser(constants.DEFAULT_ENV_DIRECTORY)], help=_('Environment file directories that are automatically ' 'added to the environment. ' 'Can be specified more than once. Files in directories are ' 'loaded in ascending sort order.') ) parser.add_argument( '--roles-file', '-r', dest='roles_file', default=roles_file, help=_( 'Roles file, overrides the default %s in the t-h-t templates ' 'directory used for deployment. May be an ' 'absolute path or the path relative to the templates dir.' ) % constants.OVERCLOUD_ROLES_FILE ) parser.add_argument( "--output-env-file", dest="output_env_file", metavar='<file path>', help=_("File to write heat environment file which specifies all " "image parameters. Any existing file will be overwritten."), ) parser.add_argument( '--dry-run', dest='dry_run', action='store_true', default=False, help=_('Do not perform any pull, modify, or push operations. ' 'The environment file will still be populated as if these ' 'operations were performed.') ) parser.add_argument( "--cleanup", dest="cleanup", metavar='<full, partial, none>', default=image_uploader.CLEANUP_FULL, help=_("Cleanup behavior for local images left after upload. " "The default 'full' will attempt to delete all local " "images. 'partial' will leave images required for " "deployment on this host. 'none' will do no cleanup.") ) return parser
def get_parser(self, prog_name): parser = super(PrepareImageFiles, self).get_parser(prog_name) try: roles_file = utils.rel_or_abs_path( constants.OVERCLOUD_ROLES_FILE, constants.TRIPLEO_HEAT_TEMPLATES) except exceptions.DeploymentError: roles_file = None defaults = kolla_builder.container_images_prepare_defaults() parser.add_argument( "--template-file", dest="template_file", default=kolla_builder.DEFAULT_TEMPLATE_FILE, metavar='<yaml template file>', help=_("YAML template file which the images config file will be " "built from.\n" "Default: %s") % kolla_builder.DEFAULT_TEMPLATE_FILE, ) parser.add_argument( "--push-destination", dest="push_destination", metavar='<location>', help=_("Location of image registry to push images to. " "If specified, a push_destination will be set for every " "image entry."), ) parser.add_argument( "--tag", dest="tag", default=defaults['tag'], metavar='<tag>', help=_("Override the default tag substitution. " "If --tag-from-label is specified, " "start discovery with this tag.\n" "Default: %s") % defaults['tag'], ) parser.add_argument( "--tag-from-label", dest="tag_from_label", metavar='<image label>', help=_("Use the value of the specified label(s) to discover the " "tag. Labels can be combined in a template format, " "for example: {version}-{release}"), ) parser.add_argument( "--namespace", dest="namespace", default=defaults['namespace'], metavar='<namespace>', help=_("Override the default namespace substitution.\n" "Default: %s") % defaults['namespace'], ) parser.add_argument( "--prefix", dest="prefix", default=defaults['name_prefix'], metavar='<prefix>', help=_("Override the default name prefix substitution.\n" "Default: %s") % defaults['name_prefix'], ) parser.add_argument( "--suffix", dest="suffix", default=defaults['name_suffix'], metavar='<suffix>', help=_("Override the default name suffix substitution.\n" "Default: %s") % defaults['name_suffix'], ) parser.add_argument( '--set', metavar='<variable=value>', action='append', help=_('Set the value of a variable in the template, even if it ' 'has no dedicated argument such as "--suffix".') ) parser.add_argument( "--exclude", dest="excludes", metavar='<regex>', default=[], action="append", help=_("Pattern to match against resulting imagename entries to " "exclude from the final output. Can be specified multiple " "times."), ) parser.add_argument( "--include", dest="includes", metavar='<regex>', default=[], action="append", help=_("Pattern to match against resulting imagename entries to " "include in final output. Can be specified multiple " "times, entries not matching any --include will be " "excluded. --exclude is ignored if --include is used."), ) parser.add_argument( "--output-images-file", dest="output_images_file", metavar='<file path>', help=_("File to write resulting image entries to, as well as " "stdout. Any existing file will be overwritten."), ) parser.add_argument( '--environment-file', '-e', metavar='<file path>', action='append', dest='environment_files', help=_('Environment files specifying which services are ' 'containerized. Entries will be filtered to only contain ' 'images used by containerized services. (Can be specified ' 'more than once.)') ) parser.add_argument( '--environment-directory', metavar='<HEAT ENVIRONMENT DIRECTORY>', action='append', dest='environment_directories', default=[os.path.expanduser(constants.DEFAULT_ENV_DIRECTORY)], help=_('Environment file directories that are automatically ' 'added to the update command. Entries will be filtered ' 'to only contain images used by containerized services. ' 'Can be specified more than once. Files in directories are ' 'loaded in ascending sort order.') ) parser.add_argument( "--output-env-file", dest="output_env_file", metavar='<file path>', help=_("File to write heat environment file which specifies all " "image parameters. Any existing file will be overwritten."), ) parser.add_argument( '--roles-file', '-r', dest='roles_file', default=roles_file, help=_( 'Roles file, overrides the default %s in the t-h-t templates ' 'directory used for deployment. May be an ' 'absolute path or the path relative to the templates dir.' ) % constants.OVERCLOUD_ROLES_FILE ) parser.add_argument( '--modify-role', dest='modify_role', help=_('Name of ansible role to run between every image upload ' 'pull and push.') ) parser.add_argument( '--modify-vars', dest='modify_vars', help=_('Ansible variable file containing variables to use when ' 'invoking the role --modify-role.') ) return parser