def run(self, context): with tf.NamedTemporaryFile() as tmp_tarball: tarball.create_tarball(self.templates_path, tmp_tarball.name) tarball.tarball_extract_to_swift_container( self.get_object_client(context), tmp_tarball.name, self.container)
def run(self, context): with tempfile.NamedTemporaryFile() as tmp_tarball: tarball.create_tarball(self.dir_to_upload, tmp_tarball.name) tarball.tarball_extract_to_swift_container( self.get_object_client(context), tmp_tarball.name, self.container)
def run(self, context): try: swift = self.get_object_client(context) # Upload template dir to tmp container container_tmp = '%s-tmp' % self.container with tempfile.NamedTemporaryFile() as tmp_tarball: tarball.create_tarball(self.templates_dir, tmp_tarball.name) tarball.tarball_extract_to_swift_container( swift, tmp_tarball.name, container_tmp) # Get all new templates: new_templates = swift.get_object(container_tmp, '')[1].splitlines() old_templates = swift.get_object(self.container, '')[1].splitlines() # Update the old container for new in new_templates: # if doesn't exist, push it: if new not in old_templates: swift.put_object(self.container, new, swift.get_object(container_tmp, new)[1]) else: content_new = swift.get_object(container_tmp, new) content_old = swift.get_object(self.container, new) if (not content_new == content_old and constants.PLAN_ENVIRONMENT not in new): swift.put_object( self.container, new, swift.get_object(container_tmp, new)[1]) except swiftexceptions.ClientException as err: msg = "Error attempting an operation on container: %s" % err return actions.Result(error=msg) except Exception as err: msg = "Error while updating plan: %s" % err return actions.Result(error=msg)
def _upload_templates(swift_client, container_name, tht_root, roles_file=None, plan_env_file=None, networks_file=None): """tarball up a given directory and upload it to Swift to be extracted""" with tempfile.NamedTemporaryFile() as tmp_tarball: tarball.create_tarball(tht_root, tmp_tarball.name) tarball.tarball_extract_to_swift_container(swift_client, tmp_tarball.name, container_name) # Optional override of the roles_data.yaml file if roles_file: _upload_file(swift_client, container_name, constants.OVERCLOUD_ROLES_FILE, utils.rel_or_abs_path(roles_file, tht_root)) # Optional override of the network_data.yaml file if networks_file: _upload_file(swift_client, container_name, constants.OVERCLOUD_NETWORKS_FILE, networks_file) # Optional override of the plan-environment.yaml file if plan_env_file: # TODO(jpalanis): Instead of overriding default file, # merging the user override plan-environment with default # plan-environment file will avoid explict merging issues. _upload_file(swift_client, container_name, constants.PLAN_ENVIRONMENT, plan_env_file)
def create_and_upload_tarball(swiftservice, tmp_dir, container, tarball_name, tarball_options='-czf', delete_after=3600, segment_size=1048576000, use_slo=True, segment_container=None, leave_segments=False, changed=None, skip_identical=False, fail_fast=True, dir_marker=False): """Create a tarball containing the tmp_dir and upload it to Swift. This method allows to upload files bigger than 5GB. It will create 2 swift containers to store the segments and one container to reference the manifest with the segment pointers """ try: with tempfile.NamedTemporaryFile() as tmp_tarball: tarball.create_tarball(tmp_dir, tmp_tarball.name, tarball_options) objs = [SwiftUploadObject(tmp_tarball, object_name=tarball_name)] options = { 'meta': [], 'header': ['X-Delete-After: ' + str(delete_after)], 'segment_size': segment_size, 'use_slo': use_slo, 'segment_container': segment_container, 'leave_segments': leave_segments, 'changed': changed, 'skip_identical': skip_identical, 'fail_fast': fail_fast, 'dir_marker': dir_marker } for r in swiftservice.upload(container, objs, options=options): if r['success']: if 'object' in r: LOG.info(r['object']) elif 'for_object' in r: LOG.info('%s segment %s' % (r['for_object'], r['segment_index'])) else: error = r['error'] if r['action'] == "create_container": LOG.warning( 'Warning: failed to create container ' "'%s'%s", container, error) elif r['action'] == "upload_object": LOG.error( "Failed to upload object %s to container %s: %s" % (container, r['object'], error)) else: LOG.error("%s" % error) except SwiftError as e: LOG.error(e.value)
def _upload_templates(swift_client, container_name, tht_root): """tarball up a given directory and upload it to Swift to be extracted""" with tempfile.NamedTemporaryFile() as tmp_tarball: tarball.create_tarball(tht_root, tmp_tarball.name) tarball.tarball_extract_to_swift_container(swift_client, tmp_tarball.name, container_name)
def run(self): tht_base_path = constants.DEFAULT_TEMPLATES_PATH with tf.NamedTemporaryFile() as tmp_tarball: tarball.create_tarball(tht_base_path, tmp_tarball.name) tarball.tarball_extract_to_swift_container( self._get_object_client(), tmp_tarball.name, self.container)
def run(self, context): try: swift = self.get_object_client(context) # Upload template dir to tmp container container_tmp = '%s-tmp' % self.container with tempfile.NamedTemporaryFile() as tmp_tarball: tarball.create_tarball(self.templates_dir, tmp_tarball.name) tarball.tarball_extract_to_swift_container( swift, tmp_tarball.name, container_tmp) # Get all new templates: new_templates = swiftutils.get_object_string(swift, container_tmp, '').splitlines() old_templates = swiftutils.get_object_string(swift, self.container, '').splitlines() exclude_user_data = [constants.PLAN_ENVIRONMENT, constants.OVERCLOUD_J2_ROLES_NAME, constants.OVERCLOUD_J2_NETWORKS_NAME, constants.OVERCLOUD_J2_EXCLUDES] # Update the old container for new in new_templates: # if doesn't exist, push it: if new not in old_templates: swiftutils.put_object_string( swift, self.container, new, swiftutils.get_object_string(swift, container_tmp, new) ) else: content_new = swiftutils.get_object_string(swift, container_tmp, new) content_old = swiftutils.get_object_string(swift, self.container, new) if (not content_new == content_old and new not in exclude_user_data): swiftutils.put_object_string( swift, self.container, new, swiftutils.get_object_string(swift, container_tmp, new) ) except swiftexceptions.ClientException as err: msg = "Error attempting an operation on container: %s" % err LOG.exception(msg) return actions.Result(error=msg) except Exception as err: msg = "Error while updating plan: %s" % err LOG.exception(msg) return actions.Result(error=msg)
def _upload_templates(swift_client, container_name, tht_root, roles_file=None): """tarball up a given directory and upload it to Swift to be extracted""" with tempfile.NamedTemporaryFile() as tmp_tarball: tarball.create_tarball(tht_root, tmp_tarball.name) tarball.tarball_extract_to_swift_container( swift_client, tmp_tarball.name, container_name) # Allow optional override of the roles_data.yaml file if roles_file: with open(roles_file) as rf: swift_client.put_object(container_name, constants.OVERCLOUD_ROLES_FILE, rf)
def run(self, context): heat = self.get_orchestration_client(context) swift = self.get_object_client(context) # Since the config-download directory is now a git repo, first download # the existing config container if it exists so we can reuse the # existing git repo. try: swiftutils.download_container(swift, self.container_config, self.config_dir) # Delete the existing container before we re-upload, otherwise # files may not be fully overwritten. swiftutils.delete_container(swift, self.container_config) except swiftexceptions.ClientException as err: if err.http_status != 404: raise # Delete downloaded tarball as it will be recreated later and we don't # want to include the old tarball in the new tarball. old_tarball_path = os.path.join(self.config_dir, '%s.tar.gz' % self.container_config) if os.path.exists(old_tarball_path): os.unlink(old_tarball_path) config = ooo_config.Config(heat) message = ('Automatic commit by Mistral GetOvercloudConfig action.\n\n' 'User: {user}\n' 'Project: {project}'.format(user=context.user_name, project=context.project_name)) config_path = config.download_config(self.container, self.config_dir, self.config_type, preserve_config_dir=True, commit_message=message) with tempfile.NamedTemporaryFile() as tmp_tarball: tarball.create_tarball(config_path, tmp_tarball.name, excludes=['.tox', '*.pyc', '*.pyo']) tarball.tarball_extract_to_swift_container( self.get_object_client(context), tmp_tarball.name, self.container_config) # Also upload the tarball to the container for use by export later with open(tmp_tarball.name, 'rb') as t: swift.put_object(self.container_config, '%s.tar.gz' % self.container_config, t) if os.path.exists(config_path): shutil.rmtree(config_path)
def create_and_upload_tarball(swiftclient, tmp_dir, container, tarball_name, delete_after=3600): """Create a tarball containing the tmp_dir and upload it to Swift.""" headers = {'X-Delete-After': delete_after} get_or_create_container(swiftclient, container) with tempfile.NamedTemporaryFile() as tmp_tarball: tarball.create_tarball(tmp_dir, tmp_tarball.name) swiftclient.put_object(container, tarball_name, tmp_tarball, headers=headers)
def _create_and_upload_tarball(self, swift, tmp_dir): """Create a tarball containing the tmp_dir and upload it to Swift.""" tarball_name = '%s.tar.gz' % self.plan headers = {'X-Delete-After': self.delete_after} # make sure the root container which holds all plan exports exists try: swift.get_container(self.exports_container) except swiftexceptions.ClientException: swift.put_container(self.exports_container) with tempfile.NamedTemporaryFile() as tmp_tarball: tarball.create_tarball(tmp_dir, tmp_tarball.name) swift.put_object(self.exports_container, tarball_name, tmp_tarball, headers=headers)
def run(self, context): heat = self.get_orchestration_client(context) swift = self.get_object_client(context) # Since the config-download directory is now a git repo, first download # the existing config container if it exists so we can reuse the # existing git repo. try: swiftutils.download_container(swift, self.container_config, self.config_dir) # Delete the existing container before we re-upload, otherwise # files may not be fully overwritten. swiftutils.delete_container(swift, self.container_config) except swiftexceptions.ClientException as err: if err.http_status != 404: raise # Delete downloaded tarball as it will be recreated later and we don't # want to include the old tarball in the new tarball. old_tarball_path = os.path.join( self.config_dir, '%s.tar.gz' % self.container_config) if os.path.exists(old_tarball_path): os.unlink(old_tarball_path) config = ooo_config.Config(heat) message = ('Automatic commit by Mistral GetOvercloudConfig action.\n\n' 'User: {user}\n' 'Project: {project}'.format(user=context.user_name, project=context.project_name)) config_path = config.download_config(self.container, self.config_dir, self.config_type, preserve_config_dir=True, commit_message=message) with tempfile.NamedTemporaryFile() as tmp_tarball: tarball.create_tarball(config_path, tmp_tarball.name, excludes=['.tox', '*.pyc', '*.pyo']) tarball.tarball_extract_to_swift_container( self.get_object_client(context), tmp_tarball.name, self.container_config) # Also upload the tarball to the container for use by export later with open(tmp_tarball.name, 'rb') as t: swift.put_object(self.container_config, '%s.tar.gz' % self.container_config, t) if os.path.exists(config_path): shutil.rmtree(config_path)
def run(self, context): heat = self.get_orchestration_client(context) swift = self.get_object_client(context) # Since the config-download directory is now a git repo, first download # the existing config container if it exists so we can reuse the # existing git repo. try: swiftutils.download_container(swift, self.container_config, self.config_dir) # Delete the existing container before we re-upload, otherwise # files may not be fully overwritten. swiftutils.delete_container(swift, self.container_config) except swiftexceptions.ClientException as err: if err.http_status != 404: raise config = ooo_config.Config(heat) message = ('Automatic commit by Mistral GetOvercloudConfig action.\n\n' 'User: {user}\n' 'Project: {project}'.format(user=context.user_name, project=context.project_name)) config_path = config.download_config(self.container, self.config_dir, preserve_config_dir=True, commit_message=message) with tempfile.NamedTemporaryFile() as tmp_tarball: tarball.create_tarball(config_path, tmp_tarball.name, excludes=['.tox', '*.pyc', '*.pyo']) tarball.tarball_extract_to_swift_container( self.get_object_client(context), tmp_tarball.name, self.container_config) if os.path.exists(config_path): shutil.rmtree(config_path)