def run(self, context):
with tf.NamedTemporaryFile() as tmp_tarball:
tarball.create_tarball(self.templates_path, tmp_tarball.name)
tarball.tarball_extract_to_swift_container(
self.get_object_client(context),
tmp_tarball.name,
self.container)
def run(self, context):
with tempfile.NamedTemporaryFile() as tmp_tarball:
tarball.create_tarball(self.dir_to_upload, tmp_tarball.name)
tarball.tarball_extract_to_swift_container(
self.get_object_client(context),
tmp_tarball.name,
self.container)
def run(self, context):
try:
swift = self.get_object_client(context)
# Upload template dir to tmp container
container_tmp = '%s-tmp' % self.container
with tempfile.NamedTemporaryFile() as tmp_tarball:
tarball.create_tarball(self.templates_dir, tmp_tarball.name)
tarball.tarball_extract_to_swift_container(
swift, tmp_tarball.name, container_tmp)
# Get all new templates:
new_templates = swift.get_object(container_tmp, '')[1].splitlines()
old_templates = swift.get_object(self.container,
'')[1].splitlines()
# Update the old container
for new in new_templates:
# if doesn't exist, push it:
if new not in old_templates:
swift.put_object(self.container, new,
swift.get_object(container_tmp, new)[1])
else:
content_new = swift.get_object(container_tmp, new)
content_old = swift.get_object(self.container, new)
if (not content_new == content_old
and constants.PLAN_ENVIRONMENT not in new):
swift.put_object(
self.container, new,
swift.get_object(container_tmp, new)[1])
except swiftexceptions.ClientException as err:
msg = "Error attempting an operation on container: %s" % err
return actions.Result(error=msg)
except Exception as err:
msg = "Error while updating plan: %s" % err
return actions.Result(error=msg)
def _upload_templates(swift_client,
container_name,
tht_root,
roles_file=None,
plan_env_file=None,
networks_file=None):
"""tarball up a given directory and upload it to Swift to be extracted"""
with tempfile.NamedTemporaryFile() as tmp_tarball:
tarball.create_tarball(tht_root, tmp_tarball.name)
tarball.tarball_extract_to_swift_container(swift_client,
tmp_tarball.name,
container_name)
# Optional override of the roles_data.yaml file
if roles_file:
_upload_file(swift_client, container_name,
constants.OVERCLOUD_ROLES_FILE,
utils.rel_or_abs_path(roles_file, tht_root))
# Optional override of the network_data.yaml file
if networks_file:
_upload_file(swift_client, container_name,
constants.OVERCLOUD_NETWORKS_FILE, networks_file)
# Optional override of the plan-environment.yaml file
if plan_env_file:
# TODO(jpalanis): Instead of overriding default file,
# merging the user override plan-environment with default
# plan-environment file will avoid explict merging issues.
_upload_file(swift_client, container_name, constants.PLAN_ENVIRONMENT,
plan_env_file)
def run(self, context):
with tf.NamedTemporaryFile() as tmp_tarball:
tarball.create_tarball(self.templates_path, tmp_tarball.name)
tarball.tarball_extract_to_swift_container(
self.get_object_client(context),
tmp_tarball.name,
self.container)
def create_and_upload_tarball(swiftservice,
tmp_dir,
container,
tarball_name,
tarball_options='-czf',
delete_after=3600,
segment_size=1048576000,
use_slo=True,
segment_container=None,
leave_segments=False,
changed=None,
skip_identical=False,
fail_fast=True,
dir_marker=False):
"""Create a tarball containing the tmp_dir and upload it to Swift.
This method allows to upload files bigger than 5GB.
It will create 2 swift containers to store the segments and
one container to reference the manifest with the segment pointers
"""
try:
with tempfile.NamedTemporaryFile() as tmp_tarball:
tarball.create_tarball(tmp_dir, tmp_tarball.name, tarball_options)
objs = [SwiftUploadObject(tmp_tarball, object_name=tarball_name)]
options = {
'meta': [],
'header': ['X-Delete-After: ' + str(delete_after)],
'segment_size': segment_size,
'use_slo': use_slo,
'segment_container': segment_container,
'leave_segments': leave_segments,
'changed': changed,
'skip_identical': skip_identical,
'fail_fast': fail_fast,
'dir_marker': dir_marker
}
for r in swiftservice.upload(container, objs, options=options):
if r['success']:
if 'object' in r:
LOG.info(r['object'])
elif 'for_object' in r:
LOG.info('%s segment %s' %
(r['for_object'], r['segment_index']))
else:
error = r['error']
if r['action'] == "create_container":
LOG.warning(
'Warning: failed to create container '
"'%s'%s", container, error)
elif r['action'] == "upload_object":
LOG.error(
"Failed to upload object %s to container %s: %s" %
(container, r['object'], error))
else:
LOG.error("%s" % error)
except SwiftError as e:
LOG.error(e.value)
def _upload_templates(swift_client, container_name, tht_root):
"""tarball up a given directory and upload it to Swift to be extracted"""
with tempfile.NamedTemporaryFile() as tmp_tarball:
tarball.create_tarball(tht_root, tmp_tarball.name)
tarball.tarball_extract_to_swift_container(swift_client,
tmp_tarball.name,
container_name)
def run(self):
tht_base_path = constants.DEFAULT_TEMPLATES_PATH
with tf.NamedTemporaryFile() as tmp_tarball:
tarball.create_tarball(tht_base_path, tmp_tarball.name)
tarball.tarball_extract_to_swift_container(
self._get_object_client(),
tmp_tarball.name,
self.container)
def run(self, context):
try:
swift = self.get_object_client(context)
# Upload template dir to tmp container
container_tmp = '%s-tmp' % self.container
with tempfile.NamedTemporaryFile() as tmp_tarball:
tarball.create_tarball(self.templates_dir, tmp_tarball.name)
tarball.tarball_extract_to_swift_container(
swift,
tmp_tarball.name,
container_tmp)
# Get all new templates:
new_templates = swiftutils.get_object_string(swift, container_tmp,
'').splitlines()
old_templates = swiftutils.get_object_string(swift, self.container,
'').splitlines()
exclude_user_data = [constants.PLAN_ENVIRONMENT,
constants.OVERCLOUD_J2_ROLES_NAME,
constants.OVERCLOUD_J2_NETWORKS_NAME,
constants.OVERCLOUD_J2_EXCLUDES]
# Update the old container
for new in new_templates:
# if doesn't exist, push it:
if new not in old_templates:
swiftutils.put_object_string(
swift,
self.container,
new,
swiftutils.get_object_string(swift, container_tmp, new)
)
else:
content_new = swiftutils.get_object_string(swift,
container_tmp,
new)
content_old = swiftutils.get_object_string(swift,
self.container,
new)
if (not content_new == content_old and
new not in exclude_user_data):
swiftutils.put_object_string(
swift,
self.container,
new,
swiftutils.get_object_string(swift, container_tmp,
new)
)
except swiftexceptions.ClientException as err:
msg = "Error attempting an operation on container: %s" % err
LOG.exception(msg)
return actions.Result(error=msg)
except Exception as err:
msg = "Error while updating plan: %s" % err
LOG.exception(msg)
return actions.Result(error=msg)
def _upload_templates(swift_client, container_name, tht_root, roles_file=None):
"""tarball up a given directory and upload it to Swift to be extracted"""
with tempfile.NamedTemporaryFile() as tmp_tarball:
tarball.create_tarball(tht_root, tmp_tarball.name)
tarball.tarball_extract_to_swift_container(
swift_client, tmp_tarball.name, container_name)
# Allow optional override of the roles_data.yaml file
if roles_file:
with open(roles_file) as rf:
swift_client.put_object(container_name,
constants.OVERCLOUD_ROLES_FILE,
rf)
def run(self, context):
heat = self.get_orchestration_client(context)
swift = self.get_object_client(context)
# Since the config-download directory is now a git repo, first download
# the existing config container if it exists so we can reuse the
# existing git repo.
try:
swiftutils.download_container(swift, self.container_config,
self.config_dir)
# Delete the existing container before we re-upload, otherwise
# files may not be fully overwritten.
swiftutils.delete_container(swift, self.container_config)
except swiftexceptions.ClientException as err:
if err.http_status != 404:
raise
# Delete downloaded tarball as it will be recreated later and we don't
# want to include the old tarball in the new tarball.
old_tarball_path = os.path.join(self.config_dir,
'%s.tar.gz' % self.container_config)
if os.path.exists(old_tarball_path):
os.unlink(old_tarball_path)
config = ooo_config.Config(heat)
message = ('Automatic commit by Mistral GetOvercloudConfig action.\n\n'
'User: {user}\n'
'Project: {project}'.format(user=context.user_name,
project=context.project_name))
config_path = config.download_config(self.container,
self.config_dir,
self.config_type,
preserve_config_dir=True,
commit_message=message)
with tempfile.NamedTemporaryFile() as tmp_tarball:
tarball.create_tarball(config_path,
tmp_tarball.name,
excludes=['.tox', '*.pyc', '*.pyo'])
tarball.tarball_extract_to_swift_container(
self.get_object_client(context), tmp_tarball.name,
self.container_config)
# Also upload the tarball to the container for use by export later
with open(tmp_tarball.name, 'rb') as t:
swift.put_object(self.container_config,
'%s.tar.gz' % self.container_config, t)
if os.path.exists(config_path):
shutil.rmtree(config_path)
def create_and_upload_tarball(swiftclient,
tmp_dir,
container,
tarball_name,
delete_after=3600):
"""Create a tarball containing the tmp_dir and upload it to Swift."""
headers = {'X-Delete-After': delete_after}
get_or_create_container(swiftclient, container)
with tempfile.NamedTemporaryFile() as tmp_tarball:
tarball.create_tarball(tmp_dir, tmp_tarball.name)
swiftclient.put_object(container,
tarball_name,
tmp_tarball,
headers=headers)
def _create_and_upload_tarball(self, swift, tmp_dir):
"""Create a tarball containing the tmp_dir and upload it to Swift."""
tarball_name = '%s.tar.gz' % self.plan
headers = {'X-Delete-After': self.delete_after}
# make sure the root container which holds all plan exports exists
try:
swift.get_container(self.exports_container)
except swiftexceptions.ClientException:
swift.put_container(self.exports_container)
with tempfile.NamedTemporaryFile() as tmp_tarball:
tarball.create_tarball(tmp_dir, tmp_tarball.name)
swift.put_object(self.exports_container, tarball_name, tmp_tarball,
headers=headers)
def run(self, context):
heat = self.get_orchestration_client(context)
swift = self.get_object_client(context)
# Since the config-download directory is now a git repo, first download
# the existing config container if it exists so we can reuse the
# existing git repo.
try:
swiftutils.download_container(swift, self.container_config,
self.config_dir)
# Delete the existing container before we re-upload, otherwise
# files may not be fully overwritten.
swiftutils.delete_container(swift, self.container_config)
except swiftexceptions.ClientException as err:
if err.http_status != 404:
raise
# Delete downloaded tarball as it will be recreated later and we don't
# want to include the old tarball in the new tarball.
old_tarball_path = os.path.join(
self.config_dir, '%s.tar.gz' % self.container_config)
if os.path.exists(old_tarball_path):
os.unlink(old_tarball_path)
config = ooo_config.Config(heat)
message = ('Automatic commit by Mistral GetOvercloudConfig action.\n\n'
'User: {user}\n'
'Project: {project}'.format(user=context.user_name,
project=context.project_name))
config_path = config.download_config(self.container, self.config_dir,
self.config_type,
preserve_config_dir=True,
commit_message=message)
with tempfile.NamedTemporaryFile() as tmp_tarball:
tarball.create_tarball(config_path, tmp_tarball.name,
excludes=['.tox', '*.pyc', '*.pyo'])
tarball.tarball_extract_to_swift_container(
self.get_object_client(context),
tmp_tarball.name,
self.container_config)
# Also upload the tarball to the container for use by export later
with open(tmp_tarball.name, 'rb') as t:
swift.put_object(self.container_config,
'%s.tar.gz' % self.container_config, t)
if os.path.exists(config_path):
shutil.rmtree(config_path)
def _create_and_upload_tarball(self, swift, tmp_dir):
"""Create a tarball containing the tmp_dir and upload it to Swift."""
tarball_name = '%s.tar.gz' % self.plan
headers = {'X-Delete-After': self.delete_after}
# make sure the root container which holds all plan exports exists
try:
swift.get_container(self.exports_container)
except swiftexceptions.ClientException:
swift.put_container(self.exports_container)
with tempfile.NamedTemporaryFile() as tmp_tarball:
tarball.create_tarball(tmp_dir, tmp_tarball.name)
swift.put_object(self.exports_container,
tarball_name,
tmp_tarball,
headers=headers)
def run(self, context):
heat = self.get_orchestration_client(context)
swift = self.get_object_client(context)
# Since the config-download directory is now a git repo, first download
# the existing config container if it exists so we can reuse the
# existing git repo.
try:
swiftutils.download_container(swift, self.container_config,
self.config_dir)
# Delete the existing container before we re-upload, otherwise
# files may not be fully overwritten.
swiftutils.delete_container(swift, self.container_config)
except swiftexceptions.ClientException as err:
if err.http_status != 404:
raise
config = ooo_config.Config(heat)
message = ('Automatic commit by Mistral GetOvercloudConfig action.\n\n'
'User: {user}\n'
'Project: {project}'.format(user=context.user_name,
project=context.project_name))
config_path = config.download_config(self.container,
self.config_dir,
preserve_config_dir=True,
commit_message=message)
with tempfile.NamedTemporaryFile() as tmp_tarball:
tarball.create_tarball(config_path,
tmp_tarball.name,
excludes=['.tox', '*.pyc', '*.pyo'])
tarball.tarball_extract_to_swift_container(
self.get_object_client(context), tmp_tarball.name,
self.container_config)
if os.path.exists(config_path):
shutil.rmtree(config_path)
def run(self, context):
with tempfile.NamedTemporaryFile() as tmp_tarball:
tarball.create_tarball(self.dir_to_upload, tmp_tarball.name)
tarball.tarball_extract_to_swift_container(
self.get_object_client(context), tmp_tarball.name,
self.container)
def run(self):
tht_base_path = constants.DEFAULT_TEMPLATES_PATH
with tf.NamedTemporaryFile() as tmp_tarball:
tarball.create_tarball(tht_base_path, tmp_tarball.name)
tarball.tarball_extract_to_swift_container(
self._get_object_client(), tmp_tarball.name, self.container)
