def load_from_dir(source_directory, filename='manifest.yaml'):
if not os.path.isdir(source_directory) or not os.path.exists(
source_directory):
raise e.PackageLoadError('Invalid package directory')
full_path = path.secure_join(source_directory, filename)
if not os.path.isfile(full_path):
raise e.PackageLoadError('Unable to find package manifest')
try:
with open(full_path) as stream:
content = yaml.safe_load(stream)
except Exception as ex:
trace = sys.exc_info()[2]
utils.reraise(
e.PackageLoadError,
e.PackageLoadError("Unable to load due to '{0}'".format(ex)),
trace)
else:
format_spec = str(content.get('Format') or 'MuranoPL/1.0')
if format_spec[0].isdigit():
format_spec = 'MuranoPL/' + format_spec
plugin_loader = get_plugin_loader()
handler = plugin_loader.get_package_handler(format_spec)
if handler is None:
raise e.PackageFormatError(
'Unsupported format {0}'.format(format_spec))
return handler(source_directory, content)
def _load_image(self, file_name, default_name, what_image):
full_path = path.secure_join(
self._source_directory, file_name or default_name)
if not os.path.isfile(full_path) and not file_name:
return
allowed_ftype = ('png', 'jpeg', 'gif')
allowed_size = 500 * 1024
try:
if imghdr.what(full_path) not in allowed_ftype:
msg = _('{0}: Unsupported Format. Only {1} allowed').format(
what_image, ', '.join(allowed_ftype))
raise exceptions.PackageLoadError(msg)
fsize = os.stat(full_path).st_size
if fsize > allowed_size:
msg = _('{0}: Uploaded image size {1} is too large. '
'Max allowed size is {2}').format(
what_image, fsize, allowed_size)
raise exceptions.PackageLoadError(msg)
with open(full_path, 'rb') as stream:
return stream.read()
except Exception as ex:
trace = sys.exc_info()[2]
six.reraise(exceptions.PackageLoadError,
exceptions.PackageLoadError(
'Unable to load {0}: {1}'.format(what_image, ex)),
trace)
def load_from_file(archive_path, target_dir=None, drop_dir=False):
if not os.path.isfile(archive_path):
raise e.PackageLoadError('Unable to find package file')
created = False
if not target_dir:
target_dir = tempfile.mkdtemp()
created = True
elif not os.path.exists(target_dir):
os.mkdir(target_dir)
created = True
else:
if os.listdir(target_dir):
raise e.PackageLoadError('Target directory is not empty')
try:
if not zipfile.is_zipfile(archive_path):
raise e.PackageFormatError("Uploaded file {0} is not a "
"zip archive".format(archive_path))
package = zipfile.ZipFile(archive_path)
package.extractall(path=target_dir)
yield load_from_dir(target_dir)
except ValueError as err:
raise e.PackageLoadError("Couldn't load package from file: "
"{0}".format(err))
finally:
if drop_dir:
if created:
shutil.rmtree(target_dir)
else:
for f in os.listdir(target_dir):
os.unlink(os.path.join(target_dir, f))
def load_from_dir(source_directory,
filename='manifest.yaml',
preload=False,
loader=yaql_yaml_loader.YaqlYamlLoader):
formats = {
'1.0': murano.packages.versions.mpl_v1,
'MuranoPL/1.0': murano.packages.versions.mpl_v1,
'Heat.HOT/1.0': murano.packages.versions.hot_v1
}
if not os.path.isdir(source_directory) or not os.path.exists(
source_directory):
raise e.PackageLoadError('Invalid package directory')
full_path = os.path.join(source_directory, filename)
if not os.path.isfile(full_path):
raise e.PackageLoadError('Unable to find package manifest')
try:
with open(full_path) as stream:
content = yaml.safe_load(stream)
except Exception as ex:
trace = sys.exc_info()[2]
raise e.PackageLoadError("Unable to load due to '{0}'".format(
str(ex))), None, trace
if content:
p_format = str(content.get('Format'))
if not p_format or p_format not in formats:
raise e.PackageFormatError('Unknown or missing format version')
package = formats[p_format].create(source_directory, content, loader)
formats[p_format].load(package, content)
if preload:
package.validate()
return package
def load_from_file(archive_path,
target_dir=None,
drop_dir=False,
loader=yaql_yaml_loader.YaqlYamlLoader):
if not os.path.isfile(archive_path):
raise e.PackageLoadError('Unable to find package file')
created = False
if not target_dir:
target_dir = tempfile.mkdtemp()
created = True
elif not os.path.exists(target_dir):
os.mkdir(target_dir)
created = True
else:
if os.listdir(target_dir):
raise e.PackageLoadError('Target directory is not empty')
try:
if not zipfile.is_zipfile(archive_path):
raise e.PackageFormatError("Uploading file should be a "
"zip' archive")
package = zipfile.ZipFile(archive_path)
package.extractall(path=target_dir)
return load_from_dir(target_dir, preload=True, loader=loader)
finally:
if drop_dir:
if created:
shutil.rmtree(target_dir)
else:
for f in os.listdir(target_dir):
os.unlink(os.path.join(target_dir, f))
def _load_image(self, file_name, default_name, what_image):
full_path = os.path.join(self._source_directory, file_name
or default_name)
if not os.path.isfile(full_path) and not file_name:
return
try:
if imghdr.what(full_path) != 'png':
raise exceptions.PackageLoadError(
'{0} is not in PNG format'.format(what_image))
with open(full_path) as stream:
return stream.read()
except Exception as ex:
trace = sys.exc_info()[2]
raise exceptions.PackageLoadError('Unable to load {0}: {1}'.format(
what_image, ex)), None, trace
def _load_logo(self, validate=False):
logo_file = self._logo or 'logo.png'
full_path = os.path.join(self._source_directory, logo_file)
if not os.path.isfile(full_path) and logo_file == 'logo.png':
self._logo_cache = None
return
try:
if validate:
if imghdr.what(full_path) != 'png':
raise e.PackageLoadError("Logo is not in PNG format")
with open(full_path) as stream:
self._logo_cache = stream.read()
except Exception as ex:
trace = sys.exc_info()[2]
raise e.PackageLoadError("Unable to load logo: " +
str(ex)), None, trace
def _get_package_by_definition(self, package_def):
package_id = package_def.id
package_name = package_def.fully_qualified_name
package_directory = os.path.join(self._cache_directory, package_name)
if os.path.exists(package_directory):
try:
return load_utils.load_from_dir(
package_directory,
preload=True,
loader=yaql_yaml_loader.YaqlYamlLoader)
except pkg_exc.PackageLoadError:
LOG.exception(
_LE('Unable to load package from cache. Clean-up...'))
shutil.rmtree(package_directory, ignore_errors=True)
try:
package_data = self._murano_client_factory().packages.download(
package_id)
except muranoclient_exc.HTTPException as e:
msg = 'Error loading package id {0}: {1}'.format(
package_id, str(e))
exc_info = sys.exc_info()
raise pkg_exc.PackageLoadError(msg), None, exc_info[2]
package_file = None
try:
with tempfile.NamedTemporaryFile(delete=False) as package_file:
package_file.write(package_data)
return load_utils.load_from_file(
package_file.name,
target_dir=package_directory,
drop_dir=False,
loader=yaql_yaml_loader.YaqlYamlLoader)
except IOError:
msg = 'Unable to extract package data for %s' % package_id
exc_info = sys.exc_info()
raise pkg_exc.PackageLoadError(msg), None, exc_info[2]
finally:
try:
if package_file:
os.remove(package_file.name)
except OSError:
pass
def load_from_dir(source_directory, filename='manifest.yaml'):
formats = {
'MuranoPL': {
('1.0.0', '1.0.0'): murano.packages.mpl_package.MuranoPlPackage,
},
'Heat.HOT': {
('1.0.0', '1.0.0'): murano.packages.hot_package.HotPackage
}
}
if not os.path.isdir(source_directory) or not os.path.exists(
source_directory):
raise e.PackageLoadError('Invalid package directory')
full_path = os.path.join(source_directory, filename)
if not os.path.isfile(full_path):
raise e.PackageLoadError('Unable to find package manifest')
try:
with open(full_path) as stream:
content = yaml.safe_load(stream)
except Exception as ex:
trace = sys.exc_info()[2]
raise e.PackageLoadError("Unable to load due to '{0}'".format(
str(ex))), None, trace
if content:
p_format_spec = str(content.get('Format') or 'MuranoPL/1.0')
if p_format_spec[0] in string.digits:
p_format_spec = 'MuranoPL/' + p_format_spec
parts = p_format_spec.split('/', 1)
if parts[0] not in formats:
raise e.PackageFormatError('Unknown or missing format version')
format_set = formats[parts[0]]
version = semantic_version.Version('0.0.0')
if len(parts) > 1:
version = semantic_version.Version.coerce(parts[1])
for key, value in format_set.iteritems():
min_version = semantic_version.Version(key[0])
max_version = semantic_version.Version(key[1])
if min_version <= version <= max_version:
return value(source_directory, content, parts[0], version)
raise e.PackageFormatError('Unsupported {0} format version {1}'.format(
parts[0], version))
def load_definition(self, name):
try:
package = self._get_package_for(name)
if package is None:
raise exceptions.NoPackageForClassFound(name)
return package.get_class(name)
# (sjmc7) This is used as a control condition for system classes;
# do not delete (although I think it needs a better solution)
except exceptions.NoPackageForClassFound:
raise
except Exception as e:
msg = "Error loading {0}: {1}".format(name, str(e))
raise pkg_exceptions.PackageLoadError(msg), None, sys.exc_info()[2]
def _get_package_by_definition(self, package_def):
package_id = package_def.id
package_directory = os.path.join(
self._cache_directory, package_def.fully_qualified_name,
getattr(package_def, 'version', '0.0.0'), package_id)
if os.path.isdir(package_directory):
try:
return load_utils.load_from_dir(package_directory)
except pkg_exc.PackageLoadError:
LOG.exception(
_LE('Unable to load package from cache. Clean-up.'))
shutil.rmtree(package_directory, ignore_errors=True)
# the package is not yet in cache, let's try and download it.
download_lock_path = os.path.join(
self._cache_directory, '{}_download.lock'.format(package_id))
download_ipc_lock = m_utils.ExclusiveInterProcessLock(
path=download_lock_path, sleep_func=eventlet.sleep)
with download_mem_locks[package_id].write_lock(), download_ipc_lock:
# NOTE(kzaitsev):
# in case there were 2 concurrent threads/processes one might have
# already downloaded this package. Check before trying to download
if os.path.isdir(package_directory):
try:
return load_utils.load_from_dir(package_directory)
except pkg_exc.PackageLoadError:
LOG.error(
_LE('Unable to load package from cache. Clean-up.'))
shutil.rmtree(package_directory, ignore_errors=True)
# attempt the download itself
try:
LOG.debug("Attempting to download package {} {}".format(
package_def.fully_qualified_name, package_id))
package_data = self.client.packages.download(package_id)
except muranoclient_exc.HTTPException as e:
msg = 'Error loading package id {0}: {1}'.format(
package_id, str(e))
exc_info = sys.exc_info()
six.reraise(pkg_exc.PackageLoadError,
pkg_exc.PackageLoadError(msg), exc_info[2])
package_file = None
try:
with tempfile.NamedTemporaryFile(delete=False) as package_file:
package_file.write(package_data)
with load_utils.load_from_file(package_file.name,
target_dir=package_directory,
drop_dir=False) as app_package:
LOG.info(
_LI("Successfully downloaded and unpacked package {} {}"
).format(package_def.fully_qualified_name,
package_id))
self._downloaded.append(app_package)
self.try_cleanup_cache(os.path.split(package_directory)[0],
current_id=package_id)
return app_package
except IOError:
msg = 'Unable to extract package data for %s' % package_id
exc_info = sys.exc_info()
six.reraise(pkg_exc.PackageLoadError,
pkg_exc.PackageLoadError(msg), exc_info[2])
finally:
try:
if package_file:
os.remove(package_file.name)
except OSError:
pass
