def load_from_dir(source_directory, filename='manifest.yaml'):
if not os.path.isdir(source_directory) or not os.path.exists(
source_directory):
raise e.PackageLoadError('Invalid package directory')
full_path = path.secure_join(source_directory, filename)
if not os.path.isfile(full_path):
raise e.PackageLoadError('Unable to find package manifest')
try:
with open(full_path) as stream:
content = yaml.safe_load(stream)
except Exception as ex:
trace = sys.exc_info()[2]
utils.reraise(
e.PackageLoadError,
e.PackageLoadError("Unable to load due to '{0}'".format(ex)),
trace)
else:
format_spec = str(content.get('Format') or 'MuranoPL/1.0')
if format_spec[0].isdigit():
format_spec = 'MuranoPL/' + format_spec
plugin_loader = get_plugin_loader()
handler = plugin_loader.get_package_handler(format_spec)
if handler is None:
raise e.PackageFormatError(
'Unsupported format {0}'.format(format_spec))
return handler(source_directory, content)
def _load_image(self, file_name, default_name, what_image):
full_path = path.secure_join(self._source_directory, file_name
or default_name)
if not os.path.isfile(full_path) and not file_name:
return
allowed_ftype = ('png', 'jpeg', 'gif')
allowed_size = 500 * 1024
try:
if imghdr.what(full_path) not in allowed_ftype:
msg = _('{0}: Unsupported Format. Only {1} allowed').format(
what_image, ', '.join(allowed_ftype))
raise exceptions.PackageLoadError(msg)
fsize = os.stat(full_path).st_size
if fsize > allowed_size:
msg = _('{0}: Uploaded image size {1} is too large. '
'Max allowed size is {2}').format(
what_image, fsize, allowed_size)
raise exceptions.PackageLoadError(msg)
with open(full_path, 'rb') as stream:
return stream.read()
except Exception as ex:
trace = sys.exc_info()[2]
utils.reraise(
exceptions.PackageLoadError,
exceptions.PackageLoadError('Unable to load {0}: {1}'.format(
what_image, ex)), trace)
def _execute(self, name, obj, *args, **kwargs):
try:
final_args = []
final_kwargs = {}
for arg in args:
if isinstance(arg, object_model.Object):
arg = object_model.build_model(arg)
final_args.append(arg)
for name, arg in kwargs.items():
if isinstance(arg, object_model.Object):
arg = object_model.build_model(arg)
final_kwargs[name] = arg
cls = obj if isinstance(obj, dsl_types.MuranoType) else obj.type
runtime_version = cls.package.runtime_version
yaql_engine = yaql_integration.choose_yaql_engine(runtime_version)
with helpers.with_object_store(self.executor.object_store):
return dsl.to_mutable(
cls.invoke(name, obj, tuple(final_args), final_kwargs),
yaql_engine)
except dsl_exception.MuranoPlException as e:
if not self.preserve_exception:
original_exception = getattr(e, 'original_exception', None)
if original_exception and not isinstance(
original_exception, dsl_exception.MuranoPlException):
exc_traceback = getattr(e, 'original_traceback',
None) or sys.exc_info()[2]
utils.reraise(type(original_exception), original_exception,
exc_traceback)
raise
def load_package(self, package_name, version_spec):
fixed_versions = self._fixations[package_name]
version = version_spec.select(fixed_versions)
if version:
version_spec = helpers.parse_version_spec(version)
packages = self._package_cache.get(package_name)
if packages:
version = version_spec.select(packages.keys())
if version:
return packages[version]
filter_opts = {
'fqn': package_name,
'version': helpers.breakdown_spec_to_query(version_spec)
}
try:
package_definition = self._get_definition(filter_opts)
self._lock_usage(package_definition)
except LookupError:
exc_info = sys.exc_info()
utils.reraise(exceptions.NoPackageFound,
exceptions.NoPackageFound(package_name), exc_info[2])
else:
package = self._get_package_by_definition(package_definition)
self._fixations[package_name].add(package.version)
self._new_fixations[package_name].add(package.version)
return self._to_dsl_package(package)
def transform(self, *args, **kwargs):
try:
return super(MuranoProperty, self).transform(*args, **kwargs)
except exceptions.ContractViolationException as e:
msg = u'[{0}.{1}{2}] {3}'.format(self.declaring_type.name,
self.name, e.path, str(e))
utils.reraise(exceptions.ContractViolationException,
exceptions.ContractViolationException(msg),
sys.exc_info()[2])
def transform(self, value, this, *args, **kwargs):
try:
if self.murano_method.usage == dsl_types.MethodUsages.Extension:
this = self.murano_method.declaring_type
return super(MuranoMethodArgument,
self).transform(value, this, *args, **kwargs)
except exceptions.ContractViolationException as e:
msg = u'[{0}::{1}({2}{3})] {4}'.format(
self.murano_method.declaring_type.name,
self.murano_method.name, self.name, e.path, str(e))
utils.reraise(exceptions.ContractViolationException,
exceptions.ContractViolationException(msg),
sys.exc_info()[2])
def parallel_select(collection, func, limit=1000):
# workaround for eventlet issue 232
# https://github.com/eventlet/eventlet/issues/232
context = get_context()
object_store = get_object_store()
def wrapper(element):
try:
with with_object_store(object_store), contextual(context):
return func(element), False, None
except Exception as e:
return e, True, sys.exc_info()[2]
gpool = eventlet.greenpool.GreenPool(limit)
result = list(gpool.imap(wrapper, collection))
try:
exception = next(t for t in result if t[1])
except StopIteration:
return map(lambda t: t[0], result)
else:
utils.reraise(type(exception[0]), exception[0], exception[2])
def load_class_package(self, class_name, version_spec):
packages = self._class_cache.get(class_name)
if packages:
version = version_spec.select(packages.keys())
if version:
return packages[version]
filter_opts = {
'class_name': class_name,
'version': helpers.breakdown_spec_to_query(version_spec)
}
try:
package_definition = self._get_definition(filter_opts)
self._lock_usage(package_definition)
except LookupError:
exc_info = sys.exc_info()
utils.reraise(exceptions.NoPackageForClassFound,
exceptions.NoPackageForClassFound(class_name),
exc_info[2])
return self._to_dsl_package(
self._get_package_by_definition(package_definition))
def _get_package_by_definition(self, package_def):
package_id = package_def.id
package_directory = os.path.join(
self._cache_directory, package_def.fully_qualified_name,
getattr(package_def, 'version', '0.0.0'), package_id)
if os.path.isdir(package_directory):
try:
return load_utils.load_from_dir(package_directory)
except pkg_exc.PackageLoadError:
LOG.exception('Unable to load package from cache. Clean-up.')
shutil.rmtree(package_directory, ignore_errors=True)
# the package is not yet in cache, let's try to download it.
download_lock_path = os.path.join(
self._cache_directory, '{}_download.lock'.format(package_id))
download_ipc_lock = m_utils.ExclusiveInterProcessLock(
path=download_lock_path, sleep_func=eventlet.sleep)
with download_mem_locks[package_id].write_lock(), download_ipc_lock:
# NOTE(kzaitsev):
# in case there were 2 concurrent threads/processes one might have
# already downloaded this package. Check before trying to download
if os.path.isdir(package_directory):
try:
return load_utils.load_from_dir(package_directory)
except pkg_exc.PackageLoadError:
LOG.error('Unable to load package from cache. Clean-up.')
shutil.rmtree(package_directory, ignore_errors=True)
# attempt the download itself
try:
LOG.debug("Attempting to download package {} {}".format(
package_def.fully_qualified_name, package_id))
package_data = self.client.packages.download(package_id)
except muranoclient_exc.HTTPException as e:
msg = 'Error loading package id {0}: {1}'.format(
package_id, str(e))
exc_info = sys.exc_info()
utils.reraise(pkg_exc.PackageLoadError,
pkg_exc.PackageLoadError(msg), exc_info[2])
package_file = None
try:
with tempfile.NamedTemporaryFile(delete=False) as package_file:
package_file.write(package_data)
with load_utils.load_from_file(package_file.name,
target_dir=package_directory,
drop_dir=False) as app_package:
LOG.info("Successfully downloaded and unpacked "
"package {} {}".format(
package_def.fully_qualified_name, package_id))
self._downloaded.append(app_package)
self.try_cleanup_cache(os.path.split(package_directory)[0],
current_id=package_id)
return app_package
except IOError:
msg = 'Unable to extract package data for %s' % package_id
exc_info = sys.exc_info()
utils.reraise(pkg_exc.PackageLoadError,
pkg_exc.PackageLoadError(msg), exc_info[2])
finally:
try:
if package_file:
os.remove(package_file.name)
except OSError:
pass