def test_heat_files_generated_empty(self):
package_dir = os.path.abspath(
os.path.join(__file__, '../../test_packages/test.hot.v1.app'))
load_utils.load_from_dir(package_dir)
files = murano.packages.hot_package.HotPackage \
._translate_files(package_dir)
msg = "heat files were not generated correctly. Expected empty dict"
self.assertEqual(files, [], msg)
def test_load_from_dir_open_file_negative(self, mock_isfile):
mock_isfile.return_value = True
source_directory = self._create_temp_dir()
random_filename = ''.join(
random.choice(string.ascii_lowercase) for i in range(20))
expected_error_msg = 'Unable to load due to'
with self.assertRaisesRegex(exceptions.PackageLoadError,
expected_error_msg):
load_utils.load_from_dir(source_directory,
filename=random_filename)
def test_heat_files_generated_empty(self):
package_dir = os.path.abspath(
os.path.join(__file__,
'../../test_packages/test.hot.v1.app')
)
load_utils.load_from_dir(package_dir)
files = murano.packages.hot_package.HotPackage \
._translate_files(package_dir)
msg = "heat files were not generated correctly. Expected empty dict"
self.assertEqual(files, [], msg)
def test_heat_files_generated(self):
package_dir = os.path.abspath(
os.path.join(__file__,
'../../test_packages/test.hot.v1.app_with_files'))
load_utils.load_from_dir(package_dir)
files = murano.packages.hot_package.HotPackage._translate_files(
package_dir)
expected_result = [
"testHeatFile", "middle_file/testHeatFile",
"middle_file/inner_file/testHeatFile",
"middle_file/inner_file2/testHeatFile"
]
msg = "hot files were not generated correctly"
self.assertEqual(expected_result, files, msg)
def _test_package(self, manifest='manifest.yaml'):
package_dir = os.path.abspath(
os.path.join(
__file__,
'../../../packages/test_packages/test.mpl.v1.app',
)
)
pkg = load_utils.load_from_dir(
package_dir, filename=manifest
)
package = {
'fully_qualified_name': pkg.full_name,
'type': pkg.package_type,
'author': pkg.author,
'supplier': pkg.supplier,
'name': pkg.display_name,
'description': pkg.description,
'is_public': True,
'tags': pkg.tags,
'logo': pkg.logo,
'supplier_logo': pkg.supplier_logo,
'ui_definition': pkg.ui,
'class_definitions': tuple(pkg.classes),
'archive': pkg.blob,
'categories': [],
}
return pkg, package
def _test_package(self):
package_dir = os.path.abspath(
os.path.join(
__file__,
'../../../packages/test_packages/test.mpl.v1.app'
)
)
pkg = load_utils.load_from_dir(
package_dir
)
package = {
'fully_qualified_name': pkg.full_name,
'type': pkg.package_type,
'author': pkg.author,
'supplier': pkg.supplier,
'name': pkg.display_name,
'description': pkg.description,
'is_public': True,
'tags': pkg.tags,
'logo': pkg.logo,
'supplier_logo': pkg.supplier_logo,
'ui_definition': pkg.raw_ui,
'class_definitions': pkg.classes,
'archive': pkg.blob,
'categories': [],
}
return pkg, package
def test_heat_files_generated(self):
package_dir = os.path.abspath(
os.path.join(__file__,
'../../test_packages/test.hot.v1.app_with_files')
)
load_utils.load_from_dir(package_dir)
files = murano.packages.hot_package.HotPackage._translate_files(
package_dir)
expected_result = [
"testHeatFile",
"middle_file/testHeatFile",
"middle_file/inner_file/testHeatFile",
"middle_file/inner_file2/testHeatFile"
]
msg = "hot files were not generated correctly"
self.assertEqual(expected_result, files, msg)
def _build_index(self):
for folder in self.search_package_folders(self._base_path):
try:
package = load_utils.load_from_dir(folder)
dsl_package = murano_package.MuranoPackage(self._root_loader, package)
for class_name in package.classes:
dsl_package.register_class(
(lambda pkg, cls: lambda: get_class(pkg, cls))(package, class_name), class_name
)
if dsl_package.name == constants.CORE_LIBRARY:
system_objects.register(dsl_package)
self.register_package(dsl_package)
except pkg_exc.PackageLoadError:
LOG.info(_LI("Unable to load package from path: {0}").format(folder))
continue
LOG.info(_LI("Loaded package from path {0}").format(folder))
def _do_import_package(_dir, categories, update=False):
LOG.debug(
"Going to import Murano package from {source}".format(source=_dir))
pkg = load_utils.load_from_dir(_dir)
LOG.debug("Checking for existing packages")
existing = db_catalog_api.package_search({'fqn': pkg.full_name},
AdminContext())
if existing:
existing_pkg = existing[0]
if update:
LOG.debug('Deleting existing package {exst_pkg_id}').format(
exst_pkg_id=existing_pkg.id)
db_catalog_api.package_delete(existing_pkg.id, AdminContext())
else:
LOG.error(
_LE("Package '{name}' exists ({pkg_id}). Use --update.").
format(name=pkg.full_name, pkg_id=existing_pkg.id))
return
package = {
'fully_qualified_name': pkg.full_name,
'type': pkg.package_type,
'author': pkg.author,
'supplier': pkg.supplier,
'name': pkg.display_name,
'description': pkg.description,
# note: we explicitly mark all the imported packages as public,
# until a parameter added to control visibility scope of a package
'is_public': True,
'tags': pkg.tags,
'logo': pkg.logo,
'supplier_logo': pkg.supplier_logo,
'ui_definition': pkg.ui,
'class_definitions': pkg.classes,
'archive': pkg.blob,
'categories': categories or []
}
# note(ruhe): the second parameter is tenant_id
# it is a required field in the DB, that's why we pass an empty string
result = db_catalog_api.package_upload(package, '')
LOG.info(
_LI("Finished import of package {res_id}").format(res_id=result.id))
def _do_import_package(_dir, categories, update=False):
LOG.debug("Going to import Murano package from {source}".format(
source=_dir))
pkg = load_utils.load_from_dir(_dir)
LOG.debug("Checking for existing packages")
existing = db_catalog_api.package_search(
{'fqn': pkg.full_name},
AdminContext())
if existing:
existing_pkg = existing[0]
if update:
LOG.debug('Deleting existing package {exst_pkg_id}').format(
exst_pkg_id=existing_pkg.id)
db_catalog_api.package_delete(existing_pkg.id, AdminContext())
else:
LOG.error(_LE("Package '{name}' exists ({pkg_id}). Use --update.")
.format(name=pkg.full_name, pkg_id=existing_pkg.id))
return
package = {
'fully_qualified_name': pkg.full_name,
'type': pkg.package_type,
'author': pkg.author,
'supplier': pkg.supplier,
'name': pkg.display_name,
'description': pkg.description,
# note: we explicitly mark all the imported packages as public,
# until a parameter added to control visibility scope of a package
'is_public': True,
'tags': pkg.tags,
'logo': pkg.logo,
'supplier_logo': pkg.supplier_logo,
'ui_definition': pkg.ui,
'class_definitions': pkg.classes,
'archive': pkg.blob,
'categories': categories or []
}
# note(ruhe): the second parameter is tenant_id
# it is a required field in the DB, that's why we pass an empty string
result = db_catalog_api.package_upload(package, '')
LOG.info(_LI("Finished import of package {res_id}").format(
res_id=result.id))
def _build_index(self):
for folder in self.search_package_folders(self._base_path):
try:
package = load_utils.load_from_dir(folder)
dsl_package = murano_package.MuranoPackage(
self._root_loader, package)
for class_name in package.classes:
dsl_package.register_class(
(lambda pkg, cls: lambda: get_class(pkg, cls))(
package, class_name), class_name)
if dsl_package.name == constants.CORE_LIBRARY:
system_objects.register(dsl_package)
self.register_package(dsl_package)
except pkg_exc.PackageLoadError:
LOG.info(
'Unable to load package from path: {0}'.format(folder))
continue
LOG.info('Loaded package from path {0}'.format(folder))
def test_supplier_info_load(self):
package_dir = os.path.abspath(
os.path.join(__file__, '../../test_packages/test.mpl.v1.app'))
package = load_utils.load_from_dir(package_dir)
self.assertNotEqual(package.supplier, None)
self.assertEqual(package.supplier['Name'], 'Supplier Name')
self.assertEqual(package.supplier['CompanyUrl'], {
'Link': 'http://example.com',
'Text': 'Example Company'
})
self.assertEqual(package.supplier['Summary'],
'Company summary goes here')
self.assertEqual(package.supplier['Description'],
'Marked up company description goes here')
self.assertEqual(package.supplier['Logo'], 'test_supplier_logo.png')
self.assertEqual(imghdr.what('', package.supplier_logo), 'png')
def _get_package_by_definition(self, package_def):
package_id = package_def.id
package_name = package_def.fully_qualified_name
package_directory = os.path.join(self._cache_directory, package_name)
if os.path.exists(package_directory):
try:
return load_utils.load_from_dir(
package_directory, preload=True,
loader=yaql_yaml_loader.YaqlYamlLoader)
except pkg_exc.PackageLoadError:
LOG.exception(_LE(
'Unable to load package from cache. Clean-up...'))
shutil.rmtree(package_directory, ignore_errors=True)
try:
package_data = self._murano_client_factory().packages.download(
package_id)
except muranoclient_exc.HTTPException as e:
msg = 'Error loading package id {0}: {1}'.format(
package_id, str(e)
)
exc_info = sys.exc_info()
raise pkg_exc.PackageLoadError(msg), None, exc_info[2]
package_file = None
try:
with tempfile.NamedTemporaryFile(delete=False) as package_file:
package_file.write(package_data)
return load_utils.load_from_file(
package_file.name,
target_dir=package_directory,
drop_dir=False,
loader=yaql_yaml_loader.YaqlYamlLoader
)
except IOError:
msg = 'Unable to extract package data for %s' % package_id
exc_info = sys.exc_info()
raise pkg_exc.PackageLoadError(msg), None, exc_info[2]
finally:
try:
if package_file:
os.remove(package_file.name)
except OSError:
pass
def _get_package_by_definition(self, package_def):
package_id = package_def.id
package_name = package_def.fully_qualified_name
package_directory = os.path.join(self._cache_directory, package_name)
if os.path.exists(package_directory):
try:
return load_utils.load_from_dir(
package_directory,
preload=True,
loader=yaql_yaml_loader.YaqlYamlLoader)
except pkg_exc.PackageLoadError:
LOG.exception(
_LE('Unable to load package from cache. Clean-up...'))
shutil.rmtree(package_directory, ignore_errors=True)
try:
package_data = self._murano_client_factory().packages.download(
package_id)
except muranoclient_exc.HTTPException as e:
msg = 'Error loading package id {0}: {1}'.format(
package_id, str(e))
exc_info = sys.exc_info()
raise pkg_exc.PackageLoadError(msg), None, exc_info[2]
package_file = None
try:
with tempfile.NamedTemporaryFile(delete=False) as package_file:
package_file.write(package_data)
return load_utils.load_from_file(
package_file.name,
target_dir=package_directory,
drop_dir=False,
loader=yaql_yaml_loader.YaqlYamlLoader)
except IOError:
msg = 'Unable to extract package data for %s' % package_id
exc_info = sys.exc_info()
raise pkg_exc.PackageLoadError(msg), None, exc_info[2]
finally:
try:
if package_file:
os.remove(package_file.name)
except OSError:
pass
def _test_package(self):
package_dir = os.path.abspath(os.path.join(__file__, "../../../packages/test_packages/test.mpl.v1.app"))
pkg = load_utils.load_from_dir(package_dir)
package = {
"fully_qualified_name": pkg.full_name,
"type": pkg.package_type,
"author": pkg.author,
"supplier": pkg.supplier,
"name": pkg.display_name,
"description": pkg.description,
"is_public": True,
"tags": pkg.tags,
"logo": pkg.logo,
"supplier_logo": pkg.supplier_logo,
"ui_definition": pkg.ui,
"class_definitions": tuple(pkg.classes),
"archive": pkg.blob,
"categories": [],
}
return pkg, package
def _build_index(self):
for entry in os.listdir(self._base_path):
folder = os.path.join(self._base_path, entry)
if not os.path.isdir(folder) or entry in self._processed_entries:
continue
try:
package = load_utils.load_from_dir(
folder, preload=True,
loader=yaql_yaml_loader.YaqlYamlLoader)
except pkg_exc.PackageLoadError:
LOG.exception(_LE('Unable to load package from path: '
'{0}').format(entry))
continue
for c in package.classes:
self._packages_by_class[c] = package
self._packages_by_name[package.full_name] = package
self._processed_entries.add(entry)
def _build_index(self):
for entry in os.listdir(self._base_path):
folder = os.path.join(self._base_path, entry)
if not os.path.isdir(folder) or entry in self._processed_entries:
continue
try:
package = load_utils.load_from_dir(
folder,
preload=True,
loader=yaql_yaml_loader.YaqlYamlLoader)
except pkg_exc.PackageLoadError:
LOG.exception('Unable to load package from path: '
'{0}'.format(entry))
continue
for c in package.classes:
self._packages_by_class[c] = package
self._packages_by_name[package.full_name] = package
self._processed_entries.add(entry)
def test_load_package_with_supplier_info(self, mock_policy_check):
package_dir = os.path.abspath(
os.path.join(
__file__,
'../../../packages/test_packages/test.mpl.v1.app'
)
)
pkg = load_utils.load_from_dir(
package_dir
)
package = {
'fully_qualified_name': pkg.full_name,
'type': pkg.package_type,
'author': pkg.author,
'supplier': pkg.supplier,
'name': pkg.display_name,
'description': pkg.description,
'is_public': True,
'tags': pkg.tags,
'logo': pkg.logo,
'supplier_logo': pkg.supplier_logo,
'ui_definition': pkg.raw_ui,
'class_definitions': pkg.classes,
'archive': pkg.blob,
'categories': []
}
saved_package = db_catalog_api.package_upload(package, '')
req = self._get('/v1/catalog/packages/%s' % saved_package.id)
result = self.controller.get(req, saved_package.id)
self.assertEqual(package['supplier'], result['supplier'])
req = self._get(
'/v1/catalog/packages/%s/supplier_logo' % saved_package.id
)
result = self.controller.get_supplier_logo(req, saved_package.id)
self.assertEqual(imghdr.what('', result), 'png')
def test_supplier_info_load(self):
package_dir = os.path.abspath(
os.path.join(__file__, '../../test_packages/test.mpl.v1.app')
)
package = load_utils.load_from_dir(package_dir)
self.assertNotEqual(package.supplier, None)
self.assertEqual('Supplier Name', package.supplier['Name'])
self.assertEqual({'Link': 'http://example.com',
'Text': 'Example Company'},
package.supplier['CompanyUrl'])
self.assertEqual(
'Company summary goes here',
package.supplier['Summary']
)
self.assertEqual(
'Marked up company description goes here',
package.supplier['Description']
)
self.assertEqual('test_supplier_logo.png', package.supplier['Logo'])
self.assertEqual('png', imghdr.what('', package.supplier_logo))
def _get_package_by_definition(self, package_def):
package_id = package_def.id
package_name = package_def.fully_qualified_name
package_directory = os.path.join(self._cache_directory, package_name)
if os.path.exists(package_directory):
try:
return load_utils.load_from_dir(
package_directory, preload=True,
loader=yaql_yaml_loader.YaqlYamlLoader)
except pkg_exc.PackageLoadError:
LOG.exception('Unable to load package from cache. Clean-up...')
shutil.rmtree(package_directory, ignore_errors=True)
try:
package_data = self._client.packages.download(package_id)
except muranoclient_exc.HTTPException:
LOG.exception('Unable to download '
'package with id {0}'.format(package_id))
raise pkg_exc.PackageLoadError()
package_file = None
try:
with tempfile.NamedTemporaryFile(delete=False) as package_file:
package_file.write(package_data)
return load_utils.load_from_file(
package_file.name,
target_dir=package_directory,
drop_dir=False,
loader=yaql_yaml_loader.YaqlYamlLoader
)
except IOError:
LOG.exception('Unable to write package file')
raise pkg_exc.PackageLoadError()
finally:
try:
if package_file:
os.remove(package_file.name)
except OSError:
pass
def test_load_from_dir_without_source_directory(self):
expected_error_msg = 'Invalid package directory'
with self.assertRaisesRegex(exceptions.PackageLoadError,
expected_error_msg):
load_utils.load_from_dir('random_test_directory')
def _get_package_by_definition(self, package_def):
package_id = package_def.id
package_directory = os.path.join(
self._cache_directory,
package_def.fully_qualified_name,
getattr(package_def, 'version', '0.0.0'),
package_id)
if os.path.isdir(package_directory):
try:
return load_utils.load_from_dir(package_directory)
except pkg_exc.PackageLoadError:
LOG.exception(
_LE('Unable to load package from cache. Clean-up.'))
shutil.rmtree(package_directory, ignore_errors=True)
# the package is not yet in cache, let's try and download it.
download_lock_path = os.path.join(
self._cache_directory, '{}_download.lock'.format(package_id))
download_ipc_lock = m_utils.ExclusiveInterProcessLock(
path=download_lock_path, sleep_func=eventlet.sleep)
with download_mem_locks[package_id].write_lock(),\
download_ipc_lock:
# NOTE(kzaitsev):
# in case there were 2 concurrent threads/processes one might have
# already downloaded this package. Check before trying to download
if os.path.isdir(package_directory):
try:
return load_utils.load_from_dir(package_directory)
except pkg_exc.PackageLoadError:
LOG.error(
_LE('Unable to load package from cache. Clean-up.'))
shutil.rmtree(package_directory, ignore_errors=True)
# attempt the download itself
try:
LOG.debug("Attempting to download package {} {}".format(
package_def.fully_qualified_name, package_id))
package_data = self._murano_client_factory().packages.download(
package_id)
except muranoclient_exc.HTTPException as e:
msg = 'Error loading package id {0}: {1}'.format(
package_id, str(e)
)
exc_info = sys.exc_info()
six.reraise(pkg_exc.PackageLoadError(msg), None, exc_info[2])
package_file = None
try:
with tempfile.NamedTemporaryFile(delete=False) as package_file:
package_file.write(package_data)
with load_utils.load_from_file(
package_file.name,
target_dir=package_directory,
drop_dir=False) as app_package:
LOG.info(_LI(
"Successfully downloaded and unpacked package {} {}")
.format(package_def.fully_qualified_name, package_id))
self._downloaded.append(app_package)
self.try_cleanup_cache(
os.path.split(package_directory)[0],
current_id=package_id)
return app_package
except IOError:
msg = 'Unable to extract package data for %s' % package_id
exc_info = sys.exc_info()
raise pkg_exc.PackageLoadError(msg), None, exc_info[2]
finally:
try:
if package_file:
os.remove(package_file.name)
except OSError:
pass
def _get_package_by_definition(self, package_def):
package_id = package_def.id
package_directory = os.path.join(
self._cache_directory, package_def.fully_qualified_name,
getattr(package_def, 'version', '0.0.0'), package_id)
if os.path.isdir(package_directory):
try:
return load_utils.load_from_dir(package_directory)
except pkg_exc.PackageLoadError:
LOG.exception(
_LE('Unable to load package from cache. Clean-up.'))
shutil.rmtree(package_directory, ignore_errors=True)
# the package is not yet in cache, let's try and download it.
download_lock_path = os.path.join(
self._cache_directory, '{}_download.lock'.format(package_id))
download_ipc_lock = m_utils.ExclusiveInterProcessLock(
path=download_lock_path, sleep_func=eventlet.sleep)
with download_mem_locks[package_id].write_lock(), download_ipc_lock:
# NOTE(kzaitsev):
# in case there were 2 concurrent threads/processes one might have
# already downloaded this package. Check before trying to download
if os.path.isdir(package_directory):
try:
return load_utils.load_from_dir(package_directory)
except pkg_exc.PackageLoadError:
LOG.error(
_LE('Unable to load package from cache. Clean-up.'))
shutil.rmtree(package_directory, ignore_errors=True)
# attempt the download itself
try:
LOG.debug("Attempting to download package {} {}".format(
package_def.fully_qualified_name, package_id))
package_data = self.client.packages.download(package_id)
except muranoclient_exc.HTTPException as e:
msg = 'Error loading package id {0}: {1}'.format(
package_id, str(e))
exc_info = sys.exc_info()
six.reraise(pkg_exc.PackageLoadError,
pkg_exc.PackageLoadError(msg), exc_info[2])
package_file = None
try:
with tempfile.NamedTemporaryFile(delete=False) as package_file:
package_file.write(package_data)
with load_utils.load_from_file(package_file.name,
target_dir=package_directory,
drop_dir=False) as app_package:
LOG.info(
_LI("Successfully downloaded and unpacked package {} {}"
).format(package_def.fully_qualified_name,
package_id))
self._downloaded.append(app_package)
self.try_cleanup_cache(os.path.split(package_directory)[0],
current_id=package_id)
return app_package
except IOError:
msg = 'Unable to extract package data for %s' % package_id
exc_info = sys.exc_info()
six.reraise(pkg_exc.PackageLoadError,
pkg_exc.PackageLoadError(msg), exc_info[2])
finally:
try:
if package_file:
os.remove(package_file.name)
except OSError:
pass
def test_load_from_dir_with_invalid_source_directory(self):
source_directory = self._create_temp_dir()
expected_error_msg = 'Unable to find package manifest'
with self.assertRaisesRegex(exceptions.PackageLoadError,
expected_error_msg):
load_utils.load_from_dir(source_directory)
