def upload(self, req, body=None):
"""Upload new file archive for the new package
together with package metadata.
"""
policy.check("upload_package", req.context)
_check_content_type(req, 'multipart/form-data')
file_obj, package_meta = _validate_body(body)
if package_meta:
try:
jsonschema.validate(package_meta, schemas.PKG_UPLOAD_SCHEMA)
except jsonschema.ValidationError as e:
msg = _("Package schema is not valid: {reason}").format(
reason=e)
LOG.exception(msg)
raise exc.HTTPBadRequest(explanation=msg)
else:
package_meta = {}
if package_meta.get('is_public'):
policy.check('publicize_package', req.context)
with tempfile.NamedTemporaryFile(delete=False) as tempf:
LOG.debug("Storing package archive in a temporary file")
content = file_obj.file.read()
if not content:
msg = _("Uploading file can't be empty")
LOG.error(msg)
raise exc.HTTPBadRequest(explanation=msg)
tempf.write(content)
package_meta['archive'] = content
try:
with load_utils.load_from_file(
tempf.name, target_dir=None,
drop_dir=True) as pkg_to_upload:
# extend dictionary for update db
for k, v in six.iteritems(PKG_PARAMS_MAP):
if hasattr(pkg_to_upload, k):
package_meta[v] = getattr(pkg_to_upload, k)
if len(package_meta['name']) > 80:
msg = _('Package name should be 80 characters maximum')
LOG.error(msg)
raise exc.HTTPBadRequest(explanation=msg)
try:
package = db_api.package_upload(
package_meta, req.context.tenant)
except db_exc.DBDuplicateEntry:
msg = _('Package with specified full '
'name is already registered')
LOG.exception(msg)
raise exc.HTTPConflict(msg)
return package.to_dict()
except pkg_exc.PackageLoadError as e:
msg = _("Couldn't load package from file: {reason}").format(
reason=e)
LOG.exception(msg)
raise exc.HTTPBadRequest(explanation=msg)
finally:
LOG.debug("Deleting package archive temporary file")
os.remove(tempf.name)
def upload(self, req, body=None):
"""Upload new file archive for the new package
together with package metadata.
"""
policy.check("upload_package", req.context)
_check_content_type(req, 'multipart/form-data')
file_obj, package_meta = _validate_body(body)
if package_meta:
try:
jsonschema.validate(package_meta, schemas.PKG_UPLOAD_SCHEMA)
except jsonschema.ValidationError as e:
LOG.exception(e)
raise exc.HTTPBadRequest(explanation=e.message)
else:
package_meta = {}
with tempfile.NamedTemporaryFile(delete=False) as tempf:
LOG.debug("Storing package archive in a temporary file")
content = file_obj.file.read()
if not content:
msg = _("Uploading file can't be empty")
LOG.error(msg)
raise exc.HTTPBadRequest(msg)
tempf.write(content)
package_meta['archive'] = content
try:
pkg_to_upload = load_utils.load_from_file(tempf.name,
target_dir=None,
drop_dir=True)
except pkg_exc.PackageLoadError as e:
LOG.exception(e)
raise exc.HTTPBadRequest(e)
finally:
LOG.debug("Deleting package archive temporary file")
os.remove(tempf.name)
# extend dictionary for update db
for k, v in PKG_PARAMS_MAP.iteritems():
if hasattr(pkg_to_upload, k):
package_meta[v] = getattr(pkg_to_upload, k)
if req.params.get('is_public', '').lower() == 'true':
policy.check('publicize_image', req.context)
package_meta['is_public'] = True
try:
package = db_api.package_upload(package_meta, req.context.tenant)
except db_exc.DBDuplicateEntry:
msg = _('Package with specified full name is already registered')
LOG.exception(msg)
raise exc.HTTPServerError(msg)
return package.to_dict()
def test_load_from_file_handle_value_error(self, mock_zipfile):
test_error_msg = 'Random error message.'
expected_error_msg = "Couldn't load package from file: {0}".\
format(test_error_msg)
mock_zipfile.is_zipfile = mock.MagicMock(
side_effect=ValueError(test_error_msg))
temp_file = self._create_temp_file()
with self.assertRaisesRegex(exceptions.PackageLoadError,
expected_error_msg):
with load_utils.load_from_file(temp_file.name):
pass
mock_zipfile.is_zipfile.assert_called_once_with(temp_file.name)
def upload(self, req, body=None):
"""Upload new file archive for the new package
together with package metadata.
"""
policy.check("upload_package", req.context)
_check_content_type(req, 'multipart/form-data')
file_obj, package_meta = _validate_body(body)
if package_meta:
try:
jsonschema.validate(package_meta, schemas.PKG_UPLOAD_SCHEMA)
except jsonschema.ValidationError as e:
LOG.exception(e)
raise exc.HTTPBadRequest(explanation=e.message)
else:
package_meta = {}
with tempfile.NamedTemporaryFile(delete=False) as tempf:
LOG.debug("Storing package archive in a temporary file")
content = file_obj.file.read()
if not content:
msg = _("Uploading file can't be empty")
LOG.error(msg)
raise exc.HTTPBadRequest(msg)
tempf.write(content)
package_meta['archive'] = content
try:
pkg_to_upload = load_utils.load_from_file(
tempf.name, target_dir=None, drop_dir=True)
except pkg_exc.PackageLoadError as e:
LOG.exception(e)
raise exc.HTTPBadRequest(e)
finally:
LOG.debug("Deleting package archive temporary file")
os.remove(tempf.name)
# extend dictionary for update db
for k, v in PKG_PARAMS_MAP.iteritems():
if hasattr(pkg_to_upload, k):
package_meta[v] = getattr(pkg_to_upload, k)
if req.params.get('is_public', '').lower() == 'true':
policy.check('publicize_image', req.context)
package_meta['is_public'] = True
try:
package = db_api.package_upload(package_meta, req.context.tenant)
except db_exc.DBDuplicateEntry:
msg = _('Package with specified full name is already registered')
LOG.exception(msg)
raise exc.HTTPServerError(msg)
return package.to_dict()
def get_ui(self, req, package_id):
if CONF.engine.packages_service == 'murano':
target = {'package_id': package_id}
policy.check("get_package", req.context, target)
package = db_api.package_get(package_id, req.context)
return package.ui_definition
else:
g_client = self._get_glare_client(req)
blob_data = g_client.artifacts.download_blob(package_id, 'archive')
with tempfile.NamedTemporaryFile() as tempf:
for chunk in blob_data:
tempf.write(chunk)
with load_utils.load_from_file(tempf.name, target_dir=None,
drop_dir=True) as pkg:
return pkg.ui
def get_ui(self, req, package_id):
if CONF.engine.packages_service == 'murano':
target = {'package_id': package_id}
policy.check("get_package", req.context, target)
package = db_api.package_get(package_id, req.context)
return package.ui_definition
else:
g_client = self._get_glare_client(req)
blob_data = g_client.artifacts.download_blob(package_id, 'archive')
with tempfile.NamedTemporaryFile() as tempf:
for chunk in blob_data:
tempf.write(chunk)
with load_utils.load_from_file(tempf.name,
target_dir=None,
drop_dir=True) as pkg:
return pkg.ui
def _get_package_by_definition(self, package_def):
package_id = package_def.id
package_name = package_def.fully_qualified_name
package_directory = os.path.join(self._cache_directory, package_name)
if os.path.exists(package_directory):
try:
return load_utils.load_from_dir(
package_directory, preload=True,
loader=yaql_yaml_loader.YaqlYamlLoader)
except pkg_exc.PackageLoadError:
LOG.exception(_LE(
'Unable to load package from cache. Clean-up...'))
shutil.rmtree(package_directory, ignore_errors=True)
try:
package_data = self._murano_client_factory().packages.download(
package_id)
except muranoclient_exc.HTTPException as e:
msg = 'Error loading package id {0}: {1}'.format(
package_id, str(e)
)
exc_info = sys.exc_info()
raise pkg_exc.PackageLoadError(msg), None, exc_info[2]
package_file = None
try:
with tempfile.NamedTemporaryFile(delete=False) as package_file:
package_file.write(package_data)
return load_utils.load_from_file(
package_file.name,
target_dir=package_directory,
drop_dir=False,
loader=yaql_yaml_loader.YaqlYamlLoader
)
except IOError:
msg = 'Unable to extract package data for %s' % package_id
exc_info = sys.exc_info()
raise pkg_exc.PackageLoadError(msg), None, exc_info[2]
finally:
try:
if package_file:
os.remove(package_file.name)
except OSError:
pass
def _get_package_by_definition(self, package_def):
package_id = package_def.id
package_name = package_def.fully_qualified_name
package_directory = os.path.join(self._cache_directory, package_name)
if os.path.exists(package_directory):
try:
return load_utils.load_from_dir(
package_directory,
preload=True,
loader=yaql_yaml_loader.YaqlYamlLoader)
except pkg_exc.PackageLoadError:
LOG.exception(
_LE('Unable to load package from cache. Clean-up...'))
shutil.rmtree(package_directory, ignore_errors=True)
try:
package_data = self._murano_client_factory().packages.download(
package_id)
except muranoclient_exc.HTTPException as e:
msg = 'Error loading package id {0}: {1}'.format(
package_id, str(e))
exc_info = sys.exc_info()
raise pkg_exc.PackageLoadError(msg), None, exc_info[2]
package_file = None
try:
with tempfile.NamedTemporaryFile(delete=False) as package_file:
package_file.write(package_data)
return load_utils.load_from_file(
package_file.name,
target_dir=package_directory,
drop_dir=False,
loader=yaql_yaml_loader.YaqlYamlLoader)
except IOError:
msg = 'Unable to extract package data for %s' % package_id
exc_info = sys.exc_info()
raise pkg_exc.PackageLoadError(msg), None, exc_info[2]
finally:
try:
if package_file:
os.remove(package_file.name)
except OSError:
pass
def _get_package_by_definition(self, package_def):
package_id = package_def.id
package_name = package_def.fully_qualified_name
package_directory = os.path.join(self._cache_directory, package_name)
if os.path.exists(package_directory):
try:
return load_utils.load_from_dir(
package_directory, preload=True,
loader=yaql_yaml_loader.YaqlYamlLoader)
except pkg_exc.PackageLoadError:
LOG.exception('Unable to load package from cache. Clean-up...')
shutil.rmtree(package_directory, ignore_errors=True)
try:
package_data = self._client.packages.download(package_id)
except muranoclient_exc.HTTPException:
LOG.exception('Unable to download '
'package with id {0}'.format(package_id))
raise pkg_exc.PackageLoadError()
package_file = None
try:
with tempfile.NamedTemporaryFile(delete=False) as package_file:
package_file.write(package_data)
return load_utils.load_from_file(
package_file.name,
target_dir=package_directory,
drop_dir=False,
loader=yaql_yaml_loader.YaqlYamlLoader
)
except IOError:
LOG.exception('Unable to write package file')
raise pkg_exc.PackageLoadError()
finally:
try:
if package_file:
os.remove(package_file.name)
except OSError:
pass
def _get_package_by_definition(self, package_def):
package_id = package_def.id
package_directory = os.path.join(
self._cache_directory,
package_def.fully_qualified_name,
getattr(package_def, 'version', '0.0.0'),
package_id)
if os.path.isdir(package_directory):
try:
return load_utils.load_from_dir(package_directory)
except pkg_exc.PackageLoadError:
LOG.exception(
_LE('Unable to load package from cache. Clean-up.'))
shutil.rmtree(package_directory, ignore_errors=True)
# the package is not yet in cache, let's try and download it.
download_lock_path = os.path.join(
self._cache_directory, '{}_download.lock'.format(package_id))
download_ipc_lock = m_utils.ExclusiveInterProcessLock(
path=download_lock_path, sleep_func=eventlet.sleep)
with download_mem_locks[package_id].write_lock(),\
download_ipc_lock:
# NOTE(kzaitsev):
# in case there were 2 concurrent threads/processes one might have
# already downloaded this package. Check before trying to download
if os.path.isdir(package_directory):
try:
return load_utils.load_from_dir(package_directory)
except pkg_exc.PackageLoadError:
LOG.error(
_LE('Unable to load package from cache. Clean-up.'))
shutil.rmtree(package_directory, ignore_errors=True)
# attempt the download itself
try:
LOG.debug("Attempting to download package {} {}".format(
package_def.fully_qualified_name, package_id))
package_data = self._murano_client_factory().packages.download(
package_id)
except muranoclient_exc.HTTPException as e:
msg = 'Error loading package id {0}: {1}'.format(
package_id, str(e)
)
exc_info = sys.exc_info()
six.reraise(pkg_exc.PackageLoadError(msg), None, exc_info[2])
package_file = None
try:
with tempfile.NamedTemporaryFile(delete=False) as package_file:
package_file.write(package_data)
with load_utils.load_from_file(
package_file.name,
target_dir=package_directory,
drop_dir=False) as app_package:
LOG.info(_LI(
"Successfully downloaded and unpacked package {} {}")
.format(package_def.fully_qualified_name, package_id))
self._downloaded.append(app_package)
self.try_cleanup_cache(
os.path.split(package_directory)[0],
current_id=package_id)
return app_package
except IOError:
msg = 'Unable to extract package data for %s' % package_id
exc_info = sys.exc_info()
raise pkg_exc.PackageLoadError(msg), None, exc_info[2]
finally:
try:
if package_file:
os.remove(package_file.name)
except OSError:
pass
def _get_package_by_definition(self, package_def):
package_id = package_def.id
package_directory = os.path.join(
self._cache_directory, package_def.fully_qualified_name,
getattr(package_def, 'version', '0.0.0'), package_id)
if os.path.isdir(package_directory):
try:
return load_utils.load_from_dir(package_directory)
except pkg_exc.PackageLoadError:
LOG.exception(
_LE('Unable to load package from cache. Clean-up.'))
shutil.rmtree(package_directory, ignore_errors=True)
# the package is not yet in cache, let's try and download it.
download_lock_path = os.path.join(
self._cache_directory, '{}_download.lock'.format(package_id))
download_ipc_lock = m_utils.ExclusiveInterProcessLock(
path=download_lock_path, sleep_func=eventlet.sleep)
with download_mem_locks[package_id].write_lock(), download_ipc_lock:
# NOTE(kzaitsev):
# in case there were 2 concurrent threads/processes one might have
# already downloaded this package. Check before trying to download
if os.path.isdir(package_directory):
try:
return load_utils.load_from_dir(package_directory)
except pkg_exc.PackageLoadError:
LOG.error(
_LE('Unable to load package from cache. Clean-up.'))
shutil.rmtree(package_directory, ignore_errors=True)
# attempt the download itself
try:
LOG.debug("Attempting to download package {} {}".format(
package_def.fully_qualified_name, package_id))
package_data = self.client.packages.download(package_id)
except muranoclient_exc.HTTPException as e:
msg = 'Error loading package id {0}: {1}'.format(
package_id, str(e))
exc_info = sys.exc_info()
six.reraise(pkg_exc.PackageLoadError,
pkg_exc.PackageLoadError(msg), exc_info[2])
package_file = None
try:
with tempfile.NamedTemporaryFile(delete=False) as package_file:
package_file.write(package_data)
with load_utils.load_from_file(package_file.name,
target_dir=package_directory,
drop_dir=False) as app_package:
LOG.info(
_LI("Successfully downloaded and unpacked package {} {}"
).format(package_def.fully_qualified_name,
package_id))
self._downloaded.append(app_package)
self.try_cleanup_cache(os.path.split(package_directory)[0],
current_id=package_id)
return app_package
except IOError:
msg = 'Unable to extract package data for %s' % package_id
exc_info = sys.exc_info()
six.reraise(pkg_exc.PackageLoadError,
pkg_exc.PackageLoadError(msg), exc_info[2])
finally:
try:
if package_file:
os.remove(package_file.name)
except OSError:
pass
def upload(self, req, body=None):
"""Upload new file archive
Upload new file archive for the new package
together with package metadata.
"""
policy.check("upload_package", req.context)
_check_content_type(req, 'multipart/form-data')
file_obj, package_meta = _validate_body(body)
if package_meta:
try:
jsonschema.validate(package_meta,
validation_schemas.PKG_UPLOAD_SCHEMA)
except jsonschema.ValidationError as e:
msg = _("Package schema is not valid: {reason}").format(
reason=e)
LOG.exception(msg)
raise exc.HTTPBadRequest(explanation=msg)
else:
package_meta = {}
if package_meta.get('is_public'):
policy.check('publicize_package', req.context)
with tempfile.NamedTemporaryFile(delete=False) as tempf:
LOG.debug("Storing package archive in a temporary file")
content = file_obj.file.read()
if not content:
msg = _("Uploading file can't be empty")
LOG.error(msg)
raise exc.HTTPBadRequest(explanation=msg)
tempf.write(content)
package_meta['archive'] = content
try:
with load_utils.load_from_file(tempf.name,
target_dir=None,
drop_dir=True) as pkg_to_upload:
# extend dictionary for update db
for k, v in six.iteritems(PKG_PARAMS_MAP):
if hasattr(pkg_to_upload, k):
package_meta[v] = getattr(pkg_to_upload, k)
if len(package_meta['name']) > 80:
msg = _('Package name should be 80 characters maximum')
LOG.error(msg)
raise exc.HTTPBadRequest(explanation=msg)
try:
package = db_api.package_upload(package_meta,
req.context.tenant)
except db_exc.DBDuplicateEntry:
msg = _('Package with specified full '
'name is already registered')
LOG.exception(msg)
raise exc.HTTPConflict(msg)
return package.to_dict()
except pkg_exc.PackageLoadError as e:
msg = _("Couldn't load package from file: {reason}").format(
reason=e)
LOG.exception(msg)
raise exc.HTTPBadRequest(explanation=msg)
finally:
LOG.debug("Deleting package archive temporary file")
os.remove(tempf.name)
def test_load_from_file_with_invalid_archive_path(self):
expected_error_msg = "Unable to find package file"
with self.assertRaisesRegex(exceptions.PackageLoadError,
expected_error_msg):
with load_utils.load_from_file('invalid file path'):
pass
