def upload(self, req, body=None): """Upload new file archive for the new package together with package metadata. """ policy.check("upload_package", req.context) _check_content_type(req, 'multipart/form-data') file_obj, package_meta = _validate_body(body) if package_meta: try: jsonschema.validate(package_meta, schemas.PKG_UPLOAD_SCHEMA) except jsonschema.ValidationError as e: msg = _("Package schema is not valid: {reason}").format( reason=e) LOG.exception(msg) raise exc.HTTPBadRequest(explanation=msg) else: package_meta = {} if package_meta.get('is_public'): policy.check('publicize_package', req.context) with tempfile.NamedTemporaryFile(delete=False) as tempf: LOG.debug("Storing package archive in a temporary file") content = file_obj.file.read() if not content: msg = _("Uploading file can't be empty") LOG.error(msg) raise exc.HTTPBadRequest(explanation=msg) tempf.write(content) package_meta['archive'] = content try: with load_utils.load_from_file( tempf.name, target_dir=None, drop_dir=True) as pkg_to_upload: # extend dictionary for update db for k, v in six.iteritems(PKG_PARAMS_MAP): if hasattr(pkg_to_upload, k): package_meta[v] = getattr(pkg_to_upload, k) if len(package_meta['name']) > 80: msg = _('Package name should be 80 characters maximum') LOG.error(msg) raise exc.HTTPBadRequest(explanation=msg) try: package = db_api.package_upload( package_meta, req.context.tenant) except db_exc.DBDuplicateEntry: msg = _('Package with specified full ' 'name is already registered') LOG.exception(msg) raise exc.HTTPConflict(msg) return package.to_dict() except pkg_exc.PackageLoadError as e: msg = _("Couldn't load package from file: {reason}").format( reason=e) LOG.exception(msg) raise exc.HTTPBadRequest(explanation=msg) finally: LOG.debug("Deleting package archive temporary file") os.remove(tempf.name)
def upload(self, req, body=None): """Upload new file archive for the new package together with package metadata. """ policy.check("upload_package", req.context) _check_content_type(req, 'multipart/form-data') file_obj, package_meta = _validate_body(body) if package_meta: try: jsonschema.validate(package_meta, schemas.PKG_UPLOAD_SCHEMA) except jsonschema.ValidationError as e: LOG.exception(e) raise exc.HTTPBadRequest(explanation=e.message) else: package_meta = {} with tempfile.NamedTemporaryFile(delete=False) as tempf: LOG.debug("Storing package archive in a temporary file") content = file_obj.file.read() if not content: msg = _("Uploading file can't be empty") LOG.error(msg) raise exc.HTTPBadRequest(msg) tempf.write(content) package_meta['archive'] = content try: pkg_to_upload = load_utils.load_from_file(tempf.name, target_dir=None, drop_dir=True) except pkg_exc.PackageLoadError as e: LOG.exception(e) raise exc.HTTPBadRequest(e) finally: LOG.debug("Deleting package archive temporary file") os.remove(tempf.name) # extend dictionary for update db for k, v in PKG_PARAMS_MAP.iteritems(): if hasattr(pkg_to_upload, k): package_meta[v] = getattr(pkg_to_upload, k) if req.params.get('is_public', '').lower() == 'true': policy.check('publicize_image', req.context) package_meta['is_public'] = True try: package = db_api.package_upload(package_meta, req.context.tenant) except db_exc.DBDuplicateEntry: msg = _('Package with specified full name is already registered') LOG.exception(msg) raise exc.HTTPServerError(msg) return package.to_dict()
def test_load_from_file_handle_value_error(self, mock_zipfile): test_error_msg = 'Random error message.' expected_error_msg = "Couldn't load package from file: {0}".\ format(test_error_msg) mock_zipfile.is_zipfile = mock.MagicMock( side_effect=ValueError(test_error_msg)) temp_file = self._create_temp_file() with self.assertRaisesRegex(exceptions.PackageLoadError, expected_error_msg): with load_utils.load_from_file(temp_file.name): pass mock_zipfile.is_zipfile.assert_called_once_with(temp_file.name)
def upload(self, req, body=None): """Upload new file archive for the new package together with package metadata. """ policy.check("upload_package", req.context) _check_content_type(req, 'multipart/form-data') file_obj, package_meta = _validate_body(body) if package_meta: try: jsonschema.validate(package_meta, schemas.PKG_UPLOAD_SCHEMA) except jsonschema.ValidationError as e: LOG.exception(e) raise exc.HTTPBadRequest(explanation=e.message) else: package_meta = {} with tempfile.NamedTemporaryFile(delete=False) as tempf: LOG.debug("Storing package archive in a temporary file") content = file_obj.file.read() if not content: msg = _("Uploading file can't be empty") LOG.error(msg) raise exc.HTTPBadRequest(msg) tempf.write(content) package_meta['archive'] = content try: pkg_to_upload = load_utils.load_from_file( tempf.name, target_dir=None, drop_dir=True) except pkg_exc.PackageLoadError as e: LOG.exception(e) raise exc.HTTPBadRequest(e) finally: LOG.debug("Deleting package archive temporary file") os.remove(tempf.name) # extend dictionary for update db for k, v in PKG_PARAMS_MAP.iteritems(): if hasattr(pkg_to_upload, k): package_meta[v] = getattr(pkg_to_upload, k) if req.params.get('is_public', '').lower() == 'true': policy.check('publicize_image', req.context) package_meta['is_public'] = True try: package = db_api.package_upload(package_meta, req.context.tenant) except db_exc.DBDuplicateEntry: msg = _('Package with specified full name is already registered') LOG.exception(msg) raise exc.HTTPServerError(msg) return package.to_dict()
def get_ui(self, req, package_id): if CONF.engine.packages_service == 'murano': target = {'package_id': package_id} policy.check("get_package", req.context, target) package = db_api.package_get(package_id, req.context) return package.ui_definition else: g_client = self._get_glare_client(req) blob_data = g_client.artifacts.download_blob(package_id, 'archive') with tempfile.NamedTemporaryFile() as tempf: for chunk in blob_data: tempf.write(chunk) with load_utils.load_from_file(tempf.name, target_dir=None, drop_dir=True) as pkg: return pkg.ui
def _get_package_by_definition(self, package_def): package_id = package_def.id package_name = package_def.fully_qualified_name package_directory = os.path.join(self._cache_directory, package_name) if os.path.exists(package_directory): try: return load_utils.load_from_dir( package_directory, preload=True, loader=yaql_yaml_loader.YaqlYamlLoader) except pkg_exc.PackageLoadError: LOG.exception(_LE( 'Unable to load package from cache. Clean-up...')) shutil.rmtree(package_directory, ignore_errors=True) try: package_data = self._murano_client_factory().packages.download( package_id) except muranoclient_exc.HTTPException as e: msg = 'Error loading package id {0}: {1}'.format( package_id, str(e) ) exc_info = sys.exc_info() raise pkg_exc.PackageLoadError(msg), None, exc_info[2] package_file = None try: with tempfile.NamedTemporaryFile(delete=False) as package_file: package_file.write(package_data) return load_utils.load_from_file( package_file.name, target_dir=package_directory, drop_dir=False, loader=yaql_yaml_loader.YaqlYamlLoader ) except IOError: msg = 'Unable to extract package data for %s' % package_id exc_info = sys.exc_info() raise pkg_exc.PackageLoadError(msg), None, exc_info[2] finally: try: if package_file: os.remove(package_file.name) except OSError: pass
def _get_package_by_definition(self, package_def): package_id = package_def.id package_name = package_def.fully_qualified_name package_directory = os.path.join(self._cache_directory, package_name) if os.path.exists(package_directory): try: return load_utils.load_from_dir( package_directory, preload=True, loader=yaql_yaml_loader.YaqlYamlLoader) except pkg_exc.PackageLoadError: LOG.exception( _LE('Unable to load package from cache. Clean-up...')) shutil.rmtree(package_directory, ignore_errors=True) try: package_data = self._murano_client_factory().packages.download( package_id) except muranoclient_exc.HTTPException as e: msg = 'Error loading package id {0}: {1}'.format( package_id, str(e)) exc_info = sys.exc_info() raise pkg_exc.PackageLoadError(msg), None, exc_info[2] package_file = None try: with tempfile.NamedTemporaryFile(delete=False) as package_file: package_file.write(package_data) return load_utils.load_from_file( package_file.name, target_dir=package_directory, drop_dir=False, loader=yaql_yaml_loader.YaqlYamlLoader) except IOError: msg = 'Unable to extract package data for %s' % package_id exc_info = sys.exc_info() raise pkg_exc.PackageLoadError(msg), None, exc_info[2] finally: try: if package_file: os.remove(package_file.name) except OSError: pass
def _get_package_by_definition(self, package_def): package_id = package_def.id package_name = package_def.fully_qualified_name package_directory = os.path.join(self._cache_directory, package_name) if os.path.exists(package_directory): try: return load_utils.load_from_dir( package_directory, preload=True, loader=yaql_yaml_loader.YaqlYamlLoader) except pkg_exc.PackageLoadError: LOG.exception('Unable to load package from cache. Clean-up...') shutil.rmtree(package_directory, ignore_errors=True) try: package_data = self._client.packages.download(package_id) except muranoclient_exc.HTTPException: LOG.exception('Unable to download ' 'package with id {0}'.format(package_id)) raise pkg_exc.PackageLoadError() package_file = None try: with tempfile.NamedTemporaryFile(delete=False) as package_file: package_file.write(package_data) return load_utils.load_from_file( package_file.name, target_dir=package_directory, drop_dir=False, loader=yaql_yaml_loader.YaqlYamlLoader ) except IOError: LOG.exception('Unable to write package file') raise pkg_exc.PackageLoadError() finally: try: if package_file: os.remove(package_file.name) except OSError: pass
def _get_package_by_definition(self, package_def): package_id = package_def.id package_directory = os.path.join( self._cache_directory, package_def.fully_qualified_name, getattr(package_def, 'version', '0.0.0'), package_id) if os.path.isdir(package_directory): try: return load_utils.load_from_dir(package_directory) except pkg_exc.PackageLoadError: LOG.exception( _LE('Unable to load package from cache. Clean-up.')) shutil.rmtree(package_directory, ignore_errors=True) # the package is not yet in cache, let's try and download it. download_lock_path = os.path.join( self._cache_directory, '{}_download.lock'.format(package_id)) download_ipc_lock = m_utils.ExclusiveInterProcessLock( path=download_lock_path, sleep_func=eventlet.sleep) with download_mem_locks[package_id].write_lock(),\ download_ipc_lock: # NOTE(kzaitsev): # in case there were 2 concurrent threads/processes one might have # already downloaded this package. Check before trying to download if os.path.isdir(package_directory): try: return load_utils.load_from_dir(package_directory) except pkg_exc.PackageLoadError: LOG.error( _LE('Unable to load package from cache. Clean-up.')) shutil.rmtree(package_directory, ignore_errors=True) # attempt the download itself try: LOG.debug("Attempting to download package {} {}".format( package_def.fully_qualified_name, package_id)) package_data = self._murano_client_factory().packages.download( package_id) except muranoclient_exc.HTTPException as e: msg = 'Error loading package id {0}: {1}'.format( package_id, str(e) ) exc_info = sys.exc_info() six.reraise(pkg_exc.PackageLoadError(msg), None, exc_info[2]) package_file = None try: with tempfile.NamedTemporaryFile(delete=False) as package_file: package_file.write(package_data) with load_utils.load_from_file( package_file.name, target_dir=package_directory, drop_dir=False) as app_package: LOG.info(_LI( "Successfully downloaded and unpacked package {} {}") .format(package_def.fully_qualified_name, package_id)) self._downloaded.append(app_package) self.try_cleanup_cache( os.path.split(package_directory)[0], current_id=package_id) return app_package except IOError: msg = 'Unable to extract package data for %s' % package_id exc_info = sys.exc_info() raise pkg_exc.PackageLoadError(msg), None, exc_info[2] finally: try: if package_file: os.remove(package_file.name) except OSError: pass
def _get_package_by_definition(self, package_def): package_id = package_def.id package_directory = os.path.join( self._cache_directory, package_def.fully_qualified_name, getattr(package_def, 'version', '0.0.0'), package_id) if os.path.isdir(package_directory): try: return load_utils.load_from_dir(package_directory) except pkg_exc.PackageLoadError: LOG.exception( _LE('Unable to load package from cache. Clean-up.')) shutil.rmtree(package_directory, ignore_errors=True) # the package is not yet in cache, let's try and download it. download_lock_path = os.path.join( self._cache_directory, '{}_download.lock'.format(package_id)) download_ipc_lock = m_utils.ExclusiveInterProcessLock( path=download_lock_path, sleep_func=eventlet.sleep) with download_mem_locks[package_id].write_lock(), download_ipc_lock: # NOTE(kzaitsev): # in case there were 2 concurrent threads/processes one might have # already downloaded this package. Check before trying to download if os.path.isdir(package_directory): try: return load_utils.load_from_dir(package_directory) except pkg_exc.PackageLoadError: LOG.error( _LE('Unable to load package from cache. Clean-up.')) shutil.rmtree(package_directory, ignore_errors=True) # attempt the download itself try: LOG.debug("Attempting to download package {} {}".format( package_def.fully_qualified_name, package_id)) package_data = self.client.packages.download(package_id) except muranoclient_exc.HTTPException as e: msg = 'Error loading package id {0}: {1}'.format( package_id, str(e)) exc_info = sys.exc_info() six.reraise(pkg_exc.PackageLoadError, pkg_exc.PackageLoadError(msg), exc_info[2]) package_file = None try: with tempfile.NamedTemporaryFile(delete=False) as package_file: package_file.write(package_data) with load_utils.load_from_file(package_file.name, target_dir=package_directory, drop_dir=False) as app_package: LOG.info( _LI("Successfully downloaded and unpacked package {} {}" ).format(package_def.fully_qualified_name, package_id)) self._downloaded.append(app_package) self.try_cleanup_cache(os.path.split(package_directory)[0], current_id=package_id) return app_package except IOError: msg = 'Unable to extract package data for %s' % package_id exc_info = sys.exc_info() six.reraise(pkg_exc.PackageLoadError, pkg_exc.PackageLoadError(msg), exc_info[2]) finally: try: if package_file: os.remove(package_file.name) except OSError: pass
def upload(self, req, body=None): """Upload new file archive Upload new file archive for the new package together with package metadata. """ policy.check("upload_package", req.context) _check_content_type(req, 'multipart/form-data') file_obj, package_meta = _validate_body(body) if package_meta: try: jsonschema.validate(package_meta, validation_schemas.PKG_UPLOAD_SCHEMA) except jsonschema.ValidationError as e: msg = _("Package schema is not valid: {reason}").format( reason=e) LOG.exception(msg) raise exc.HTTPBadRequest(explanation=msg) else: package_meta = {} if package_meta.get('is_public'): policy.check('publicize_package', req.context) with tempfile.NamedTemporaryFile(delete=False) as tempf: LOG.debug("Storing package archive in a temporary file") content = file_obj.file.read() if not content: msg = _("Uploading file can't be empty") LOG.error(msg) raise exc.HTTPBadRequest(explanation=msg) tempf.write(content) package_meta['archive'] = content try: with load_utils.load_from_file(tempf.name, target_dir=None, drop_dir=True) as pkg_to_upload: # extend dictionary for update db for k, v in six.iteritems(PKG_PARAMS_MAP): if hasattr(pkg_to_upload, k): package_meta[v] = getattr(pkg_to_upload, k) if len(package_meta['name']) > 80: msg = _('Package name should be 80 characters maximum') LOG.error(msg) raise exc.HTTPBadRequest(explanation=msg) try: package = db_api.package_upload(package_meta, req.context.tenant) except db_exc.DBDuplicateEntry: msg = _('Package with specified full ' 'name is already registered') LOG.exception(msg) raise exc.HTTPConflict(msg) return package.to_dict() except pkg_exc.PackageLoadError as e: msg = _("Couldn't load package from file: {reason}").format( reason=e) LOG.exception(msg) raise exc.HTTPBadRequest(explanation=msg) finally: LOG.debug("Deleting package archive temporary file") os.remove(tempf.name)
def test_load_from_file_with_invalid_archive_path(self): expected_error_msg = "Unable to find package file" with self.assertRaisesRegex(exceptions.PackageLoadError, expected_error_msg): with load_utils.load_from_file('invalid file path'): pass