def _settings(self):
"""
Return a SystemSettings object instance.
"""
with self._real_settings_lock:
if self._real_settings is None:
self._real_settings = SystemSettings()
const_debug_write(__name__, "SystemSettings loaded")
# add our SystemSettings plugin
# Make sure we connect Entropy Client plugin
# AFTER client db init
self._real_settings.add_plugin(self._settings_client_plugin)
return self._real_settings
def __init__(self, repository_id):
"""
object constructor, repository_id must be a valid repository
identifier.
@param repository_id: valid repository identifier
@type repository_id: string
"""
super(UGCErrorReport, self).__init__("#fake#")
self.__system_settings = SystemSettings()
self._entropy = Client()
if repository_id not in self.__system_settings['repositories']['order']:
raise AttributeError('invalid repository_id provided')
self.__repository_id = repository_id
self._factory = self._entropy.WebServices()
try:
self._webserv = self._factory.new(self.__repository_id)
except WebService.UnsupportedService:
raise AttributeError('Web Services not supported by %s' % (
self.__repository_id,))
try:
available = self._webserv.service_available()
except WebService.WebServiceException:
available = False
if not available:
raise AttributeError('Web Services not supported by %s (2)' % (
self.__repository_id,))
def _settings(self):
"""
Get SystemSettings instance
"""
if self.__settings is None:
self.__settings = SystemSettings()
return self.__settings
def _ensure_package_sets_dir():
sets_dir = SystemSettings.packages_sets_directory()
if not os.path.isdir(sets_dir):
if os.path.lexists(sets_dir):
os.remove(sets_dir)
os.makedirs(sets_dir, 0o775)
const_setup_perms(sets_dir, etpConst['entropygid'],
recursion = False)
def _ensure_package_sets_dir():
sets_dir = SystemSettings.packages_sets_directory()
if not os.path.isdir(sets_dir):
if os.path.lexists(sets_dir):
os.remove(sets_dir)
os.makedirs(sets_dir, 0o775)
const_setup_perms(sets_dir, etpConst['entropygid'],
recursion = False)
def __init__(self, repository_id):
"""
Object constructor.
@param repository_id: the repository identifier
@type repository_id: string
"""
self._repository_id = repository_id
self._settings = SystemSettings()
def update_entropy_repositories(self):
progressQ.send_message(_("Downloading software repositories..."))
settings = SystemSettings()
chroot = ROOT_PATH
root = etpSys['rootdir']
if chroot != root:
self._change_entropy_chroot(chroot)
repos = list(settings['repositories']['available'].keys())
try:
# fetch_security = False => avoid spamming stdout
try:
repo_intf = self._backend.entropy.Repositories(
repos, fetch_security=False)
except AttributeError as err:
log.error("No repositories in repositories.conf")
return False
except Exception as err:
log.error("Unhandled exception: %s" % (err, ))
return False
try:
update_rc = repo_intf.sync()
except Exception as err:
log.error("Sync error: %s" % (err, ))
return False
if repo_intf.sync_errors or (update_rc != 0):
log.error("Cannot download repositories atm")
return False
return update_rc == 0
finally:
self._backend.entropy.close_repositories()
settings.clear()
if chroot != root:
self._change_entropy_chroot(root)
def update_entropy_repositories(self):
progressQ.send_message(_("Downloading software repositories..."))
settings = SystemSettings()
chroot = ROOT_PATH
root = etpSys['rootdir']
if chroot != root:
self._change_entropy_chroot(chroot)
repos = list(settings['repositories']['available'].keys())
try:
# fetch_security = False => avoid spamming stdout
try:
repo_intf = self._backend.entropy.Repositories(
repos, fetch_security=False)
except AttributeError as err:
log.error("No repositories in repositories.conf")
return False
except Exception as err:
log.error("Unhandled exception: %s" % (err,))
return False
try:
update_rc = repo_intf.sync()
except Exception as err:
log.error("Sync error: %s" % (err,))
return False
if repo_intf.sync_errors or (update_rc != 0):
log.error("Cannot download repositories atm")
return False
return update_rc == 0
finally:
self._backend.entropy.close_repositories()
settings.clear()
if chroot != root:
self._change_entropy_chroot(root)
def log_message(self, message):
"""
Log message string to logfile.
@param message: message string to log
@type message: string
"""
with LogFile(level=SystemSettings()['system']['log_level'],
filename=etpConst['entropylogfile'],
header="[spm]") as log:
log.write(message)
def _settings(self):
"""
Return a SystemSettings object instance.
"""
with self._real_settings_lock:
if self._real_settings is None:
self._real_settings = SystemSettings()
const_debug_write(__name__, "SystemSettings loaded")
# add our SystemSettings plugin
# Make sure we connect Entropy Client plugin
# AFTER client db init
self._real_settings.add_plugin(
self._settings_client_plugin)
return self._real_settings
def add(self, set_name, set_atoms):
"""
Add a user-defined package set to Entropy Client (changes are permanent)
@param set_name: package set name
@type set_name: string
@param set_atoms: list of package names in given set
@type set_atoms: list (set)
@raise entropy.exceptions.InvalidPackageSet: if package set data
passed is invalid (non ASCII chars, invalid set_name).
The encapsulated error string will contain a mnemonic reason.
"""
def _ensure_package_sets_dir():
sets_dir = SystemSettings.packages_sets_directory()
if not os.path.isdir(sets_dir):
if os.path.lexists(sets_dir):
os.remove(sets_dir)
os.makedirs(sets_dir, 0o775)
const_setup_perms(sets_dir, etpConst['entropygid'],
recursion = False)
if not const_isunicode(set_name):
raise InvalidPackageSet("%s %s" % (
set_name, "must be unicode",))
if set_name.startswith(etpConst['packagesetprefix']):
raise InvalidPackageSet("%s %s '%s'" % (
set_name, "cannot start with", etpConst['packagesetprefix'],))
set_match = self.match(set_name)
if set_match:
raise InvalidPackageSet(_("Name already taken"))
_ensure_package_sets_dir()
set_file = os.path.join(SystemSettings.packages_sets_directory(),
set_name)
set_file_tmp = set_file + ".sets_add_tmp"
enc = etpConst['conf_encoding']
try:
with codecs.open(set_file_tmp, "w", encoding=enc) as f:
for x in set_atoms:
f.write(x)
f.write("\n")
os.rename(set_file_tmp, set_file)
except (OSError, IOError) as err:
raise InvalidPackageSet(_("Cannot create the element"))
self._settings['system_package_sets'][set_name] = set(set_atoms)
def add(self, set_name, set_atoms):
"""
Add a user-defined package set to Entropy Client (changes are permanent)
@param set_name: package set name
@type set_name: string
@param set_atoms: list of package names in given set
@type set_atoms: list (set)
@raise entropy.exceptions.InvalidPackageSet: if package set data
passed is invalid (non ASCII chars, invalid set_name).
The encapsulated error string will contain a mnemonic reason.
"""
def _ensure_package_sets_dir():
sets_dir = SystemSettings.packages_sets_directory()
if not os.path.isdir(sets_dir):
if os.path.lexists(sets_dir):
os.remove(sets_dir)
os.makedirs(sets_dir, 0o775)
const_setup_perms(sets_dir, etpConst['entropygid'],
recursion = False)
if not const_isunicode(set_name):
raise InvalidPackageSet("%s %s" % (
set_name, "must be unicode",))
if set_name.startswith(etpConst['packagesetprefix']):
raise InvalidPackageSet("%s %s '%s'" % (
set_name, "cannot start with", etpConst['packagesetprefix'],))
set_match = self.match(set_name)
if set_match:
raise InvalidPackageSet(_("Name already taken"))
_ensure_package_sets_dir()
set_file = os.path.join(SystemSettings.packages_sets_directory(),
set_name)
set_file_tmp = set_file + ".sets_add_tmp"
enc = etpConst['conf_encoding']
try:
with codecs.open(set_file_tmp, "w", encoding=enc) as f:
for x in set_atoms:
f.write(x)
f.write("\n")
os.rename(set_file_tmp, set_file)
except (OSError, IOError) as err:
raise InvalidPackageSet(_("Cannot create the element"))
self._settings['system_package_sets'][set_name] = set(set_atoms)
def build_application_store_url(app, sub_page):
"""
take rigo.models.application.Application object
and build up HTTP Entropy Application Store URL
pointing to exact Application page.
sub_page is used to load a specific part of the page,
for example "ugc" can be passed to load URL/ugc page.
"""
settings = SystemSettings()
details = app.get_details()
pkg_id, pkg_repo = details.pkg
branch = settings['repositories']['branch']
product = settings['repositories']['product']
url = "%s/show/%s,%s,%s,%s,%s,%s/%s" % (
etpConst['packages_website_url'], details.pkgname, pkg_id, pkg_repo,
etpConst['currentarch'], branch, product, sub_page)
return url
def _clear_resources_after_lock(self):
"""
Clear resources that could have become stale after
the Entropy Lock acquisition.
"""
cacher = EntropyCacher()
with cacher:
SystemSettings().clear()
cacher.discard()
with EntropyResourcesLock._POST_ACQUIRE_HOOK_LOCK:
callables = list(
EntropyResourcesLock._POST_ACQUIRE_HOOKS.values())
for callab in callables:
callab()
cacher.sync()
def __init__(self, entropy_client, repo_identifiers = None,
force = False, fetch_security = True, gpg = True):
"""
Entropy Client Repositories management interface constructor.
@param entropy_client: a valid entropy.client.interfaces.client.Client
instance
@type entropy_client: entropy.client.interfaces.client.Client
@keyword repo_identifiers: list of repository identifiers you want to
take into consideration
@type repo_identifiers: list
@
"""
if repo_identifiers is None:
repo_identifiers = []
self._entropy = entropy_client
self._settings = SystemSettings()
self._pkg_size_warning_th = 512*1024000 # 500mb
repo_ids = repo_identifiers
self.force = force
self.sync_errors = False
self.updated = False
self.new_entropy = False
self.need_packages_cleanup = False
self.updated_repos = set()
self.fetch_security = fetch_security
self.already_updated = 0
self.not_available = 0
self._gpg_feature = gpg
env_gpg = os.getenv('ETP_DISBLE_GPG')
if env_gpg is not None:
self._gpg_feature = False
if not repo_ids:
avail_repos = self._settings['repositories']['available'].keys()
repo_ids.extend(list(avail_repos))
# filter out package repositories
self.repo_ids = self._entropy.filter_repositories(repo_ids)
def remove(self, set_name):
"""
Remove a user-defined package set from Entropy Client
(changes are permanent)
@param set_name: package set name
@type set_name: string
@raise entropy.exceptions.InvalidPackageSet: if package set data
passed is invalid (non ASCII chars, invalid set_name).
The encapsulated error string will contain a mnemonic reason.
"""
if not const_isunicode(set_name):
raise InvalidPackageSet("%s %s" % (
set_name, "must be unicode",))
if set_name.startswith(etpConst['packagesetprefix']):
raise InvalidPackageSet("InvalidPackageSet: %s %s '%s'" % (
set_name, _("cannot start with"),
etpConst['packagesetprefix'],))
set_match = self.match(set_name)
if not set_match:
raise InvalidPackageSet(_("Already removed"))
set_id, set_x, set_y = set_match
if set_id != etpConst['userpackagesetsid']:
raise InvalidPackageSet(_("Not defined by user"))
set_file = os.path.join(SystemSettings.packages_sets_directory(),
set_name)
try:
os.remove(set_file)
except OSError as err:
if err.errno != errno.ENOENT:
raise InvalidPackageSet(_("Set not found or unable to remove"))
self._settings['system_package_sets'].pop(set_name, None)
def remove(self, set_name):
"""
Remove a user-defined package set from Entropy Client
(changes are permanent)
@param set_name: package set name
@type set_name: string
@raise entropy.exceptions.InvalidPackageSet: if package set data
passed is invalid (non ASCII chars, invalid set_name).
The encapsulated error string will contain a mnemonic reason.
"""
if not const_isunicode(set_name):
raise InvalidPackageSet("%s %s" % (
set_name, "must be unicode",))
if set_name.startswith(etpConst['packagesetprefix']):
raise InvalidPackageSet("InvalidPackageSet: %s %s '%s'" % (
set_name, _("cannot start with"),
etpConst['packagesetprefix'],))
set_match = self.match(set_name)
if not set_match:
raise InvalidPackageSet(_("Already removed"))
set_id, set_x, set_y = set_match
if set_id != etpConst['userpackagesetsid']:
raise InvalidPackageSet(_("Not defined by user"))
set_file = os.path.join(SystemSettings.packages_sets_directory(),
set_name)
try:
os.remove(set_file)
except OSError as err:
if err.errno != errno.ENOENT:
raise InvalidPackageSet(_("Set not found or unable to remove"))
self._settings['system_package_sets'].pop(set_name, None)
def __init__(self, entropy_client, quiet=False):
self._quiet = quiet
self._entropy = entropy_client
self._settings = SystemSettings()
dict.__init__(self)
self._load()
class Client(Singleton, TextInterface, CalculatorsMixin, RepositoryMixin,
MiscMixin, MatchMixin):
def init_singleton(self,
indexing=True,
installed_repo=None,
xcache=True,
user_xcache=False,
repo_validation=True,
url_fetcher=None,
multiple_url_fetcher=None,
**kwargs):
"""
Entropy Client Singleton interface. Your hitchhikers' guide to the
Galaxy.
@keyword indexing: enable metadata indexing (default is True)
@type indexing: bool
@keyword installed_repo: open installed packages repository? (default
is True). Accepted values: True = open, False = open but consider
it not available, -1 = do not even try to open
@type installed_repo: bool or int
@keyword xcache: enable on-disk cache (default is True)
@type xcache: bool
@keyword user_xcache: enable on-disk cache even for users not in the
entropy group (default is False). Dangerous, could lead to cache
inconsistencies.
@type user_xcache: bool
@keyword repo_validation: validate all the available repositories
and automatically exclude the faulty ones
@type repo_validation: bool
@keyword url_fetcher: override default entropy.fetchers.UrlFetcher
class usage. Provide your own implementation of UrlFetcher using
this argument.
@type url_fetcher: class or None
@keyword multiple_url_fetcher: override default
entropy.fetchers.MultipleUrlFetcher class usage. Provide your own
implementation of MultipleUrlFetcher using this argument.
"""
self.__post_acquire_hook_idx = None
self.__instance_destroyed = False
self._repo_error_messages_cache = set()
self._repodb_cache = {}
self._repodb_cache_mutex = threading.RLock()
self._memory_db_instances = {}
self._real_installed_repository = None
self._real_installed_repository_lock = threading.RLock()
self._treeupdates_repos = set()
self._can_run_sys_set_hooks = False
const_debug_write(__name__, "debug enabled")
self.safe_mode = 0
self._indexing = indexing
self._repo_validation = repo_validation
self._real_cacher = None
self._real_cacher_lock = threading.RLock()
# setup package settings (masking and other stuff)
self._real_settings = None
self._real_settings_lock = threading.RLock()
self._real_settings_client_plg = None
self._real_settings_client_plg_lock = threading.RLock()
self._real_logger = None
self._real_logger_lock = threading.RLock()
self._real_enabled_repos = None
self._real_enabled_repos_lock = threading.RLock()
self._multiple_url_fetcher = multiple_url_fetcher
self._url_fetcher = url_fetcher
if url_fetcher is None:
self._url_fetcher = UrlFetcher
if multiple_url_fetcher is None:
self._multiple_url_fetcher = MultipleUrlFetcher
self._do_open_installed_repo = True
self._installed_repo_enable = True
if installed_repo in (True, None, 1):
self._installed_repo_enable = True
elif installed_repo in (False, 0):
self._installed_repo_enable = False
elif installed_repo == -1:
self._installed_repo_enable = False
self._do_open_installed_repo = False
self.xcache = xcache
shell_xcache = os.getenv("ETP_NOCACHE")
if shell_xcache:
self.xcache = False
# now if we are on live, we should disable it
# are we running on a livecd? (/proc/cmdline has "cdroot")
if entropy.tools.islive():
self.xcache = False
elif (not entropy.tools.is_user_in_entropy_group()
) and not user_xcache:
self.xcache = False
# Add Entropy Resources Lock post-acquire hook that cleans
# repository caches.
hook_ref = EntropyResourcesLock.add_post_acquire_hook(
self._resources_post_hook)
self.__post_acquire_hook_idx = hook_ref
# enable System Settings hooks
self._can_run_sys_set_hooks = True
const_debug_write(__name__, "singleton loaded")
@property
def _settings(self):
"""
Return a SystemSettings object instance.
"""
with self._real_settings_lock:
if self._real_settings is None:
self._real_settings = SystemSettings()
const_debug_write(__name__, "SystemSettings loaded")
# add our SystemSettings plugin
# Make sure we connect Entropy Client plugin
# AFTER client db init
self._real_settings.add_plugin(self._settings_client_plugin)
return self._real_settings
@property
def _settings_client_plugin(self):
"""
Return the SystemSettings Entropy Client plugin.
"""
with self._real_settings_client_plg_lock:
if self._real_settings_client_plg is None:
plugin = ClientSystemSettingsPlugin(self)
self._real_settings_client_plg = plugin
return self._real_settings_client_plg
@property
def _cacher(self):
"""
Return an EntropyCacher object instance.
"""
with self._real_cacher_lock:
if self._real_cacher is None:
real_cacher = EntropyCacher()
const_debug_write(__name__, "EntropyCacher loaded")
# needs to be started here otherwise repository
# cache will be always dropped
if self.xcache:
real_cacher.start()
else:
# disable STASHING_CACHE or we leak
EntropyCacher.STASHING_CACHE = False
self._real_cacher = real_cacher
return self._real_cacher
@property
def logger(self):
"""
Return the Entropy Client Logger instance.
"""
with self._real_logger_lock:
if self._real_logger is None:
real_logger = LogFile(
level=self._settings['system']['log_level'],
filename=etpConst['entropylogfile'],
header="[client]")
const_debug_write(__name__, "Logger loaded")
self._real_logger = real_logger
return self._real_logger
@property
def _enabled_repos(self):
with self._real_enabled_repos_lock:
if self._real_enabled_repos is None:
real_enabled_repos = []
if self._repo_validation:
self._validate_repositories(
enabled_repos=real_enabled_repos)
else:
real_enabled_repos.extend(
self._settings['repositories']['order'])
self._real_enabled_repos = real_enabled_repos
return self._real_enabled_repos
def _resources_post_hook(self):
"""
Hook running after Entropy Resources Lock acquisition.
This method takes care of the repository memory caches, by
invalidating it.
"""
with self._real_installed_repository_lock:
if self._real_installed_repository is not None:
self._real_installed_repository.clearCache()
with self._repodb_cache_mutex:
for repo in self._repodb_cache.values():
repo.clearCache()
def destroy(self, _from_shutdown=False):
"""
Destroy this Singleton instance, closing repositories, removing
SystemSettings plugins added during instance initialization.
This method should be always called when instance is not used anymore.
"""
self.__instance_destroyed = True
if self.__post_acquire_hook_idx is not None:
EntropyResourcesLock.remove_post_acquire_hook(
self.__post_acquire_hook_idx)
self.__post_acquire_hook_idx = None
if hasattr(self, '_installed_repository'):
inst_repo = self.installed_repository()
if inst_repo is not None:
inst_repo.close(_token=InstalledPackagesRepository.NAME)
if hasattr(self, '_real_logger_lock'):
with self._real_logger_lock:
if self._real_logger is not None:
self._real_logger.close()
if not _from_shutdown:
if hasattr(self, '_real_settings') and \
hasattr(self._real_settings, 'remove_plugin'):
# shutdown() will terminate the whole process
# so there is no need to remove plugins from
# SystemSettings, it wouldn't make any diff.
if self._real_settings is not None:
try:
self._real_settings.remove_plugin(
ClientSystemSettingsPlugin.ID)
except KeyError:
pass
self.close_repositories(mask_clear=False)
def shutdown(self):
"""
This method should be called when the whole process is going to be
killed. It calls destroy() and stops any running thread
"""
self._cacher.sync() # enforce, destroy() may kill the current content
self.destroy(_from_shutdown=True)
self._cacher.stop()
entropy.tools.kill_threads()
@sharedinstlock
def repository_packages_spm_sync(self,
repository_identifier,
repo_db,
force=False):
"""
Service method used to sync package names with Source Package Manager
via metadata stored in Repository dbs collected at server-time.
Source Package Manager can change package names, categories or slot
and Entropy repositories must be kept in sync.
In other words, it checks for /usr/portage/profiles/updates changes,
of course indirectly, since there is no way entropy.client can directly
depend on Portage.
@param repository_identifier: repository identifier which repo_db
parameter is bound
@type repository_identifier: string
@param repo_db: repository database instance
@type repo_db: entropy.db.EntropyRepository
@return: bool stating if changes have been made
@rtype: bool
"""
inst_repo = self.installed_repository()
if not inst_repo:
# nothing to do if client db is not availabe
return False
self._treeupdates_repos.add(repository_identifier)
do_rescan = False
shell_rescan = os.getenv("ETP_TREEUPDATES_RESCAN")
if shell_rescan:
do_rescan = True
# check database digest
stored_digest = repo_db.retrieveRepositoryUpdatesDigest(
repository_identifier)
if stored_digest == -1:
do_rescan = True
# check stored value in client database
client_digest = "0"
if not do_rescan:
client_digest = \
inst_repo.retrieveRepositoryUpdatesDigest(
repository_identifier)
if do_rescan or (str(stored_digest) != str(client_digest)) or force:
# reset database tables
inst_repo.clearTreeupdatesEntries(repository_identifier)
# load updates
update_actions = repo_db.retrieveTreeUpdatesActions(
repository_identifier)
# now filter the required actions
update_actions = inst_repo.filterTreeUpdatesActions(update_actions)
if update_actions:
mytxt = "%s: %s." % (
bold(_("ATTENTION")),
red(_("forcing packages metadata update")),
)
self.output(mytxt,
importance=1,
level="info",
header=darkred(" * "))
mytxt = "%s %s." % (
red(_("Updating system database using repository")),
blue(repository_identifier),
)
self.output(mytxt,
importance=1,
level="info",
header=darkred(" * "))
# run stuff
inst_repo.runTreeUpdatesActions(update_actions)
# store new digest into database
inst_repo.setRepositoryUpdatesDigest(repository_identifier,
stored_digest)
# store new actions
inst_repo.addRepositoryUpdatesActions(
InstalledPackagesRepository.NAME, update_actions,
self._settings['repositories']['branch'])
inst_repo.commit()
# clear client cache
inst_repo.clearCache()
return True
def is_destroyed(self):
return self.__instance_destroyed
def clear_cache(self):
"""
Clear all the Entropy default cache directory. This function is
fault tolerant and will never return any exception.
"""
with self._cacher:
# no data is written while holding self._cacher by the balls
# drop all the buffers then remove on-disk data
self._cacher.discard()
# clear repositories live cache
inst_repo = self.installed_repository()
if inst_repo is not None:
inst_repo.clearCache()
with self._repodb_cache_mutex:
for repo in self._repodb_cache.values():
repo.clearCache()
cache_dir = self._cacher.current_directory()
try:
shutil.rmtree(cache_dir, True)
except (shutil.Error, IOError, OSError):
return
try:
os.makedirs(cache_dir, 0o775)
except (IOError, OSError):
return
try:
const_setup_perms(cache_dir, etpConst['entropygid'])
except (IOError, OSError):
return
def QA(self):
"""
Load Entropy QA interface object
@rtype: entropy.qa.QAInterface
"""
qa_intf = QAInterface()
qa_intf.output = self.output
qa_intf.ask_question = self.ask_question
qa_intf.input_box = self.input_box
qa_intf.set_title = self.set_title
return qa_intf
def Settings(self):
"""
Return SystemSettings instance object
"""
return self._settings
def ClientSettings(self):
"""
Return SystemSettings Entropy Client plugin metadata dictionary
"""
p_id = ClientSystemSettingsPlugin.ID
return self._settings[p_id]
def Cacher(self):
"""
Return EntropyCacher instance object
@return: EntropyCacher instance object
@rtype: entropy.cache.EntropyCacher
"""
return self._cacher
def PackageActionFactory(self):
"""
Load Entropy PackageActionFactory instance object
"""
return PackageActionFactory(self)
def ConfigurationUpdates(self):
"""
Return Entropy Configuration File Updates management object.
"""
return ConfigurationUpdates(self)
def Spm(self):
"""
Load Source Package Manager instance object
"""
return get_spm(self)
def Spm_class(self):
"""
Load Source Package Manager default plugin class
"""
return get_spm_default_class()
def Repositories(self, *args, **kwargs):
"""
Load Entropy Repositories manager instance object
@return: Repository instance object
@rtype: entropy.client.interfaces.repository.Repository
"""
client_data = self.ClientSettings()['misc']
kwargs['gpg'] = client_data['gpg']
return Repository(self, *args, **kwargs)
def Security(self, *args, **kwargs):
"""
Load Entropy Security Advisories interface object
@return: Repository Security instance object
@rtype: entropy.security.System
"""
return System(self, *args, **kwargs)
def RepositorySecurity(self, keystore_dir=None):
"""
Load Entropy Repository Security interface object
@return: Repository Repository Security instance object
@rtype: entropy.security.Repository
@raise RepositorySecurity.GPGError: GPGError based instances in case
of problems.
"""
if keystore_dir is None:
keystore_dir = etpConst['etpclientgpgdir']
return RepositorySecurity(keystore_dir=keystore_dir)
def Sets(self):
"""
Load Package Sets interface object
@return: Sets instance object
@rtype: entropy.client.interfaces.sets.Sets
"""
return Sets(self)
def WebServices(self):
"""
Load the Entropy Web Services Factory interface, that can be used
to obtain a WebService object that is able to communicate with
repository remote services, if available.
@return: WebServicesFactory instance object
@rtype: entropy.client.services.interfaces.WebServicesFactory
"""
return ClientWebServiceFactory(self)
def RepositoryWebServices(self):
"""
Load the Repository Entropy Web Services Factory interface, that can
be used to obtain a RepositoryWebService object that is able to
communicate with repository remote services, querying for package
metadata and general repository status.
@return: RepositoryWebServiceFactory instance object
@rtype: entropy.client.services.interfaces.RepositoryWebServiceFactory
"""
return RepositoryWebServiceFactory(self)
def handle_exception(exc_class, exc_instance, exc_tb):
# restore original exception handler, to avoid loops
uninstall_exception_handler()
_text = TextInterface()
if exc_class is SystemDatabaseError:
_text.output(darkred(
_("Installed packages repository corrupted. "
"Please re-generate it")),
importance=1,
level="error")
os._exit(101)
generic_exc_classes = (OnlineMirrorError, RepositoryError,
PermissionDenied, FileNotFound, SPMError,
SystemError)
if exc_class in generic_exc_classes:
_text.output("%s: %s" % (
exc_instance,
darkred(_("Cannot continue")),
),
importance=1,
level="error")
os._exit(1)
if exc_class is SystemExit:
return
if issubclass(exc_class, IOError): # in Python 3.3+ it's BrokenPipeError
if exc_instance.errno == errno.EPIPE:
return
if exc_class is KeyboardInterrupt:
os._exit(1)
t_back = entropy.tools.get_traceback(tb_obj=exc_tb)
if const_debug_enabled():
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
sys.stdin = sys.__stdin__
entropy.tools.print_exception(tb_data=exc_tb)
pdb.set_trace()
if exc_class in (IOError, OSError):
if exc_instance.errno == errno.ENOSPC:
print_generic(t_back)
_text.output("%s: %s" % (
exc_instance,
darkred(_("Your hard drive is full! Your fault!")),
),
importance=1,
level="error")
os._exit(5)
elif exc_instance.errno == errno.ENOMEM:
print_generic(t_back)
_text.output("%s: %s" % (
exc_instance,
darkred(_("No more memory dude! Your fault!")),
),
importance=1,
level="error")
os._exit(5)
_text.output(darkred(
_("Hi. My name is Bug Reporter. "
"I am sorry to inform you that the program crashed. "
"Well, you know, shit happens.")),
importance=1,
level="error")
_text.output(darkred(
_("But there's something you could "
"do to help me to be a better application.")),
importance=1,
level="error")
_text.output(darkred(
_("-- BUT, DO NOT SUBMIT THE SAME REPORT MORE THAN ONCE --")),
importance=1,
level="error")
_text.output(darkred(
_("Now I am showing you what happened. "
"Don't panic, I'm here to help you.")),
importance=1,
level="error")
entropy.tools.print_exception(tb_data=exc_tb)
exception_data = entropy.tools.print_exception(silent=True,
tb_data=exc_tb,
all_frame_data=True)
exception_tback_raw = const_convert_to_rawstring(t_back)
error_fd, error_file = None, None
try:
error_fd, error_file = const_mkstemp(prefix="entropy.error.report.",
suffix=".txt")
with os.fdopen(error_fd, "wb") as ferror:
ferror.write(
const_convert_to_rawstring("\nRevision: %s\n\n" %
(etpConst['entropyversion'], )))
ferror.write(exception_tback_raw)
ferror.write(const_convert_to_rawstring("\n\n"))
ferror.write(const_convert_to_rawstring(''.join(exception_data)))
ferror.write(const_convert_to_rawstring("\n"))
except (OSError, IOError) as err:
_text.output("%s: %s" %
(err,
darkred(
_("Oh well, I cannot even write to TMPDIR. "
"So, please copy the error and "
"mail [email protected]."))),
importance=1,
level="error")
os._exit(1)
finally:
if error_fd is not None:
try:
os.close(error_fd)
except OSError:
pass
_text.output("", level="error")
ask_msg = _("Erm... Can I send the error, "
"along with some other information\nabout your "
"hardware to my creators so they can fix me? "
"(Your IP will be logged)")
rc = _text.ask_question(ask_msg)
if rc == _("No"):
_text.output(darkgreen(_("Ok, ok ok ok... Sorry!")), level="error")
os._exit(2)
_text.output(darkgreen(
_("If you want to be contacted back "
"(and actively supported), also answer "
"the questions below:")),
level="error")
try:
name = readtext(_("Your Full name:"))
email = readtext(_("Your E-Mail address:"))
description = readtext(_("What you were doing:"))
except EOFError:
os._exit(2)
try:
from entropy.client.interfaces.qa import UGCErrorReport
from entropy.core.settings.base import SystemSettings
_settings = SystemSettings()
repository_id = _settings['repositories']['default_repository']
error = UGCErrorReport(repository_id)
except (
OnlineMirrorError,
AttributeError,
ImportError,
):
error = None
result = None
if error is not None:
error.prepare(exception_tback_raw, name, email,
'\n'.join([x for x in exception_data]), description)
result = error.submit()
if result:
_text.output(darkgreen(
_("Thank you very much. The error has been "
"reported and hopefully, the problem will "
"be solved as soon as possible.")),
level="error")
else:
_text.output(darkred(
_("Ugh. Cannot send the report. "
"Please mail the file below "
"to [email protected].")),
level="error")
_text.output("", level="error")
_text.output("==> %s" % (error_file, ), level="error")
_text.output("", level="error")
class Client(Singleton, TextInterface, LoadersMixin, CacheMixin,
CalculatorsMixin, RepositoryMixin, MiscMixin,
MatchMixin, NoticeBoardMixin):
def init_singleton(self, indexing = True, installed_repo = None,
xcache = True, user_xcache = False, repo_validation = True,
url_fetcher = None, multiple_url_fetcher = None, **kwargs):
"""
Entropy Client Singleton interface. Your hitchhikers' guide to the
Galaxy.
@keyword indexing: enable metadata indexing (default is True)
@type indexing: bool
@keyword installed_repo: open installed packages repository? (default
is True). Accepted values: True = open, False = open but consider
it not available, -1 = do not even try to open
@type installed_repo: bool or int
@keyword xcache: enable on-disk cache (default is True)
@type xcache: bool
@keyword user_xcache: enable on-disk cache even for users not in the
entropy group (default is False). Dangerous, could lead to cache
inconsistencies.
@type user_xcache: bool
@keyword repo_validation: validate all the available repositories
and automatically exclude the faulty ones
@type repo_validation: bool
@keyword url_fetcher: override default entropy.fetchers.UrlFetcher
class usage. Provide your own implementation of UrlFetcher using
this argument.
@type url_fetcher: class or None
@keyword multiple_url_fetcher: override default
entropy.fetchers.MultipleUrlFetcher class usage. Provide your own
implementation of MultipleUrlFetcher using this argument.
"""
self.__post_acquire_hook_idx = None
self.__instance_destroyed = False
self._repo_error_messages_cache = set()
self._repodb_cache = {}
self._repodb_cache_mutex = threading.RLock()
self._memory_db_instances = {}
self._real_installed_repository = None
self._real_installed_repository_lock = threading.RLock()
self._treeupdates_repos = set()
self._can_run_sys_set_hooks = False
const_debug_write(__name__, "debug enabled")
self.safe_mode = 0
self._indexing = indexing
self._repo_validation = repo_validation
self._real_cacher = None
self._real_cacher_lock = threading.Lock()
# setup package settings (masking and other stuff)
self._real_settings = None
self._real_settings_lock = threading.Lock()
self._real_settings_client_plg = None
self._real_settings_client_plg_lock = threading.Lock()
self._real_logger = None
self._real_logger_lock = threading.Lock()
self._real_enabled_repos = None
self._real_enabled_repos_lock = threading.RLock()
# class init
LoadersMixin.__init__(self)
self._multiple_url_fetcher = multiple_url_fetcher
self._url_fetcher = url_fetcher
if url_fetcher is None:
self._url_fetcher = UrlFetcher
if multiple_url_fetcher is None:
self._multiple_url_fetcher = MultipleUrlFetcher
self._do_open_installed_repo = True
self._installed_repo_enable = True
if installed_repo in (True, None, 1):
self._installed_repo_enable = True
elif installed_repo in (False, 0):
self._installed_repo_enable = False
elif installed_repo == -1:
self._installed_repo_enable = False
self._do_open_installed_repo = False
self.xcache = xcache
shell_xcache = os.getenv("ETP_NOCACHE")
if shell_xcache:
self.xcache = False
# now if we are on live, we should disable it
# are we running on a livecd? (/proc/cmdline has "cdroot")
if entropy.tools.islive():
self.xcache = False
elif (not entropy.tools.is_user_in_entropy_group()) and not user_xcache:
self.xcache = False
# Add Entropy Resources Lock post-acquire hook that cleans
# repository caches.
hook_ref = EntropyResourcesLock.add_post_acquire_hook(
self._resources_post_hook)
self.__post_acquire_hook_idx = hook_ref
# enable System Settings hooks
self._can_run_sys_set_hooks = True
const_debug_write(__name__, "singleton loaded")
@property
def _settings(self):
"""
Return a SystemSettings object instance.
"""
if self._real_settings is None:
# once != None, will be always != None
with self._real_settings_lock:
if self._real_settings is None:
self._real_settings = SystemSettings()
const_debug_write(__name__, "SystemSettings loaded")
# add our SystemSettings plugin
# Make sure we connect Entropy Client plugin
# AFTER client db init
self._real_settings.add_plugin(
self._settings_client_plugin)
return self._real_settings
@property
def _settings_client_plugin(self):
"""
Return the SystemSettings Entropy Client plugin.
"""
if self._real_settings_client_plg is None:
# once != None, will be always != None
with self._real_settings_client_plg_lock:
if self._real_settings_client_plg is None:
plugin = ClientSystemSettingsPlugin(self)
self._real_settings_client_plg = plugin
return self._real_settings_client_plg
@property
def _cacher(self):
"""
Return an EntropyCacher object instance.
"""
if self._real_cacher is None:
# once != None, will be always != None
with self._real_cacher_lock:
if self._real_cacher is None:
self._real_cacher = EntropyCacher()
const_debug_write(__name__, "EntropyCacher loaded")
# needs to be started here otherwise repository
# cache will be always dropped
if self.xcache:
self._real_cacher.start()
else:
# disable STASHING_CACHE or we leak
EntropyCacher.STASHING_CACHE = False
return self._real_cacher
@property
def logger(self):
"""
Return the Entropy Client Logger instance.
"""
if self._real_logger is None:
# once != None, will be always != None
with self._real_logger_lock:
if self._real_logger is None:
self._real_logger = LogFile(
level = self._settings['system']['log_level'],
filename = etpConst['entropylogfile'],
header = "[client]")
const_debug_write(__name__, "Logger loaded")
return self._real_logger
@property
def _enabled_repos(self):
if self._real_enabled_repos is None:
with self._real_enabled_repos_lock:
if self._real_enabled_repos is None:
self._real_enabled_repos = []
if self._repo_validation:
self._validate_repositories(
enabled_repos = self._real_enabled_repos)
else:
self._real_enabled_repos.extend(
self._settings['repositories']['order'])
return self._real_enabled_repos
def _resources_post_hook(self):
"""
Hook running after Entropy Resources Lock acquisition.
This method takes care of the repository memory caches, by
invalidating it.
"""
with self._real_installed_repository_lock:
if self._real_installed_repository is not None:
self._real_installed_repository.clearCache()
with self._repodb_cache_mutex:
for repo in self._repodb_cache.values():
repo.clearCache()
def destroy(self, _from_shutdown = False):
"""
Destroy this Singleton instance, closing repositories, removing
SystemSettings plugins added during instance initialization.
This method should be always called when instance is not used anymore.
"""
self.__instance_destroyed = True
if self.__post_acquire_hook_idx is not None:
EntropyResourcesLock.remove_post_acquire_hook(
self.__post_acquire_hook_idx)
self.__post_acquire_hook_idx = None
if hasattr(self, '_installed_repository'):
inst_repo = self.installed_repository()
if inst_repo is not None:
inst_repo.close(_token = InstalledPackagesRepository.NAME)
if hasattr(self, '_real_logger_lock'):
with self._real_logger_lock:
if self._real_logger is not None:
self._real_logger.close()
if not _from_shutdown:
if hasattr(self, '_real_settings') and \
hasattr(self._real_settings, 'remove_plugin'):
# shutdown() will terminate the whole process
# so there is no need to remove plugins from
# SystemSettings, it wouldn't make any diff.
if self._real_settings is not None:
try:
self._real_settings.remove_plugin(
ClientSystemSettingsPlugin.ID)
except KeyError:
pass
self.close_repositories(mask_clear = False)
def shutdown(self):
"""
This method should be called when the whole process is going to be
killed. It calls destroy() and stops any running thread
"""
self._cacher.sync() # enforce, destroy() may kill the current content
self.destroy(_from_shutdown = True)
self._cacher.stop()
entropy.tools.kill_threads()
def repository_packages_spm_sync(self, repository_identifier, repo_db,
force = False):
"""
Service method used to sync package names with Source Package Manager
via metadata stored in Repository dbs collected at server-time.
Source Package Manager can change package names, categories or slot
and Entropy repositories must be kept in sync.
In other words, it checks for /usr/portage/profiles/updates changes,
of course indirectly, since there is no way entropy.client can directly
depend on Portage.
@param repository_identifier: repository identifier which repo_db
parameter is bound
@type repository_identifier: string
@param repo_db: repository database instance
@type repo_db: entropy.db.EntropyRepository
@return: bool stating if changes have been made
@rtype: bool
"""
inst_repo = self.installed_repository()
if not inst_repo:
# nothing to do if client db is not availabe
return False
self._treeupdates_repos.add(repository_identifier)
do_rescan = False
shell_rescan = os.getenv("ETP_TREEUPDATES_RESCAN")
if shell_rescan:
do_rescan = True
# check database digest
stored_digest = repo_db.retrieveRepositoryUpdatesDigest(
repository_identifier)
if stored_digest == -1:
do_rescan = True
# check stored value in client database
client_digest = "0"
if not do_rescan:
client_digest = \
inst_repo.retrieveRepositoryUpdatesDigest(
repository_identifier)
if do_rescan or (str(stored_digest) != str(client_digest)) or force:
# reset database tables
inst_repo.clearTreeupdatesEntries(
repository_identifier)
# load updates
update_actions = repo_db.retrieveTreeUpdatesActions(
repository_identifier)
# now filter the required actions
update_actions = inst_repo.filterTreeUpdatesActions(
update_actions)
if update_actions:
mytxt = "%s: %s." % (
bold(_("ATTENTION")),
red(_("forcing packages metadata update")),
)
self.output(
mytxt,
importance = 1,
level = "info",
header = darkred(" * ")
)
mytxt = "%s %s." % (
red(_("Updating system database using repository")),
blue(repository_identifier),
)
self.output(
mytxt,
importance = 1,
level = "info",
header = darkred(" * ")
)
# run stuff
inst_repo.runTreeUpdatesActions(
update_actions)
# store new digest into database
inst_repo.setRepositoryUpdatesDigest(
repository_identifier, stored_digest)
# store new actions
inst_repo.addRepositoryUpdatesActions(
InstalledPackagesRepository.NAME, update_actions,
self._settings['repositories']['branch'])
inst_repo.commit()
# clear client cache
inst_repo.clearCache()
return True
def is_destroyed(self):
return self.__instance_destroyed
def init_singleton(self, indexing = True, installed_repo = None,
xcache = True, user_xcache = False, repo_validation = True,
url_fetcher = None, multiple_url_fetcher = None, **kwargs):
"""
Entropy Client Singleton interface. Your hitchhikers' guide to the
Galaxy.
@keyword indexing: enable metadata indexing (default is True)
@type indexing: bool
@keyword installed_repo: open installed packages repository? (default
is True). Accepted values: True = open, False = open but consider
it not available, -1 = do not even try to open
@type installed_repo: bool or int
@keyword xcache: enable on-disk cache (default is True)
@type xcache: bool
@keyword user_xcache: enable on-disk cache even for users not in the
entropy group (default is False). Dangerous, could lead to cache
inconsistencies.
@type user_xcache: bool
@keyword repo_validation: validate all the available repositories
and automatically exclude the faulty ones
@type repo_validation: bool
@keyword url_fetcher: override default entropy.fetchers.UrlFetcher
class usage. Provide your own implementation of UrlFetcher using
this argument.
@type url_fetcher: class or None
@keyword multiple_url_fetcher: override default
entropy.fetchers.MultipleUrlFetcher class usage. Provide your own
implementation of MultipleUrlFetcher using this argument.
"""
self.__instance_destroyed = False
self._repo_error_messages_cache = set()
self._repodb_cache = {}
self._repodb_cache_mutex = threading.RLock()
self._memory_db_instances = {}
self._installed_repository = None
self._treeupdates_repos = set()
self._can_run_sys_set_hooks = False
const_debug_write(__name__, "debug enabled")
self.sys_settings_client_plugin_id = \
etpConst['system_settings_plugins_ids']['client_plugin']
self._enabled_repos = []
self.safe_mode = 0
self._indexing = indexing
self._repo_validation = repo_validation
# setup package settings (masking and other stuff)
self._settings = SystemSettings()
const_debug_write(__name__, "SystemSettings loaded")
# class init
LoadersMixin.__init__(self)
self.logger = LogFile(
level = self._settings['system']['log_level'],
filename = etpConst['entropylogfile'], header = "[client]")
self._multiple_url_fetcher = multiple_url_fetcher
self._url_fetcher = url_fetcher
if url_fetcher is None:
self._url_fetcher = UrlFetcher
if multiple_url_fetcher is None:
self._multiple_url_fetcher = MultipleUrlFetcher
self._cacher = EntropyCacher()
# backward compatibility, will be removed after 2011
if "noclientdb" in kwargs:
noclientdb = kwargs.get("noclientdb")
self._do_open_installed_repo = True
self._installed_repo_enable = True
if noclientdb in (True, 1):
self._installed_repo_enable = False
elif noclientdb in (False, 0):
self._installed_repo_enable = True
elif noclientdb == 2:
self._installed_repo_enable = False
self._do_open_installed_repo = False
else:
self._do_open_installed_repo = True
self._installed_repo_enable = True
if installed_repo in (True, None, 1):
self._installed_repo_enable = True
elif installed_repo in (False, 0):
self._installed_repo_enable = False
elif installed_repo == -1:
self._installed_repo_enable = False
self._do_open_installed_repo = False
self.xcache = xcache
shell_xcache = os.getenv("ETP_NOCACHE")
if shell_xcache:
self.xcache = False
do_validate_repo_cache = False
# now if we are on live, we should disable it
# are we running on a livecd? (/proc/cmdline has "cdroot")
if entropy.tools.islive():
self.xcache = False
elif (not entropy.tools.is_user_in_entropy_group()) and not user_xcache:
self.xcache = False
elif not user_xcache:
do_validate_repo_cache = True
if not self.xcache and (entropy.tools.is_user_in_entropy_group()):
self.clear_cache()
if self._do_open_installed_repo:
self._open_installed_repository()
# create our SystemSettings plugin
self.sys_settings_client_plugin = ClientSystemSettingsPlugin(
self.sys_settings_client_plugin_id, self)
# needs to be started here otherwise repository cache will be
# always dropped
if self.xcache:
self._cacher.start()
else:
# disable STASHING_CACHE or we leak
EntropyCacher.STASHING_CACHE = False
if do_validate_repo_cache:
self._validate_repositories_cache()
if self._repo_validation:
self._validate_repositories()
else:
self._enabled_repos.extend(self._settings['repositories']['order'])
# add our SystemSettings plugin
# Make sure we connect Entropy Client plugin AFTER client db init
self._settings.add_plugin(self.sys_settings_client_plugin)
# enable System Settings hooks
self._can_run_sys_set_hooks = True
const_debug_write(__name__, "singleton loaded")
def config(cls, repository_id):
"""
Return the WebService configuration for the given repository.
The object returned is a dictionary containing the following
items:
- url: the Entropy WebService base URL (or None, if not supported)
- update_eapi: the maximum supported EAPI for repository updates.
- repo_eapi: the maximum supported EAPI for User Generate Content.
@param repository_id: repository identifier
@type repository_id: string
"""
settings = SystemSettings()
_repository_data = settings['repositories']['available'].get(
repository_id)
if _repository_data is None:
const_debug_write(__name__, "WebService.config error: no repo")
return None
web_services_conf = _repository_data.get('webservices_config')
if web_services_conf is None:
const_debug_write(__name__, "WebService.config error: no metadata")
return None
data = {
'url': None,
'_url_obj': None,
'update_eapi': None,
'repo_eapi': None,
}
content = []
try:
content += entropy.tools.generic_file_content_parser(
web_services_conf, encoding=etpConst['conf_encoding'])
except (OSError, IOError) as err:
const_debug_write(__name__,
"WebService.config error: %s" % (err, ))
return None
if not content:
const_debug_write(__name__,
"WebService.config error: empty config")
return None
remote_url = content.pop(0)
if remote_url == "-": # as per specs
remote_url = None
elif not remote_url:
remote_url = None
data['url'] = remote_url
if data['url']:
url_obj = entropy.tools.spliturl(data['url'])
if url_obj.scheme in WebService.SUPPORTED_URL_SCHEMAS:
data['_url_obj'] = url_obj
else:
data['url'] = None
for line in content:
for k in ("UPDATE_EAPI", "REPO_EAPI"):
if line.startswith(k + "="):
try:
data[k.lower()] = int(line.split("=", 1)[-1])
except (IndexError, ValueError):
pass
return data
def __init__(self, entropy_client):
self._entropy = entropy_client
self._settings = SystemSettings()
"""
@author: Fabio Erculiani <*****@*****.**>
@contact: [email protected]
@copyright: Fabio Erculiani
@license: GPL-2
B{Entropy Source Package Manager Plugins factory module}.
"""
from entropy.core import EntropyPluginFactory
from entropy.core.settings.base import SystemSettings
from entropy.spm.plugins.skel import SpmPlugin
import entropy.spm.plugins.interfaces as plugs
_settings = SystemSettings()
_USER_PLUG = _settings['system'].get('spm_backend')
FACTORY = EntropyPluginFactory(SpmPlugin, plugs,
default_plugin_name = _USER_PLUG)
get_available_plugins = FACTORY.get_available_plugins
def get_default_class():
"""
Return default Source Package Manager plugin class.
@return: default Source Package Manager plugin class
@raise SystemError: if no default plugin class has been specified.
This usually means a programming error.
"""
def __init__(self, url_path_list, checksum = True,
show_speed = True, resume = True,
abort_check_func = None, disallow_redirect = False,
url_fetcher_class = None, timeout = None,
download_context_func = None,
pre_download_hook = None, post_download_hook = None):
"""
@param url_path_list: list of tuples composed by url and
path to save, for eg. [(url,path_to_save,),...]
@type url_path_list: list
@keyword checksum: return md5 hash instead of status code
@type checksum: bool
@keyword show_speed: show download speed
@type show_speed: bool
@keyword resume: enable resume support
@type resume: bool
@keyword abort_check_func: callback used to stop download, it has to
raise an exception that has to be caught by provider application.
This exception will be considered an "abort" request.
@type abort_check_func: callable
@keyword disallow_redirect: disallow automatic HTTP redirects
@type disallow_redirect: bool
@keyword thread_stop_func: callback used to stop download, it has to
raise an exception that has to be caught by provider application.
This exception will be considered a "stop" request.
@type thread_stop_func: callable
@param url_fetcher_class: UrlFetcher based class to use
@type url_fetcher_class: subclass of UrlFetcher
@keyword timeout: custom request timeout value (in seconds), if None
the value is read from Entropy configuration files.
@type timeout: int
@keyword download_context_func: if not None, it must be a function
exposing a context manager and taking a path (the download path)
as argument. This can be used to implement locking on files to be
downloaded.
@type download_context_func: callable
@keyword pre_download_hook: hook called before starting the download
process, inside the download_context_func context. This can be
used to verify if the download is actually needed or just return.
If the returned value is not None, the download method will return
that value. The function takes a path (the download path) and the
download id as arguments.
@type pre_download_hook: callable
@keyword post_download_hook: hook called after the download is complete,
inside the download_context_func context. This can be used to verify
the integrity of the downloaded data.
The function takes a path (the download path) and the download
status and the download id as arguments.
@type post_download_hook: callable
"""
self._progress_data = {}
self._url_path_list = url_path_list
self.__system_settings = SystemSettings()
self.__resume = resume
self.__checksum = checksum
self.__show_speed = show_speed
self.__abort_check_func = abort_check_func
self.__disallow_redirect = disallow_redirect
self.__timeout = timeout
self.__download_context_func = download_context_func
self.__pre_download_hook = pre_download_hook
self.__post_download_hook = post_download_hook
# important to have a declaration here
self.__data_transfer = 0
self.__average = 0
self.__old_average = 0
self.__time_remaining_secs = 0
self.__url_fetcher = url_fetcher_class
if self.__url_fetcher == None:
self.__url_fetcher = UrlFetcher
def __init__(self, url, path_to_save, checksum = True,
show_speed = True, resume = True,
abort_check_func = None, disallow_redirect = False,
thread_stop_func = None, speed_limit = None,
timeout = None, download_context_func = None,
pre_download_hook = None, post_download_hook = None):
"""
Entropy URL downloader constructor.
@param url: download URL (do not URL-encode it!)
@type url: string
@param path_to_save: file path where to save downloaded data
@type path_to_save: string
@keyword checksum: return md5 hash instead of status code
@type checksum: bool
@keyword show_speed: show download speed
@type show_speed: bool
@keyword resume: enable resume support
@type resume: bool
@keyword abort_check_func: callback used to stop download, it has to
raise an exception that has to be caught by provider application.
This exception will be considered an "abort" request.
@type abort_check_func: callable
@keyword disallow_redirect: disallow automatic HTTP redirects
@type disallow_redirect: bool
@keyword thread_stop_func: callback used to stop download, it has to
raise an exception that has to be caught by provider application.
This exception will be considered a "stop" request.
@type thread_stop_func: callable
@keyword speed_limit: speed limit in kb/sec
@type speed_limit: int
@keyword timeout: custom request timeout value (in seconds), if None
the value is read from Entropy configuration files.
@type timeout: int
@keyword download_context_func: if not None, it must be a function
exposing a context manager and taking a path (the download path)
as argument. This can be used to implement locking on files to be
downloaded.
@type download_context_func: callable
@keyword pre_download_hook: hook called before starting the download
process, inside the download_context_func context. This can be
used to verify if the download is actually needed or just return.
If the returned value is not None, the download method will return
that value. The function takes a path (the download path) and the
download id as arguments.
@type pre_download_hook: callable
@keyword post_download_hook: hook called after the download is complete,
inside the download_context_func context. This can be used to verify
the integrity of the downloaded data.
The function takes a path (the download path) and the download
status and the download id as arguments.
@type post_download_hook: callable
"""
self.__supported_uris = {
'file': self._urllib_download,
'http': self._urllib_download,
'https': self._urllib_download,
'ftp': self._urllib_download,
'ftps': self._urllib_download,
'rsync': self._rsync_download,
'ssh': self._rsync_download,
}
self.__system_settings = SystemSettings()
if speed_limit == None:
speed_limit = \
self.__system_settings['repositories']['transfer_limit']
if timeout is None:
self.__timeout = \
self.__system_settings['repositories']['timeout']
else:
self.__timeout = timeout
self.__th_id = 0
if download_context_func is None:
@contextlib.contextmanager
def download_context_func(path):
yield
self.__download_context_func = download_context_func
self.__pre_download_hook = pre_download_hook
self.__post_download_hook = post_download_hook
self.__resume = resume
self.__url = url
self.__path_to_save = path_to_save
self.__checksum = checksum
self.__show_speed = show_speed
self.__abort_check_func = abort_check_func
self.__thread_stop_func = thread_stop_func
self.__disallow_redirect = disallow_redirect
self.__speedlimit = speed_limit # kbytes/sec
self._init_vars()
self.__init_urllib()
def _settings(self):
"""
Return a SystemSettings instance.
"""
return SystemSettings()
def __init__(self,
entropy_interface,
uris,
files_to_upload,
download=False,
remove=False,
txc_basedir=None,
local_basedir=None,
critical_files=None,
handlers_data=None,
repo=None,
copy_herustic_support=False):
if critical_files is None:
critical_files = []
if handlers_data is None:
handlers_data = {}
self._entropy = entropy_interface
if not isinstance(uris, list):
raise AttributeError("uris must be a list instance")
if not isinstance(files_to_upload, (list, dict)):
raise AttributeError(
"files_to_upload must be a list or dict instance")
self.uris = uris
if isinstance(files_to_upload, list):
self.myfiles = files_to_upload[:]
else:
self.myfiles = sorted([x for x in files_to_upload])
self._settings = SystemSettings()
self.sys_settings_plugin_id = \
etpConst['system_settings_plugins_ids']['server_plugin']
srv_set = self._settings[self.sys_settings_plugin_id]['server']
# server-side speed limit
self.speed_limit = srv_set['sync_speed_limit']
self.download = download
self.remove = remove
self.repo = repo
if self.repo == None:
self.repo = self._entropy.repository()
if self.remove:
self.download = False
self._copy_herustic = copy_herustic_support
if self._copy_herustic and (self.download or self.remove):
raise AttributeError(
"copy_herustic_support can be enabled only for uploads")
if not txc_basedir:
raise AttributeError("invalid txc_basedir passed")
self.txc_basedir = txc_basedir
if not local_basedir:
# default to database directory
self.local_basedir = os.path.dirname(
self._entropy._get_local_repository_file(self.repo))
else:
self.local_basedir = local_basedir
self.critical_files = critical_files
self.handlers_data = handlers_data.copy()
