def _load_catalogs_tsafe(self, refresh=False):
self._installed_plugins_treeview_tracker = copy.deepcopy(self.config['plugins.installed'])
for plugin in list(self._installed_plugins_treeview_tracker.keys()):
# Remove plugins already found to be locally installed.
if not self._installed_plugins_treeview_tracker[plugin]:
self._installed_plugins_treeview_tracker.pop(plugin)
if refresh:
gui_utilities.glib_idle_add_once(self._model.clear)
expiration = datetime.timedelta(seconds=smoke_zephyr.utilities.parse_timespan(self.config.get('cache.age', '4h')))
self._update_status_bar_tsafe('Loading, catalogs...')
self._load_catalog_local_tsafe()
catalog_cache = self.catalog_plugins.get_cache()
now = datetime.datetime.utcnow()
for catalog_url in self.config['catalogs']:
catalog_cache_dict = catalog_cache.get_catalog_by_url(catalog_url)
if not refresh and catalog_cache_dict and catalog_cache_dict['created'] + expiration > now:
catalog = self._load_catalog_from_cache_tsafe(catalog_cache_dict)
if catalog is not None:
continue
catalog_cache_dict = None
self.logger.debug("downloading catalog: {}".format(catalog_url))
self._update_status_bar_tsafe("Loading, downloading catalog: {}".format(catalog_url))
catalog = self._load_catalog_from_url_tsafe(catalog_url)
if catalog is None and catalog_cache_dict is not None:
self.logger.warning('failing over to loading the catalog from the cache')
self._load_catalog_from_cache_tsafe(catalog_cache_dict)
if self._installed_plugins_treeview_tracker:
self._load_missing_plugins_tsafe()
self._update_status_bar_tsafe('Loading completed')
self._installed_plugins_treeview_tracker = None
def _add_catalog_to_tree_tsafe(self, catalog): """ Create a :py:class:`._ModelNode` instance to representing the catalog, its data and add it to the TreeView model. :param catalog: The catalog to add to the TreeView. :type catalog: :py:class:`.Catalog` """ catalog_node = _ModelNode( id=catalog.id, installed=None, enabled=True, title=catalog.id, compatibility=None, version=None, visible_enabled=False, visible_installed=False, sensitive_installed=False, type=_ROW_TYPE_CATALOG ) for repo in catalog.repositories.values(): repo_node = _ModelNode( id=repo.id, installed=None, enabled=True, title=repo.title, compatibility=None, version=None, visible_enabled=False, visible_installed=False, sensitive_installed=False, type=_ROW_TYPE_REPOSITORY ) catalog_node.children.append(repo_node) plugin_collection = self.catalog_plugins.get_collection(catalog.id, repo.id) for plugin_info in plugin_collection.values(): installed = False enabled = False plugin_name = plugin_info['name'] install_src = self.config['plugins.installed'].get(plugin_name) if install_src and repo.id == install_src['repo_id'] and catalog.id == install_src['catalog_id']: installed = True # plugin was added to treeview so it is removed from the temporary tracking dict self._installed_plugins_treeview_tracker.pop(plugin_name) enabled = plugin_name in self.config['plugins.enabled'] repo_node.children.append(_ModelNamedRow( id=plugin_name, installed=installed, enabled=enabled, title=plugin_info['title'], compatibility='Yes' if self.catalog_plugins.is_compatible(catalog.id, repo.id, plugin_name) else 'No', version=plugin_info['version'], visible_enabled=True, visible_installed=True, sensitive_installed=True, type=_ROW_TYPE_PLUGIN )) gui_utilities.glib_idle_add_once(self.__store_add_node, catalog_node)
def _reload_plugin_tsafe(self, model_row, named_row, enabled=None):
self._update_status_bar_tsafe('Reloading plugin...')
pm = self.application.plugin_manager
if enabled is None:
enabled = named_row.id in pm.enabled_plugins
pm.unload(named_row.id)
try:
klass = pm.load(named_row.id, reload_module=True)
except Exception as error:
self._on_plugin_load_error_tsafe(named_row.id, error)
klass = None
else:
if enabled:
pm.enable(named_row.id)
self.__load_errors.pop(named_row.id, None)
gui_utilities.glib_idle_add_once(self.__reload_plugin_post, model_row, named_row, klass)
return klass
def _plugin_install_tsafe(self, catalog_model, repo_model, model_row, named_row):
self.__installing_plugin = named_row.id
self.logger.debug("installing plugin '{0}'".format(named_row.id))
self._update_status_bar_tsafe("Installing plugin {}...".format(named_row.title))
_show_dialog_error_tsafe = functools.partial(gui_utilities.glib_idle_add_once, gui_utilities.show_dialog_error, 'Failed To Install', self.window)
try:
self.catalog_plugins.install_plugin(catalog_model.id, repo_model.id, named_row.id, self.plugin_path)
except requests.exceptions.ConnectionError:
self.logger.warning("failed to download plugin {}".format(named_row.id))
_show_dialog_error_tsafe("Failed to download {} plugin, check your internet connection.".format(named_row.id))
self._update_status_bar_tsafe("Installing plugin {} failed.".format(named_row.title))
self.__installing_plugin = None
return
except Exception:
self.logger.warning("failed to install plugin {}".format(named_row.id), exc_info=True)
_show_dialog_error_tsafe("Failed to install {} plugin.".format(named_row.id))
self._update_status_bar_tsafe("Installing plugin {} failed.".format(named_row.title))
self.__installing_plugin = None
return
self.config['plugins.installed'][named_row.id] = {'catalog_id': catalog_model.id, 'repo_id': repo_model.id, 'plugin_id': named_row.id}
self.logger.info("installed plugin '{}' from catalog:{}, repository:{}".format(named_row.id, catalog_model.id, repo_model.id))
plugin = self._reload_plugin_tsafe(model_row, named_row)
if self.config['plugins.pip.install_dependencies']:
packages = smoke_zephyr.requirements.check_requirements(tuple(plugin.req_packages.keys()))
if packages:
self.logger.debug("installing missing or incompatible packages from PyPi for plugin '{0}'".format(named_row.id))
self._update_status_bar_tsafe(
"Installing {:,} dependenc{} for plugin {} from PyPi.".format(len(packages), 'y' if len(packages) == 1 else 'ies', named_row.title)
)
pip_results = self._pip_install(packages)
if pip_results is None:
self.logger.warning('pip install failed')
_show_dialog_error_tsafe(
"Failed to run pip to install package(s) for plugin {}.".format(named_row.id)
)
elif pip_results.status:
self.logger.warning('pip install failed, exit status: ' + str(pip_results.status))
_show_dialog_error_tsafe(
"Failed to install pip package(s) for plugin {}.".format(named_row.id)
)
else:
plugin = self._reload_plugin_tsafe(model_row, named_row)
self.__installing_plugin = None
gui_utilities.glib_idle_add_once(self.__plugin_install_post, catalog_model, repo_model, model_row, named_row)
def _load_url_treeview_tsafe(self, hostname=None, refresh=False):
if refresh or not self._url_information['created']:
self._url_information['data'] = self.application.rpc.graphql_find_file('get_site_templates.graphql')
self._url_information['created'] = datetime.datetime.utcnow()
url_information = self._url_information['data']
if not url_information:
return
rows = []
domains = []
for edge in url_information['siteTemplates']['edges']:
template = edge['node']
for page in template['metadata']['pages']:
if hostname and template['hostname'] and not template['hostname'].startswith(hostname):
continue
page = page.strip('/')
resource = '/' + '/'.join((template.get('path', '').strip('/'), page)).lstrip('/')
domains.append(template['hostname'])
rows.append(_ModelNamedRow(
hostname=template['hostname'],
page=page,
url=self._build_url(template['hostname'], resource, 'http'),
classifiers=template['metadata']['classifiers'],
authors=template['metadata']['authors'],
description=template['metadata']['description'].strip('\n'),
created=utilities.format_datetime(utilities.datetime_utc_to_local(template['created']))
))
if self._server_uses_ssl:
rows.append(_ModelNamedRow(
hostname=template['hostname'],
page=page,
url=self._build_url(template['hostname'], resource, 'https'),
classifiers=template['metadata']['classifiers'],
authors=template['metadata']['authors'],
description=template['metadata']['description'].strip('\n'),
created=utilities.format_datetime(utilities.datetime_utc_to_local(template['created']))
))
gui_utilities.glib_idle_add_once(self.gobjects['treeselection_url_selector'].unselect_all)
gui_utilities.glib_idle_add_store_extend(self._url_model, rows, clear=True)
# make domain list unique in case multiple pages are advertised for the domains
domains = [[domain] for domain in set(domains)]
gui_utilities.glib_idle_add_store_extend(self._hostname_list_store, domains, clear=True)
def _show_dialog_error_tsafe(self, title, message):
gui_utilities.glib_idle_add_once(gui_utilities.show_dialog_error,
title, self.window, message)
def _plugin_install_tsafe(self, catalog_model, repo_model, model_row,
named_row):
self.__installing_plugin = named_row.id
self.logger.debug("installing plugin '{0}'".format(named_row.id))
self._update_status_bar_tsafe("Installing plugin {}...".format(
named_row.title))
_show_dialog_error_tsafe = functools.partial(
gui_utilities.glib_idle_add_once, gui_utilities.show_dialog_error,
'Failed To Install', self.window)
try:
self.catalog_plugins.install_plugin(catalog_model.id,
repo_model.id, named_row.id,
self.plugin_path)
except requests.exceptions.ConnectionError:
self.logger.warning("failed to download plugin {}".format(
named_row.id))
_show_dialog_error_tsafe(
"Failed to download {} plugin, check your internet connection."
.format(named_row.id))
self._update_status_bar_tsafe(
"Installing plugin {} failed.".format(named_row.title))
self.__installing_plugin = None
return
except Exception:
self.logger.warning("failed to install plugin {}".format(
named_row.id),
exc_info=True)
_show_dialog_error_tsafe("Failed to install {} plugin.".format(
named_row.id))
self._update_status_bar_tsafe(
"Installing plugin {} failed.".format(named_row.title))
self.__installing_plugin = None
return
self.config['plugins.installed'][named_row.id] = {
'catalog_id': catalog_model.id,
'repo_id': repo_model.id,
'plugin_id': named_row.id
}
self.logger.info(
"installed plugin '{}' from catalog:{}, repository:{}".format(
named_row.id, catalog_model.id, repo_model.id))
plugin = self._reload_plugin_tsafe(model_row, named_row)
if self.config['plugins.pip.install_dependencies']:
try:
packages = smoke_zephyr.requirements.check_requirements(
tuple(plugin.req_packages.keys()))
except ValueError:
self.logger.warning(
"requirements check failed for plugin '{}', can not automatically install requirements"
.format(named_row.id))
packages = None
if packages:
self.logger.debug(
"installing missing or incompatible packages from PyPi for plugin '{0}'"
.format(named_row.id))
self._update_status_bar_tsafe(
"Installing {:,} dependenc{} for plugin {} from PyPi.".
format(len(packages), 'y' if len(packages) == 1 else 'ies',
named_row.title))
if self.application.plugin_manager.library_path:
pip_results = self.application.plugin_manager.install_packages(
packages)
else:
self.logger.warning(
'no library path to install plugin dependencies')
_show_dialog_error_tsafe(
"Failed to run pip to install package(s) for plugin {}."
.format(named_row.id))
# set pip results to none to safely complete and cleanly release installing lock.
pip_results = None
if pip_results is None:
self.logger.warning('pip install failed')
_show_dialog_error_tsafe(
"Failed to run pip to install package(s) for plugin {}."
.format(named_row.id))
elif pip_results.status:
self.logger.warning('pip install failed, exit status: ' +
str(pip_results.status))
_show_dialog_error_tsafe(
"Failed to install pip package(s) for plugin {}.".
format(named_row.id))
else:
plugin = self._reload_plugin_tsafe(model_row, named_row)
self.__installing_plugin = None
gui_utilities.glib_idle_add_once(self.__plugin_install_post,
catalog_model, repo_model, model_row,
named_row)
def _update_status_bar_tsafe(self, string_to_set): gui_utilities.glib_idle_add_once(self._update_status_bar, string_to_set)
def _show_dialog_error_tsafe(self, title, message): gui_utilities.glib_idle_add_once(gui_utilities.show_dialog_error, title, self.window, message)
def _update_status_bar_tsafe(self, string_to_set):
gui_utilities.glib_idle_add_once(self._update_status_bar,
string_to_set)
def _load_plugins(self):
"""
Load the plugins which are available into the treeview to make them
visible to the user.
"""
self.logger.debug('loading plugins')
self._update_status_bar('Loading plugins...', idle=True)
store = self._model
store.clear()
pm = self.application.plugin_manager
self._module_errors = {}
pm.load_all(on_error=self._on_plugin_load_error)
model = (_LOCAL_REPOSITORY_ID, None, True, _LOCAL_REPOSITORY_TITLE, None, None, False, False, False, _ROW_TYPE_CATALOG)
catalog_row = gui_utilities.glib_idle_add_wait(self._store_append, store, None, model)
models = []
for name, plugin in pm.loaded_plugins.items():
if self.config['plugins.installed'].get(name):
continue
self.config['plugins.installed'][name] = None
models.append(self._named_model(
id=plugin.name,
installed=True,
enabled=plugin.name in pm.enabled_plugins,
title=plugin.title,
compatibility='Yes' if plugin.is_compatible else 'No',
version=plugin.version,
visible_enabled=True,
visible_installed=True,
sensitive_installed=False,
type=_ROW_TYPE_PLUGIN
))
gui_utilities.glib_idle_add_once(self._store_extend, store, catalog_row, models)
del models
for name in self._module_errors.keys():
model = (name, True, False, "{0} (Load Failed)".format(name), 'No', 'Unknown', True, True, False, _ROW_TYPE_PLUGIN)
gui_utilities.glib_idle_add_once(self._store_append, store, catalog_row, model)
self.logger.debug('loading catalog into plugin treeview')
for catalog_id in self.catalog_plugins.catalog_ids():
model = (catalog_id, None, True, catalog_id, None, None, False, False, False, _ROW_TYPE_CATALOG)
catalog_row = gui_utilities.glib_idle_add_wait(self._store_append, store, None, model)
for repo in self.catalog_plugins.get_repositories(catalog_id):
model = (repo.id, None, True, repo.title, None, None, False, False, False, _ROW_TYPE_REPOSITORY)
repo_row = gui_utilities.glib_idle_add_wait(self._store_append, store, catalog_row, model)
plugin_collections = self.catalog_plugins.get_collection(catalog_id, repo.id)
self._add_plugins_to_tree(catalog_id, repo, store, repo_row, plugin_collections)
catalog_cache = self.catalog_plugins.get_cache()
for catalog_id in catalog_cache:
if self.catalog_plugins.catalogs.get(catalog_id, None):
continue
named_catalog = catalog_cache[catalog_id]
model = (catalog_id, None, True, catalog_id, None, None, False, False, False, _ROW_TYPE_CATALOG)
catalog_row = gui_utilities.glib_idle_add_wait(self._store_append, store, None, model)
for repo in named_catalog.repositories:
model = (repo.id, None, True, repo.title, None, None, False, False, False, _ROW_TYPE_REPOSITORY)
repo_row = gui_utilities.glib_idle_add_wait(self._store_append, store, catalog_row, model)
self._add_plugins_offline(catalog_id, repo.id, store, repo_row)
gui_utilities.glib_idle_add_once(self._treeview_unselect)
self._update_status_bar('Loading completed', idle=True)
def _update_status_bar(self, string_to_set, idle=False): if idle: gui_utilities.glib_idle_add_once(self.__update_status_bar, string_to_set) else: self.__update_status_bar(string_to_set)
def _load_url_treeview_tsafe(self, hostname=None, refresh=False):
if refresh or not self._url_information['created']:
self._url_information[
'data'] = self.application.rpc.graphql_find_file(
'get_site_templates.graphql')
self._url_information['created'] = datetime.datetime.utcnow()
url_information = self._url_information['data']
if not url_information:
return
rows = []
domains = []
for edge in url_information['siteTemplates']['edges']:
template = edge['node']
for page in template['metadata']['pages']:
if hostname and template['hostname'] and not template[
'hostname'].startswith(hostname):
continue
page = page.strip('/')
resource = '/' + '/'.join(
(template.get('path', '').strip('/'), page)).lstrip('/')
domains.append(template['hostname'])
rows.append(
_ModelNamedRow(
hostname=template['hostname'],
page=page,
url=self._build_url(template['hostname'], resource,
'http'),
classifiers=template['metadata']['classifiers'],
authors=template['metadata']['authors'],
description=template['metadata']['description'].strip(
'\n'),
created=utilities.format_datetime(
utilities.datetime_utc_to_local(
template['created']))))
if self._server_uses_ssl:
rows.append(
_ModelNamedRow(
hostname=template['hostname'],
page=page,
url=self._build_url(template['hostname'], resource,
'https'),
classifiers=template['metadata']['classifiers'],
authors=template['metadata']['authors'],
description=template['metadata']
['description'].strip('\n'),
created=utilities.format_datetime(
utilities.datetime_utc_to_local(
template['created']))))
gui_utilities.glib_idle_add_once(
self.gobjects['treeselection_url_selector'].unselect_all)
gui_utilities.glib_idle_add_store_extend(self._url_model,
rows,
clear=True)
# make domain list unique in case multiple pages are advertised for the domains
domains = [[domain] for domain in set(domains)]
gui_utilities.glib_idle_add_store_extend(self._hostname_list_store,
domains,
clear=True)