def testEscape(self): self.assertEqual("", model.escape("")) self.assertEqual("hello", model.escape("hello")) self.assertEqual("%20", model.escape(" ")) self.assertEqual("file%3a%2f%2ffoo%7ebar", model.escape("file://foo~bar")) self.assertEqual("file%3a%2f%2ffoo%25bar", model.escape("file://foo%bar")) self.assertEqual("file:##foo%7ebar", model._pretty_escape("file://foo~bar")) self.assertEqual("file:##foo%25bar", model._pretty_escape("file://foo%bar"))
def update_from_cache(interface, iface_cache = None): """Read a cached interface and any native feeds or user overrides. @param interface: the interface object to update @type interface: L{model.Interface} @return: True if cached version and user overrides loaded OK. False if upstream not cached. Local interfaces (starting with /) are always considered to be cached, although they are not actually stored in the cache. Internal: use L{iface_cache.IfaceCache.get_interface} instread. @rtype: bool""" interface.reset() if iface_cache is None: from zeroinstall.injector import policy iface_cache = policy.get_deprecated_singleton_config().iface_cache # Add the distribution package manager's version, if any path = basedir.load_first_data(config_site, 'native_feeds', model._pretty_escape(interface.uri)) if path: # Resolve any symlinks info(_("Adding native packager feed '%s'"), path) interface.extra_feeds.append(Feed(os.path.realpath(path), None, False)) update_user_overrides(interface) main_feed = iface_cache.get_feed(interface.uri, force = True) if main_feed: update_user_feed_overrides(main_feed) return main_feed is not None
def update_from_cache(interface, iface_cache=None): """Read a cached interface and any native feeds or user overrides. @param interface: the interface object to update @type interface: L{model.Interface} @return: True if cached version and user overrides loaded OK. False if upstream not cached. Local interfaces (starting with /) are always considered to be cached, although they are not actually stored in the cache. Internal: use L{iface_cache.IfaceCache.get_interface} instread. @rtype: bool""" interface.reset() if iface_cache is None: from zeroinstall.injector import policy iface_cache = policy.get_deprecated_singleton_config().iface_cache # Add the distribution package manager's version, if any path = basedir.load_first_data(config_site, 'native_feeds', model._pretty_escape(interface.uri)) if path: # Resolve any symlinks info(_("Adding native packager feed '%s'"), path) interface.extra_feeds.append(Feed(os.path.realpath(path), None, False)) update_user_overrides(interface) main_feed = iface_cache.get_feed(interface.uri, force=True) if main_feed: update_user_feed_overrides(main_feed) return main_feed is not None
def get_inputs(): for sel in sels.selections.values(): logger.info("Checking %s", sel.feed) if sel.feed.startswith('distribution:'): # If the package has changed version, we'll detect that below # with get_unavailable_selections. pass elif os.path.isabs(sel.feed): # Local feed yield sel.feed else: # Cached feed cached = basedir.load_first_cache(namespaces.config_site, 'interfaces', model.escape(sel.feed)) if cached: yield cached else: raise IOError("Input %s missing; update" % sel.feed) # Per-feed configuration yield basedir.load_first_config(namespaces.config_site, namespaces.config_prog, 'interfaces', model._pretty_escape(sel.interface)) # Global configuration yield basedir.load_first_config(namespaces.config_site, namespaces.config_prog, 'global')
def get_inputs(): for sel in sels.selections.values(): logger.info("Checking %s", sel.feed) if sel.feed.startswith('distribution:'): # If the package has changed version, we'll detect that below # with get_unavailable_selections. pass elif os.path.isabs(sel.feed): # Local feed yield sel.feed else: # Cached feed cached = basedir.load_first_cache( namespaces.config_site, 'interfaces', model.escape(sel.feed)) if cached: yield cached else: raise IOError("Input %s missing; update" % sel.feed) # Per-feed configuration yield basedir.load_first_config( namespaces.config_site, namespaces.config_prog, 'interfaces', model._pretty_escape(sel.interface)) # Global configuration yield basedir.load_first_config(namespaces.config_site, namespaces.config_prog, 'global')
def update_user_overrides(interface): """Update an interface with user-supplied information. Sets preferred stability and updates extra_feeds. @param interface: the interface object to update @type interface: L{model.Interface} """ user = basedir.load_first_config(config_site, config_prog, 'interfaces', model._pretty_escape(interface.uri)) if user is None: # For files saved by 0launch < 0.49 user = basedir.load_first_config(config_site, config_prog, 'user_overrides', escape(interface.uri)) if not user: return try: root = qdom.parse(file(user)) except Exception as ex: warn(_("Error reading '%(user)s': %(exception)s"), {'user': user, 'exception': ex}) raise stability_policy = root.getAttribute('stability-policy') if stability_policy: interface.set_stability_policy(stability_levels[str(stability_policy)]) for item in root.childNodes: if item.uri != XMLNS_IFACE: continue if item.name == 'feed': feed_src = item.getAttribute('src') if not feed_src: raise InvalidInterface(_('Missing "src" attribute in <feed>')) interface.extra_feeds.append(Feed(feed_src, item.getAttribute('arch'), True, langs = item.getAttribute('langs')))
def get_last_check_attempt(self, url): """Return the time of the most recent update attempt for a feed. @see: L{mark_as_checking} @return: The time, or None if none is recorded @rtype: float | None""" timestamp_path = basedir.load_first_cache(config_site, config_prog, 'last-check-attempt', model._pretty_escape(url)) if timestamp_path: return os.stat(timestamp_path).st_mtime return None
def _atomic_save(doc, parent, uri): import tempfile tmp_fd, tmp_name = tempfile.mkstemp(dir=parent) try: tmp_file = os.fdopen(tmp_fd, 'w') doc.writexml(tmp_file, addindent=" ", newl='\n') tmp_file.close() path = os.path.join(parent, model._pretty_escape(uri)) support.portable_rename(tmp_name, path) except: os.unlink(tmp_name) raise
def mark_as_checking(self, url): """Touch a 'last_check_attempt_timestamp' file for this feed. If url is a local path, nothing happens. This prevents us from repeatedly trying to download a failing feed many times in a short period.""" if os.path.isabs(url): return feeds_dir = basedir.save_cache_path(config_site, config_prog, 'last-check-attempt') timestamp_path = os.path.join(feeds_dir, model._pretty_escape(url)) fd = os.open(timestamp_path, os.O_WRONLY | os.O_CREAT, 0644) os.close(fd) os.utime(timestamp_path, None) # In case file already exists
def get_last_check_attempt(self, url): """Return the time of the most recent update attempt for a feed. @type url: str @return: The time, or None if none is recorded @rtype: float | None @see: L{mark_as_checking}""" timestamp_path = basedir.load_first_cache( config_site, config_prog, "last-check-attempt", model._pretty_escape(url) ) if timestamp_path: return os.stat(timestamp_path).st_mtime return None
def _atomic_save(doc, parent, uri): import tempfile tmp_fd, tmp_name = tempfile.mkstemp(dir = parent) try: tmp_file = os.fdopen(tmp_fd, 'w') doc.writexml(tmp_file, addindent = " ", newl = '\n') tmp_file.close() path = os.path.join(parent, model._pretty_escape(uri)) os.rename(tmp_name, path) except: os.unlink(tmp_name) raise
def mark_as_checking(self, url): """Touch a 'last_check_attempt_timestamp' file for this feed. If url is a local path, nothing happens. This prevents us from repeatedly trying to download a failing feed many times in a short period.""" if url.startswith('/'): return feeds_dir = basedir.save_cache_path(config_site, config_prog, 'last-check-attempt') timestamp_path = os.path.join(feeds_dir, model._pretty_escape(url)) fd = os.open(timestamp_path, os.O_WRONLY | os.O_CREAT, 0644) os.close(fd) os.utime(timestamp_path, None) # In case file already exists
def update_user_overrides(interface, known_site_feeds=frozenset()): """Update an interface with user-supplied information. Sets preferred stability and updates extra_feeds. @param interface: the interface object to update @type interface: L{model.Interface} @param known_site_feeds: feeds to ignore (for backwards compatibility) """ user = basedir.load_first_config(config_site, config_prog, 'interfaces', model._pretty_escape(interface.uri)) if user is None: # For files saved by 0launch < 0.49 user = basedir.load_first_config(config_site, config_prog, 'user_overrides', escape(interface.uri)) if not user: return try: with open(user, 'rb') as stream: root = qdom.parse(stream) except Exception as ex: logger.warn(_("Error reading '%(user)s': %(exception)s"), { 'user': user, 'exception': ex }) raise stability_policy = root.getAttribute('stability-policy') if stability_policy: interface.set_stability_policy(stability_levels[str(stability_policy)]) for item in root.childNodes: if item.uri != XMLNS_IFACE: continue if item.name == 'feed': feed_src = item.getAttribute('src') if not feed_src: raise InvalidInterface(_('Missing "src" attribute in <feed>')) # (note: 0install 1.9..1.12 used a different scheme and the "site-package" attribute; # we deliberately use a different attribute name to avoid confusion) if item.getAttribute('is-site-package'): # Site packages are detected earlier. This test isn't completely reliable, # since older versions will remove the attribute when saving the config # (hence the next test). continue if feed_src in known_site_feeds: continue interface.extra_feeds.append( Feed(feed_src, item.getAttribute('arch'), True, langs=item.getAttribute('langs')))
def delete(self): if not os.path.isabs(self.uri): cached_iface = basedir.load_first_cache(namespaces.config_site, "interfaces", model.escape(self.uri)) if cached_iface: if SAFE_MODE: print("Delete", cached_iface) else: os.unlink(cached_iface) user_overrides = basedir.load_first_config( namespaces.config_site, namespaces.config_prog, "interfaces", model._pretty_escape(self.uri) ) if user_overrides: if SAFE_MODE: print("Delete", user_overrides) else: os.unlink(user_overrides)
def update_user_feed_overrides(feed): """Update a feed with user-supplied information. Sets last_checked and user_stability ratings. @param feed: feed to update @since 0.49 """ user = basedir.load_first_config(config_site, config_prog, 'feeds', model._pretty_escape(feed.url)) if user is None: # For files saved by 0launch < 0.49 user = basedir.load_first_config(config_site, config_prog, 'user_overrides', escape(feed.url)) if not user: return try: with open(user, 'rb') as stream: root = qdom.parse(stream) except Exception as ex: logger.warn(_("Error reading '%(user)s': %(exception)s"), { 'user': user, 'exception': ex }) raise last_checked = root.getAttribute('last-checked') if last_checked: feed.last_checked = int(last_checked) for item in root.childNodes: if item.uri != XMLNS_IFACE: continue if item.name == 'implementation': id = item.getAttribute('id') assert id is not None impl = feed.implementations.get(id, None) if not impl: logger.debug( _("Ignoring user-override for unknown implementation %(id)s in %(interface)s" ), { 'id': id, 'interface': feed }) continue user_stability = item.getAttribute('user-stability') if user_stability: impl.user_stability = stability_levels[str(user_stability)]
def delete(self): if not os.path.isabs(self.uri): cached_iface = basedir.load_first_cache(namespaces.config_site, 'interfaces', model.escape(self.uri)) if cached_iface: if SAFE_MODE: print("Delete", cached_iface) else: os.unlink(cached_iface) user_overrides = basedir.load_first_config(namespaces.config_site, namespaces.config_prog, 'interfaces', model._pretty_escape(self.uri)) if user_overrides: if SAFE_MODE: print("Delete", user_overrides) else: os.unlink(user_overrides)
def get_inputs(): for sel in sels.selections.values(): logger.info("Checking %s", sel.feed) feed = iface_cache.get_feed(sel.feed) if not feed: raise IOError("Input %s missing; update" % sel.feed) else: if feed.local_path: yield feed.local_path else: yield (feed.url, feed.last_modified) # Per-feed configuration yield basedir.load_first_config(namespaces.config_site, namespaces.config_prog, 'interfaces', model._pretty_escape(sel.interface)) # Global configuration yield basedir.load_first_config(namespaces.config_site, namespaces.config_prog, 'global')
def update_user_overrides(interface, known_site_feeds=frozenset()): """Update an interface with user-supplied information. Sets preferred stability and updates extra_feeds. @param interface: the interface object to update @type interface: L{model.Interface} @param known_site_feeds: feeds to ignore (for backwards compatibility) """ user = basedir.load_first_config(config_site, config_prog, "interfaces", model._pretty_escape(interface.uri)) if user is None: # For files saved by 0launch < 0.49 user = basedir.load_first_config(config_site, config_prog, "user_overrides", escape(interface.uri)) if not user: return try: with open(user, "rb") as stream: root = qdom.parse(stream) except Exception as ex: logger.warn(_("Error reading '%(user)s': %(exception)s"), {"user": user, "exception": ex}) raise stability_policy = root.getAttribute("stability-policy") if stability_policy: interface.set_stability_policy(stability_levels[str(stability_policy)]) for item in root.childNodes: if item.uri != XMLNS_IFACE: continue if item.name == "feed": feed_src = item.getAttribute("src") if not feed_src: raise InvalidInterface(_('Missing "src" attribute in <feed>')) # (note: 0install 1.9..1.12 used a different scheme and the "site-package" attribute; # we deliberately use a different attribute name to avoid confusion) if item.getAttribute("is-site-package"): # Site packages are detected earlier. This test isn't completely reliable, # since older versions will remove the attribute when saving the config # (hence the next test). continue if feed_src in known_site_feeds: continue interface.extra_feeds.append( Feed(feed_src, item.getAttribute("arch"), True, langs=item.getAttribute("langs")) )
def update_from_cache(interface, iface_cache=None): """Read a cached interface and any native feeds or user overrides. @param interface: the interface object to update @type interface: L{model.Interface} @return: True if cached version and user overrides loaded OK. False if upstream not cached. Local interfaces (starting with /) are always considered to be cached, although they are not actually stored in the cache. Internal: use L{iface_cache.IfaceCache.get_interface} instread. @rtype: bool""" interface.reset() if iface_cache is None: import warnings warnings.warn("iface_cache should be specified", DeprecationWarning, 2) from zeroinstall.injector import policy iface_cache = policy.get_deprecated_singleton_config().iface_cache # Add the distribution package manager's version, if any path = basedir.load_first_data(config_site, 'native_feeds', model._pretty_escape(interface.uri)) if path: # Resolve any symlinks logger.info(_("Adding native packager feed '%s'"), path) interface.extra_feeds.append(Feed(os.path.realpath(path), None, False)) # Add locally-compiled binaries, if any escaped_uri = model.escape_interface_uri(interface.uri) known_site_feeds = set() for path in basedir.load_data_paths(config_site, 'site-packages', *escaped_uri): try: _add_site_packages(interface, path, known_site_feeds) except Exception as ex: logger.warn("Error loading site packages from {path}: {ex}".format( path=path, ex=ex)) update_user_overrides(interface, known_site_feeds) main_feed = iface_cache.get_feed(interface.uri, force=True) if main_feed: update_user_feed_overrides(main_feed) return main_feed is not None
def update_user_overrides(interface): """Update an interface with user-supplied information. Sets preferred stability and updates extra_feeds. @param interface: the interface object to update @type interface: L{model.Interface} """ user = basedir.load_first_config(config_site, config_prog, 'interfaces', model._pretty_escape(interface.uri)) if user is None: # For files saved by 0launch < 0.49 user = basedir.load_first_config(config_site, config_prog, 'user_overrides', escape(interface.uri)) if not user: return try: root = qdom.parse(file(user)) except Exception, ex: warn(_("Error reading '%(user)s': %(exception)s"), {'user': user, 'exception': ex}) raise
def get_inputs(): for sel in sels.selections.values(): logger.info("Checking %s", sel.feed) feed = iface_cache.get_feed(sel.feed) if not feed: raise IOError("Input %s missing; update" % sel.feed) else: if feed.local_path: yield feed.local_path else: yield (feed.url, feed.last_modified) # Per-feed configuration yield basedir.load_first_config( namespaces.config_site, namespaces.config_prog, 'interfaces', model._pretty_escape(sel.interface)) # Global configuration yield basedir.load_first_config(namespaces.config_site, namespaces.config_prog, 'global')
def update_user_feed_overrides(feed): """Update a feed with user-supplied information. Sets last_checked and user_stability ratings. @param feed: feed to update @since 0.49 """ user = basedir.load_first_config(config_site, config_prog, 'feeds', model._pretty_escape(feed.url)) if user is None: # For files saved by 0launch < 0.49 user = basedir.load_first_config(config_site, config_prog, 'user_overrides', escape(feed.url)) if not user: return try: root = qdom.parse(file(user)) except Exception, ex: warn(_("Error reading '%(user)s': %(exception)s"), {'user': user, 'exception': ex}) raise
def update_user_feed_overrides(feed): """Update a feed with user-supplied information. Sets last_checked and user_stability ratings. @param feed: feed to update @since 0.49 """ user = basedir.load_first_config(config_site, config_prog, "feeds", model._pretty_escape(feed.url)) if user is None: # For files saved by 0launch < 0.49 user = basedir.load_first_config(config_site, config_prog, "user_overrides", escape(feed.url)) if not user: return try: with open(user, "rb") as stream: root = qdom.parse(stream) except Exception as ex: logger.warn(_("Error reading '%(user)s': %(exception)s"), {"user": user, "exception": ex}) raise last_checked = root.getAttribute("last-checked") if last_checked: feed.last_checked = int(last_checked) for item in root.childNodes: if item.uri != XMLNS_IFACE: continue if item.name == "implementation": id = item.getAttribute("id") assert id is not None impl = feed.implementations.get(id, None) if not impl: logger.debug( _("Ignoring user-override for unknown implementation %(id)s in %(interface)s"), {"id": id, "interface": feed}, ) continue user_stability = item.getAttribute("user-stability") if user_stability: impl.user_stability = stability_levels[str(user_stability)]
def update_from_cache(interface, iface_cache=None): """Read a cached interface and any native feeds or user overrides. @param interface: the interface object to update @type interface: L{model.Interface} @return: True if cached version and user overrides loaded OK. False if upstream not cached. Local interfaces (starting with /) are always considered to be cached, although they are not actually stored in the cache. Internal: use L{iface_cache.IfaceCache.get_interface} instread. @rtype: bool""" interface.reset() if iface_cache is None: import warnings warnings.warn("iface_cache should be specified", DeprecationWarning, 2) from zeroinstall.injector import policy iface_cache = policy.get_deprecated_singleton_config().iface_cache # Add the distribution package manager's version, if any path = basedir.load_first_data(config_site, "native_feeds", model._pretty_escape(interface.uri)) if path: # Resolve any symlinks logger.info(_("Adding native packager feed '%s'"), path) interface.extra_feeds.append(Feed(os.path.realpath(path), None, False)) # Add locally-compiled binaries, if any escaped_uri = model.escape_interface_uri(interface.uri) known_site_feeds = set() for path in basedir.load_data_paths(config_site, "site-packages", *escaped_uri): try: _add_site_packages(interface, path, known_site_feeds) except Exception as ex: logger.warn("Error loading site packages from {path}: {ex}".format(path=path, ex=ex)) update_user_overrides(interface, known_site_feeds) main_feed = iface_cache.get_feed(interface.uri, force=True) if main_feed: update_user_feed_overrides(main_feed) return main_feed is not None
def update_from_cache(interface): """Read a cached interface and any native feeds or user overrides. @param interface: the interface object to update @type interface: L{model.Interface} @return: True if cached version and user overrides loaded OK. False if upstream not cached. Local interfaces (starting with /) are always considered to be cached, although they are not actually stored in the cache. @rtype: bool""" interface.reset() main_feed = None if interface.uri.startswith('/'): debug(_("Loading local interface file '%s'"), interface.uri) update(interface, interface.uri, local=True) cached = True else: cached = basedir.load_first_cache(config_site, 'interfaces', escape(interface.uri)) if cached: debug( _("Loading cached information for %(interface)s from %(cached)s" ), { 'interface': interface, 'cached': cached }) main_feed = update(interface, cached) # Add the distribution package manager's version, if any path = basedir.load_first_data(config_site, 'native_feeds', model._pretty_escape(interface.uri)) if path: # Resolve any symlinks info(_("Adding native packager feed '%s'"), path) interface.extra_feeds.append(Feed(os.path.realpath(path), None, False)) update_user_overrides(interface, main_feed) return bool(cached)
def update_user_feed_overrides(feed): """Update a feed with user-supplied information. Sets last_checked and user_stability ratings. @param feed: feed to update @since 0.49 """ user = basedir.load_first_config(config_site, config_prog, 'feeds', model._pretty_escape(feed.url)) if user is None: # For files saved by 0launch < 0.49 user = basedir.load_first_config(config_site, config_prog, 'user_overrides', escape(feed.url)) if not user: return try: root = qdom.parse(file(user)) except Exception as ex: warn(_("Error reading '%(user)s': %(exception)s"), {'user': user, 'exception': ex}) raise last_checked = root.getAttribute('last-checked') if last_checked: feed.last_checked = int(last_checked) for item in root.childNodes: if item.uri != XMLNS_IFACE: continue if item.name == 'implementation': id = item.getAttribute('id') assert id is not None impl = feed.implementations.get(id, None) if not impl: debug(_("Ignoring user-override for unknown implementation %(id)s in %(interface)s"), {'id': id, 'interface': feed}) continue user_stability = item.getAttribute('user-stability') if user_stability: impl.user_stability = stability_levels[str(user_stability)]
def check(str): self.assertEqual(str, model.unescape(model.escape(str))) self.assertEqual(str, model.unescape(model._pretty_escape(str))) self.assertEqual(str, escaping.ununderscore_escape(escaping.underscore_escape(str)))
def _populate_model(self): # Find cached implementations unowned = {} # Impl ID -> Store duplicates = [] # TODO for s in self.iface_cache.stores.stores: if os.path.isdir(s.dir): for id in os.listdir(s.dir): if id in unowned: duplicates.append(id) unowned[id] = s ok_feeds = [] error_feeds = [] # Look through cached feeds for implementation owners all_interfaces = self.iface_cache.list_all_interfaces() all_feeds = {} for uri in all_interfaces: try: iface = self.iface_cache.get_interface(uri) except Exception as ex: error_feeds.append((uri, str(ex), 0)) else: all_feeds.update(self.iface_cache.get_feeds(iface)) for url, feed in all_feeds.items(): if not feed: continue yield feed_size = 0 try: if url != feed.url: # (e.g. for .new feeds) raise Exception('Incorrect URL for feed (%s vs %s)' % (url, feed.url)) if os.path.isabs(url): cached_feed = url feed_type = LocalFeed else: feed_type = RemoteFeed cached_feed = basedir.load_first_cache(namespaces.config_site, 'interfaces', model.escape(url)) user_overrides = basedir.load_first_config(namespaces.config_site, namespaces.config_prog, 'interfaces', model._pretty_escape(url)) feed_size = size_if_exists(cached_feed) + size_if_exists(user_overrides) except Exception as ex: error_feeds.append((url, str(ex), feed_size)) else: cached_feed = feed_type(feed, feed_size) for impl in feed.implementations.values(): if impl.local_path: cached_feed.in_cache.append(LocalImplementation(impl)) for digest in impl.digests: if digest in unowned: cached_dir = unowned[digest].dir impl_path = os.path.join(cached_dir, digest) impl_size = get_size(impl_path) cached_feed.in_cache.append(KnownImplementation(cached_feed, cached_dir, impl, impl_size, digest)) del unowned[digest] cached_feed.in_cache.sort() ok_feeds.append(cached_feed) if error_feeds: iter = SECTION_INVALID_INTERFACES.append_to(self.raw_model) for uri, ex, size in error_feeds: item = InvalidFeed(uri, ex, size) item.append_to(self.raw_model, iter) unowned_sizes = [] local_dir = os.path.join(basedir.xdg_cache_home, '0install.net', 'implementations') for id in unowned: if unowned[id].dir == local_dir: impl = UnusedImplementation(local_dir, id) unowned_sizes.append((impl.size, impl)) if unowned_sizes: iter = SECTION_UNOWNED_IMPLEMENTATIONS.append_to(self.raw_model) for size, item in unowned_sizes: item.append_to(self.raw_model, iter) if ok_feeds: iter = SECTION_INTERFACES.append_to(self.raw_model) for item in ok_feeds: yield item.append_to(self.raw_model, iter) self._update_sizes()
def check(str): self.assertEqual(str, model.unescape(model.escape(str))) self.assertEqual(str, model.unescape(model._pretty_escape(str)))