def get_inputs(): for sel in sels.selections.values(): logger.info("Checking %s", sel.feed) if sel.feed.startswith('distribution:'): # If the package has changed version, we'll detect that below # with get_unavailable_selections. pass elif os.path.isabs(sel.feed): # Local feed yield sel.feed else: # Cached feed cached = basedir.load_first_cache(namespaces.config_site, 'interfaces', model.escape(sel.feed)) if cached: yield cached else: raise IOError("Input %s missing; update" % sel.feed) # Per-feed configuration yield basedir.load_first_config(namespaces.config_site, namespaces.config_prog, 'interfaces', model._pretty_escape(sel.interface)) # Global configuration yield basedir.load_first_config(namespaces.config_site, namespaces.config_prog, 'global')
def get_inputs(): for sel in sels.selections.values(): logger.info("Checking %s", sel.feed) if sel.feed.startswith('distribution:'): # If the package has changed version, we'll detect that below # with get_unavailable_selections. pass elif os.path.isabs(sel.feed): # Local feed yield sel.feed else: # Cached feed cached = basedir.load_first_cache( namespaces.config_site, 'interfaces', model.escape(sel.feed)) if cached: yield cached else: raise IOError("Input %s missing; update" % sel.feed) # Per-feed configuration yield basedir.load_first_config( namespaces.config_site, namespaces.config_prog, 'interfaces', model._pretty_escape(sel.interface)) # Global configuration yield basedir.load_first_config(namespaces.config_site, namespaces.config_prog, 'global')
def get_icon_path(self, iface): """Get the path of a cached icon for an interface. @param iface: interface whose icon we want @return: the path of the cached icon, or None if not cached. @rtype: str""" return basedir.load_first_cache(config_site, 'interface_icons', escape(iface.uri))
def get_last_check_attempt(self, url): """Return the time of the most recent update attempt for a feed. @see: L{mark_as_checking} @return: The time, or None if none is recorded @rtype: float | None""" timestamp_path = basedir.load_first_cache(config_site, config_prog, 'last-check-attempt', model._pretty_escape(url)) if timestamp_path: return os.stat(timestamp_path).st_mtime return None
def get_local_feed_file(url): if os.path.isabs(url): logging.debug("local feed: %s" % (url,)) return url else: cached = basedir.load_first_cache(namespaces.config_site, 'interfaces', reader.escape(url)) if not cached: raise RuntimeError("not cached") logging.debug("cached feed: %s" % (cached,)) return cached
def delete(self): if not os.path.isabs(self.uri): cached_iface = basedir.load_first_cache(namespaces.config_site, "interfaces", model.escape(self.uri)) if cached_iface: # print "Delete", cached_iface os.unlink(cached_iface) user_overrides = basedir.load_first_config( namespaces.config_site, namespaces.config_prog, "user_overrides", model.escape(self.uri) ) if user_overrides: # print "Delete", user_overrides os.unlink(user_overrides)
def _populate_model(self): # Find cached implementations unowned = {} # Impl ID -> Store duplicates = [] # TODO for s in self.iface_cache.stores.stores: if os.path.isdir(s.dir): for id in os.listdir(s.dir): if id in unowned: duplicates.append(id) unowned[id] = s ok_interfaces = [] error_interfaces = [] # Look through cached interfaces for implementation owners all = self.iface_cache.list_all_interfaces() all.sort() for uri in all: iface_size = 0 try: if uri.startswith('/'): cached_iface = uri else: cached_iface = basedir.load_first_cache( namespaces.config_site, 'interfaces', model.escape(uri)) user_overrides = basedir.load_first_config( namespaces.config_site, namespaces.config_prog, 'user_overrides', model.escape(uri)) iface_size = size_if_exists(cached_iface) + size_if_exists( user_overrides) iface = self.iface_cache.get_interface(uri) except Exception, ex: error_interfaces.append((uri, str(ex), iface_size)) else: cached_iface = ValidInterface(iface, iface_size) for impl in iface.implementations.values(): if impl.id.startswith('/') or impl.id.startswith('.'): cached_iface.in_cache.append(LocalImplementation(impl)) if impl.id in unowned: cached_dir = unowned[impl.id].dir impl_path = os.path.join(cached_dir, impl.id) impl_size = get_size(impl_path) cached_iface.in_cache.append( KnownImplementation(cached_iface, cached_dir, impl, impl_size)) del unowned[impl.id] cached_iface.in_cache.sort() ok_interfaces.append(cached_iface)
def testSymlink(self): old_out = sys.stdout try: sys.stdout = StringIO() self.child = server.handle_requests(("HelloWorld.tar.bz2", "HelloSym.tgz")) policy = Policy(os.path.abspath("RecipeSymlink.xml"), config=self.config) try: download_and_execute(policy, []) assert False except model.SafeException, ex: if 'Attempt to unpack dir over symlink "HelloWorld"' not in str(ex): raise self.assertEquals(None, basedir.load_first_cache("0install.net", "implementations", "main"))
def delete(self): if not self.uri.startswith('/'): cached_iface = basedir.load_first_cache(namespaces.config_site, 'interfaces', model.escape(self.uri)) if cached_iface: #print "Delete", cached_iface os.unlink(cached_iface) user_overrides = basedir.load_first_config(namespaces.config_site, namespaces.config_prog, 'user_overrides', model.escape(self.uri)) if user_overrides: #print "Delete", user_overrides os.unlink(user_overrides)
def testSymlink(self): old_out = sys.stdout try: sys.stdout = StringIO() run_server(('HelloWorld.tar.bz2', 'HelloSym.tgz')) driver = Driver(requirements = Requirements(os.path.abspath('RecipeSymlink.xml')), config = self.config) try: download_and_execute(driver, []) assert False except model.SafeException as ex: if 'Attempt to unpack dir over symlink "HelloWorld"' not in str(ex): raise self.assertEqual(None, basedir.load_first_cache('0install.net', 'implementations', 'main')) finally: sys.stdout = old_out
def delete(self): if not os.path.isabs(self.uri): cached_iface = basedir.load_first_cache(namespaces.config_site, "interfaces", model.escape(self.uri)) if cached_iface: if SAFE_MODE: print("Delete", cached_iface) else: os.unlink(cached_iface) user_overrides = basedir.load_first_config( namespaces.config_site, namespaces.config_prog, "interfaces", model._pretty_escape(self.uri) ) if user_overrides: if SAFE_MODE: print("Delete", user_overrides) else: os.unlink(user_overrides)
def _populate_model(self): # Find cached implementations unowned = {} # Impl ID -> Store duplicates = [] # TODO for s in self.iface_cache.stores.stores: if os.path.isdir(s.dir): for id in os.listdir(s.dir): if id in unowned: duplicates.append(id) unowned[id] = s ok_interfaces = [] error_interfaces = [] # Look through cached interfaces for implementation owners all = self.iface_cache.list_all_interfaces() all.sort() for uri in all: iface_size = 0 try: if os.path.isabs(uri): cached_iface = uri else: cached_iface = basedir.load_first_cache(namespaces.config_site, "interfaces", model.escape(uri)) user_overrides = basedir.load_first_config( namespaces.config_site, namespaces.config_prog, "user_overrides", model.escape(uri) ) iface_size = size_if_exists(cached_iface) + size_if_exists(user_overrides) iface = self.iface_cache.get_interface(uri) except Exception, ex: error_interfaces.append((uri, str(ex), iface_size)) else: cached_iface = ValidInterface(iface, iface_size) for impl in iface.implementations.values(): if impl.local_path: cached_iface.in_cache.append(LocalImplementation(impl)) if impl.id in unowned: cached_dir = unowned[impl.id].dir impl_path = os.path.join(cached_dir, impl.id) impl_size = get_size(impl_path) cached_iface.in_cache.append(KnownImplementation(cached_iface, cached_dir, impl, impl_size)) del unowned[impl.id] cached_iface.in_cache.sort() ok_interfaces.append(cached_iface)
def testExecutable(self): child = subprocess.Popen([local_0launch, '--', runexec, 'user-arg-run'], stdout = subprocess.PIPE, universal_newlines = True) stdout, _ = child.communicate() assert 'Runner: script=A test script: args=foo-arg -- var user-arg-run' in stdout, stdout assert 'Runner: script=A test script: args=command-arg -- path user-arg-run' in stdout, stdout # Check runenv.py is updated correctly from zeroinstall.support import basedir runenv = basedir.load_first_cache(namespaces.config_site, namespaces.config_prog, 'runenv.py') os.chmod(runenv, 0o700) with open(runenv, 'wb') as s: s.write(b'#!/\n') child = subprocess.Popen([local_0launch, '--', runexec, 'user-arg-run'], stdout = subprocess.PIPE, universal_newlines = True) stdout, _ = child.communicate() assert 'Runner: script=A test script: args=foo-arg -- var user-arg-run' in stdout, stdout assert 'Runner: script=A test script: args=command-arg -- path user-arg-run' in stdout, stdout
def load_feed_from_cache(url, selections_ok = False): """Load a feed. If the feed is remote, load from the cache. If local, load it directly. @return: the feed, or None if it's remote and not cached.""" try: if os.path.isabs(url): debug(_("Loading local feed file '%s'"), url) return load_feed(url, local = True, selections_ok = selections_ok) else: cached = basedir.load_first_cache(config_site, 'interfaces', escape(url)) if cached: debug(_("Loading cached information for %(interface)s from %(cached)s"), {'interface': url, 'cached': cached}) return load_feed(cached, local = False) else: return None except InvalidInterface, ex: ex.feed_url = url raise
def delete(self): if not os.path.isabs(self.uri): cached_iface = basedir.load_first_cache(namespaces.config_site, 'interfaces', model.escape(self.uri)) if cached_iface: if SAFE_MODE: print("Delete", cached_iface) else: os.unlink(cached_iface) user_overrides = basedir.load_first_config(namespaces.config_site, namespaces.config_prog, 'interfaces', model._pretty_escape(self.uri)) if user_overrides: if SAFE_MODE: print("Delete", user_overrides) else: os.unlink(user_overrides)
def load_feed_from_cache(url, selections_ok = False): """Load a feed. If the feed is remote, load from the cache. If local, load it directly. @return: the feed, or None if it's remote and not cached.""" try: if os.path.isabs(url): debug(_("Loading local feed file '%s'"), url) return load_feed(url, local = True, selections_ok = selections_ok) else: cached = basedir.load_first_cache(config_site, 'interfaces', escape(url)) if cached: debug(_("Loading cached information for %(interface)s from %(cached)s"), {'interface': url, 'cached': cached}) return load_feed(cached, local = False) else: return None except InvalidInterface as ex: ex.feed_url = url raise
def count_impls(url): if url not in cached_counts: cached = basedir.load_first_cache(namespaces.config_site, 'interfaces', model.escape(url)) if cached: with open(cached) as stream: cached_doc = qdom.parse(stream) def count(elem): c = 0 if elem.uri != namespaces.XMLNS_IFACE: return 0 if elem.name == 'implementation' or elem.name == 'package-implementation': c += 1 else: for child in elem.childNodes: c += count(child) return c cached_counts[url] = count(cached_doc) else: cached_counts[url] = 0 return cached_counts[url]
def get_cached_signatures(self, uri): """Verify the cached interface using GPG. Only new-style XML-signed interfaces retain their signatures in the cache. @param uri: the feed to check @type uri: str @return: a list of signatures, or None @rtype: [L{gpg.Signature}] or None @since: 0.25""" import gpg if os.path.isabs(uri): old_iface = uri else: old_iface = basedir.load_first_cache(config_site, 'interfaces', escape(uri)) if old_iface is None: return None try: return gpg.check_stream(file(old_iface))[1] except SafeException, ex: debug(_("No signatures (old-style interface): %s") % ex) return None
def get_cached_signatures(self, uri): """Verify the cached interface using GPG. Only new-style XML-signed interfaces retain their signatures in the cache. @param uri: the feed to check @type uri: str @return: a list of signatures, or None @rtype: [L{gpg.Signature}] or None @since: 0.25""" import gpg if uri.startswith('/'): old_iface = uri else: old_iface = basedir.load_first_cache(config_site, 'interfaces', escape(uri)) if old_iface is None: return None try: return gpg.check_stream(file(old_iface))[1] except SafeException, ex: debug(_("No signatures (old-style interface): %s") % ex) return None
def get_cached_signatures(self, uri): """Verify the cached interface using GPG. Only new-style XML-signed interfaces retain their signatures in the cache. @param uri: the feed to check @type uri: str @return: a list of signatures, or None @rtype: [L{gpg.Signature}] or None @since: 0.25""" from . import gpg if os.path.isabs(uri): old_iface = uri else: old_iface = basedir.load_first_cache(config_site, 'interfaces', escape(uri)) if old_iface is None: return None try: with open(old_iface, 'rb') as stream: return gpg.check_stream(stream)[1] except SafeException as ex: logger.info(_("No signatures (old-style interface): %s") % ex) return None
def load_feed_from_cache(url): """Load a feed. If the feed is remote, load from the cache. If local, load it directly. @type url: str @return: the feed, or None if it's remote and not cached. @rtype: L{ZeroInstallFeed} | None""" try: if os.path.isabs(url): logger.debug(_("Loading local feed file '%s'"), url) return load_feed(url, local=True) else: cached = basedir.load_first_cache(config_site, "interfaces", escape(url)) if cached: logger.debug( _("Loading cached information for %(interface)s from %(cached)s"), {"interface": url, "cached": cached}, ) return load_feed(cached, local=False) else: return None except InvalidInterface as ex: ex.feed_url = url raise
def export_feeds(export_dir, feeds, keys_used): """Copy each feed (and icon) in feeds from the cache to export_dir. Add all signing key fingerprints to keys_used.""" for feed in feeds: if feed.startswith('/'): info("Skipping local feed %s", feed) continue if feed.startswith('distribution:'): info("Skipping distribution feed %s", feed) continue print "Exporting feed", feed # Store feed cached = basedir.load_first_cache(namespaces.config_site, 'interfaces', model.escape(feed)) if cached: feed_dir = os.path.join(export_dir, get_feed_path(feed)) feed_dst = os.path.join(feed_dir, 'latest.xml') if not os.path.isdir(feed_dir): os.makedirs(feed_dir) shutil.copyfile(cached, feed_dst) info("Exported feed %s", feed) icon_path = iface_cache.iface_cache.get_icon_path(iface_cache.iface_cache.get_interface(feed)) if icon_path: icon_dst = os.path.join(feed_dir, 'icon.png') shutil.copyfile(icon_path, icon_dst) # Get the keys stream = file(cached) unused, sigs = gpg.check_stream(stream) stream.close() for x in sigs: if isinstance(x, gpg.ValidSig): keys_used.add(x.fingerprint) else: warn("Signature problem: %s" % x) else: warn("Feed not cached: %s", feed)
def update_from_cache(interface): """Read a cached interface and any native feeds or user overrides. @param interface: the interface object to update @type interface: L{model.Interface} @return: True if cached version and user overrides loaded OK. False if upstream not cached. Local interfaces (starting with /) are always considered to be cached, although they are not actually stored in the cache. @rtype: bool""" interface.reset() main_feed = None if interface.uri.startswith('/'): debug(_("Loading local interface file '%s'"), interface.uri) update(interface, interface.uri, local=True) cached = True else: cached = basedir.load_first_cache(config_site, 'interfaces', escape(interface.uri)) if cached: debug( _("Loading cached information for %(interface)s from %(cached)s" ), { 'interface': interface, 'cached': cached }) main_feed = update(interface, cached) # Add the distribution package manager's version, if any path = basedir.load_first_data(config_site, 'native_feeds', model._pretty_escape(interface.uri)) if path: # Resolve any symlinks info(_("Adding native packager feed '%s'"), path) interface.extra_feeds.append(Feed(os.path.realpath(path), None, False)) update_user_overrides(interface, main_feed) return bool(cached)
def testExecutable(self): child = subprocess.Popen( [local_0launch, '--', runexec, 'user-arg-run'], stdout=subprocess.PIPE, universal_newlines=True) stdout, _ = child.communicate() assert 'Runner: script=A test script: args=foo-arg -- var user-arg-run' in stdout, stdout assert 'Runner: script=A test script: args=command-arg -- path user-arg-run' in stdout, stdout # Check runenv.py is updated correctly from zeroinstall.support import basedir runenv = basedir.load_first_cache(namespaces.config_site, namespaces.config_prog, 'runenv.py') os.chmod(runenv, 0o700) with open(runenv, 'wb') as s: s.write(b'#!/\n') child = subprocess.Popen( [local_0launch, '--', runexec, 'user-arg-run'], stdout=subprocess.PIPE, universal_newlines=True) stdout, _ = child.communicate() assert 'Runner: script=A test script: args=foo-arg -- var user-arg-run' in stdout, stdout assert 'Runner: script=A test script: args=command-arg -- path user-arg-run' in stdout, stdout
def _populate_model(self): # Find cached implementations unowned = {} # Impl ID -> Store duplicates = [] # TODO for s in self.iface_cache.stores.stores: if os.path.isdir(s.dir): for id in os.listdir(s.dir): if id in unowned: duplicates.append(id) unowned[id] = s ok_feeds = [] error_feeds = [] # Look through cached feeds for implementation owners all_interfaces = self.iface_cache.list_all_interfaces() all_feeds = {} for uri in all_interfaces: try: iface = self.iface_cache.get_interface(uri) except Exception as ex: error_feeds.append((uri, str(ex), 0)) else: all_feeds.update(self.iface_cache.get_feeds(iface)) for url, feed in all_feeds.items(): if not feed: continue yield feed_size = 0 try: if url != feed.url: # (e.g. for .new feeds) raise Exception('Incorrect URL for feed (%s vs %s)' % (url, feed.url)) if os.path.isabs(url): cached_feed = url feed_type = LocalFeed else: feed_type = RemoteFeed cached_feed = basedir.load_first_cache(namespaces.config_site, 'interfaces', model.escape(url)) user_overrides = basedir.load_first_config(namespaces.config_site, namespaces.config_prog, 'interfaces', model._pretty_escape(url)) feed_size = size_if_exists(cached_feed) + size_if_exists(user_overrides) except Exception as ex: error_feeds.append((url, str(ex), feed_size)) else: cached_feed = feed_type(feed, feed_size) for impl in feed.implementations.values(): if impl.local_path: cached_feed.in_cache.append(LocalImplementation(impl)) for digest in impl.digests: if digest in unowned: cached_dir = unowned[digest].dir impl_path = os.path.join(cached_dir, digest) impl_size = get_size(impl_path) cached_feed.in_cache.append(KnownImplementation(cached_feed, cached_dir, impl, impl_size, digest)) del unowned[digest] cached_feed.in_cache.sort() ok_feeds.append(cached_feed) if error_feeds: iter = SECTION_INVALID_INTERFACES.append_to(self.raw_model) for uri, ex, size in error_feeds: item = InvalidFeed(uri, ex, size) item.append_to(self.raw_model, iter) unowned_sizes = [] local_dir = os.path.join(basedir.xdg_cache_home, '0install.net', 'implementations') for id in unowned: if unowned[id].dir == local_dir: impl = UnusedImplementation(local_dir, id) unowned_sizes.append((impl.size, impl)) if unowned_sizes: iter = SECTION_UNOWNED_IMPLEMENTATIONS.append_to(self.raw_model) for size, item in unowned_sizes: item.append_to(self.raw_model, iter) if ok_feeds: iter = SECTION_INTERFACES.append_to(self.raw_model) for item in ok_feeds: yield item.append_to(self.raw_model, iter) self._update_sizes()
def testSymlink(self): run_server(('HelloWorld.tar.bz2', 'HelloSym.tgz')) out, err = self.run_ocaml(['download', os.path.abspath('RecipeSymlink.xml')]) assert "Exit status: 1" in err, err assert 'Attempt to unpack dir over symlink "HelloWorld"' in err, err self.assertEqual(None, basedir.load_first_cache('0install.net', 'implementations', 'main'))