def get_inputs():
for sel in sels.selections.values():
logger.info("Checking %s", sel.feed)
if sel.feed.startswith('distribution:'):
# If the package has changed version, we'll detect that below
# with get_unavailable_selections.
pass
elif os.path.isabs(sel.feed):
# Local feed
yield sel.feed
else:
# Cached feed
cached = basedir.load_first_cache(namespaces.config_site, 'interfaces', model.escape(sel.feed))
if cached:
yield cached
else:
raise IOError("Input %s missing; update" % sel.feed)
# Per-feed configuration
yield basedir.load_first_config(namespaces.config_site, namespaces.config_prog,
'interfaces', model._pretty_escape(sel.interface))
# Global configuration
yield basedir.load_first_config(namespaces.config_site, namespaces.config_prog, 'global')
def get_inputs():
for sel in sels.selections.values():
logger.info("Checking %s", sel.feed)
if sel.feed.startswith('distribution:'):
# If the package has changed version, we'll detect that below
# with get_unavailable_selections.
pass
elif os.path.isabs(sel.feed):
# Local feed
yield sel.feed
else:
# Cached feed
cached = basedir.load_first_cache(
namespaces.config_site, 'interfaces',
model.escape(sel.feed))
if cached:
yield cached
else:
raise IOError("Input %s missing; update" %
sel.feed)
# Per-feed configuration
yield basedir.load_first_config(
namespaces.config_site, namespaces.config_prog,
'interfaces', model._pretty_escape(sel.interface))
# Global configuration
yield basedir.load_first_config(namespaces.config_site,
namespaces.config_prog,
'global')
def get_icon_path(self, iface):
"""Get the path of a cached icon for an interface.
@param iface: interface whose icon we want
@return: the path of the cached icon, or None if not cached.
@rtype: str"""
return basedir.load_first_cache(config_site, 'interface_icons',
escape(iface.uri))
def get_icon_path(self, iface): """Get the path of a cached icon for an interface. @param iface: interface whose icon we want @return: the path of the cached icon, or None if not cached. @rtype: str""" return basedir.load_first_cache(config_site, 'interface_icons', escape(iface.uri))
def get_last_check_attempt(self, url):
"""Return the time of the most recent update attempt for a feed.
@see: L{mark_as_checking}
@return: The time, or None if none is recorded
@rtype: float | None"""
timestamp_path = basedir.load_first_cache(config_site, config_prog, 'last-check-attempt', model._pretty_escape(url))
if timestamp_path:
return os.stat(timestamp_path).st_mtime
return None
def get_local_feed_file(url):
if os.path.isabs(url):
logging.debug("local feed: %s" % (url,))
return url
else:
cached = basedir.load_first_cache(namespaces.config_site, 'interfaces', reader.escape(url))
if not cached:
raise RuntimeError("not cached")
logging.debug("cached feed: %s" % (cached,))
return cached
def get_last_check_attempt(self, url):
"""Return the time of the most recent update attempt for a feed.
@see: L{mark_as_checking}
@return: The time, or None if none is recorded
@rtype: float | None"""
timestamp_path = basedir.load_first_cache(config_site, config_prog,
'last-check-attempt',
model._pretty_escape(url))
if timestamp_path:
return os.stat(timestamp_path).st_mtime
return None
def delete(self):
if not os.path.isabs(self.uri):
cached_iface = basedir.load_first_cache(namespaces.config_site, "interfaces", model.escape(self.uri))
if cached_iface:
# print "Delete", cached_iface
os.unlink(cached_iface)
user_overrides = basedir.load_first_config(
namespaces.config_site, namespaces.config_prog, "user_overrides", model.escape(self.uri)
)
if user_overrides:
# print "Delete", user_overrides
os.unlink(user_overrides)
def _populate_model(self):
# Find cached implementations
unowned = {} # Impl ID -> Store
duplicates = [] # TODO
for s in self.iface_cache.stores.stores:
if os.path.isdir(s.dir):
for id in os.listdir(s.dir):
if id in unowned:
duplicates.append(id)
unowned[id] = s
ok_interfaces = []
error_interfaces = []
# Look through cached interfaces for implementation owners
all = self.iface_cache.list_all_interfaces()
all.sort()
for uri in all:
iface_size = 0
try:
if uri.startswith('/'):
cached_iface = uri
else:
cached_iface = basedir.load_first_cache(
namespaces.config_site, 'interfaces',
model.escape(uri))
user_overrides = basedir.load_first_config(
namespaces.config_site, namespaces.config_prog,
'user_overrides', model.escape(uri))
iface_size = size_if_exists(cached_iface) + size_if_exists(
user_overrides)
iface = self.iface_cache.get_interface(uri)
except Exception, ex:
error_interfaces.append((uri, str(ex), iface_size))
else:
cached_iface = ValidInterface(iface, iface_size)
for impl in iface.implementations.values():
if impl.id.startswith('/') or impl.id.startswith('.'):
cached_iface.in_cache.append(LocalImplementation(impl))
if impl.id in unowned:
cached_dir = unowned[impl.id].dir
impl_path = os.path.join(cached_dir, impl.id)
impl_size = get_size(impl_path)
cached_iface.in_cache.append(
KnownImplementation(cached_iface, cached_dir, impl,
impl_size))
del unowned[impl.id]
cached_iface.in_cache.sort()
ok_interfaces.append(cached_iface)
def testSymlink(self):
old_out = sys.stdout
try:
sys.stdout = StringIO()
self.child = server.handle_requests(("HelloWorld.tar.bz2", "HelloSym.tgz"))
policy = Policy(os.path.abspath("RecipeSymlink.xml"), config=self.config)
try:
download_and_execute(policy, [])
assert False
except model.SafeException, ex:
if 'Attempt to unpack dir over symlink "HelloWorld"' not in str(ex):
raise
self.assertEquals(None, basedir.load_first_cache("0install.net", "implementations", "main"))
def delete(self):
if not self.uri.startswith('/'):
cached_iface = basedir.load_first_cache(namespaces.config_site,
'interfaces',
model.escape(self.uri))
if cached_iface:
#print "Delete", cached_iface
os.unlink(cached_iface)
user_overrides = basedir.load_first_config(namespaces.config_site,
namespaces.config_prog,
'user_overrides',
model.escape(self.uri))
if user_overrides:
#print "Delete", user_overrides
os.unlink(user_overrides)
def testSymlink(self):
old_out = sys.stdout
try:
sys.stdout = StringIO()
run_server(('HelloWorld.tar.bz2', 'HelloSym.tgz'))
driver = Driver(requirements = Requirements(os.path.abspath('RecipeSymlink.xml')), config = self.config)
try:
download_and_execute(driver, [])
assert False
except model.SafeException as ex:
if 'Attempt to unpack dir over symlink "HelloWorld"' not in str(ex):
raise
self.assertEqual(None, basedir.load_first_cache('0install.net', 'implementations', 'main'))
finally:
sys.stdout = old_out
def testSymlink(self):
old_out = sys.stdout
try:
sys.stdout = StringIO()
run_server(('HelloWorld.tar.bz2', 'HelloSym.tgz'))
driver = Driver(requirements = Requirements(os.path.abspath('RecipeSymlink.xml')), config = self.config)
try:
download_and_execute(driver, [])
assert False
except model.SafeException as ex:
if 'Attempt to unpack dir over symlink "HelloWorld"' not in str(ex):
raise
self.assertEqual(None, basedir.load_first_cache('0install.net', 'implementations', 'main'))
finally:
sys.stdout = old_out
def delete(self):
if not os.path.isabs(self.uri):
cached_iface = basedir.load_first_cache(namespaces.config_site, "interfaces", model.escape(self.uri))
if cached_iface:
if SAFE_MODE:
print("Delete", cached_iface)
else:
os.unlink(cached_iface)
user_overrides = basedir.load_first_config(
namespaces.config_site, namespaces.config_prog, "interfaces", model._pretty_escape(self.uri)
)
if user_overrides:
if SAFE_MODE:
print("Delete", user_overrides)
else:
os.unlink(user_overrides)
def _populate_model(self):
# Find cached implementations
unowned = {} # Impl ID -> Store
duplicates = [] # TODO
for s in self.iface_cache.stores.stores:
if os.path.isdir(s.dir):
for id in os.listdir(s.dir):
if id in unowned:
duplicates.append(id)
unowned[id] = s
ok_interfaces = []
error_interfaces = []
# Look through cached interfaces for implementation owners
all = self.iface_cache.list_all_interfaces()
all.sort()
for uri in all:
iface_size = 0
try:
if os.path.isabs(uri):
cached_iface = uri
else:
cached_iface = basedir.load_first_cache(namespaces.config_site, "interfaces", model.escape(uri))
user_overrides = basedir.load_first_config(
namespaces.config_site, namespaces.config_prog, "user_overrides", model.escape(uri)
)
iface_size = size_if_exists(cached_iface) + size_if_exists(user_overrides)
iface = self.iface_cache.get_interface(uri)
except Exception, ex:
error_interfaces.append((uri, str(ex), iface_size))
else:
cached_iface = ValidInterface(iface, iface_size)
for impl in iface.implementations.values():
if impl.local_path:
cached_iface.in_cache.append(LocalImplementation(impl))
if impl.id in unowned:
cached_dir = unowned[impl.id].dir
impl_path = os.path.join(cached_dir, impl.id)
impl_size = get_size(impl_path)
cached_iface.in_cache.append(KnownImplementation(cached_iface, cached_dir, impl, impl_size))
del unowned[impl.id]
cached_iface.in_cache.sort()
ok_interfaces.append(cached_iface)
def testExecutable(self): child = subprocess.Popen([local_0launch, '--', runexec, 'user-arg-run'], stdout = subprocess.PIPE, universal_newlines = True) stdout, _ = child.communicate() assert 'Runner: script=A test script: args=foo-arg -- var user-arg-run' in stdout, stdout assert 'Runner: script=A test script: args=command-arg -- path user-arg-run' in stdout, stdout # Check runenv.py is updated correctly from zeroinstall.support import basedir runenv = basedir.load_first_cache(namespaces.config_site, namespaces.config_prog, 'runenv.py') os.chmod(runenv, 0o700) with open(runenv, 'wb') as s: s.write(b'#!/\n') child = subprocess.Popen([local_0launch, '--', runexec, 'user-arg-run'], stdout = subprocess.PIPE, universal_newlines = True) stdout, _ = child.communicate() assert 'Runner: script=A test script: args=foo-arg -- var user-arg-run' in stdout, stdout assert 'Runner: script=A test script: args=command-arg -- path user-arg-run' in stdout, stdout
def load_feed_from_cache(url, selections_ok = False):
"""Load a feed. If the feed is remote, load from the cache. If local, load it directly.
@return: the feed, or None if it's remote and not cached."""
try:
if os.path.isabs(url):
debug(_("Loading local feed file '%s'"), url)
return load_feed(url, local = True, selections_ok = selections_ok)
else:
cached = basedir.load_first_cache(config_site, 'interfaces', escape(url))
if cached:
debug(_("Loading cached information for %(interface)s from %(cached)s"), {'interface': url, 'cached': cached})
return load_feed(cached, local = False)
else:
return None
except InvalidInterface, ex:
ex.feed_url = url
raise
def delete(self):
if not os.path.isabs(self.uri):
cached_iface = basedir.load_first_cache(namespaces.config_site,
'interfaces', model.escape(self.uri))
if cached_iface:
if SAFE_MODE:
print("Delete", cached_iface)
else:
os.unlink(cached_iface)
user_overrides = basedir.load_first_config(namespaces.config_site,
namespaces.config_prog,
'interfaces', model._pretty_escape(self.uri))
if user_overrides:
if SAFE_MODE:
print("Delete", user_overrides)
else:
os.unlink(user_overrides)
def load_feed_from_cache(url, selections_ok = False):
"""Load a feed. If the feed is remote, load from the cache. If local, load it directly.
@return: the feed, or None if it's remote and not cached."""
try:
if os.path.isabs(url):
debug(_("Loading local feed file '%s'"), url)
return load_feed(url, local = True, selections_ok = selections_ok)
else:
cached = basedir.load_first_cache(config_site, 'interfaces', escape(url))
if cached:
debug(_("Loading cached information for %(interface)s from %(cached)s"), {'interface': url, 'cached': cached})
return load_feed(cached, local = False)
else:
return None
except InvalidInterface as ex:
ex.feed_url = url
raise
def count_impls(url): if url not in cached_counts: cached = basedir.load_first_cache(namespaces.config_site, 'interfaces', model.escape(url)) if cached: with open(cached) as stream: cached_doc = qdom.parse(stream) def count(elem): c = 0 if elem.uri != namespaces.XMLNS_IFACE: return 0 if elem.name == 'implementation' or elem.name == 'package-implementation': c += 1 else: for child in elem.childNodes: c += count(child) return c cached_counts[url] = count(cached_doc) else: cached_counts[url] = 0 return cached_counts[url]
def get_cached_signatures(self, uri):
"""Verify the cached interface using GPG.
Only new-style XML-signed interfaces retain their signatures in the cache.
@param uri: the feed to check
@type uri: str
@return: a list of signatures, or None
@rtype: [L{gpg.Signature}] or None
@since: 0.25"""
import gpg
if os.path.isabs(uri):
old_iface = uri
else:
old_iface = basedir.load_first_cache(config_site, 'interfaces', escape(uri))
if old_iface is None:
return None
try:
return gpg.check_stream(file(old_iface))[1]
except SafeException, ex:
debug(_("No signatures (old-style interface): %s") % ex)
return None
def get_cached_signatures(self, uri):
"""Verify the cached interface using GPG.
Only new-style XML-signed interfaces retain their signatures in the cache.
@param uri: the feed to check
@type uri: str
@return: a list of signatures, or None
@rtype: [L{gpg.Signature}] or None
@since: 0.25"""
import gpg
if uri.startswith('/'):
old_iface = uri
else:
old_iface = basedir.load_first_cache(config_site, 'interfaces',
escape(uri))
if old_iface is None:
return None
try:
return gpg.check_stream(file(old_iface))[1]
except SafeException, ex:
debug(_("No signatures (old-style interface): %s") % ex)
return None
def get_cached_signatures(self, uri):
"""Verify the cached interface using GPG.
Only new-style XML-signed interfaces retain their signatures in the cache.
@param uri: the feed to check
@type uri: str
@return: a list of signatures, or None
@rtype: [L{gpg.Signature}] or None
@since: 0.25"""
from . import gpg
if os.path.isabs(uri):
old_iface = uri
else:
old_iface = basedir.load_first_cache(config_site, 'interfaces',
escape(uri))
if old_iface is None:
return None
try:
with open(old_iface, 'rb') as stream:
return gpg.check_stream(stream)[1]
except SafeException as ex:
logger.info(_("No signatures (old-style interface): %s") % ex)
return None
def load_feed_from_cache(url):
"""Load a feed. If the feed is remote, load from the cache. If local, load it directly.
@type url: str
@return: the feed, or None if it's remote and not cached.
@rtype: L{ZeroInstallFeed} | None"""
try:
if os.path.isabs(url):
logger.debug(_("Loading local feed file '%s'"), url)
return load_feed(url, local=True)
else:
cached = basedir.load_first_cache(config_site, "interfaces", escape(url))
if cached:
logger.debug(
_("Loading cached information for %(interface)s from %(cached)s"),
{"interface": url, "cached": cached},
)
return load_feed(cached, local=False)
else:
return None
except InvalidInterface as ex:
ex.feed_url = url
raise
def export_feeds(export_dir, feeds, keys_used):
"""Copy each feed (and icon) in feeds from the cache to export_dir.
Add all signing key fingerprints to keys_used."""
for feed in feeds:
if feed.startswith('/'):
info("Skipping local feed %s", feed)
continue
if feed.startswith('distribution:'):
info("Skipping distribution feed %s", feed)
continue
print "Exporting feed", feed
# Store feed
cached = basedir.load_first_cache(namespaces.config_site,
'interfaces',
model.escape(feed))
if cached:
feed_dir = os.path.join(export_dir, get_feed_path(feed))
feed_dst = os.path.join(feed_dir, 'latest.xml')
if not os.path.isdir(feed_dir):
os.makedirs(feed_dir)
shutil.copyfile(cached, feed_dst)
info("Exported feed %s", feed)
icon_path = iface_cache.iface_cache.get_icon_path(iface_cache.iface_cache.get_interface(feed))
if icon_path:
icon_dst = os.path.join(feed_dir, 'icon.png')
shutil.copyfile(icon_path, icon_dst)
# Get the keys
stream = file(cached)
unused, sigs = gpg.check_stream(stream)
stream.close()
for x in sigs:
if isinstance(x, gpg.ValidSig):
keys_used.add(x.fingerprint)
else:
warn("Signature problem: %s" % x)
else:
warn("Feed not cached: %s", feed)
def update_from_cache(interface):
"""Read a cached interface and any native feeds or user overrides.
@param interface: the interface object to update
@type interface: L{model.Interface}
@return: True if cached version and user overrides loaded OK.
False if upstream not cached. Local interfaces (starting with /) are
always considered to be cached, although they are not actually stored in the cache.
@rtype: bool"""
interface.reset()
main_feed = None
if interface.uri.startswith('/'):
debug(_("Loading local interface file '%s'"), interface.uri)
update(interface, interface.uri, local=True)
cached = True
else:
cached = basedir.load_first_cache(config_site, 'interfaces',
escape(interface.uri))
if cached:
debug(
_("Loading cached information for %(interface)s from %(cached)s"
), {
'interface': interface,
'cached': cached
})
main_feed = update(interface, cached)
# Add the distribution package manager's version, if any
path = basedir.load_first_data(config_site, 'native_feeds',
model._pretty_escape(interface.uri))
if path:
# Resolve any symlinks
info(_("Adding native packager feed '%s'"), path)
interface.extra_feeds.append(Feed(os.path.realpath(path), None, False))
update_user_overrides(interface, main_feed)
return bool(cached)
def testExecutable(self):
child = subprocess.Popen(
[local_0launch, '--', runexec, 'user-arg-run'],
stdout=subprocess.PIPE,
universal_newlines=True)
stdout, _ = child.communicate()
assert 'Runner: script=A test script: args=foo-arg -- var user-arg-run' in stdout, stdout
assert 'Runner: script=A test script: args=command-arg -- path user-arg-run' in stdout, stdout
# Check runenv.py is updated correctly
from zeroinstall.support import basedir
runenv = basedir.load_first_cache(namespaces.config_site,
namespaces.config_prog, 'runenv.py')
os.chmod(runenv, 0o700)
with open(runenv, 'wb') as s:
s.write(b'#!/\n')
child = subprocess.Popen(
[local_0launch, '--', runexec, 'user-arg-run'],
stdout=subprocess.PIPE,
universal_newlines=True)
stdout, _ = child.communicate()
assert 'Runner: script=A test script: args=foo-arg -- var user-arg-run' in stdout, stdout
assert 'Runner: script=A test script: args=command-arg -- path user-arg-run' in stdout, stdout
def _populate_model(self):
# Find cached implementations
unowned = {} # Impl ID -> Store
duplicates = [] # TODO
for s in self.iface_cache.stores.stores:
if os.path.isdir(s.dir):
for id in os.listdir(s.dir):
if id in unowned:
duplicates.append(id)
unowned[id] = s
ok_feeds = []
error_feeds = []
# Look through cached feeds for implementation owners
all_interfaces = self.iface_cache.list_all_interfaces()
all_feeds = {}
for uri in all_interfaces:
try:
iface = self.iface_cache.get_interface(uri)
except Exception as ex:
error_feeds.append((uri, str(ex), 0))
else:
all_feeds.update(self.iface_cache.get_feeds(iface))
for url, feed in all_feeds.items():
if not feed: continue
yield
feed_size = 0
try:
if url != feed.url:
# (e.g. for .new feeds)
raise Exception('Incorrect URL for feed (%s vs %s)' % (url, feed.url))
if os.path.isabs(url):
cached_feed = url
feed_type = LocalFeed
else:
feed_type = RemoteFeed
cached_feed = basedir.load_first_cache(namespaces.config_site,
'interfaces', model.escape(url))
user_overrides = basedir.load_first_config(namespaces.config_site,
namespaces.config_prog,
'interfaces', model._pretty_escape(url))
feed_size = size_if_exists(cached_feed) + size_if_exists(user_overrides)
except Exception as ex:
error_feeds.append((url, str(ex), feed_size))
else:
cached_feed = feed_type(feed, feed_size)
for impl in feed.implementations.values():
if impl.local_path:
cached_feed.in_cache.append(LocalImplementation(impl))
for digest in impl.digests:
if digest in unowned:
cached_dir = unowned[digest].dir
impl_path = os.path.join(cached_dir, digest)
impl_size = get_size(impl_path)
cached_feed.in_cache.append(KnownImplementation(cached_feed, cached_dir, impl, impl_size, digest))
del unowned[digest]
cached_feed.in_cache.sort()
ok_feeds.append(cached_feed)
if error_feeds:
iter = SECTION_INVALID_INTERFACES.append_to(self.raw_model)
for uri, ex, size in error_feeds:
item = InvalidFeed(uri, ex, size)
item.append_to(self.raw_model, iter)
unowned_sizes = []
local_dir = os.path.join(basedir.xdg_cache_home, '0install.net', 'implementations')
for id in unowned:
if unowned[id].dir == local_dir:
impl = UnusedImplementation(local_dir, id)
unowned_sizes.append((impl.size, impl))
if unowned_sizes:
iter = SECTION_UNOWNED_IMPLEMENTATIONS.append_to(self.raw_model)
for size, item in unowned_sizes:
item.append_to(self.raw_model, iter)
if ok_feeds:
iter = SECTION_INTERFACES.append_to(self.raw_model)
for item in ok_feeds:
yield
item.append_to(self.raw_model, iter)
self._update_sizes()
def testSymlink(self):
run_server(('HelloWorld.tar.bz2', 'HelloSym.tgz'))
out, err = self.run_ocaml(['download', os.path.abspath('RecipeSymlink.xml')])
assert "Exit status: 1" in err, err
assert 'Attempt to unpack dir over symlink "HelloWorld"' in err, err
self.assertEqual(None, basedir.load_first_cache('0install.net', 'implementations', 'main'))