def _populate_model(self):
# Find cached implementations
unowned = {} # Impl ID -> Store
duplicates = [] # TODO
for s in self.iface_cache.stores.stores:
if os.path.isdir(s.dir):
for id in os.listdir(s.dir):
if id in unowned:
duplicates.append(id)
unowned[id] = s
ok_interfaces = []
error_interfaces = []
# Look through cached interfaces for implementation owners
all = self.iface_cache.list_all_interfaces()
all.sort()
for uri in all:
iface_size = 0
try:
if uri.startswith('/'):
cached_iface = uri
else:
cached_iface = basedir.load_first_cache(
namespaces.config_site, 'interfaces',
model.escape(uri))
user_overrides = basedir.load_first_config(
namespaces.config_site, namespaces.config_prog,
'user_overrides', model.escape(uri))
iface_size = size_if_exists(cached_iface) + size_if_exists(
user_overrides)
iface = self.iface_cache.get_interface(uri)
except Exception, ex:
error_interfaces.append((uri, str(ex), iface_size))
else:
cached_iface = ValidInterface(iface, iface_size)
for impl in iface.implementations.values():
if impl.id.startswith('/') or impl.id.startswith('.'):
cached_iface.in_cache.append(LocalImplementation(impl))
if impl.id in unowned:
cached_dir = unowned[impl.id].dir
impl_path = os.path.join(cached_dir, impl.id)
impl_size = get_size(impl_path)
cached_iface.in_cache.append(
KnownImplementation(cached_iface, cached_dir, impl,
impl_size))
del unowned[impl.id]
cached_iface.in_cache.sort()
ok_interfaces.append(cached_iface)
def testEscape(self):
self.assertEqual("", model.escape(""))
self.assertEqual("hello", model.escape("hello"))
self.assertEqual("%20", model.escape(" "))
self.assertEqual("file%3a%2f%2ffoo%7ebar",
model.escape("file://foo~bar"))
self.assertEqual("file%3a%2f%2ffoo%25bar",
model.escape("file://foo%bar"))
self.assertEqual("file:##foo%7ebar",
model._pretty_escape("file://foo~bar"))
self.assertEqual("file:##foo%25bar",
model._pretty_escape("file://foo%bar"))
def testEscape(self):
self.assertEqual("", model.escape(""))
self.assertEqual("hello", model.escape("hello"))
self.assertEqual("%20", model.escape(" "))
self.assertEqual("file%3a%2f%2ffoo%7ebar",
model.escape("file://foo~bar"))
self.assertEqual("file%3a%2f%2ffoo%25bar",
model.escape("file://foo%bar"))
self.assertEqual("file:##foo%7ebar",
model._pretty_escape("file://foo~bar"))
self.assertEqual("file:##foo%25bar",
model._pretty_escape("file://foo%bar"))
def delete(self):
if not self.uri.startswith('/'):
cached_iface = basedir.load_first_cache(namespaces.config_site,
'interfaces',
model.escape(self.uri))
if cached_iface:
#print "Delete", cached_iface
os.unlink(cached_iface)
user_overrides = basedir.load_first_config(namespaces.config_site,
namespaces.config_prog,
'user_overrides',
model.escape(self.uri))
if user_overrides:
#print "Delete", user_overrides
os.unlink(user_overrides)
def get_icon_path(self, iface):
"""Get the path of a cached icon for an interface.
@param iface: interface whose icon we want
@return: the path of the cached icon, or None if not cached.
@rtype: str"""
return basedir.load_first_cache(config_site, 'interface_icons',
escape(iface.uri))
def cache_iface(self, name, data):
cached_ifaces = basedir.save_cache_path('0install.net',
'interfaces')
f = open(os.path.join(cached_ifaces, model.escape(name)), 'w')
f.write(data)
f.close()
def cache_iface(self, name, data):
cached_ifaces = basedir.save_cache_path('0install.net',
'interfaces')
f = open(os.path.join(cached_ifaces, model.escape(name)), 'w')
f.write(data)
f.close()
def update_user_overrides(interface):
"""Update an interface with user-supplied information.
Sets preferred stability and updates extra_feeds.
@param interface: the interface object to update
@type interface: L{model.Interface}
"""
user = basedir.load_first_config(config_site, config_prog,
'interfaces', model._pretty_escape(interface.uri))
if user is None:
# For files saved by 0launch < 0.49
user = basedir.load_first_config(config_site, config_prog,
'user_overrides', escape(interface.uri))
if not user:
return
try:
root = qdom.parse(file(user))
except Exception as ex:
warn(_("Error reading '%(user)s': %(exception)s"), {'user': user, 'exception': ex})
raise
stability_policy = root.getAttribute('stability-policy')
if stability_policy:
interface.set_stability_policy(stability_levels[str(stability_policy)])
for item in root.childNodes:
if item.uri != XMLNS_IFACE: continue
if item.name == 'feed':
feed_src = item.getAttribute('src')
if not feed_src:
raise InvalidInterface(_('Missing "src" attribute in <feed>'))
interface.extra_feeds.append(Feed(feed_src, item.getAttribute('arch'), True, langs = item.getAttribute('langs')))
def _import_new_feed(self, feed_url, new_xml, modified_time):
"""Write new_xml into the cache.
@param feed_url: the URL for the feed being updated
@param new_xml: the data to write
@param modified_time: when new_xml was modified
@raises ReplayAttack: if the new mtime is older than the current one
"""
assert modified_time
assert isinstance(new_xml, bytes), repr(new_xml)
upstream_dir = basedir.save_cache_path(config_site, 'interfaces')
cached = os.path.join(upstream_dir, escape(feed_url))
old_modified = None
if os.path.exists(cached):
with open(cached, 'rb') as stream:
old_xml = stream.read()
if old_xml == new_xml:
logger.debug(_("No change"))
# Update in-memory copy, in case someone else updated the disk copy
self.get_feed(feed_url, force=True)
return
old_modified = int(os.stat(cached).st_mtime)
# Do we need to write this temporary file now?
try:
with open(cached + '.new', 'wb') as stream:
stream.write(new_xml)
os.utime(cached + '.new', (modified_time, modified_time))
new_mtime = reader.check_readable(feed_url, cached + '.new')
assert new_mtime == modified_time
old_modified = self._get_signature_date(feed_url) or old_modified
if old_modified:
if new_mtime < old_modified:
raise ReplayAttack(
_("New feed's modification time is "
"before old version!\nInterface: %(iface)s\nOld time: %(old_time)s\nNew time: %(new_time)s\n"
"Refusing update.") % {
'iface': feed_url,
'old_time': _pretty_time(old_modified),
'new_time': _pretty_time(new_mtime)
})
if new_mtime == old_modified:
# You used to have to update the modification time manually.
# Now it comes from the signature, this check isn't useful
# and often causes problems when the stored format changes
# (e.g., when we stopped writing last-modified attributes)
pass
#raise SafeException("Interface has changed, but modification time "
# "hasn't! Refusing update.")
except:
os.unlink(cached + '.new')
raise
portable_rename(cached + '.new', cached)
logger.debug(_("Saved as %s") % cached)
self.get_feed(feed_url, force=True)
def testTimes(self):
iface_cache = self.config.iface_cache
with tempfile.TemporaryFile() as stream:
stream.write(data.thomas_key)
stream.seek(0)
gpg.import_key(stream)
upstream_dir = basedir.save_cache_path(config_site, 'interfaces')
cached = os.path.join(upstream_dir, model.escape('http://foo'))
with open(cached, 'wb') as stream:
stream.write(data.foo_signed_xml)
signed = iface_cache._get_signature_date('http://foo')
assert signed == None
trust.trust_db.trust_key(
'92429807C9853C0744A68B9AAE07828059A53CC1')
signed = iface_cache._get_signature_date('http://foo')
self.assertEqual(1154850229, signed)
with open(cached, 'w+b') as stream:
stream.seek(0)
stream.write(b'Hello')
# When the signature is invalid, we just return None.
# This is because versions < 0.22 used to corrupt the signatue
# by adding an attribute to the XML
signed = iface_cache._get_signature_date('http://foo')
assert signed == None
def get_inputs():
for sel in sels.selections.values():
logger.info("Checking %s", sel.feed)
if sel.feed.startswith('distribution:'):
# If the package has changed version, we'll detect that below
# with get_unavailable_selections.
pass
elif os.path.isabs(sel.feed):
# Local feed
yield sel.feed
else:
# Cached feed
cached = basedir.load_first_cache(
namespaces.config_site, 'interfaces',
model.escape(sel.feed))
if cached:
yield cached
else:
raise IOError("Input %s missing; update" %
sel.feed)
# Per-feed configuration
yield basedir.load_first_config(
namespaces.config_site, namespaces.config_prog,
'interfaces', model._pretty_escape(sel.interface))
# Global configuration
yield basedir.load_first_config(namespaces.config_site,
namespaces.config_prog,
'global')
def get_icon_path(self, iface): """Get the path of a cached icon for an interface. @param iface: interface whose icon we want @return: the path of the cached icon, or None if not cached. @rtype: str""" return basedir.load_first_cache(config_site, 'interface_icons', escape(iface.uri))
def get_inputs():
for sel in sels.selections.values():
logger.info("Checking %s", sel.feed)
if sel.feed.startswith('distribution:'):
# If the package has changed version, we'll detect that below
# with get_unavailable_selections.
pass
elif os.path.isabs(sel.feed):
# Local feed
yield sel.feed
else:
# Cached feed
cached = basedir.load_first_cache(namespaces.config_site, 'interfaces', model.escape(sel.feed))
if cached:
yield cached
else:
raise IOError("Input %s missing; update" % sel.feed)
# Per-feed configuration
yield basedir.load_first_config(namespaces.config_site, namespaces.config_prog,
'interfaces', model._pretty_escape(sel.interface))
# Global configuration
yield basedir.load_first_config(namespaces.config_site, namespaces.config_prog, 'global')
def testXMLupdate(self):
iface_cache = self.config.iface_cache
trust.trust_db.trust_key(
'92429807C9853C0744A68B9AAE07828059A53CC1')
with tempfile.TemporaryFile() as stream:
stream.write(data.thomas_key)
stream.seek(0)
gpg.import_key(stream)
iface = iface_cache.get_interface('http://foo')
with tempfile.TemporaryFile() as src:
src.write(data.foo_signed_xml)
src.seek(0)
pending = PendingFeed(iface.uri, src)
assert iface_cache.update_feed_if_trusted(iface.uri, pending.sigs, pending.new_xml)
iface_cache.__init__()
feed = iface_cache.get_feed('http://foo')
assert feed.last_modified == 1154850229
# mtimes are unreliable because copying often changes them -
# check that we extract the time from the signature when upgrading
upstream_dir = basedir.save_cache_path(config_site, 'interfaces')
cached = os.path.join(upstream_dir, model.escape(feed.url))
os.utime(cached, None)
iface_cache.__init__()
feed = iface_cache.get_feed('http://foo')
assert feed.last_modified > 1154850229
with tempfile.TemporaryFile() as src:
src.write(data.new_foo_signed_xml)
src.seek(0)
pending = PendingFeed(feed.url, src)
old_stdout = sys.stdout
sys.stdout = StringIO()
try:
assert iface_cache.update_feed_if_trusted(feed.url, pending.sigs, pending.new_xml, dry_run = True)
finally:
sys.stdout = old_stdout
assert iface_cache.update_feed_if_trusted(feed.url, pending.sigs, pending.new_xml)
# Can't 'update' to an older copy
with tempfile.TemporaryFile() as src:
src.write(data.foo_signed_xml)
src.seek(0)
try:
pending = PendingFeed(feed.url, src)
assert iface_cache.update_feed_if_trusted(feed.url, pending.sigs, pending.new_xml)
assert 0
except model.SafeException:
pass
def _import_new_feed(self, feed_url, new_xml, modified_time):
"""Write new_xml into the cache.
@param feed_url: the URL for the feed being updated
@param new_xml: the data to write
@param modified_time: when new_xml was modified
@raises ReplayAttack: if the new mtime is older than the current one
"""
assert modified_time
assert isinstance(new_xml, bytes), repr(new_xml)
upstream_dir = basedir.save_cache_path(config_site, 'interfaces')
cached = os.path.join(upstream_dir, escape(feed_url))
old_modified = None
if os.path.exists(cached):
with open(cached, 'rb') as stream:
old_xml = stream.read()
if old_xml == new_xml:
logger.debug(_("No change"))
# Update in-memory copy, in case someone else updated the disk copy
self.get_feed(feed_url, force = True)
return
old_modified = int(os.stat(cached).st_mtime)
# Do we need to write this temporary file now?
try:
with open(cached + '.new', 'wb') as stream:
stream.write(new_xml)
os.utime(cached + '.new', (modified_time, modified_time))
new_mtime = reader.check_readable(feed_url, cached + '.new')
assert new_mtime == modified_time
old_modified = self._get_signature_date(feed_url) or old_modified
if old_modified:
if new_mtime < old_modified:
raise ReplayAttack(_("New feed's modification time is "
"before old version!\nInterface: %(iface)s\nOld time: %(old_time)s\nNew time: %(new_time)s\n"
"Refusing update.")
% {'iface': feed_url, 'old_time': _pretty_time(old_modified), 'new_time': _pretty_time(new_mtime)})
if new_mtime == old_modified:
# You used to have to update the modification time manually.
# Now it comes from the signature, this check isn't useful
# and often causes problems when the stored format changes
# (e.g., when we stopped writing last-modified attributes)
pass
#raise SafeException("Interface has changed, but modification time "
# "hasn't! Refusing update.")
except:
os.unlink(cached + '.new')
raise
portable_rename(cached + '.new', cached)
logger.debug(_("Saved as %s") % cached)
self.get_feed(feed_url, force = True)
def delete(self):
if not os.path.isabs(self.uri):
cached_iface = basedir.load_first_cache(namespaces.config_site, "interfaces", model.escape(self.uri))
if cached_iface:
# print "Delete", cached_iface
os.unlink(cached_iface)
user_overrides = basedir.load_first_config(
namespaces.config_site, namespaces.config_prog, "user_overrides", model.escape(self.uri)
)
if user_overrides:
# print "Delete", user_overrides
os.unlink(user_overrides)
def _import_new_interface(self, interface, new_xml, modified_time):
"""Write new_xml into the cache.
@param interface: updated once the new XML is written
@param new_xml: the data to write
@param modified_time: when new_xml was modified
@raises ReplayAttack: if the new mtime is older than the current one
"""
assert modified_time
upstream_dir = basedir.save_cache_path(config_site, 'interfaces')
cached = os.path.join(upstream_dir, escape(interface.uri))
if os.path.exists(cached):
old_xml = file(cached).read()
if old_xml == new_xml:
debug(_("No change"))
reader.update_from_cache(interface)
return
stream = file(cached + '.new', 'w')
stream.write(new_xml)
stream.close()
os.utime(cached + '.new', (modified_time, modified_time))
new_mtime = reader.check_readable(interface.uri, cached + '.new')
assert new_mtime == modified_time
old_modified = self._get_signature_date(interface.uri)
if old_modified is None:
old_modified = interface.last_modified
if old_modified:
if new_mtime < old_modified:
os.unlink(cached + '.new')
raise ReplayAttack(
_("New interface's modification time is "
"before old version!\nOld time: %(old_time)s\nNew time: %(new_time)s\n"
"Refusing update.") % {
'old_time': _pretty_time(old_modified),
'new_time': _pretty_time(new_mtime)
})
if new_mtime == old_modified:
# You used to have to update the modification time manually.
# Now it comes from the signature, this check isn't useful
# and often causes problems when the stored format changes
# (e.g., when we stopped writing last-modified attributes)
pass
#raise SafeException("Interface has changed, but modification time "
# "hasn't! Refusing update.")
os.rename(cached + '.new', cached)
debug(_("Saved as %s") % cached)
reader.update_from_cache(interface)
def update_user_overrides(interface, known_site_feeds=frozenset()):
"""Update an interface with user-supplied information.
Sets preferred stability and updates extra_feeds.
@param interface: the interface object to update
@type interface: L{model.Interface}
@param known_site_feeds: feeds to ignore (for backwards compatibility)
"""
user = basedir.load_first_config(config_site, config_prog, 'interfaces',
model._pretty_escape(interface.uri))
if user is None:
# For files saved by 0launch < 0.49
user = basedir.load_first_config(config_site,
config_prog, 'user_overrides',
escape(interface.uri))
if not user:
return
try:
with open(user, 'rb') as stream:
root = qdom.parse(stream)
except Exception as ex:
logger.warn(_("Error reading '%(user)s': %(exception)s"), {
'user': user,
'exception': ex
})
raise
stability_policy = root.getAttribute('stability-policy')
if stability_policy:
interface.set_stability_policy(stability_levels[str(stability_policy)])
for item in root.childNodes:
if item.uri != XMLNS_IFACE: continue
if item.name == 'feed':
feed_src = item.getAttribute('src')
if not feed_src:
raise InvalidInterface(_('Missing "src" attribute in <feed>'))
# (note: 0install 1.9..1.12 used a different scheme and the "site-package" attribute;
# we deliberately use a different attribute name to avoid confusion)
if item.getAttribute('is-site-package'):
# Site packages are detected earlier. This test isn't completely reliable,
# since older versions will remove the attribute when saving the config
# (hence the next test).
continue
if feed_src in known_site_feeds:
continue
interface.extra_feeds.append(
Feed(feed_src,
item.getAttribute('arch'),
True,
langs=item.getAttribute('langs')))
def download_and_add_icon(): stream = dl.tempfile yield dl.downloaded try: tasks.check(dl.downloaded) if dl.unmodified: return stream.seek(0) import shutil icons_cache = basedir.save_cache_path(config_site, 'interface_icons') icon_file = file(os.path.join(icons_cache, escape(interface.uri)), 'w') shutil.copyfileobj(stream, icon_file) except Exception as ex: self.handler.report_error(ex)
def testXMLupdate(self):
iface_cache = self.config.iface_cache
trust.trust_db.trust_key('92429807C9853C0744A68B9AAE07828059A53CC1')
stream = tempfile.TemporaryFile()
stream.write(data.thomas_key)
stream.seek(0)
gpg.import_key(stream)
iface = iface_cache.get_interface('http://foo')
src = tempfile.TemporaryFile()
src.write(data.foo_signed_xml)
src.seek(0)
pending = PendingFeed(iface.uri, src)
assert iface_cache.update_feed_if_trusted(iface.uri, pending.sigs,
pending.new_xml)
iface_cache.__init__()
feed = iface_cache.get_feed('http://foo')
assert feed.last_modified == 1154850229
# mtimes are unreliable because copying often changes them -
# check that we extract the time from the signature when upgrading
upstream_dir = basedir.save_cache_path(config_site, 'interfaces')
cached = os.path.join(upstream_dir, model.escape(feed.url))
os.utime(cached, None)
iface_cache.__init__()
feed = iface_cache.get_feed('http://foo')
assert feed.last_modified > 1154850229
src = tempfile.TemporaryFile()
src.write(data.new_foo_signed_xml)
src.seek(0)
pending = PendingFeed(feed.url, src)
assert iface_cache.update_feed_if_trusted(feed.url, pending.sigs,
pending.new_xml)
# Can't 'update' to an older copy
src = tempfile.TemporaryFile()
src.write(data.foo_signed_xml)
src.seek(0)
try:
pending = PendingFeed(feed.url, src)
assert iface_cache.update_feed_if_trusted(feed.url, pending.sigs,
pending.new_xml)
assert 0
except model.SafeException:
pass
def download_and_add_icon():
stream = dl.tempfile
yield dl.downloaded
try:
tasks.check(dl.downloaded)
if dl.unmodified: return
stream.seek(0)
import shutil
icons_cache = basedir.save_cache_path(config_site,
'interface_icons')
icon_file = file(
os.path.join(icons_cache, escape(interface.uri)), 'w')
shutil.copyfileobj(stream, icon_file)
except Exception, ex:
self.handler.report_error(ex)
def update_user_feed_overrides(feed):
"""Update a feed with user-supplied information.
Sets last_checked and user_stability ratings.
@param feed: feed to update
@since 0.49
"""
user = basedir.load_first_config(config_site, config_prog, 'feeds',
model._pretty_escape(feed.url))
if user is None:
# For files saved by 0launch < 0.49
user = basedir.load_first_config(config_site, config_prog,
'user_overrides', escape(feed.url))
if not user:
return
try:
with open(user, 'rb') as stream:
root = qdom.parse(stream)
except Exception as ex:
logger.warn(_("Error reading '%(user)s': %(exception)s"), {
'user': user,
'exception': ex
})
raise
last_checked = root.getAttribute('last-checked')
if last_checked:
feed.last_checked = int(last_checked)
for item in root.childNodes:
if item.uri != XMLNS_IFACE: continue
if item.name == 'implementation':
id = item.getAttribute('id')
assert id is not None
impl = feed.implementations.get(id, None)
if not impl:
logger.debug(
_("Ignoring user-override for unknown implementation %(id)s in %(interface)s"
), {
'id': id,
'interface': feed
})
continue
user_stability = item.getAttribute('user-stability')
if user_stability:
impl.user_stability = stability_levels[str(user_stability)]
def delete(self):
if not os.path.isabs(self.uri):
cached_iface = basedir.load_first_cache(namespaces.config_site,
'interfaces', model.escape(self.uri))
if cached_iface:
if SAFE_MODE:
print("Delete", cached_iface)
else:
os.unlink(cached_iface)
user_overrides = basedir.load_first_config(namespaces.config_site,
namespaces.config_prog,
'interfaces', model._pretty_escape(self.uri))
if user_overrides:
if SAFE_MODE:
print("Delete", user_overrides)
else:
os.unlink(user_overrides)
def delete(self):
if not os.path.isabs(self.uri):
cached_iface = basedir.load_first_cache(namespaces.config_site,
'interfaces', model.escape(self.uri))
if cached_iface:
if SAFE_MODE:
print("Delete", cached_iface)
else:
os.unlink(cached_iface)
user_overrides = basedir.load_first_config(namespaces.config_site,
namespaces.config_prog,
'interfaces', model._pretty_escape(self.uri))
if user_overrides:
if SAFE_MODE:
print("Delete", user_overrides)
else:
os.unlink(user_overrides)
def load_feed_from_cache(url, selections_ok = False):
"""Load a feed. If the feed is remote, load from the cache. If local, load it directly.
@return: the feed, or None if it's remote and not cached."""
try:
if os.path.isabs(url):
debug(_("Loading local feed file '%s'"), url)
return load_feed(url, local = True, selections_ok = selections_ok)
else:
cached = basedir.load_first_cache(config_site, 'interfaces', escape(url))
if cached:
debug(_("Loading cached information for %(interface)s from %(cached)s"), {'interface': url, 'cached': cached})
return load_feed(cached, local = False)
else:
return None
except InvalidInterface, ex:
ex.feed_url = url
raise
def load_feed_from_cache(url, selections_ok = False):
"""Load a feed. If the feed is remote, load from the cache. If local, load it directly.
@return: the feed, or None if it's remote and not cached."""
try:
if os.path.isabs(url):
debug(_("Loading local feed file '%s'"), url)
return load_feed(url, local = True, selections_ok = selections_ok)
else:
cached = basedir.load_first_cache(config_site, 'interfaces', escape(url))
if cached:
debug(_("Loading cached information for %(interface)s from %(cached)s"), {'interface': url, 'cached': cached})
return load_feed(cached, local = False)
else:
return None
except InvalidInterface as ex:
ex.feed_url = url
raise
def update_user_overrides(interface, known_site_feeds=frozenset()):
"""Update an interface with user-supplied information.
Sets preferred stability and updates extra_feeds.
@param interface: the interface object to update
@type interface: L{model.Interface}
@param known_site_feeds: feeds to ignore (for backwards compatibility)
"""
user = basedir.load_first_config(config_site, config_prog, "interfaces", model._pretty_escape(interface.uri))
if user is None:
# For files saved by 0launch < 0.49
user = basedir.load_first_config(config_site, config_prog, "user_overrides", escape(interface.uri))
if not user:
return
try:
with open(user, "rb") as stream:
root = qdom.parse(stream)
except Exception as ex:
logger.warn(_("Error reading '%(user)s': %(exception)s"), {"user": user, "exception": ex})
raise
stability_policy = root.getAttribute("stability-policy")
if stability_policy:
interface.set_stability_policy(stability_levels[str(stability_policy)])
for item in root.childNodes:
if item.uri != XMLNS_IFACE:
continue
if item.name == "feed":
feed_src = item.getAttribute("src")
if not feed_src:
raise InvalidInterface(_('Missing "src" attribute in <feed>'))
# (note: 0install 1.9..1.12 used a different scheme and the "site-package" attribute;
# we deliberately use a different attribute name to avoid confusion)
if item.getAttribute("is-site-package"):
# Site packages are detected earlier. This test isn't completely reliable,
# since older versions will remove the attribute when saving the config
# (hence the next test).
continue
if feed_src in known_site_feeds:
continue
interface.extra_feeds.append(
Feed(feed_src, item.getAttribute("arch"), True, langs=item.getAttribute("langs"))
)
def download_and_add_icon(): stream = dl.tempfile try: yield dl.downloaded tasks.check(dl.downloaded) if dl.unmodified: return stream.seek(0) import shutil, tempfile icons_cache = basedir.save_cache_path(config_site, 'interface_icons') tmp_file = tempfile.NamedTemporaryFile(dir = icons_cache, delete = False) shutil.copyfileobj(stream, tmp_file) tmp_file.close() icon_file = os.path.join(icons_cache, escape(interface.uri)) portable_rename(tmp_file.name, icon_file) finally: stream.close()
def count_impls(url): if url not in cached_counts: cached = basedir.load_first_cache(namespaces.config_site, 'interfaces', model.escape(url)) if cached: with open(cached) as stream: cached_doc = qdom.parse(stream) def count(elem): c = 0 if elem.uri != namespaces.XMLNS_IFACE: return 0 if elem.name == 'implementation' or elem.name == 'package-implementation': c += 1 else: for child in elem.childNodes: c += count(child) return c cached_counts[url] = count(cached_doc) else: cached_counts[url] = 0 return cached_counts[url]
def download_and_add_icon(): stream = dl.tempfile try: yield dl.downloaded tasks.check(dl.downloaded) if dl.unmodified: return stream.seek(0) import shutil, tempfile icons_cache = basedir.save_cache_path(config_site, 'interface_icons') tmp_file = tempfile.NamedTemporaryFile(dir = icons_cache, delete = False) shutil.copyfileobj(stream, tmp_file) tmp_file.close() icon_file = os.path.join(icons_cache, escape(interface.uri)) portable_rename(tmp_file.name, icon_file) finally: stream.close()
def import_feed(self, url, contents):
"""contents can be a path or an Element."""
iface_cache = self.config.iface_cache
iface_cache.get_interface(url)
if isinstance(contents, qdom.Element):
feed = model.ZeroInstallFeed(contents)
else:
feed = reader.load_feed(contents)
iface_cache._feeds[url] = feed
xml = qdom.to_UTF8(feed.feed_element)
upstream_dir = basedir.save_cache_path(namespaces.config_site, "interfaces")
cached = os.path.join(upstream_dir, model.escape(url))
with open(cached, "wb") as stream:
stream.write(xml)
return feed
def update_user_overrides(interface, main_feed=None):
"""Update an interface with user-supplied information.
@param interface: the interface object to update
@type interface: L{model.Interface}
@param main_feed: feed to update with last_checked information
@note: feed updates shouldn't really be here. main_feed may go away in future.
"""
user = basedir.load_first_config(config_site, config_prog,
'user_overrides', escape(interface.uri))
if not user:
return
try:
root = qdom.parse(file(user))
except Exception, ex:
warn(_("Error reading '%(user)s': %(exception)s"), {
'user': user,
'exception': ex
})
raise
def get_cached_signatures(self, uri):
"""Verify the cached interface using GPG.
Only new-style XML-signed interfaces retain their signatures in the cache.
@param uri: the feed to check
@type uri: str
@return: a list of signatures, or None
@rtype: [L{gpg.Signature}] or None
@since: 0.25"""
import gpg
if os.path.isabs(uri):
old_iface = uri
else:
old_iface = basedir.load_first_cache(config_site, 'interfaces', escape(uri))
if old_iface is None:
return None
try:
return gpg.check_stream(file(old_iface))[1]
except SafeException, ex:
debug(_("No signatures (old-style interface): %s") % ex)
return None
def update_user_feed_overrides(feed):
"""Update a feed with user-supplied information.
Sets last_checked and user_stability ratings.
@param feed: feed to update
@since 0.49
"""
user = basedir.load_first_config(config_site, config_prog,
'feeds', model._pretty_escape(feed.url))
if user is None:
# For files saved by 0launch < 0.49
user = basedir.load_first_config(config_site, config_prog,
'user_overrides', escape(feed.url))
if not user:
return
try:
root = qdom.parse(file(user))
except Exception, ex:
warn(_("Error reading '%(user)s': %(exception)s"), {'user': user, 'exception': ex})
raise
def update_user_overrides(interface):
"""Update an interface with user-supplied information.
Sets preferred stability and updates extra_feeds.
@param interface: the interface object to update
@type interface: L{model.Interface}
"""
user = basedir.load_first_config(config_site, config_prog,
'interfaces', model._pretty_escape(interface.uri))
if user is None:
# For files saved by 0launch < 0.49
user = basedir.load_first_config(config_site, config_prog,
'user_overrides', escape(interface.uri))
if not user:
return
try:
root = qdom.parse(file(user))
except Exception, ex:
warn(_("Error reading '%(user)s': %(exception)s"), {'user': user, 'exception': ex})
raise
def update_user_feed_overrides(feed):
"""Update a feed with user-supplied information.
Sets last_checked and user_stability ratings.
@param feed: feed to update
@since 0.49
"""
user = basedir.load_first_config(config_site, config_prog, "feeds", model._pretty_escape(feed.url))
if user is None:
# For files saved by 0launch < 0.49
user = basedir.load_first_config(config_site, config_prog, "user_overrides", escape(feed.url))
if not user:
return
try:
with open(user, "rb") as stream:
root = qdom.parse(stream)
except Exception as ex:
logger.warn(_("Error reading '%(user)s': %(exception)s"), {"user": user, "exception": ex})
raise
last_checked = root.getAttribute("last-checked")
if last_checked:
feed.last_checked = int(last_checked)
for item in root.childNodes:
if item.uri != XMLNS_IFACE:
continue
if item.name == "implementation":
id = item.getAttribute("id")
assert id is not None
impl = feed.implementations.get(id, None)
if not impl:
logger.debug(
_("Ignoring user-override for unknown implementation %(id)s in %(interface)s"),
{"id": id, "interface": feed},
)
continue
user_stability = item.getAttribute("user-stability")
if user_stability:
impl.user_stability = stability_levels[str(user_stability)]
def save_interface(interface):
user_overrides = basedir.save_config_path(config_site, config_prog, 'user_overrides')
impl = minidom.getDOMImplementation()
doc = impl.createDocument(XMLNS_IFACE, 'interface-preferences', None)
root = doc.documentElement
root.setAttributeNS(XMLNS_NAMESPACE, 'xmlns', XMLNS_IFACE)
root.setAttribute('uri', interface.uri)
if interface.stability_policy:
root.setAttribute('stability-policy', str(interface.stability_policy))
if interface.last_checked:
root.setAttribute('last-checked', str(interface.last_checked))
impls = interface.implementations.values()
impls.sort()
for impl in impls:
_add_impl(root, impl)
for feed in interface.extra_feeds:
if feed.user_override:
elem = doc.createElementNS(XMLNS_IFACE, 'feed')
root.appendChild(elem)
elem.setAttribute('src', feed.uri)
if feed.arch:
elem.setAttribute('arch', feed.arch)
import tempfile
tmp_fd, tmp_name = tempfile.mkstemp(dir = user_overrides)
try:
tmp_file = os.fdopen(tmp_fd, 'w')
doc.writexml(tmp_file, addindent = " ", newl = '\n')
tmp_file.close()
path = os.path.join(user_overrides, escape(interface.uri))
os.rename(tmp_name, path)
os.chmod(path, 0660)
except:
os.unlink(tmp_name)
raise
def get_cached_signatures(self, uri):
"""Verify the cached interface using GPG.
Only new-style XML-signed interfaces retain their signatures in the cache.
@param uri: the feed to check
@type uri: str
@return: a list of signatures, or None
@rtype: [L{gpg.Signature}] or None
@since: 0.25"""
import gpg
if uri.startswith('/'):
old_iface = uri
else:
old_iface = basedir.load_first_cache(config_site, 'interfaces',
escape(uri))
if old_iface is None:
return None
try:
return gpg.check_stream(file(old_iface))[1]
except SafeException, ex:
debug(_("No signatures (old-style interface): %s") % ex)
return None
def get_cached_signatures(self, uri):
"""Verify the cached interface using GPG.
Only new-style XML-signed interfaces retain their signatures in the cache.
@param uri: the feed to check
@type uri: str
@return: a list of signatures, or None
@rtype: [L{gpg.Signature}] or None
@since: 0.25"""
from . import gpg
if os.path.isabs(uri):
old_iface = uri
else:
old_iface = basedir.load_first_cache(config_site, 'interfaces',
escape(uri))
if old_iface is None:
return None
try:
with open(old_iface, 'rb') as stream:
return gpg.check_stream(stream)[1]
except SafeException as ex:
logger.info(_("No signatures (old-style interface): %s") % ex)
return None
def load_feed_from_cache(url):
"""Load a feed. If the feed is remote, load from the cache. If local, load it directly.
@type url: str
@return: the feed, or None if it's remote and not cached.
@rtype: L{ZeroInstallFeed} | None"""
try:
if os.path.isabs(url):
logger.debug(_("Loading local feed file '%s'"), url)
return load_feed(url, local=True)
else:
cached = basedir.load_first_cache(config_site, "interfaces", escape(url))
if cached:
logger.debug(
_("Loading cached information for %(interface)s from %(cached)s"),
{"interface": url, "cached": cached},
)
return load_feed(cached, local=False)
else:
return None
except InvalidInterface as ex:
ex.feed_url = url
raise
def export_feeds(export_dir, feeds, keys_used):
"""Copy each feed (and icon) in feeds from the cache to export_dir.
Add all signing key fingerprints to keys_used."""
for feed in feeds:
if feed.startswith('/'):
info("Skipping local feed %s", feed)
continue
if feed.startswith('distribution:'):
info("Skipping distribution feed %s", feed)
continue
print "Exporting feed", feed
# Store feed
cached = basedir.load_first_cache(namespaces.config_site,
'interfaces',
model.escape(feed))
if cached:
feed_dir = os.path.join(export_dir, get_feed_path(feed))
feed_dst = os.path.join(feed_dir, 'latest.xml')
if not os.path.isdir(feed_dir):
os.makedirs(feed_dir)
shutil.copyfile(cached, feed_dst)
info("Exported feed %s", feed)
icon_path = iface_cache.iface_cache.get_icon_path(iface_cache.iface_cache.get_interface(feed))
if icon_path:
icon_dst = os.path.join(feed_dir, 'icon.png')
shutil.copyfile(icon_path, icon_dst)
# Get the keys
stream = file(cached)
unused, sigs = gpg.check_stream(stream)
stream.close()
for x in sigs:
if isinstance(x, gpg.ValidSig):
keys_used.add(x.fingerprint)
else:
warn("Signature problem: %s" % x)
else:
warn("Feed not cached: %s", feed)
def update_user_feed_overrides(feed):
"""Update a feed with user-supplied information.
Sets last_checked and user_stability ratings.
@param feed: feed to update
@since 0.49
"""
user = basedir.load_first_config(config_site, config_prog,
'feeds', model._pretty_escape(feed.url))
if user is None:
# For files saved by 0launch < 0.49
user = basedir.load_first_config(config_site, config_prog,
'user_overrides', escape(feed.url))
if not user:
return
try:
root = qdom.parse(file(user))
except Exception as ex:
warn(_("Error reading '%(user)s': %(exception)s"), {'user': user, 'exception': ex})
raise
last_checked = root.getAttribute('last-checked')
if last_checked:
feed.last_checked = int(last_checked)
for item in root.childNodes:
if item.uri != XMLNS_IFACE: continue
if item.name == 'implementation':
id = item.getAttribute('id')
assert id is not None
impl = feed.implementations.get(id, None)
if not impl:
debug(_("Ignoring user-override for unknown implementation %(id)s in %(interface)s"), {'id': id, 'interface': feed})
continue
user_stability = item.getAttribute('user-stability')
if user_stability:
impl.user_stability = stability_levels[str(user_stability)]
def update_from_cache(interface):
"""Read a cached interface and any native feeds or user overrides.
@param interface: the interface object to update
@type interface: L{model.Interface}
@return: True if cached version and user overrides loaded OK.
False if upstream not cached. Local interfaces (starting with /) are
always considered to be cached, although they are not actually stored in the cache.
@rtype: bool"""
interface.reset()
main_feed = None
if interface.uri.startswith('/'):
debug(_("Loading local interface file '%s'"), interface.uri)
update(interface, interface.uri, local=True)
cached = True
else:
cached = basedir.load_first_cache(config_site, 'interfaces',
escape(interface.uri))
if cached:
debug(
_("Loading cached information for %(interface)s from %(cached)s"
), {
'interface': interface,
'cached': cached
})
main_feed = update(interface, cached)
# Add the distribution package manager's version, if any
path = basedir.load_first_data(config_site, 'native_feeds',
model._pretty_escape(interface.uri))
if path:
# Resolve any symlinks
info(_("Adding native packager feed '%s'"), path)
interface.extra_feeds.append(Feed(os.path.realpath(path), None, False))
update_user_overrides(interface, main_feed)
return bool(cached)
def _import_new_feed(self, feed_url, new_xml, modified_time, dry_run):
"""Write new_xml into the cache.
@param feed_url: the URL for the feed being updated
@type feed_url: str
@param new_xml: the data to write
@type new_xml: str
@param modified_time: when new_xml was modified
@type modified_time: int
@type dry_run: bool
@raises ReplayAttack: if the new mtime is older than the current one"""
assert modified_time
assert isinstance(new_xml, bytes), repr(new_xml)
upstream_dir = basedir.save_cache_path(config_site, "interfaces")
cached = os.path.join(upstream_dir, escape(feed_url))
old_modified = None
if os.path.exists(cached):
with open(cached, "rb") as stream:
old_xml = stream.read()
if old_xml == new_xml:
logger.debug(_("No change"))
# Update in-memory copy, in case someone else updated the disk copy
self.get_feed(feed_url, force=True)
return
old_modified = int(os.stat(cached).st_mtime)
if dry_run:
print(_("[dry-run] would cache feed {url} as {cached}").format(url=feed_url, cached=cached))
from io import BytesIO
from zeroinstall.injector import qdom
root = qdom.parse(BytesIO(new_xml), filter_for_version=True)
feed = model.ZeroInstallFeed(root)
reader.update_user_feed_overrides(feed)
self._feeds[feed_url] = feed
return
# Do we need to write this temporary file now?
try:
with open(cached + ".new", "wb") as stream:
stream.write(new_xml)
os.utime(cached + ".new", (modified_time, modified_time))
new_mtime = reader.check_readable(feed_url, cached + ".new")
assert new_mtime == modified_time
old_modified = self._get_signature_date(feed_url) or old_modified
if old_modified:
if new_mtime < old_modified:
raise ReplayAttack(
_(
"New feed's modification time is "
"before old version!\nInterface: %(iface)s\nOld time: %(old_time)s\nNew time: %(new_time)s\n"
"Refusing update."
)
% {
"iface": feed_url,
"old_time": _pretty_time(old_modified),
"new_time": _pretty_time(new_mtime),
}
)
if new_mtime == old_modified:
# You used to have to update the modification time manually.
# Now it comes from the signature, this check isn't useful
# and often causes problems when the stored format changes
# (e.g., when we stopped writing last-modified attributes)
pass
# raise SafeException("Interface has changed, but modification time "
# "hasn't! Refusing update.")
except:
os.unlink(cached + ".new")
raise
portable_rename(cached + ".new", cached)
logger.debug(_("Saved as %s") % cached)
self.get_feed(feed_url, force=True)
def check(str): self.assertEqual(str, model.unescape(model.escape(str))) self.assertEqual(str, model.unescape(model._pretty_escape(str)))
def check(str): self.assertEqual(str, model.unescape(model.escape(str))) self.assertEqual(str, model.unescape(model._pretty_escape(str))) self.assertEqual(str, escaping.ununderscore_escape(escaping.underscore_escape(str)))
def check(str):
self.assertEqual(str, model.unescape(model.escape(str)))
self.assertEqual(str, model.unescape(model._pretty_escape(str)))
def testCompletion(self):
shell = None
complete = lambda *a: self.complete(*a, shell=shell)
for shell in ['zsh','fish']:
assert 'select\n' in complete(["s"], 1)
assert 'select\n' in complete([], 1)
assert 'select\n' in complete(["", "bar"], 1)
assert '' == complete(["", "bar"], 2)
assert '' == complete(["unknown", "bar"], 2)
#self.assertEqual('', complete(["--", "s"], 2))
assert '--help\n' in complete(["-"], 1)
assert '--help\n' in complete(["--"], 1)
assert '--help\n' in complete(["--h"], 1)
assert '-h\n' in complete(["-h"], 1)
assert '-hv\n' in complete(["-hv"], 1)
assert '' == complete(["-hi"], 1)
#assert '--message' not in complete(["--m"], 1)
assert '--message' in complete(["--m", "select"], 1)
assert '--message' in complete(["select", "--m"], 2)
assert '--help' in complete(["select", "foo", "--h"], 3)
assert '--help' not in complete(["run", "foo", "--h"], 3)
#assert '--help' not in complete(["select", "--version", "--h"], 3)
# Fall back to file completion for the program's arguments
self.assertEqual('file\n', complete(["run", "foo", ""], 3))
# Option value completion
assert 'file\n' in complete(["select", "--with-store"], 3)
assert 'Linux\n' in complete(["select", "--os"], 3)
assert 'x86_64\n' in complete(["select", "--cpu"], 3)
assert 'sha256new\n' in complete(["digest", "--algorithm"], 3)
# Option=value complete
for shell in ['zsh','fish']:
assert 'file\n' in complete(["select", "--with-store="], 2)
assert 'filter --cpu=x86_64\n' in complete(["select", "--cpu="], 2)
for shell in ['bash']:
assert 'file\n' in complete(["select", "--with-store", "="], 3)
assert 'file\n' in complete(["select", "--with-store", "=", "foo"], 4)
assert 'filter x86_64 \n' in complete(["select", "--cpu", "="], 3)
from zeroinstall.support import basedir
from zeroinstall.injector.namespaces import config_site
d = basedir.save_cache_path(config_site, "interfaces")
with open(os.path.join(d, model.escape('http://example.com/foo')), 'wb') as stream:
stream.write(b"<?xml version='1.0'?>"
b"<interface uri='http://example.com/foo' xmlns='http://zero-install.sourceforge.net/2004/injector/interface'>"
b"<name>-</name><summary>-</summary>"
b"<implementation version='1.2' id='12'/>"
b"<implementation version='1.5' id='15'/>"
b"</interface>")
for shell in ['bash']:
assert 'filter select \n' in complete(["sel"], 1)
self.assertEqual('prefix http://example.com/\nfile\n', complete(["select", "ht"], 2))
self.assertEqual('prefix //example.com/\nfile\n', complete(["select", "http:"], 2))
self.assertEqual('prefix //example.com/\nfile\n', complete(["select", "http:/"], 2))
self.assertEqual('filter //example.com/foo \n', complete(["select", "http://example.com/"], 2))
for shell in ['zsh','fish']:
# Check options are ignored correctly
self.assertEqual('prefix http://example.com/\nfile\n', complete(["select", "--with-store=.", "http:"], 3))
self.assertEqual('prefix http://example.com/\nfile\n', complete(["select", "http:", "--with-store=."], 2))
self.assertEqual('prefix http://example.com/\nfile\n', complete(["select", "--with-store", ".", "http:"], 4))
self.assertEqual('prefix http://example.com/\nfile\n', complete(["select", "http:", "--with-store", "."], 2))
# Version completion
self.assertEqual('filter 1.2\nfilter 1.5\n', complete(["select", "--before", "", "http://example.com/foo"], 3))
self.assertEqual('filter 1.2\nfilter 1.5\n', complete(["select", "--version", "", "http://example.com/foo"], 3))
self.assertEqual('filter 1.2..!1.2\nfilter 1.2..!1.5\n', complete(["select", "--version", "1.2..", "http://example.com/foo"], 3))
self.assertEqual('prefix http://example.com/\nfile\n', complete(["select", "--version-for", "http:", "", ], 3))
self.assertEqual('filter 1.2\nfilter 1.5\n', complete(["select", "--version-for", "http://example.com/foo", "", ], 4))
# -- before argument
self.assertEqual('prefix http://example.com/\nfile\n', complete(["select", "--", "http:"], 3))
def testCompletion(self):
shell = None
complete = lambda *a: self.complete(*a, shell=shell)
for shell in ['zsh', 'fish']:
assert 'select\n' in complete(["s"], 1)
assert 'select\n' in complete([], 1)
assert 'select\n' in complete(["", "bar"], 1)
assert '' == complete(["", "bar"], 2)
assert '' == complete(["unknown", "bar"], 2)
#self.assertEqual('', complete(["--", "s"], 2))
assert '--help\n' in complete(["-"], 1)
assert '--help\n' in complete(["--"], 1)
assert '--help\n' in complete(["--h"], 1)
assert '-h\n' in complete(["-h"], 1)
assert '-hv\n' in complete(["-hv"], 1)
assert '' == complete(["-hi"], 1)
#assert '--message' not in complete(["--m"], 1)
assert '--message' in complete(["--m", "select"], 1)
assert '--message' in complete(["select", "--m"], 2)
assert '--help' in complete(["select", "foo", "--h"], 3)
assert '--help' not in complete(["run", "foo", "--h"], 3)
#assert '--help' not in complete(["select", "--version", "--h"], 3)
# Fall back to file completion for the program's arguments
self.assertEqual('file\n', complete(["run", "foo", ""], 3))
# Option value completion
assert 'file\n' in complete(["select", "--with-store"], 3)
assert 'Linux\n' in complete(["select", "--os"], 3)
assert 'x86_64\n' in complete(["select", "--cpu"], 3)
assert 'sha256new\n' in complete(["digest", "--algorithm"], 3)
# Option=value complete
for shell in ['zsh', 'fish']:
assert 'file\n' in complete(["select", "--with-store="], 2)
assert 'filter --cpu=x86_64\n' in complete(["select", "--cpu="], 2)
for shell in ['bash']:
assert 'file\n' in complete(["select", "--with-store", "="], 3)
assert 'file\n' in complete(["select", "--with-store", "=", "foo"],
4)
assert 'filter x86_64 \n' in complete(["select", "--cpu", "="], 3)
from zeroinstall.support import basedir
from zeroinstall.injector.namespaces import config_site
d = basedir.save_cache_path(config_site, "interfaces")
with open(os.path.join(d, model.escape('http://example.com/foo')),
'wb') as stream:
stream.write(
b"<?xml version='1.0'?>"
b"<interface uri='http://example.com/foo' xmlns='http://zero-install.sourceforge.net/2004/injector/interface'>"
b"<name>-</name><summary>-</summary>"
b"<implementation version='1.2' id='12'/>"
b"<implementation version='1.5' id='15'/>"
b"</interface>")
for shell in ['bash']:
assert 'filter select \n' in complete(["sel"], 1)
self.assertEqual('prefix http://example.com/\nfile\n',
complete(["select", "ht"], 2))
self.assertEqual('prefix //example.com/\nfile\n',
complete(["select", "http:"], 2))
self.assertEqual('prefix //example.com/\nfile\n',
complete(["select", "http:/"], 2))
self.assertEqual('filter //example.com/foo \n',
complete(["select", "http://example.com/"], 2))
for shell in ['zsh', 'fish']:
# Check options are ignored correctly
self.assertEqual(
'prefix http://example.com/\nfile\n',
complete(["select", "--with-store=.", "http:"], 3))
self.assertEqual(
'prefix http://example.com/\nfile\n',
complete(["select", "http:", "--with-store=."], 2))
self.assertEqual(
'prefix http://example.com/\nfile\n',
complete(["select", "--with-store", ".", "http:"], 4))
self.assertEqual(
'prefix http://example.com/\nfile\n',
complete(["select", "http:", "--with-store", "."], 2))
# Version completion
self.assertEqual(
'filter 1.2\nfilter 1.5\n',
complete(["select", "--before", "", "http://example.com/foo"],
3))
self.assertEqual(
'filter 1.2\nfilter 1.5\n',
complete(["select", "--version", "", "http://example.com/foo"],
3))
self.assertEqual(
'filter 1.2..!1.2\nfilter 1.2..!1.5\n',
complete(
["select", "--version", "1.2..", "http://example.com/foo"],
3))
self.assertEqual(
'prefix http://example.com/\nfile\n',
complete([
"select",
"--version-for",
"http:",
"",
], 3))
self.assertEqual(
'filter 1.2\nfilter 1.5\n',
complete([
"select",
"--version-for",
"http://example.com/foo",
"",
], 4))
# -- before argument
self.assertEqual('prefix http://example.com/\nfile\n',
complete(["select", "--", "http:"], 3))
def check(str): self.assertEqual(str, model.unescape(model.escape(str))) self.assertEqual(str, model.unescape(model._pretty_escape(str))) self.assertEqual(str, escaping.ununderscore_escape(escaping.underscore_escape(str)))
def _populate_model(self):
# Find cached implementations
unowned = {} # Impl ID -> Store
duplicates = [] # TODO
for s in self.iface_cache.stores.stores:
if os.path.isdir(s.dir):
for id in os.listdir(s.dir):
if id in unowned:
duplicates.append(id)
unowned[id] = s
ok_interfaces = []
error_interfaces = []
# Look through cached interfaces for implementation owners
all = self.iface_cache.list_all_interfaces()
all.sort()
for uri in all:
iface_size = 0
try:
if os.path.isabs(uri):
cached_iface = uri
else:
cached_iface = basedir.load_first_cache(namespaces.config_site, "interfaces", model.escape(uri))
user_overrides = basedir.load_first_config(
namespaces.config_site, namespaces.config_prog, "user_overrides", model.escape(uri)
)
iface_size = size_if_exists(cached_iface) + size_if_exists(user_overrides)
iface = self.iface_cache.get_interface(uri)
except Exception, ex:
error_interfaces.append((uri, str(ex), iface_size))
else:
cached_iface = ValidInterface(iface, iface_size)
for impl in iface.implementations.values():
if impl.local_path:
cached_iface.in_cache.append(LocalImplementation(impl))
if impl.id in unowned:
cached_dir = unowned[impl.id].dir
impl_path = os.path.join(cached_dir, impl.id)
impl_size = get_size(impl_path)
cached_iface.in_cache.append(KnownImplementation(cached_iface, cached_dir, impl, impl_size))
del unowned[impl.id]
cached_iface.in_cache.sort()
ok_interfaces.append(cached_iface)
def _populate_model(self):
# Find cached implementations
unowned = {} # Impl ID -> Store
duplicates = [] # TODO
for s in self.iface_cache.stores.stores:
if os.path.isdir(s.dir):
for id in os.listdir(s.dir):
if id in unowned:
duplicates.append(id)
unowned[id] = s
ok_feeds = []
error_feeds = []
# Look through cached feeds for implementation owners
all_interfaces = self.iface_cache.list_all_interfaces()
all_feeds = {}
for uri in all_interfaces:
try:
iface = self.iface_cache.get_interface(uri)
except Exception as ex:
error_feeds.append((uri, str(ex), 0))
else:
all_feeds.update(self.iface_cache.get_feeds(iface))
for url, feed in all_feeds.items():
if not feed: continue
yield
feed_size = 0
try:
if url != feed.url:
# (e.g. for .new feeds)
raise Exception('Incorrect URL for feed (%s vs %s)' % (url, feed.url))
if os.path.isabs(url):
cached_feed = url
feed_type = LocalFeed
else:
feed_type = RemoteFeed
cached_feed = basedir.load_first_cache(namespaces.config_site,
'interfaces', model.escape(url))
user_overrides = basedir.load_first_config(namespaces.config_site,
namespaces.config_prog,
'interfaces', model._pretty_escape(url))
feed_size = size_if_exists(cached_feed) + size_if_exists(user_overrides)
except Exception as ex:
error_feeds.append((url, str(ex), feed_size))
else:
cached_feed = feed_type(feed, feed_size)
for impl in feed.implementations.values():
if impl.local_path:
cached_feed.in_cache.append(LocalImplementation(impl))
for digest in impl.digests:
if digest in unowned:
cached_dir = unowned[digest].dir
impl_path = os.path.join(cached_dir, digest)
impl_size = get_size(impl_path)
cached_feed.in_cache.append(KnownImplementation(cached_feed, cached_dir, impl, impl_size, digest))
del unowned[digest]
cached_feed.in_cache.sort()
ok_feeds.append(cached_feed)
if error_feeds:
iter = SECTION_INVALID_INTERFACES.append_to(self.raw_model)
for uri, ex, size in error_feeds:
item = InvalidFeed(uri, ex, size)
item.append_to(self.raw_model, iter)
unowned_sizes = []
local_dir = os.path.join(basedir.xdg_cache_home, '0install.net', 'implementations')
for id in unowned:
if unowned[id].dir == local_dir:
impl = UnusedImplementation(local_dir, id)
unowned_sizes.append((impl.size, impl))
if unowned_sizes:
iter = SECTION_UNOWNED_IMPLEMENTATIONS.append_to(self.raw_model)
for size, item in unowned_sizes:
item.append_to(self.raw_model, iter)
if ok_feeds:
iter = SECTION_INTERFACES.append_to(self.raw_model)
for item in ok_feeds:
yield
item.append_to(self.raw_model, iter)
self._update_sizes()
def do_install(self, archive_stream, progress_bar, archive_offset):
# Step 1. Import GPG keys
# Maybe GPG has never been run before. Let it initialse, or we'll get an error code
# from the first import... (ignore return value here)
subprocess.call([get_gpg(), '--check-trustdb', '-q'])
key_dir = os.path.join(mydir, 'keys')
for key in os.listdir(key_dir):
check_call([get_gpg(), '--import', '-q', os.path.join(key_dir, key)])
# Step 2. Import feeds and trust their signing keys
for root, dirs, files in os.walk(os.path.join(mydir, 'feeds')):
if 'latest.xml' in files:
feed_path = os.path.join(root, 'latest.xml')
icon_path = os.path.join(root, 'icon.png')
# Get URI
feed_stream = file(feed_path)
doc = qdom.parse(feed_stream)
uri = doc.getAttribute('uri')
assert uri, "Missing 'uri' attribute on root element in '%s'" % feed_path
domain = trust.domain_from_url(uri)
feed_stream.seek(0)
stream, sigs = gpg.check_stream(feed_stream)
for s in sigs:
if not trust.trust_db.is_trusted(s.fingerprint, domain):
print "Adding key %s to trusted list for %s" % (s.fingerprint, domain)
trust.trust_db.trust_key(s.fingerprint, domain)
oldest_sig = min([s.get_timestamp() for s in sigs])
try:
config.iface_cache.update_feed_from_network(uri, stream.read(), oldest_sig)
except iface_cache.ReplayAttack:
# OK, the user has a newer copy already
pass
if feed_stream != stream:
feed_stream.close()
stream.close()
if os.path.exists(icon_path):
icons_cache = basedir.save_cache_path(namespaces.config_site, 'interface_icons')
icon_file = os.path.join(icons_cache, model.escape(uri))
if not os.path.exists(icon_file):
shutil.copyfile(icon_path, icon_file)
# Step 3. Solve to find out which implementations we actually need
archive_stream.seek(archive_offset)
extract_impls = {} # Impls we need but which are compressed (ID -> Impl)
tmp = tempfile.mkdtemp(prefix = '0export-')
try:
# Create a "fake store" with the implementation in the archive
archive = tarfile.open(name=archive_stream.name, mode='r|', fileobj=archive_stream)
fake_store = FakeStore()
for tarmember in archive:
if tarmember.name.startswith('implementations'):
impl = os.path.basename(tarmember.name).split('.')[0]
fake_store.impls.add(impl)
bootstrap_store = zerostore.Store(os.path.join(mydir, 'implementations'))
stores = config.stores
toplevel_uris = [uri.strip() for uri in file(os.path.join(mydir, 'toplevel_uris'))]
ZEROINSTALL_URI = "@ZEROINSTALL_URI@"
for uri in [ZEROINSTALL_URI] + toplevel_uris:
# This is so the solver treats versions in the setup archive as 'cached',
# meaning that it will prefer using them to doing a download
stores.stores.append(bootstrap_store)
stores.stores.append(fake_store)
# Shouldn't need to download anything, but we might not have all feeds
r = requirements.Requirements(uri)
d = driver.Driver(config = config, requirements = r)
config.network_use = model.network_minimal
download_feeds = d.solve_with_downloads()
h.wait_for_blocker(download_feeds)
assert d.solver.ready, d.solver.get_failure_reason()
# Add anything chosen from the setup store to the main store
stores.stores.remove(fake_store)
stores.stores.remove(bootstrap_store)
for iface, impl in d.get_uncached_implementations():
print >>sys.stderr, "Need to import", impl
if impl.id in fake_store.impls:
# Delay extraction
extract_impls[impl.id] = impl
else:
impl_src = os.path.join(mydir, 'implementations', impl.id)
if os.path.isdir(impl_src):
stores.add_dir_to_cache(impl.id, impl_src)
else:
print >>sys.stderr, "Required impl %s (for %s) not present" % (impl, iface)
# Remember where we copied 0launch to, because we'll need it after
# the temporary directory is deleted.
if uri == ZEROINSTALL_URI:
global copied_0launch_in_cache
impl = d.solver.selections.selections[uri]
if not impl.id.startswith('package:'):
copied_0launch_in_cache = impl.get_path(stores = config.stores)
# (else we selected the distribution version of Zero Install)
finally:
shutil.rmtree(tmp)
# Count total number of bytes to extract
extract_total = 0
for impl in extract_impls.values():
impl_info = archive.getmember('implementations/' + impl.id + '.tar.bz2')
extract_total += impl_info.size
self.sent = 0
# Actually extract+import implementations in archive
archive_stream.seek(archive_offset)
archive = tarfile.open(name=archive_stream.name, mode='r|',
fileobj=archive_stream)
for tarmember in archive:
if not tarmember.name.startswith('implementations'):
continue
impl_id = tarmember.name.split('/')[1].split('.')[0]
if impl_id not in extract_impls:
print "Skip", impl_id
continue
print "Extracting", impl_id
tmp = tempfile.mkdtemp(prefix = '0export-')
try:
impl_stream = archive.extractfile(tarmember)
self.child = subprocess.Popen('bunzip2|tar xf -', shell = True, stdin = subprocess.PIPE, cwd = tmp)
mainloop = gobject.MainLoop(gobject.main_context_default())
def pipe_ready(src, cond):
data = impl_stream.read(4096)
if not data:
mainloop.quit()
self.child.stdin.close()
return False
self.sent += len(data)
if progress_bar:
progress_bar.set_fraction(float(self.sent) / extract_total)
self.child.stdin.write(data)
return True
gobject.io_add_watch(self.child.stdin, gobject.IO_OUT | gobject.IO_HUP, pipe_ready, priority = gobject.PRIORITY_LOW)
mainloop.run()
self.child.wait()
if self.child.returncode:
raise Exception("Failed to unpack archive (code %d)" % self.child.returncode)
stores.add_dir_to_cache(impl_id, tmp)
finally:
shutil.rmtree(tmp)
return toplevel_uris
