def set_selections(self, sels, set_last_checked = True): """Store a new set of selections. We include today's date in the filename so that we keep a history of previous selections (max one per day), in case we want to to roll back later.""" date = time.strftime('%Y-%m-%d') sels_file = os.path.join(self.path, 'selections-{date}.xml'.format(date = date)) dom = sels.toDOM() tmp = tempfile.NamedTemporaryFile(prefix = 'selections.xml-', dir = self.path, delete = False, mode = 'wt') try: dom.writexml(tmp, addindent=" ", newl="\n", encoding = 'utf-8') except: tmp.close() os.unlink(tmp.name) raise tmp.close() portable_rename(tmp.name, sels_file) sels_latest = os.path.join(self.path, 'selections.xml') if os.path.exists(sels_latest): os.unlink(sels_latest) os.symlink(os.path.basename(sels_file), sels_latest) if set_last_checked: self.set_last_checked()
def write_to_git(feed_path, new_xml, commit_msg, config, new_file = False): did_git_add = False git_path = relpath(feed_path, 'feeds') try: with open(feed_path + '.new', 'wb') as stream: stream.write(new_xml) support.portable_rename(feed_path + '.new', feed_path) # Commit if new_file: subprocess.check_call(['git', 'add', git_path], cwd = 'feeds') did_git_add = True # (this must be last in the try block) scm.commit('feeds', [git_path], commit_msg, key = config.GPG_SIGNING_KEY if getattr(config, 'SIGN_COMMITS', True) else None) except Exception as ex: # Roll-back (we didn't commit to Git yet) print(ex) print("Error updating feed {feed}; rolling-back...".format(feed = git_path)) if new_file: if os.path.exists(feed_path): os.unlink(feed_path) if did_git_add: subprocess.check_call(['git', 'rm', '--', git_path], cwd = 'feeds') else: subprocess.check_call(['git', 'checkout', 'HEAD', '--', git_path], cwd = 'feeds') raise
def save(self): d = basedir.save_config_path(config_site, config_prog) db_file = os.path.join(d, 'trustdb.xml') if self._dry_run: print( _("[dry-run] would update trust database {file}").format( file=db_file)) return from xml.dom import minidom import tempfile doc = minidom.Document() root = doc.createElementNS(XMLNS_TRUST, 'trusted-keys') root.setAttribute('xmlns', XMLNS_TRUST) doc.appendChild(root) for fingerprint in self.keys: keyelem = doc.createElementNS(XMLNS_TRUST, 'key') root.appendChild(keyelem) keyelem.setAttribute('fingerprint', fingerprint) for domain in self.keys[fingerprint]: domainelem = doc.createElementNS(XMLNS_TRUST, 'domain') domainelem.setAttribute('value', domain) keyelem.appendChild(domainelem) with tempfile.NamedTemporaryFile(dir=d, prefix='trust-', delete=False, mode='wt') as tmp: doc.writexml(tmp, indent="", addindent=" ", newl="\n", encoding='utf-8') support.portable_rename(tmp.name, db_file)
def _import_new_feed(self, feed_url, new_xml, modified_time): """Write new_xml into the cache. @param feed_url: the URL for the feed being updated @param new_xml: the data to write @param modified_time: when new_xml was modified @raises ReplayAttack: if the new mtime is older than the current one """ assert modified_time assert isinstance(new_xml, bytes), repr(new_xml) upstream_dir = basedir.save_cache_path(config_site, 'interfaces') cached = os.path.join(upstream_dir, escape(feed_url)) old_modified = None if os.path.exists(cached): with open(cached, 'rb') as stream: old_xml = stream.read() if old_xml == new_xml: logger.debug(_("No change")) # Update in-memory copy, in case someone else updated the disk copy self.get_feed(feed_url, force=True) return old_modified = int(os.stat(cached).st_mtime) # Do we need to write this temporary file now? try: with open(cached + '.new', 'wb') as stream: stream.write(new_xml) os.utime(cached + '.new', (modified_time, modified_time)) new_mtime = reader.check_readable(feed_url, cached + '.new') assert new_mtime == modified_time old_modified = self._get_signature_date(feed_url) or old_modified if old_modified: if new_mtime < old_modified: raise ReplayAttack( _("New feed's modification time is " "before old version!\nInterface: %(iface)s\nOld time: %(old_time)s\nNew time: %(new_time)s\n" "Refusing update.") % { 'iface': feed_url, 'old_time': _pretty_time(old_modified), 'new_time': _pretty_time(new_mtime) }) if new_mtime == old_modified: # You used to have to update the modification time manually. # Now it comes from the signature, this check isn't useful # and often causes problems when the stored format changes # (e.g., when we stopped writing last-modified attributes) pass #raise SafeException("Interface has changed, but modification time " # "hasn't! Refusing update.") except: os.unlink(cached + '.new') raise portable_rename(cached + '.new', cached) logger.debug(_("Saved as %s") % cached) self.get_feed(feed_url, force=True)
def save(self): d = basedir.save_config_path(config_site, config_prog) db_file = os.path.join(d, 'trustdb.xml') if self._dry_run: print(_("[dry-run] would update trust database {file}").format(file = db_file)) return from xml.dom import minidom import tempfile doc = minidom.Document() root = doc.createElementNS(XMLNS_TRUST, 'trusted-keys') root.setAttribute('xmlns', XMLNS_TRUST) doc.appendChild(root) for fingerprint in self.keys: keyelem = doc.createElementNS(XMLNS_TRUST, 'key') root.appendChild(keyelem) keyelem.setAttribute('fingerprint', fingerprint) for domain in self.keys[fingerprint]: domainelem = doc.createElementNS(XMLNS_TRUST, 'domain') domainelem.setAttribute('value', domain) keyelem.appendChild(domainelem) with tempfile.NamedTemporaryFile(dir = d, prefix = 'trust-', delete = False, mode = 'wt') as tmp: doc.writexml(tmp, indent = "", addindent = " ", newl = "\n", encoding = 'utf-8') support.portable_rename(tmp.name, db_file)
def set_selections(self, sels): """Store a new set of selections. We include today's date in the filename so that we keep a history of previous selections (max one per day), in case we want to to roll back later.""" date = time.strftime('%Y-%m-%d') sels_file = os.path.join(self.path, 'selections-{date}.xml'.format(date=date)) dom = sels.toDOM() tmp = tempfile.NamedTemporaryFile(prefix='selections.xml-', dir=self.path, delete=False, mode='wt') try: dom.writexml(tmp, addindent=" ", newl="\n", encoding='utf-8') except: tmp.close() os.unlink(tmp.name) raise tmp.close() portable_rename(tmp.name, sels_file) sels_latest = os.path.join(self.path, 'selections.xml') if os.path.exists(sels_latest): os.unlink(sels_latest) os.symlink(os.path.basename(sels_file), sels_latest) self.set_last_checked()
def handle(args): cmd.find_config() config = cmd.load_config() path = join(basedir.save_config_path('0install.net', '0repo'), 'repositories.json') if os.path.exists(path): with open(path, 'rb') as stream: db = json.load(stream) else: db = {} existing = db.get(config.REPOSITORY_BASE_URL, None) entry = {'type': 'local', 'path': os.getcwd()} if existing and existing == entry: print("Already registered in {path} (no changes made):\n{base}: {json}".format( path = path, base = config.REPOSITORY_BASE_URL, json = json.dumps(entry))) return db[config.REPOSITORY_BASE_URL] = entry with open(path + '.new', 'wb') as stream: json.dump(db, stream) support.portable_rename(path + '.new', path) if existing: print("Updated entry in {path} to:".format(path = path)) else: print("Created new entry in {path}:".format(path = path)) print("{base}: {json}".format(base = config.REPOSITORY_BASE_URL, json = json.dumps(entry)))
def build_slave(src_feed, archive_file, archive_dir_public_url, target_feed): try: COMPILE = [os.environ['0COMPILE']] except KeyError: # (build slave has an old 0release) COMPILE = [ '0launch', '--not-before=1.2', 'http://0install.net/2006/interfaces/0compile.xml' ] feed = support.load_feed(src_feed) src_feed = os.path.abspath(src_feed) archive_file = os.path.abspath(archive_file) target_feed = os.path.abspath(target_feed) impl, = feed.implementations.values() tmpdir = tempfile.mkdtemp(prefix='0release-') try: os.chdir(tmpdir) depdir = os.path.join(tmpdir, 'dependencies') os.mkdir(depdir) support.unpack_tarball(archive_file) portable_rename(impl.download_sources[0].extract, os.path.join(depdir, impl.id)) config = ConfigParser.RawConfigParser() config.add_section('compile') config.set('compile', 'download-base-url', archive_dir_public_url) config.set('compile', 'version-modifier', '') config.set('compile', 'interface', src_feed) config.set('compile', 'selections', '') config.set('compile', 'metadir', '0install') stream = open(os.path.join(tmpdir, '0compile.properties'), 'w') try: config.write(stream) finally: stream.close() support.check_call(COMPILE + ['build'], cwd=tmpdir) support.check_call(COMPILE + ['publish', '--target-feed', target_feed], cwd=tmpdir) # TODO: run unit-tests feed = support.load_feed(target_feed) impl = support.get_singleton_impl(feed) archive_file = support.get_archive_basename(impl) shutil.move(archive_file, os.path.join(os.path.dirname(target_feed), archive_file)) except: print "\nLeaving temporary directory %s for inspection...\n" % tmpdir raise else: shutil.rmtree(tmpdir)
def save_all(self): with open(self.path + '.new', 'wt') as stream: stream.write("# Records the absolute URL of all known archives.\n" "# To relocate archives, edit this file to contain the new addresses and run '0repo'.\n" "# Each line is 'basename SHA1 URL'\n") for basename, e in sorted(self.entries.items()): stream.write('%s %s %s\n' % (basename, e.sha1, e.url)) support.portable_rename(self.path + '.new', self.path)
def build_public_feeds(config): feeds = [] for dirpath, dirnames, filenames in os.walk('feeds'): for f in filenames: if f.endswith('.xml') and not f.startswith('.'): source_path = join(dirpath, f) public_rel_path = paths.get_public_rel_path(config, relpath(source_path, 'feeds')) target_path = join("public", public_rel_path) new_doc = generate_public_xml(config, source_path) changed = True if os.path.exists(target_path): with open(target_path, 'rb') as stream: old_doc = minidom.parse(stream) if xmltools.nodes_equal(old_doc.documentElement, new_doc.documentElement): #print("%s unchanged" % source_path) changed = False feeds.append(PublicFeed(abspath(source_path), public_rel_path, new_doc, changed)) if config.GPG_SIGNING_KEY: key_path = export_key(join('public', 'keys'), config.GPG_SIGNING_KEY) other_files = [relpath(key_path, 'public')] else: other_files = [] for public_feed in feeds: target_path = join('public', public_feed.public_rel_path) target_dir = dirname(target_path) if not os.path.isdir(target_dir): os.makedirs(target_dir) if config.GPG_SIGNING_KEY and config.GPG_PUBLIC_KEY_DIRECTORY: key_symlink_rel_path = join(dirname(public_feed.public_rel_path), config.GPG_PUBLIC_KEY_DIRECTORY, basename(key_path)) other_files.append(key_symlink_rel_path) key_symlink_path = join('public', key_symlink_rel_path) if not os.path.exists(key_symlink_path): if os.name == 'nt': import shutil shutil.copyfile(key_path, key_symlink_path) else: os.symlink(relpath(key_path, dirname(key_symlink_path)), key_symlink_path) os.stat(key_symlink_path) if not public_feed.changed: continue path_to_resources = relpath(join('public', 'resources'), dirname(target_path)) new_xml = (feed_header % path_to_resources).encode('utf-8') + public_feed.doc.documentElement.toxml('utf-8') + '\n' signed_xml = sign_xml(config, new_xml) with open(target_path + '.new', 'wb') as stream: stream.write(signed_xml) support.portable_rename(target_path + '.new', target_path) print("Updated", target_path) return feeds, other_files
def _import_new_feed(self, feed_url, new_xml, modified_time): """Write new_xml into the cache. @param feed_url: the URL for the feed being updated @param new_xml: the data to write @param modified_time: when new_xml was modified @raises ReplayAttack: if the new mtime is older than the current one """ assert modified_time assert isinstance(new_xml, bytes), repr(new_xml) upstream_dir = basedir.save_cache_path(config_site, 'interfaces') cached = os.path.join(upstream_dir, escape(feed_url)) old_modified = None if os.path.exists(cached): with open(cached, 'rb') as stream: old_xml = stream.read() if old_xml == new_xml: logger.debug(_("No change")) # Update in-memory copy, in case someone else updated the disk copy self.get_feed(feed_url, force = True) return old_modified = int(os.stat(cached).st_mtime) # Do we need to write this temporary file now? try: with open(cached + '.new', 'wb') as stream: stream.write(new_xml) os.utime(cached + '.new', (modified_time, modified_time)) new_mtime = reader.check_readable(feed_url, cached + '.new') assert new_mtime == modified_time old_modified = self._get_signature_date(feed_url) or old_modified if old_modified: if new_mtime < old_modified: raise ReplayAttack(_("New feed's modification time is " "before old version!\nInterface: %(iface)s\nOld time: %(old_time)s\nNew time: %(new_time)s\n" "Refusing update.") % {'iface': feed_url, 'old_time': _pretty_time(old_modified), 'new_time': _pretty_time(new_mtime)}) if new_mtime == old_modified: # You used to have to update the modification time manually. # Now it comes from the signature, this check isn't useful # and often causes problems when the stored format changes # (e.g., when we stopped writing last-modified attributes) pass #raise SafeException("Interface has changed, but modification time " # "hasn't! Refusing update.") except: os.unlink(cached + '.new') raise portable_rename(cached + '.new', cached) logger.debug(_("Saved as %s") % cached) self.get_feed(feed_url, force = True)
def build_public_feeds(config): feeds = [] for dirpath, dirnames, filenames in os.walk('feeds'): for f in filenames: if f.endswith('.xml') and not f.startswith('.'): source_path = join(dirpath, f) public_rel_path = paths.get_public_rel_path(config, relpath(source_path, 'feeds')) target_path = join("public", public_rel_path) new_doc = generate_public_xml(config, source_path) changed = True if os.path.exists(target_path): with open(target_path, 'rb') as stream: old_doc = minidom.parse(stream) if xmltools.nodes_equal(old_doc.documentElement, new_doc.documentElement): #print("%s unchanged" % source_path) changed = False feeds.append(PublicFeed(abspath(source_path), public_rel_path, new_doc, changed)) key_path = export_key(join('public', 'keys'), config.GPG_SIGNING_KEY) other_files = [relpath(key_path, 'public')] for public_feed in feeds: target_path = join('public', public_feed.public_rel_path) target_dir = dirname(target_path) if not os.path.isdir(target_dir): os.makedirs(target_dir) if config.GPG_PUBLIC_KEY_DIRECTORY: key_symlink_rel_path = join(dirname(public_feed.public_rel_path), config.GPG_PUBLIC_KEY_DIRECTORY, basename(key_path)) other_files.append(key_symlink_rel_path) key_symlink_path = join('public', key_symlink_rel_path) if not os.path.exists(key_symlink_path): if os.name == 'nt': import shutil shutil.copyfile(key_path, key_symlink_path) else: os.symlink(relpath(key_path, dirname(key_symlink_path)), key_symlink_path) os.stat(key_symlink_path) if not public_feed.changed: continue path_to_resources = relpath(join('public', 'resources'), dirname(target_path)) new_xml = (feed_header % path_to_resources).encode('utf-8') + public_feed.doc.documentElement.toxml('utf-8') + '\n' signed_xml = sign_xml(config, new_xml) with open(target_path + '.new', 'wb') as stream: stream.write(signed_xml) support.portable_rename(target_path + '.new', target_path) print("Updated", target_path) return feeds, other_files
def backup_if_exists(name): if not os.path.exists(name): return backup = name + '~' if os.path.exists(backup): print "(deleting old backup %s)" % backup if os.path.isdir(backup): ro_rmtree(backup) else: os.unlink(backup) portable_rename(name, backup) print "(renamed old %s as %s; will delete on next run)" % (name, backup)
def _atomic_save(doc, parent, uri): import tempfile tmp_fd, tmp_name = tempfile.mkstemp(dir=parent) try: tmp_file = os.fdopen(tmp_fd, 'w') doc.writexml(tmp_file, addindent=" ", newl='\n') tmp_file.close() path = os.path.join(parent, model._pretty_escape(uri)) support.portable_rename(tmp_name, path) except: os.unlink(tmp_name) raise
def save(self): tmp_name = release_status_file + '.new' tmp = file(tmp_name, 'w') try: lines = ["%s=%s\n" % (name, getattr(self, name)) for name in self.__slots__ if getattr(self, name)] tmp.write(''.join(lines)) tmp.close() portable_rename(tmp_name, release_status_file) info("Wrote status to %s", release_status_file) except: os.unlink(tmp_name) raise
def _atomic_save(doc, parent, uri): import tempfile tmp_fd, tmp_name = tempfile.mkstemp(dir = parent) try: tmp_file = os.fdopen(tmp_fd, 'w') doc.writexml(tmp_file, addindent = " ", newl = '\n') tmp_file.close() path = os.path.join(parent, model._pretty_escape(uri)) support.portable_rename(tmp_name, path) except: os.unlink(tmp_name) raise
def sign_xml(path, data, key): tmp = write_tmp(path, data) sigtmp = tmp + '.sig' try: run_gpg(key, '--detach-sign', '--output', sigtmp, tmp) finally: os.unlink(tmp) with open(sigtmp, 'rb') as stream: encoded = base64.encodestring(stream.read()) os.unlink(sigtmp) sig = b"<!-- Base64 Signature\n" + encoded + b"\n-->\n" support.portable_rename(write_tmp(path, data + sig), path)
def build_slave(src_feed, archive_file, archive_dir_public_url, target_feed): try: COMPILE = [os.environ['0COMPILE']] except KeyError: # (build slave has an old 0release) COMPILE = ['0launch', '--not-before=0.30', 'http://0install.net/2006/interfaces/0compile.xml'] feed = support.load_feed(src_feed) src_feed = os.path.abspath(src_feed) archive_file = os.path.abspath(archive_file) target_feed = os.path.abspath(target_feed) impl, = feed.implementations.values() tmpdir = tempfile.mkdtemp(prefix = '0release-') try: os.chdir(tmpdir) depdir = os.path.join(tmpdir, 'dependencies') os.mkdir(depdir) support.unpack_tarball(archive_file) portable_rename(impl.download_sources[0].extract, os.path.join(depdir, impl.id)) config = ConfigParser.RawConfigParser() config.add_section('compile') config.set('compile', 'download-base-url', archive_dir_public_url) config.set('compile', 'version-modifier', '') config.set('compile', 'interface', src_feed) config.set('compile', 'selections', '') config.set('compile', 'metadir', '0install') stream = open(os.path.join(tmpdir, '0compile.properties'), 'w') try: config.write(stream) finally: stream.close() support.check_call(COMPILE + ['build'], cwd = tmpdir) support.check_call(COMPILE + ['publish', '--target-feed', target_feed], cwd = tmpdir) # TODO: run unit-tests feed = support.load_feed(target_feed) impl = support.get_singleton_impl(feed) archive_file = support.get_archive_basename(impl) shutil.move(archive_file, os.path.join(os.path.dirname(target_feed), archive_file)) except: print "\nLeaving temporary directory %s for inspection...\n" % tmpdir raise else: shutil.rmtree(tmpdir)
def set_requirements(self, requirements): import json tmp = tempfile.NamedTemporaryFile(prefix = 'tmp-requirements-', dir = self.path, delete = False, mode = 'wt') try: json.dump(dict((key, getattr(requirements, key)) for key in requirements.__slots__), tmp) except: tmp.close() os.unlink(tmp.name) raise tmp.close() reqs_file = os.path.join(self.path, 'requirements.json') portable_rename(tmp.name, reqs_file)
def write_catalog(config, feeds, dir_rel_path): cat_ns = namespace.Namespace() cat_ns.register_namespace(XMLNS_CATALOG, "c") impl = minidom.getDOMImplementation() cat_doc = impl.createDocument(XMLNS_CATALOG, "c:catalog", None) cat_root = cat_doc.documentElement cat_root.setAttributeNS(XMLNS_NAMESPACE, 'xmlns:c', XMLNS_CATALOG) cat_root.setAttributeNS(XMLNS_NAMESPACE, 'xmlns', XMLNS_IFACE) custom_tags = {} for (name, ns, tags) in getattr(config, 'ADDITIONAL_CATALOG_TAGS', []): cat_ns.register_namespace(ns, name) cat_root.setAttributeNS(XMLNS_NAMESPACE, 'xmlns:' + name, ns) custom_tags[ns] = tags feed_roots = [feed.doc.documentElement for feed in feeds] def get_name(feed_root): return feed_root.getElementsByTagName('name')[0].firstChild.wholeText is_excluded_from_catalog = getattr(config, 'is_excluded_from_catalog', _default_is_excluded_from_catalog) for feed_root in sorted(feed_roots, key=get_name): if is_excluded_from_catalog(feed_root, dir_rel_path): continue elem = cat_doc.createElementNS(XMLNS_IFACE, "interface") elem.setAttribute('uri', feed_root.getAttribute("uri")) for feed_elem in feed_root.childNodes: ns = feed_elem.namespaceURI if ((ns == XMLNS_IFACE and feed_elem.localName in catalog_names) or (ns in custom_tags and feed_elem.localName in custom_tags[ns])): elem.appendChild(cat_ns.import_node(cat_doc, feed_elem)) cat_root.appendChild(elem) catalog_file = join('public', dir_rel_path, 'catalog.xml') need_update = True if os.path.exists(catalog_file): with open(catalog_file, 'rb') as stream: old_catalog = minidom.parse(stream) need_update = not xmltools.nodes_equal(old_catalog.documentElement, cat_doc.documentElement) if need_update: path_to_resources = relpath('resources', dir_rel_path).replace(os.sep, '/').encode() new_data = build.sign_xml(config, (catalog_header % path_to_resources) + cat_doc.documentElement.toxml(encoding = 'utf-8') + b'\n') with open(catalog_file + '.new', 'wb') as stream: stream.write(new_data) support.portable_rename(catalog_file + '.new', catalog_file) print("Updated " + catalog_file) return join(dir_rel_path, 'catalog.xml')
def write_catalog(config, feeds): cat_ns = namespace.Namespace() cat_ns.register_namespace(XMLNS_CATALOG, "c") impl = minidom.getDOMImplementation() cat_doc = impl.createDocument(XMLNS_CATALOG, "c:catalog", None) cat_root = cat_doc.documentElement cat_root.setAttributeNS(XMLNS_NAMESPACE, 'xmlns:c', XMLNS_CATALOG) cat_root.setAttributeNS(XMLNS_NAMESPACE, 'xmlns', XMLNS_IFACE) custom_tags = {} for (name, ns, tags) in getattr(config, 'ADDITIONAL_CATALOG_TAGS', []): cat_ns.register_namespace(ns, name) cat_root.setAttributeNS(XMLNS_NAMESPACE, 'xmlns:' + name, ns) custom_tags[ns] = tags feed_roots = [feed.doc.documentElement for feed in feeds] def get_name(feed_root): return feed_root.getElementsByTagName('name')[0].firstChild.wholeText def is_replaced(feed_root): return feed_root.getElementsByTagName('replaced-by').length > 0 for feed_root in sorted(feed_roots, key=get_name): if is_replaced(feed_root): continue elem = cat_doc.createElementNS(XMLNS_IFACE, "interface") elem.setAttribute('uri', feed_root.getAttribute("uri")) for feed_elem in feed_root.childNodes: ns = feed_elem.namespaceURI if ((ns == XMLNS_IFACE and feed_elem.localName in catalog_names) or (ns in custom_tags and feed_elem.localName in custom_tags[ns])): elem.appendChild(cat_ns.import_node(cat_doc, feed_elem)) cat_root.appendChild(elem) catalog_file = join('public', 'catalog.xml') need_update = True if os.path.exists(catalog_file): with open(catalog_file, 'rb') as stream: old_catalog = minidom.parse(stream) need_update = not xmltools.nodes_equal(old_catalog.documentElement, cat_doc.documentElement) if need_update: new_data = build.sign_xml(config, catalog_header + cat_doc.documentElement.toxml(encoding = 'utf-8') + '\n') with open(catalog_file + '.new', 'wb') as stream: stream.write(new_data) support.portable_rename(catalog_file + '.new', catalog_file) print("Updated catalog.xml") return ['catalog.xml']
def build_binaries(self): if not self.targets: return print "Source package, so generating binaries..." archive_file = support.get_archive_basename(self.src_impl) for target in self.targets: start = self.get('builder-' + target, 'start', None) command = self.config.get('builder-' + target, 'build') stop = self.get('builder-' + target, 'stop', None) binary_feed = 'binary-' + target + '.xml' if os.path.exists(binary_feed): print "Feed %s already exists; not rebuilding" % binary_feed else: print "\nBuilding binary with builder '%s' ...\n" % target if start: support.show_and_run(start, []) try: args = [ os.path.basename(self.src_feed_name), archive_file, self.archive_dir_public_url, binary_feed + '.new' ] if not command: assert target == 'host', 'Missing build command' support.check_call( [sys.executable, sys.argv[0], '--build-slave'] + args) else: support.show_and_run(command, args) finally: if stop: support.show_and_run(stop, []) bin_feed = support.load_feed(binary_feed + '.new') bin_impl = support.get_singleton_impl(bin_feed) bin_archive_file = support.get_archive_basename(bin_impl) bin_size = bin_impl.download_sources[0].size assert os.path.exists( bin_archive_file ), "Compiled binary '%s' not found!" % os.path.abspath( bin_archive_file) assert os.path.getsize( bin_archive_file ) == bin_size, "Compiled binary '%s' has wrong size!" % os.path.abspath( bin_archive_file) portable_rename(binary_feed + '.new', binary_feed)
def save_globals(self): """Write global settings.""" parser = ConfigParser.ConfigParser() parser.add_section('global') parser.set('global', 'help_with_testing', str(self.help_with_testing)) parser.set('global', 'network_use', self.network_use) parser.set('global', 'freshness', str(self.freshness)) parser.set('global', 'auto_approve_keys', str(self.auto_approve_keys)) path = basedir.save_config_path(config_site, config_prog) path = os.path.join(path, 'global') with open(path + '.new', 'wt') as stream: parser.write(stream) support.portable_rename(path + '.new', path)
def save(self): tmp_name = release_status_file + '.new' tmp = file(tmp_name, 'w') try: lines = [ "%s=%s\n" % (name, getattr(self, name)) for name in self.__slots__ if getattr(self, name) ] tmp.write(''.join(lines)) tmp.close() portable_rename(tmp_name, release_status_file) info("Wrote status to %s", release_status_file) except: os.unlink(tmp_name) raise
def set_requirements(self, requirements): reqs_file = os.path.join(self.path, 'requirements.json') if self.config.handler.dry_run: print(_("[dry-run] would write {file}").format(file = reqs_file)) else: import json tmp = tempfile.NamedTemporaryFile(prefix = 'tmp-requirements-', dir = self.path, delete = False, mode = 'wt') try: json.dump(dict((key, getattr(requirements, key)) for key in requirements.__slots__), tmp) except: tmp.close() os.unlink(tmp.name) raise tmp.close() portable_rename(tmp.name, reqs_file)
def flush(self): # Wipe the cache try: info = os.stat(self.source) mtime = int(info.st_mtime) size = info.st_size except Exception as ex: logger.warning("Failed to stat %s: %s", self.source, ex) mtime = size = 0 self.cache = {} import tempfile tmp = tempfile.NamedTemporaryFile(mode = 'wt', dir = self.cache_dir, delete = False) tmp.write("mtime=%d\nsize=%d\nformat=%d\n\n" % (mtime, size, self.format)) tmp.close() portable_rename(tmp.name, os.path.join(self.cache_dir, self.cache_leaf)) self._load_cache()
def set_selections(self, sels, set_last_checked=True): """Store a new set of selections. We include today's date in the filename so that we keep a history of previous selections (max one per day), in case we want to to roll back later. @type sels: L{zeroinstall.injector.selections.Selections} @type set_last_checked: bool""" date = time.strftime('%Y-%m-%d') sels_file = os.path.join(self.path, 'selections-{date}.xml'.format(date=date)) dom = sels.toDOM() if self.config.handler.dry_run: print( _("[dry-run] would write selections to {file}").format( file=sels_file)) else: tmp = tempfile.NamedTemporaryFile(prefix='selections.xml-', dir=self.path, delete=False, mode='wt') try: dom.writexml(tmp, addindent=" ", newl="\n", encoding='utf-8') except: tmp.close() os.unlink(tmp.name) raise tmp.close() portable_rename(tmp.name, sels_file) sels_latest = os.path.join(self.path, 'selections.xml') if self.config.handler.dry_run: print( _("[dry-run] would update {link} to point to new selections file" ).format(link=sels_latest)) else: if os.path.exists(sels_latest): os.unlink(sels_latest) if os.name == "nt": import shutil shutil.copyfile(sels_file, sels_latest) else: os.symlink(os.path.basename(sels_file), sels_latest) if set_last_checked: self.set_last_checked()
def set_requirements(self, requirements): import json tmp = tempfile.NamedTemporaryFile(prefix='tmp-requirements-', dir=self.path, delete=False, mode='wt') try: json.dump( dict((key, getattr(requirements, key)) for key in requirements.__slots__), tmp) except: tmp.close() os.unlink(tmp.name) raise tmp.close() reqs_file = os.path.join(self.path, 'requirements.json') portable_rename(tmp.name, reqs_file)
def process_archives(parent): for elem in parent.childNodes: if elem.namespaceURI != namespaces.XMLNS_IFACE: continue if elem.localName in ("archive", "file"): # Download the archive if missing href = elem.getAttribute("href") assert href, "missing href on <archive>" local_copy = os.path.join(template_dir, os.path.basename(href)) if not os.path.exists(local_copy): print("Downloading {href} to {local_copy}".format(**locals())) req = request.urlopen(href) with open(local_copy + ".part", "wb") as local_stream: shutil.copyfileobj(req, local_stream) support.portable_rename(local_copy + ".part", local_copy) req.close() # Set the size attribute elem.setAttribute("size", str(os.stat(local_copy).st_size)) if elem.localName == "archive": if not elem.hasAttribute("extract"): # Unpack (a rather inefficient way to guess the 'extract' attribute) tmpdir = unpack.unpack_to_tmp(href, local_copy, elem.getAttribute("type")) try: unpack_dir = os.path.join(tmpdir, "unpacked") # Set the extract attribute extract = unpack.guess_extract(unpack_dir) if extract: elem.setAttribute("extract", extract) unpack_dir = os.path.join(unpack_dir, extract) assert os.path.isdir(unpack_dir), "Not a directory: {dir}".format(dir=unpack_dir) finally: support.ro_rmtree(tmpdir) else: extract = elem.getAttribute("extract") if extract == "": # Remove empty element elem.removeAttribute("extract") elif elem.localName == "recipe": process_archives(elem)
def download_and_add_icon(): stream = dl.tempfile try: yield dl.downloaded tasks.check(dl.downloaded) if dl.unmodified: return stream.seek(0) import shutil, tempfile icons_cache = basedir.save_cache_path(config_site, 'interface_icons') tmp_file = tempfile.NamedTemporaryFile(dir = icons_cache, delete = False) shutil.copyfileobj(stream, tmp_file) tmp_file.close() icon_file = os.path.join(icons_cache, escape(interface.uri)) portable_rename(tmp_file.name, icon_file) finally: stream.close()
def _write_cache(self, cache): #cache.sort() # Might be useful later; currently we don't care import tempfile fd, tmpname = tempfile.mkstemp(prefix='zeroinstall-cache-tmp', dir=self.cache_dir) try: stream = os.fdopen(fd, 'wt') stream.write('version: 2\n') stream.write('mtime: %d\n' % int(self._status_details.st_mtime)) stream.write('size: %d\n' % self._status_details.st_size) stream.write('\n') for line in cache: stream.write(line + '\n') stream.close() portable_rename(tmpname, os.path.join(self.cache_dir, self.cache_leaf)) except: os.unlink(tmpname) raise
def flush(self): # Wipe the cache try: info = os.stat(self.source) mtime = int(info.st_mtime) size = info.st_size except Exception as ex: logger.warn("Failed to stat %s: %s", self.source, ex) mtime = size = 0 self.cache = {} import tempfile tmp = tempfile.NamedTemporaryFile(mode='wt', dir=self.cache_dir, delete=False) tmp.write("mtime=%d\nsize=%d\nformat=%d\n\n" % (mtime, size, self.format)) tmp.close() portable_rename(tmp.name, os.path.join(self.cache_dir, self.cache_leaf)) self._load_cache()
def _write_cache(self, cache): #cache.sort() # Might be useful later; currently we don't care import tempfile fd, tmpname = tempfile.mkstemp(prefix = 'zeroinstall-cache-tmp', dir = self.cache_dir) try: stream = os.fdopen(fd, 'wt') stream.write('version: 2\n') stream.write('mtime: %d\n' % int(self._status_details.st_mtime)) stream.write('size: %d\n' % self._status_details.st_size) stream.write('\n') for line in cache: stream.write(line + '\n') stream.close() portable_rename(tmpname, os.path.join(self.cache_dir, self.cache_leaf)) except: os.unlink(tmpname) raise
def set_requirements(self, requirements): """@type requirements: L{zeroinstall.injector.requirements.Requirements}""" reqs_file = os.path.join(self.path, 'requirements.json') if self.config.handler.dry_run: print(_("[dry-run] would write {file}").format(file=reqs_file)) else: import json tmp = tempfile.NamedTemporaryFile(prefix='tmp-requirements-', dir=self.path, delete=False, mode='wt') try: json.dump( dict((key, getattr(requirements, key)) for key in requirements.__slots__), tmp) except: tmp.close() os.unlink(tmp.name) raise tmp.close() portable_rename(tmp.name, reqs_file)
def build_binaries(self): if not self.targets: return print "Source package, so generating binaries..." archive_file = support.get_archive_basename(self.src_impl) for target in self.targets: start = self.get('builder-' + target, 'start', None) command = self.config.get('builder-' + target, 'build') stop = self.get('builder-' + target, 'stop', None) binary_feed = 'binary-' + target + '.xml' if os.path.exists(binary_feed): print "Feed %s already exists; not rebuilding" % binary_feed else: print "\nBuilding binary with builder '%s' ...\n" % target if start: support.show_and_run(start, []) try: args = [os.path.basename(self.src_feed_name), archive_file, self.archive_dir_public_url, binary_feed + '.new'] if not command: assert target == 'host', 'Missing build command' support.check_call([sys.executable, sys.argv[0], '--build-slave'] + args) else: support.show_and_run(command, args) finally: if stop: support.show_and_run(stop, []) bin_feed = support.load_feed(binary_feed + '.new') bin_impl = support.get_singleton_impl(bin_feed) bin_archive_file = support.get_archive_basename(bin_impl) bin_size = bin_impl.download_sources[0].size assert os.path.exists(bin_archive_file), "Compiled binary '%s' not found!" % os.path.abspath(bin_archive_file) assert os.path.getsize(bin_archive_file) == bin_size, "Compiled binary '%s' has wrong size!" % os.path.abspath(bin_archive_file) portable_rename(binary_feed + '.new', binary_feed)
def set_selections(self, sels, set_last_checked = True): """Store a new set of selections. We include today's date in the filename so that we keep a history of previous selections (max one per day), in case we want to to roll back later. @type sels: L{zeroinstall.injector.selections.Selections} @type set_last_checked: bool""" date = time.strftime('%Y-%m-%d') sels_file = os.path.join(self.path, 'selections-{date}.xml'.format(date = date)) dom = sels.toDOM() if self.config.handler.dry_run: print(_("[dry-run] would write selections to {file}").format(file = sels_file)) else: tmp = tempfile.NamedTemporaryFile(prefix = 'selections.xml-', dir = self.path, delete = False, mode = 'wt') try: dom.writexml(tmp, addindent=" ", newl="\n", encoding = 'utf-8') except: tmp.close() os.unlink(tmp.name) raise tmp.close() portable_rename(tmp.name, sels_file) sels_latest = os.path.join(self.path, 'selections.xml') if self.config.handler.dry_run: print(_("[dry-run] would update {link} to point to new selections file").format(link = sels_latest)) else: if os.path.exists(sels_latest): os.unlink(sels_latest) if os.name == "nt": import shutil shutil.copyfile(sels_file, sels_latest) else: os.symlink(os.path.basename(sels_file), sels_latest) if set_last_checked: self.set_last_checked()
def handle(args): cmd.find_config() config = cmd.load_config() path = join(basedir.save_config_path('0install.net', '0repo'), 'repositories.json') if os.path.exists(path): with open(path, 'rb') as stream: db = json.load(stream) else: db = {} existing = db.get(config.REPOSITORY_BASE_URL, None) entry = {'type': 'local', 'path': os.getcwd()} if existing and existing == entry: print( "Already registered in {path} (no changes made):\n{base}: {json}". format(path=path, base=config.REPOSITORY_BASE_URL, json=json.dumps(entry))) return db[config.REPOSITORY_BASE_URL] = entry with open(path + '.new', 'w') as stream: json.dump(db, stream) support.portable_rename(path + '.new', path) if existing: print("Updated entry in {path} to:".format(path=path)) else: print("Created new entry in {path}:".format(path=path)) print("{base}: {json}".format(base=config.REPOSITORY_BASE_URL, json=json.dumps(entry)))
def sign_unsigned(path, data, key): support.portable_rename(write_tmp(path, data), path)
def _import_new_feed(self, feed_url, new_xml, modified_time, dry_run): """Write new_xml into the cache. @param feed_url: the URL for the feed being updated @type feed_url: str @param new_xml: the data to write @type new_xml: str @param modified_time: when new_xml was modified @type modified_time: int @type dry_run: bool @raises ReplayAttack: if the new mtime is older than the current one""" assert modified_time assert isinstance(new_xml, bytes), repr(new_xml) upstream_dir = basedir.save_cache_path(config_site, "interfaces") cached = os.path.join(upstream_dir, escape(feed_url)) old_modified = None if os.path.exists(cached): with open(cached, "rb") as stream: old_xml = stream.read() if old_xml == new_xml: logger.debug(_("No change")) # Update in-memory copy, in case someone else updated the disk copy self.get_feed(feed_url, force=True) return old_modified = int(os.stat(cached).st_mtime) if dry_run: print(_("[dry-run] would cache feed {url} as {cached}").format(url=feed_url, cached=cached)) from io import BytesIO from zeroinstall.injector import qdom root = qdom.parse(BytesIO(new_xml), filter_for_version=True) feed = model.ZeroInstallFeed(root) reader.update_user_feed_overrides(feed) self._feeds[feed_url] = feed return # Do we need to write this temporary file now? try: with open(cached + ".new", "wb") as stream: stream.write(new_xml) os.utime(cached + ".new", (modified_time, modified_time)) new_mtime = reader.check_readable(feed_url, cached + ".new") assert new_mtime == modified_time old_modified = self._get_signature_date(feed_url) or old_modified if old_modified: if new_mtime < old_modified: raise ReplayAttack( _( "New feed's modification time is " "before old version!\nInterface: %(iface)s\nOld time: %(old_time)s\nNew time: %(new_time)s\n" "Refusing update." ) % { "iface": feed_url, "old_time": _pretty_time(old_modified), "new_time": _pretty_time(new_mtime), } ) if new_mtime == old_modified: # You used to have to update the modification time manually. # Now it comes from the signature, this check isn't useful # and often causes problems when the stored format changes # (e.g., when we stopped writing last-modified attributes) pass # raise SafeException("Interface has changed, but modification time " # "hasn't! Refusing update.") except: os.unlink(cached + ".new") raise portable_rename(cached + ".new", cached) logger.debug(_("Saved as %s") % cached) self.get_feed(feed_url, force=True)
def process(config, xml_file, delete_on_success): # Step 1 : check everything looks sensible, reject if not with open(xml_file, 'rb') as stream: xml_text = stream.read() sig_index = xml_text.rfind('\n<!-- Base64 Signature') if sig_index != -1: stream.seek(0) stream, sigs = gpg.check_stream(stream) else: sig_index = len(xml_text) sigs = [] root = qdom.parse(BytesIO(xml_text)) master = get_feed_url(root, xml_file) import_master = 'uri' in root.attrs if not import_master: root.attrs['uri'] = master # (hack so we can parse it here without setting local_path) # Check signatures are valid if config.CONTRIBUTOR_GPG_KEYS is not None: for sig in sigs: if isinstance(sig, gpg.ValidSig) and sig.fingerprint in config.CONTRIBUTOR_GPG_KEYS: break else: raise SafeException("No trusted signatures on feed {path}; signatures were: {sigs}".format( path = xml_file, sigs = ', '.join([str(s) for s in sigs]))) feed = model.ZeroInstallFeed(root) # Perform custom checks defined by the repository owner for impl in feed.implementations.values(): problem = config.check_new_impl(impl) if problem: raise SafeException("{problem} in {xml_file}\n(this check was configured in {config}: check_new_impl())".format( problem = problem, xml_file = xml_file, config = config.__file__)) feeds_rel_path = paths.get_feeds_rel_path(config, master) feed_path = join("feeds", feeds_rel_path) feed_dir = dirname(feed_path) if not os.path.isdir(feed_dir): os.makedirs(feed_dir) scm.ensure_no_uncommitted_changes(feed_path) if import_master: if os.path.exists(feed_path): with open(feed_path, 'rb') as stream: existing = stream.read() if existing == xml_text[:sig_index]: print("Already imported {feed}; skipping".format(feed = feed_path)) if delete_on_success: os.unlink(xml_file) return None else: raise SafeException("Can't import '{url}'; non-identical feed {path} already exists.\n\n" "To ADD new versions to this feed, remove the a 'uri' attribute from " "the root element in {new}.\n\n" "To EDIT the feed, just edit {path} directly rather than trying to add it again.\n\n" "To RE-UPLOAD the archives, do that manually and then edit archives.db." .format(url = feed.url, new = xml_file, path = feed_path)) # Calculate commit message if import_master: name = basename(xml_file) if name == 'feed.xml': name = basename(dirname(xml_file)) action = 'Imported {file}'.format(file = name) else: versions = set(i.get_version() for i in feed.implementations.values()) action = 'Added {name} {versions}'.format(name = feed.get_name(), versions = ', '.join(versions)) commit_msg = '%s\n\n%s' % (action, xml_text.decode('utf-8')) # Calculate new XML new_file = not os.path.exists(feed_path) git_path = relpath(feed_path, 'feeds') if import_master: assert new_file new_xml = xml_text[:sig_index] elif new_file: new_xml = create_from_local(master, xml_file) else: # Merge into existing feed try: new_doc = merge.merge_files(master, feed_path, xml_file) except merge.DuplicateIDException as ex: # Did we already import this XML? Compare with the last Git log entry. msg, previous_commit_xml = get_last_commit(git_path) if previous_commit_xml == xml_text: print("Already merged this into {feed}; skipping".format(feed = feed_path)) return msg raise ex new_xml = None # (will regenerate from new_doc below) # Step 2 : upload archives to hosting processed_archives = archives.process_archives(config, incoming_dir = dirname(xml_file), feed = feed) # Step 3 : merge XML into feeds directory # Prompt about existing testing versions if new_xml is None: new_versions = frozenset(impl.get_version() for impl in feed.implementations.values()) if len(new_versions) == 1: ask_if_previous_still_testing(new_doc, list(new_versions)[0]) new_xml = formatting.format_doc(new_doc) did_git_add = False try: with open(feed_path + '.new', 'wb') as stream: stream.write(new_xml) support.portable_rename(feed_path + '.new', feed_path) # Commit if new_file: subprocess.check_call(['git', 'add', git_path], cwd = 'feeds') did_git_add = True # (this must be last in the try block) scm.commit('feeds', [git_path], commit_msg, key = config.GPG_SIGNING_KEY) except Exception as ex: # Roll-back (we didn't commit to Git yet) print(ex) print("Error updating feed {feed}; rolling-back...".format(feed = xml_file)) if new_file: if os.path.exists(feed_path): os.unlink(feed_path) if did_git_add: subprocess.check_call(['git', 'rm', '--', git_path], cwd = 'feeds') else: subprocess.check_call(['git', 'checkout', 'HEAD', '--', git_path], cwd = 'feeds') raise # Delete XML and archives from incoming directory if delete_on_success: os.unlink(xml_file) for archive in processed_archives: os.unlink(archive.incoming_path) return commit_msg.split('\n', 1)[0]
def process(config, xml_file, delete_on_success): # Step 1 : check everything looks sensible, reject if not with open(xml_file, 'rb') as stream: xml_text = stream.read() sig_index = xml_text.rfind('\n<!-- Base64 Signature') if sig_index != -1: stream.seek(0) stream, sigs = gpg.check_stream(stream) else: sig_index = len(xml_text) sigs = [] root = qdom.parse(BytesIO(xml_text)) master = get_feed_url(root, xml_file) import_master = 'uri' in root.attrs if not import_master: root.attrs['uri'] = master # (hack so we can parse it here without setting local_path) # Check signatures are valid if config.CONTRIBUTOR_GPG_KEYS is not None: for sig in sigs: if isinstance(sig, gpg.ValidSig) and sig.fingerprint in config.CONTRIBUTOR_GPG_KEYS: break else: raise SafeException("No trusted signatures on feed {path}; signatures were: {sigs}".format( path = xml_file, sigs = ', '.join([str(s) for s in sigs]))) feed = model.ZeroInstallFeed(root) # Perform custom checks defined by the repository owner for impl in feed.implementations.values(): problem = config.check_new_impl(impl) if problem: raise SafeException("{problem} in {xml_file}\n(this check was configured in {config}: check_new_impl())".format( problem = problem, xml_file = xml_file, config = config.__file__)) feeds_rel_path = paths.get_feeds_rel_path(config, master) feed_path = join("feeds", feeds_rel_path) feed_dir = dirname(feed_path) if not os.path.isdir(feed_dir): os.makedirs(feed_dir) scm.ensure_no_uncommitted_changes(feed_path) if import_master: if os.path.exists(feed_path): with open(feed_path, 'rb') as stream: existing = stream.read() if existing == xml_text[:sig_index]: print("Already imported {feed}; skipping".format(feed = feed_path)) if delete_on_success: os.unlink(xml_file) return None else: raise SafeException("Can't import '{url}'; non-identical feed {path} already exists.\n\n" "To ADD new versions to this feed, remove the a 'uri' attribute from " "the root element in {new}.\n\n" "To EDIT the feed, just edit {path} directly rather than trying to add it again.\n\n" "To RE-UPLOAD the archives, do that manually and then edit archives.db." .format(url = feed.url, new = xml_file, path = feed_path)) # Calculate commit message if import_master: name = basename(xml_file) if name == 'feed.xml': name = basename(dirname(xml_file)) action = 'Imported {file}'.format(file = name) else: versions = set(i.get_version() for i in feed.implementations.values()) action = 'Added {name} {versions}'.format(name = feed.get_name(), versions = ', '.join(versions)) commit_msg = '%s\n\n%s' % (action, xml_text.decode('utf-8')) # Calculate new XML new_file = not os.path.exists(feed_path) git_path = relpath(feed_path, 'feeds') if import_master: assert new_file new_xml = xml_text[:sig_index] elif new_file: new_xml = create_from_local(master, xml_file) else: # Merge into existing feed try: new_doc = merge.merge_files(master, feed_path, xml_file) except merge.DuplicateIDException as ex: # Did we already import this XML? Compare with the last Git log entry. msg, previous_commit_xml = get_last_commit(git_path) if previous_commit_xml == xml_text: print("Already merged this into {feed}; skipping".format(feed = feed_path)) return msg raise ex new_xml = None # (will regenerate from new_doc below) # Step 2 : upload archives to hosting processed_archives = archives.process_archives(config, incoming_dir = dirname(xml_file), feed = feed) # Step 3 : merge XML into feeds directory # Regenerate merged feed if new_xml is None: new_versions = frozenset(impl.get_version() for impl in feed.implementations.values()) if len(new_versions) == 1 and getattr(config, 'TRACK_TESTING_IMPLS', True): ask_if_previous_still_testing(new_doc, list(new_versions)[0]) new_xml = formatting.format_doc(new_doc) did_git_add = False try: with open(feed_path + '.new', 'wb') as stream: stream.write(new_xml) support.portable_rename(feed_path + '.new', feed_path) # Commit if new_file: subprocess.check_call(['git', 'add', git_path], cwd = 'feeds') did_git_add = True # (this must be last in the try block) scm.commit('feeds', [git_path], commit_msg, key = config.GPG_SIGNING_KEY) except Exception as ex: # Roll-back (we didn't commit to Git yet) print(ex) print("Error updating feed {feed}; rolling-back...".format(feed = xml_file)) if new_file: if os.path.exists(feed_path): os.unlink(feed_path) if did_git_add: subprocess.check_call(['git', 'rm', '--', git_path], cwd = 'feeds') else: subprocess.check_call(['git', 'checkout', 'HEAD', '--', git_path], cwd = 'feeds') raise # Delete XML and archives from incoming directory if delete_on_success: os.unlink(xml_file) for archive in processed_archives: os.unlink(archive.incoming_path) return commit_msg.split('\n', 1)[0]