def run(self): """Pull a remote and delete it. All pulls in our code follow the pattern pull + delete. :raise: PayloadInstallationError if the pull fails """ # pull requires this for some reason mainctx = create_new_context() mainctx.push_thread_default() cancellable = None # Variable substitute the ref: https://pagure.io/atomic-wg/issue/299 ref = RpmOstree.varsubst_basearch(self._data.ref) self.report_progress( _("Starting pull of {branch_name} from {source}").format( branch_name=ref, source=self._data.remote)) progress = OSTree.AsyncProgress.new() progress.connect('changed', self._pull_progress_cb) pull_opts = {'refs': Variant('as', [ref])} # If we're doing a kickstart, we can at least use the content as a reference: # See <https://github.com/rhinstaller/anaconda/issues/1117> # The first path here is used by <https://pagure.io/fedora-lorax-templates> # and the second by <https://github.com/projectatomic/rpm-ostree-toolbox/> # FIXME extend tests to cover this part of code if OSTree.check_version(2017, 8): for path in ['/ostree/repo', '/install/ostree/repo']: if os.path.isdir(path + '/objects'): pull_opts['localcache-repos'] = Variant('as', [path]) break sysroot_file = Gio.File.new_for_path(conf.target.physical_root) sysroot = OSTree.Sysroot.new(sysroot_file) sysroot.load(cancellable) repo = sysroot.get_repo(None)[1] # We don't support resuming from interrupted installs repo.set_disable_fsync(True) try: repo.pull_with_options(self._data.remote, Variant('a{sv}', pull_opts), progress, cancellable) except GError as e: raise PayloadInstallationError( "Failed to pull from repository: %s" % e) from e log.info("ostree pull: %s", progress.get_status() or "") self.report_progress(_("Preparing deployment of {}").format(ref)) # Now that we have the data pulled, delete the remote for now. This will allow a remote # configuration defined in the tree (if any) to override what's in the kickstart. # Otherwise, we'll re-add it in post. Ideally, ostree would support a pull without adding # a remote, but that would get quite complex. repo.remote_delete(self._data.remote, None) mainctx.pop_thread_default()
def run(self): # Variable substitute the ref: https://pagure.io/atomic-wg/issue/299 ref = RpmOstree.varsubst_basearch(self._data.ref) self.report_progress(_("Deployment starting: {}").format(ref)) safe_exec_with_redirect("ostree", [ "admin", "--sysroot=" + self._sysroot, "os-init", self._data.osname ]) log.info("ostree admin deploy starting") safe_exec_with_redirect("ostree", [ "admin", "--sysroot=" + self._sysroot, "deploy", "--os=" + self._data.osname, self._data.remote + ':' + ref ]) log.info("ostree admin deploy complete") self.report_progress(_("Deployment complete: {}").format(ref))
def install(self): mainctx = create_new_context() mainctx.push_thread_default() cancellable = None gi.require_version("OSTree", "1.0") gi.require_version("RpmOstree", "1.0") from gi.repository import OSTree, RpmOstree ostreesetup = self.data.ostreesetup log.info("executing ostreesetup=%r", ostreesetup) # Initialize the filesystem - this will create the repo as well self._safe_exec_with_redirect("ostree", [ "admin", "--sysroot=" + conf.target.physical_root, "init-fs", conf.target.physical_root ]) # Here, we use the physical root as sysroot, because we haven't # yet made a deployment. sysroot_file = Gio.File.new_for_path(conf.target.physical_root) sysroot = OSTree.Sysroot.new(sysroot_file) sysroot.load(cancellable) repo = sysroot.get_repo(None)[1] # We don't support resuming from interrupted installs repo.set_disable_fsync(True) self._remoteOptions = {} if hasattr(ostreesetup, 'nogpg') and ostreesetup.nogpg: self._remoteOptions['gpg-verify'] = Variant('b', False) if not conf.payload.verify_ssl: self._remoteOptions['tls-permissive'] = Variant('b', True) repo.remote_change(None, OSTree.RepoRemoteChange.ADD_IF_NOT_EXISTS, ostreesetup.remote, ostreesetup.url, Variant('a{sv}', self._remoteOptions), cancellable) # Variable substitute the ref: https://pagure.io/atomic-wg/issue/299 ref = RpmOstree.varsubst_basearch(ostreesetup.ref) progressQ.send_message( _("Starting pull of %(branchName)s from %(source)s") % { "branchName": ref, "source": ostreesetup.remote }) progress = OSTree.AsyncProgress.new() progress.connect('changed', self._pull_progress_cb) pull_opts = {'refs': Variant('as', [ref])} # If we're doing a kickstart, we can at least use the content as a reference: # See <https://github.com/rhinstaller/anaconda/issues/1117> # The first path here is used by <https://pagure.io/fedora-lorax-templates> # and the second by <https://github.com/projectatomic/rpm-ostree-toolbox/> if OSTree.check_version(2017, 8): for path in ['/ostree/repo', '/install/ostree/repo']: if os.path.isdir(path + '/objects'): pull_opts['localcache-repos'] = Variant('as', [path]) break try: repo.pull_with_options(ostreesetup.remote, Variant('a{sv}', pull_opts), progress, cancellable) except GError as e: exn = PayloadInstallError("Failed to pull from repository: %s" % e) log.error(str(exn)) if errors.errorHandler.cb(exn) == errors.ERROR_RAISE: progressQ.send_quit(1) util.ipmi_abort(scripts=self.data.scripts) sys.exit(1) log.info("ostree pull: %s", progress.get_status() or "") progressQ.send_message(_("Preparing deployment of %s") % (ref, )) # Now that we have the data pulled, delete the remote for now. # This will allow a remote configuration defined in the tree # (if any) to override what's in the kickstart. Otherwise, # we'll re-add it in post. Ideally, ostree would support a # pull without adding a remote, but that would get quite # complex. repo.remote_delete(self.data.ostreesetup.remote, None) self._safe_exec_with_redirect("ostree", [ "admin", "--sysroot=" + conf.target.physical_root, "os-init", ostreesetup.osname ]) admin_deploy_args = [ "admin", "--sysroot=" + conf.target.physical_root, "deploy", "--os=" + ostreesetup.osname ] admin_deploy_args.append(ostreesetup.remote + ':' + ref) log.info("ostree admin deploy starting") progressQ.send_message(_("Deployment starting: %s") % (ref, )) self._safe_exec_with_redirect("ostree", admin_deploy_args) log.info("ostree admin deploy complete") progressQ.send_message(_("Deployment complete: %s") % (ref, )) # Reload now that we've deployed, find the path to the new deployment sysroot.load(None) deployments = sysroot.get_deployments() assert len(deployments) > 0 deployment = deployments[0] deployment_path = sysroot.get_deployment_directory(deployment) util.set_system_root(deployment_path.get_path()) try: self._copy_bootloader_data() except (OSError, RuntimeError) as e: exn = PayloadInstallError("Failed to copy bootloader data: %s" % e) log.error(str(exn)) if errors.errorHandler.cb(exn) == errors.ERROR_RAISE: progressQ.send_quit(1) util.ipmi_abort(scripts=self.data.scripts) sys.exit(1) mainctx.pop_thread_default()
def get_basearch(): try: return get_basearch.saved except AttributeError: get_basearch.saved = RpmOstree.get_basearch() return get_basearch.saved
def install(self): mainctx = GLib.MainContext.new() mainctx.push_thread_default() cancellable = None gi.require_version("OSTree", "1.0") gi.require_version("RpmOstree", "1.0") from gi.repository import OSTree, RpmOstree ostreesetup = self.data.ostreesetup log.info("executing ostreesetup=%r", ostreesetup) # Initialize the filesystem - this will create the repo as well self._safeExecWithRedirect("ostree", ["admin", "--sysroot=" + iutil.getTargetPhysicalRoot(), "init-fs", iutil.getTargetPhysicalRoot()]) # Here, we use the physical root as sysroot, because we haven't # yet made a deployment. sysroot_file = Gio.File.new_for_path(iutil.getTargetPhysicalRoot()) sysroot = OSTree.Sysroot.new(sysroot_file) sysroot.load(cancellable) repo = sysroot.get_repo(None)[1] # We don't support resuming from interrupted installs repo.set_disable_fsync(True) self._remoteOptions = {} if hasattr(ostreesetup, 'nogpg') and ostreesetup.nogpg: self._remoteOptions['gpg-verify'] = GLib.Variant('b', False) if flags.noverifyssl: self._remoteOptions['tls-permissive'] = GLib.Variant('b', True) repo.remote_change(None, OSTree.RepoRemoteChange.ADD_IF_NOT_EXISTS, ostreesetup.remote, ostreesetup.url, GLib.Variant('a{sv}', self._remoteOptions), cancellable) # Variable substitute the ref: https://pagure.io/atomic-wg/issue/299 ref = RpmOstree.varsubst_basearch(ostreesetup.ref) progressQ.send_message(_("Starting pull of %(branchName)s from %(source)s") % \ {"branchName": ref, "source": ostreesetup.remote}) progress = OSTree.AsyncProgress.new() progress.connect('changed', self._pullProgressCb) pull_opts = {'refs': GLib.Variant('as', [ref])} # If we're doing a kickstart, we can at least use the content as a reference: # See <https://github.com/rhinstaller/anaconda/issues/1117> # The first path here is used by <https://pagure.io/fedora-lorax-templates> # and the second by <https://github.com/projectatomic/rpm-ostree-toolbox/> if OSTree.check_version(2017, 8): for path in ['/ostree/repo', '/install/ostree/repo']: if os.path.isdir(path + '/objects'): pull_opts['localcache-repos'] = GLib.Variant('as', [path]) break try: repo.pull_with_options(ostreesetup.remote, GLib.Variant('a{sv}', pull_opts), progress, cancellable) except GLib.GError as e: exn = PayloadInstallError("Failed to pull from repository: %s" % e) log.error(str(exn)) if errors.errorHandler.cb(exn) == errors.ERROR_RAISE: progressQ.send_quit(1) iutil.ipmi_abort(scripts=self.data.scripts) sys.exit(1) log.info("ostree pull: " + (progress.get_status() or "")) progressQ.send_message(_("Preparing deployment of %s") % (ref, )) # Now that we have the data pulled, delete the remote for now. # This will allow a remote configuration defined in the tree # (if any) to override what's in the kickstart. Otherwise, # we'll re-add it in post. Ideally, ostree would support a # pull without adding a remote, but that would get quite # complex. repo.remote_delete(self.data.ostreesetup.remote, None) self._safeExecWithRedirect("ostree", ["admin", "--sysroot=" + iutil.getTargetPhysicalRoot(), "os-init", ostreesetup.osname]) admin_deploy_args = ["admin", "--sysroot=" + iutil.getTargetPhysicalRoot(), "deploy", "--os=" + ostreesetup.osname] admin_deploy_args.append(ostreesetup.remote + ':' + ref) log.info("ostree admin deploy starting") progressQ.send_message(_("Deployment starting: %s") % (ref, )) self._safeExecWithRedirect("ostree", admin_deploy_args) log.info("ostree admin deploy complete") progressQ.send_message(_("Deployment complete: %s") % (ref, )) # Reload now that we've deployed, find the path to the new deployment sysroot.load(None) deployments = sysroot.get_deployments() assert len(deployments) > 0 deployment = deployments[0] deployment_path = sysroot.get_deployment_directory(deployment) iutil.setSysroot(deployment_path.get_path()) try: self._copyBootloaderData() except (OSError, RuntimeError) as e: exn = PayloadInstallError("Failed to copy bootloader data: %s" % e) log.error(str(exn)) if errors.errorHandler.cb(exn) == errors.ERROR_RAISE: progressQ.send_quit(1) iutil.ipmi_abort(scripts=self.data.scripts) sys.exit(1) mainctx.pop_thread_default()
def oscontainer_build(containers_storage, tmpdir, src, ref, image_name_and_tag, base_image, push=False, tls_verify=True, add_directories=[], cert_dir="", authfile="", digestfile=None, display_name=None, labeled_pkgs=[]): r = OSTree.Repo.new(Gio.File.new_for_path(src)) r.open(None) [_, rev] = r.resolve_rev(ref, True) if ref != rev: print("Resolved {} = {}".format(ref, rev)) [_, ostree_commit, _] = r.load_commit(rev) ostree_commitmeta = ostree_commit.get_child_value(0) versionv = ostree_commitmeta.lookup_value( "version", GLib.VariantType.new("s")) if versionv: ostree_version = versionv.get_string() else: ostree_version = None podman_base_argv = ['podman'] buildah_base_argv = ['buildah'] if containers_storage is not None: podman_base_argv.append(f"--root={containers_storage}") buildah_base_argv.append(f"--root={containers_storage}") if os.environ.get('container') is not None: print("Using nested container mode due to container environment variable") podman_base_argv.extend(NESTED_BUILD_ARGS) buildah_base_argv.extend(NESTED_BUILD_ARGS) else: print("Skipping nested container mode") # In general, we just stick with the default tmpdir set up. But if a # workdir is provided, then we want to be sure that all the heavy I/O work # that happens stays in there since e.g. we might be inside a tiny supermin # appliance. if tmpdir is not None: os.environ['TMPDIR'] = tmpdir bid = run_get_string(buildah_base_argv + ['from', base_image]) mnt = run_get_string(buildah_base_argv + ['mount', bid]) try: dest_repo = os.path.join(mnt, 'srv/repo') subprocess.check_call(['mkdir', '-p', dest_repo]) subprocess.check_call([ "ostree", "--repo=" + dest_repo, "init", "--mode=archive"]) # Note that oscontainers don't have refs; we also disable fsync # because the repo will be put into a container image and the build # process should handle its own fsync (or choose not to). print("Copying ostree commit into container: {} ...".format(rev)) run_verbose(["ostree", "--repo=" + dest_repo, "pull-local", "--disable-fsync", src, rev]) for d in add_directories: with os.scandir(d) as it: for entry in it: dest = os.path.join(mnt, entry.name) subprocess.check_call(['/usr/lib/coreos-assembler/cp-reflink', entry.path, dest]) print(f"Copied in content from: {d}") # We use /noentry to trick `podman create` into not erroring out # on a container with no cmd/entrypoint. It won't actually be run. config = ['--entrypoint', '["/noentry"]', '-l', OSCONTAINER_COMMIT_LABEL + '=' + rev] if ostree_version is not None: config += ['-l', 'version=' + ostree_version] base_pkgs = RpmOstree.db_query_all(r, rev, None) for pkg in base_pkgs: name = pkg.get_name() if name in labeled_pkgs: config += ['-l', f"com.coreos.rpm.{name}={pkg.get_evr()}.{pkg.get_arch()}"] # Generate pkglist.txt in to the oscontainer at / pkg_list_dest = os.path.join(mnt, 'pkglist.txt') # should already be sorted, but just re-sort to be sure nevras = sorted([pkg.get_nevra() for pkg in base_pkgs]) with open(pkg_list_dest, 'w') as f: for nevra in nevras: f.write(nevra) f.write('\n') meta = {} builddir = None if os.path.isfile('builds/builds.json'): with open('builds/builds.json') as fb: builds = json.load(fb)['builds'] latest_build = builds[0]['id'] arch = cmdlib.get_basearch() builddir = f"builds/{latest_build}/{arch}" metapath = f"{builddir}/meta.json" with open(metapath) as f: meta = json.load(f) rhcos_commit = meta['coreos-assembler.container-config-git']['commit'] imagegit = meta.get('coreos-assembler.container-image-git') if imagegit is not None: cosa_commit = imagegit['commit'] config += ['-l', f"com.coreos.coreos-assembler-commit={cosa_commit}"] config += ['-l', f"com.coreos.redhat-coreos-commit={rhcos_commit}"] if 'extensions' in meta: tarball = os.path.abspath(os.path.join(builddir, meta['extensions']['path'])) dest_dir = os.path.join(mnt, 'extensions') os.makedirs(dest_dir, exist_ok=True) run_verbose(["tar", "-xf", tarball], cwd=dest_dir) with open(os.path.join(dest_dir, 'extensions.json')) as f: extensions = json.load(f) extensions_label = ';'.join([ext for (ext, obj) in extensions['extensions'].items() if obj.get('kind', 'os-extension') == 'os-extension']) config += ['-l', f"com.coreos.os-extensions={extensions_label}"] for pkgname in meta['extensions']['manifest']: if pkgname in labeled_pkgs: evra = meta['extensions']['manifest'][pkgname] config += ['-l', f"com.coreos.rpm.{pkgname}={evra}"] if display_name is not None: config += ['-l', 'io.openshift.build.version-display-names=machine-os=' + display_name, '-l', 'io.openshift.build.versions=machine-os=' + ostree_version] run_verbose(buildah_base_argv + ['config'] + config + [bid]) print("Committing container...") iid = run_get_string(buildah_base_argv + ['commit', bid, image_name_and_tag]) print("{} {}".format(image_name_and_tag, iid)) finally: subprocess.call(buildah_base_argv + ['umount', bid], stdout=subprocess.DEVNULL) subprocess.call(buildah_base_argv + ['rm', bid], stdout=subprocess.DEVNULL) if push: print("Pushing container") podCmd = podman_base_argv + ['push'] if not tls_verify: tls_arg = '--tls-verify=false' else: tls_arg = '--tls-verify' podCmd.append(tls_arg) if authfile != "": podCmd.append("--authfile={}".format(authfile)) if cert_dir != "": podCmd.append("--cert-dir={}".format(cert_dir)) podCmd.append(image_name_and_tag) if digestfile is not None: podCmd.append(f'--digestfile={digestfile}') run_verbose(podCmd) elif digestfile is not None: inspect = run_get_json(podman_base_argv + ['inspect', image_name_and_tag])[0] with open(digestfile, 'w') as f: f.write(inspect['Digest'])
#!/usr/bin/env python import sys from gi.repository import Gio, OSTree, RpmOstree repopath, ref = sys.argv[1:3] r = OSTree.Repo.new(Gio.File.new_for_path(repopath)) r.open(None) qr = RpmOstree.db_query_all(r, ref, None) print "Package list: " for p in qr: print p.get_nevra() _, removed, added, modold, modnew = RpmOstree.db_diff(r, ref + '^', ref, None) for p in removed: print "D " + p.get_nevra() for p in added: print "A " + p.get_nevra() for o, n in zip(modold, modnew): print "M {0} {1} -> {2}".format(o.get_name(), o.get_evr(), n.get_evr())
#!/usr/bin/env python import sys from gi.repository import Gio, OSTree, RpmOstree repopath, ref = sys.argv[1:3] r = OSTree.Repo.new(Gio.File.new_for_path(repopath)) r.open(None) qr = RpmOstree.db_query_all(r, ref, None) print "Package list: " for p in qr: print p.get_nevra() _,removed,added,modold,modnew = RpmOstree.db_diff(r, ref + '^', ref, None) for p in removed: print "D " + p.get_nevra() for p in added: print "A " + p.get_nevra() for o,n in zip(modold, modnew): print "M {0} {1} -> {2}".format(o.get_name(), o.get_evr(), n.get_evr())
def oscontainer_build(containers_storage, src, ref, image_name_and_tag, base_image, push=False, tls_verify=True, cert_dir="", authfile="", inspect_out=None): r = OSTree.Repo.new(Gio.File.new_for_path(src)) r.open(None) [_, rev] = r.resolve_rev(ref, True) if ref != rev: print("Resolved {} = {}".format(ref, rev)) [_, ostree_commit, _] = r.load_commit(rev) ostree_commitmeta = ostree_commit.get_child_value(0) versionv = ostree_commitmeta.lookup_value("version", GLib.VariantType.new("s")) if versionv: ostree_version = versionv.get_string() else: ostree_version = None rootarg = '--root=' + containers_storage bid = run_get_string(['buildah', rootarg, 'from', base_image]) mnt = run_get_string(['buildah', rootarg, 'mount', bid]) try: dest_repo = os.path.join(mnt, 'srv/repo') subprocess.check_call(['mkdir', '-p', dest_repo]) subprocess.check_call( ["ostree", "--repo=" + dest_repo, "init", "--mode=archive"]) # Note that oscontainers don't have refs print("Copying ostree commit into container: {} ...".format(rev)) run_verbose(["ostree", "--repo=" + dest_repo, "pull-local", src, rev]) # Generate pkglist.txt in to the oscontainer at / pkg_list_dest = os.path.join(mnt, 'pkglist.txt') pkgs = RpmOstree.db_query_all(r, rev, None) # should already be sorted, but just re-sort to be sure nevras = sorted([pkg.get_nevra() for pkg in pkgs]) with open(pkg_list_dest, 'w') as f: for nevra in nevras: f.write(nevra) f.write('\n') # We use /noentry to trick `podman create` into not erroring out # on a container with no cmd/entrypoint. It won't actually be run. config = [ '--entrypoint', '["/noentry"]', '-l', OSCONTAINER_COMMIT_LABEL + '=' + rev ] if ostree_version is not None: config += ['-l', 'version=' + ostree_version] run_verbose(['buildah', rootarg, 'config'] + config + [bid]) print("Committing container...") iid = run_get_string( ['buildah', rootarg, 'commit', bid, image_name_and_tag]) print("{} {}".format(image_name_and_tag, iid)) finally: subprocess.call(['buildah', rootarg, 'umount', bid], stdout=subprocess.DEVNULL) subprocess.call(['buildah', rootarg, 'rm', bid], stdout=subprocess.DEVNULL) if push: print("Pushing container") podCmd = ['podman', rootarg, 'push'] if not tls_verify: tls_arg = '--tls-verify=false' else: tls_arg = '--tls-verify' podCmd.append(tls_arg) if authfile != "": podCmd.append("--authfile={}".format(authfile)) if cert_dir != "": podCmd.append("--cert-dir={}".format(cert_dir)) podCmd.append(image_name_and_tag) run_verbose(podCmd) skopeoCmd = ['skopeo', 'inspect'] if authfile != "": skopeoCmd.append("--authfile={}".format(authfile)) skopeoCmd.append("docker://" + image_name_and_tag) inspect = run_get_json_retry(skopeoCmd) else: inspect = run_get_json( ['podman', rootarg, 'inspect', image_name_and_tag])[0] if inspect_out is not None: with open(inspect_out, 'w') as f: json.dump(inspect, f)
#!/usr/bin/env python import sys from gi.repository import Gio, OSTree, RpmOstree repopath, ref = sys.argv[1:3] r = OSTree.Repo.new(Gio.File.new_for_path(repopath)) r.open(None) q = RpmOstree.db_query(r, ref, None, None) print "Package list: " for p in q.get_packages(): print p