def run(self):
"""Pull a remote and delete it.
All pulls in our code follow the pattern pull + delete.
:raise: PayloadInstallationError if the pull fails
"""
# pull requires this for some reason
mainctx = create_new_context()
mainctx.push_thread_default()
cancellable = None
# Variable substitute the ref: https://pagure.io/atomic-wg/issue/299
ref = RpmOstree.varsubst_basearch(self._data.ref)
self.report_progress(
_("Starting pull of {branch_name} from {source}").format(
branch_name=ref, source=self._data.remote))
progress = OSTree.AsyncProgress.new()
progress.connect('changed', self._pull_progress_cb)
pull_opts = {'refs': Variant('as', [ref])}
# If we're doing a kickstart, we can at least use the content as a reference:
# See <https://github.com/rhinstaller/anaconda/issues/1117>
# The first path here is used by <https://pagure.io/fedora-lorax-templates>
# and the second by <https://github.com/projectatomic/rpm-ostree-toolbox/>
# FIXME extend tests to cover this part of code
if OSTree.check_version(2017, 8):
for path in ['/ostree/repo', '/install/ostree/repo']:
if os.path.isdir(path + '/objects'):
pull_opts['localcache-repos'] = Variant('as', [path])
break
sysroot_file = Gio.File.new_for_path(conf.target.physical_root)
sysroot = OSTree.Sysroot.new(sysroot_file)
sysroot.load(cancellable)
repo = sysroot.get_repo(None)[1]
# We don't support resuming from interrupted installs
repo.set_disable_fsync(True)
try:
repo.pull_with_options(self._data.remote,
Variant('a{sv}', pull_opts), progress,
cancellable)
except GError as e:
raise PayloadInstallationError(
"Failed to pull from repository: %s" % e) from e
log.info("ostree pull: %s", progress.get_status() or "")
self.report_progress(_("Preparing deployment of {}").format(ref))
# Now that we have the data pulled, delete the remote for now. This will allow a remote
# configuration defined in the tree (if any) to override what's in the kickstart.
# Otherwise, we'll re-add it in post. Ideally, ostree would support a pull without adding
# a remote, but that would get quite complex.
repo.remote_delete(self._data.remote, None)
mainctx.pop_thread_default()
def run(self):
# Variable substitute the ref: https://pagure.io/atomic-wg/issue/299
ref = RpmOstree.varsubst_basearch(self._data.ref)
self.report_progress(_("Deployment starting: {}").format(ref))
safe_exec_with_redirect("ostree", [
"admin", "--sysroot=" + self._sysroot, "os-init", self._data.osname
])
log.info("ostree admin deploy starting")
safe_exec_with_redirect("ostree", [
"admin", "--sysroot=" + self._sysroot, "deploy",
"--os=" + self._data.osname, self._data.remote + ':' + ref
])
log.info("ostree admin deploy complete")
self.report_progress(_("Deployment complete: {}").format(ref))
def install(self):
mainctx = create_new_context()
mainctx.push_thread_default()
cancellable = None
gi.require_version("OSTree", "1.0")
gi.require_version("RpmOstree", "1.0")
from gi.repository import OSTree, RpmOstree
ostreesetup = self.data.ostreesetup
log.info("executing ostreesetup=%r", ostreesetup)
# Initialize the filesystem - this will create the repo as well
self._safe_exec_with_redirect("ostree", [
"admin", "--sysroot=" + conf.target.physical_root, "init-fs",
conf.target.physical_root
])
# Here, we use the physical root as sysroot, because we haven't
# yet made a deployment.
sysroot_file = Gio.File.new_for_path(conf.target.physical_root)
sysroot = OSTree.Sysroot.new(sysroot_file)
sysroot.load(cancellable)
repo = sysroot.get_repo(None)[1]
# We don't support resuming from interrupted installs
repo.set_disable_fsync(True)
self._remoteOptions = {}
if hasattr(ostreesetup, 'nogpg') and ostreesetup.nogpg:
self._remoteOptions['gpg-verify'] = Variant('b', False)
if not conf.payload.verify_ssl:
self._remoteOptions['tls-permissive'] = Variant('b', True)
repo.remote_change(None, OSTree.RepoRemoteChange.ADD_IF_NOT_EXISTS,
ostreesetup.remote, ostreesetup.url,
Variant('a{sv}', self._remoteOptions), cancellable)
# Variable substitute the ref: https://pagure.io/atomic-wg/issue/299
ref = RpmOstree.varsubst_basearch(ostreesetup.ref)
progressQ.send_message(
_("Starting pull of %(branchName)s from %(source)s") % {
"branchName": ref,
"source": ostreesetup.remote
})
progress = OSTree.AsyncProgress.new()
progress.connect('changed', self._pull_progress_cb)
pull_opts = {'refs': Variant('as', [ref])}
# If we're doing a kickstart, we can at least use the content as a reference:
# See <https://github.com/rhinstaller/anaconda/issues/1117>
# The first path here is used by <https://pagure.io/fedora-lorax-templates>
# and the second by <https://github.com/projectatomic/rpm-ostree-toolbox/>
if OSTree.check_version(2017, 8):
for path in ['/ostree/repo', '/install/ostree/repo']:
if os.path.isdir(path + '/objects'):
pull_opts['localcache-repos'] = Variant('as', [path])
break
try:
repo.pull_with_options(ostreesetup.remote,
Variant('a{sv}', pull_opts), progress,
cancellable)
except GError as e:
exn = PayloadInstallError("Failed to pull from repository: %s" % e)
log.error(str(exn))
if errors.errorHandler.cb(exn) == errors.ERROR_RAISE:
progressQ.send_quit(1)
util.ipmi_abort(scripts=self.data.scripts)
sys.exit(1)
log.info("ostree pull: %s", progress.get_status() or "")
progressQ.send_message(_("Preparing deployment of %s") % (ref, ))
# Now that we have the data pulled, delete the remote for now.
# This will allow a remote configuration defined in the tree
# (if any) to override what's in the kickstart. Otherwise,
# we'll re-add it in post. Ideally, ostree would support a
# pull without adding a remote, but that would get quite
# complex.
repo.remote_delete(self.data.ostreesetup.remote, None)
self._safe_exec_with_redirect("ostree", [
"admin", "--sysroot=" + conf.target.physical_root, "os-init",
ostreesetup.osname
])
admin_deploy_args = [
"admin", "--sysroot=" + conf.target.physical_root, "deploy",
"--os=" + ostreesetup.osname
]
admin_deploy_args.append(ostreesetup.remote + ':' + ref)
log.info("ostree admin deploy starting")
progressQ.send_message(_("Deployment starting: %s") % (ref, ))
self._safe_exec_with_redirect("ostree", admin_deploy_args)
log.info("ostree admin deploy complete")
progressQ.send_message(_("Deployment complete: %s") % (ref, ))
# Reload now that we've deployed, find the path to the new deployment
sysroot.load(None)
deployments = sysroot.get_deployments()
assert len(deployments) > 0
deployment = deployments[0]
deployment_path = sysroot.get_deployment_directory(deployment)
util.set_system_root(deployment_path.get_path())
try:
self._copy_bootloader_data()
except (OSError, RuntimeError) as e:
exn = PayloadInstallError("Failed to copy bootloader data: %s" % e)
log.error(str(exn))
if errors.errorHandler.cb(exn) == errors.ERROR_RAISE:
progressQ.send_quit(1)
util.ipmi_abort(scripts=self.data.scripts)
sys.exit(1)
mainctx.pop_thread_default()
def get_basearch():
try:
return get_basearch.saved
except AttributeError:
get_basearch.saved = RpmOstree.get_basearch()
return get_basearch.saved
def install(self):
mainctx = GLib.MainContext.new()
mainctx.push_thread_default()
cancellable = None
gi.require_version("OSTree", "1.0")
gi.require_version("RpmOstree", "1.0")
from gi.repository import OSTree, RpmOstree
ostreesetup = self.data.ostreesetup
log.info("executing ostreesetup=%r", ostreesetup)
# Initialize the filesystem - this will create the repo as well
self._safeExecWithRedirect("ostree",
["admin", "--sysroot=" + iutil.getTargetPhysicalRoot(),
"init-fs", iutil.getTargetPhysicalRoot()])
# Here, we use the physical root as sysroot, because we haven't
# yet made a deployment.
sysroot_file = Gio.File.new_for_path(iutil.getTargetPhysicalRoot())
sysroot = OSTree.Sysroot.new(sysroot_file)
sysroot.load(cancellable)
repo = sysroot.get_repo(None)[1]
# We don't support resuming from interrupted installs
repo.set_disable_fsync(True)
self._remoteOptions = {}
if hasattr(ostreesetup, 'nogpg') and ostreesetup.nogpg:
self._remoteOptions['gpg-verify'] = GLib.Variant('b', False)
if flags.noverifyssl:
self._remoteOptions['tls-permissive'] = GLib.Variant('b', True)
repo.remote_change(None, OSTree.RepoRemoteChange.ADD_IF_NOT_EXISTS,
ostreesetup.remote, ostreesetup.url,
GLib.Variant('a{sv}', self._remoteOptions),
cancellable)
# Variable substitute the ref: https://pagure.io/atomic-wg/issue/299
ref = RpmOstree.varsubst_basearch(ostreesetup.ref)
progressQ.send_message(_("Starting pull of %(branchName)s from %(source)s") % \
{"branchName": ref, "source": ostreesetup.remote})
progress = OSTree.AsyncProgress.new()
progress.connect('changed', self._pullProgressCb)
pull_opts = {'refs': GLib.Variant('as', [ref])}
# If we're doing a kickstart, we can at least use the content as a reference:
# See <https://github.com/rhinstaller/anaconda/issues/1117>
# The first path here is used by <https://pagure.io/fedora-lorax-templates>
# and the second by <https://github.com/projectatomic/rpm-ostree-toolbox/>
if OSTree.check_version(2017, 8):
for path in ['/ostree/repo', '/install/ostree/repo']:
if os.path.isdir(path + '/objects'):
pull_opts['localcache-repos'] = GLib.Variant('as', [path])
break
try:
repo.pull_with_options(ostreesetup.remote,
GLib.Variant('a{sv}', pull_opts),
progress, cancellable)
except GLib.GError as e:
exn = PayloadInstallError("Failed to pull from repository: %s" % e)
log.error(str(exn))
if errors.errorHandler.cb(exn) == errors.ERROR_RAISE:
progressQ.send_quit(1)
iutil.ipmi_abort(scripts=self.data.scripts)
sys.exit(1)
log.info("ostree pull: " + (progress.get_status() or ""))
progressQ.send_message(_("Preparing deployment of %s") % (ref, ))
# Now that we have the data pulled, delete the remote for now.
# This will allow a remote configuration defined in the tree
# (if any) to override what's in the kickstart. Otherwise,
# we'll re-add it in post. Ideally, ostree would support a
# pull without adding a remote, but that would get quite
# complex.
repo.remote_delete(self.data.ostreesetup.remote, None)
self._safeExecWithRedirect("ostree",
["admin", "--sysroot=" + iutil.getTargetPhysicalRoot(),
"os-init", ostreesetup.osname])
admin_deploy_args = ["admin", "--sysroot=" + iutil.getTargetPhysicalRoot(),
"deploy", "--os=" + ostreesetup.osname]
admin_deploy_args.append(ostreesetup.remote + ':' + ref)
log.info("ostree admin deploy starting")
progressQ.send_message(_("Deployment starting: %s") % (ref, ))
self._safeExecWithRedirect("ostree", admin_deploy_args)
log.info("ostree admin deploy complete")
progressQ.send_message(_("Deployment complete: %s") % (ref, ))
# Reload now that we've deployed, find the path to the new deployment
sysroot.load(None)
deployments = sysroot.get_deployments()
assert len(deployments) > 0
deployment = deployments[0]
deployment_path = sysroot.get_deployment_directory(deployment)
iutil.setSysroot(deployment_path.get_path())
try:
self._copyBootloaderData()
except (OSError, RuntimeError) as e:
exn = PayloadInstallError("Failed to copy bootloader data: %s" % e)
log.error(str(exn))
if errors.errorHandler.cb(exn) == errors.ERROR_RAISE:
progressQ.send_quit(1)
iutil.ipmi_abort(scripts=self.data.scripts)
sys.exit(1)
mainctx.pop_thread_default()
def oscontainer_build(containers_storage, tmpdir, src, ref, image_name_and_tag,
base_image, push=False, tls_verify=True,
add_directories=[], cert_dir="", authfile="", digestfile=None,
display_name=None, labeled_pkgs=[]):
r = OSTree.Repo.new(Gio.File.new_for_path(src))
r.open(None)
[_, rev] = r.resolve_rev(ref, True)
if ref != rev:
print("Resolved {} = {}".format(ref, rev))
[_, ostree_commit, _] = r.load_commit(rev)
ostree_commitmeta = ostree_commit.get_child_value(0)
versionv = ostree_commitmeta.lookup_value(
"version", GLib.VariantType.new("s"))
if versionv:
ostree_version = versionv.get_string()
else:
ostree_version = None
podman_base_argv = ['podman']
buildah_base_argv = ['buildah']
if containers_storage is not None:
podman_base_argv.append(f"--root={containers_storage}")
buildah_base_argv.append(f"--root={containers_storage}")
if os.environ.get('container') is not None:
print("Using nested container mode due to container environment variable")
podman_base_argv.extend(NESTED_BUILD_ARGS)
buildah_base_argv.extend(NESTED_BUILD_ARGS)
else:
print("Skipping nested container mode")
# In general, we just stick with the default tmpdir set up. But if a
# workdir is provided, then we want to be sure that all the heavy I/O work
# that happens stays in there since e.g. we might be inside a tiny supermin
# appliance.
if tmpdir is not None:
os.environ['TMPDIR'] = tmpdir
bid = run_get_string(buildah_base_argv + ['from', base_image])
mnt = run_get_string(buildah_base_argv + ['mount', bid])
try:
dest_repo = os.path.join(mnt, 'srv/repo')
subprocess.check_call(['mkdir', '-p', dest_repo])
subprocess.check_call([
"ostree", "--repo=" + dest_repo, "init", "--mode=archive"])
# Note that oscontainers don't have refs; we also disable fsync
# because the repo will be put into a container image and the build
# process should handle its own fsync (or choose not to).
print("Copying ostree commit into container: {} ...".format(rev))
run_verbose(["ostree", "--repo=" + dest_repo, "pull-local", "--disable-fsync", src, rev])
for d in add_directories:
with os.scandir(d) as it:
for entry in it:
dest = os.path.join(mnt, entry.name)
subprocess.check_call(['/usr/lib/coreos-assembler/cp-reflink', entry.path, dest])
print(f"Copied in content from: {d}")
# We use /noentry to trick `podman create` into not erroring out
# on a container with no cmd/entrypoint. It won't actually be run.
config = ['--entrypoint', '["/noentry"]',
'-l', OSCONTAINER_COMMIT_LABEL + '=' + rev]
if ostree_version is not None:
config += ['-l', 'version=' + ostree_version]
base_pkgs = RpmOstree.db_query_all(r, rev, None)
for pkg in base_pkgs:
name = pkg.get_name()
if name in labeled_pkgs:
config += ['-l', f"com.coreos.rpm.{name}={pkg.get_evr()}.{pkg.get_arch()}"]
# Generate pkglist.txt in to the oscontainer at /
pkg_list_dest = os.path.join(mnt, 'pkglist.txt')
# should already be sorted, but just re-sort to be sure
nevras = sorted([pkg.get_nevra() for pkg in base_pkgs])
with open(pkg_list_dest, 'w') as f:
for nevra in nevras:
f.write(nevra)
f.write('\n')
meta = {}
builddir = None
if os.path.isfile('builds/builds.json'):
with open('builds/builds.json') as fb:
builds = json.load(fb)['builds']
latest_build = builds[0]['id']
arch = cmdlib.get_basearch()
builddir = f"builds/{latest_build}/{arch}"
metapath = f"{builddir}/meta.json"
with open(metapath) as f:
meta = json.load(f)
rhcos_commit = meta['coreos-assembler.container-config-git']['commit']
imagegit = meta.get('coreos-assembler.container-image-git')
if imagegit is not None:
cosa_commit = imagegit['commit']
config += ['-l', f"com.coreos.coreos-assembler-commit={cosa_commit}"]
config += ['-l', f"com.coreos.redhat-coreos-commit={rhcos_commit}"]
if 'extensions' in meta:
tarball = os.path.abspath(os.path.join(builddir, meta['extensions']['path']))
dest_dir = os.path.join(mnt, 'extensions')
os.makedirs(dest_dir, exist_ok=True)
run_verbose(["tar", "-xf", tarball], cwd=dest_dir)
with open(os.path.join(dest_dir, 'extensions.json')) as f:
extensions = json.load(f)
extensions_label = ';'.join([ext for (ext, obj) in extensions['extensions'].items()
if obj.get('kind', 'os-extension') == 'os-extension'])
config += ['-l', f"com.coreos.os-extensions={extensions_label}"]
for pkgname in meta['extensions']['manifest']:
if pkgname in labeled_pkgs:
evra = meta['extensions']['manifest'][pkgname]
config += ['-l', f"com.coreos.rpm.{pkgname}={evra}"]
if display_name is not None:
config += ['-l', 'io.openshift.build.version-display-names=machine-os=' + display_name,
'-l', 'io.openshift.build.versions=machine-os=' + ostree_version]
run_verbose(buildah_base_argv + ['config'] + config + [bid])
print("Committing container...")
iid = run_get_string(buildah_base_argv + ['commit', bid, image_name_and_tag])
print("{} {}".format(image_name_and_tag, iid))
finally:
subprocess.call(buildah_base_argv + ['umount', bid], stdout=subprocess.DEVNULL)
subprocess.call(buildah_base_argv + ['rm', bid], stdout=subprocess.DEVNULL)
if push:
print("Pushing container")
podCmd = podman_base_argv + ['push']
if not tls_verify:
tls_arg = '--tls-verify=false'
else:
tls_arg = '--tls-verify'
podCmd.append(tls_arg)
if authfile != "":
podCmd.append("--authfile={}".format(authfile))
if cert_dir != "":
podCmd.append("--cert-dir={}".format(cert_dir))
podCmd.append(image_name_and_tag)
if digestfile is not None:
podCmd.append(f'--digestfile={digestfile}')
run_verbose(podCmd)
elif digestfile is not None:
inspect = run_get_json(podman_base_argv + ['inspect', image_name_and_tag])[0]
with open(digestfile, 'w') as f:
f.write(inspect['Digest'])
#!/usr/bin/env python
import sys
from gi.repository import Gio, OSTree, RpmOstree
repopath, ref = sys.argv[1:3]
r = OSTree.Repo.new(Gio.File.new_for_path(repopath))
r.open(None)
qr = RpmOstree.db_query_all(r, ref, None)
print "Package list: "
for p in qr:
print p.get_nevra()
_, removed, added, modold, modnew = RpmOstree.db_diff(r, ref + '^', ref, None)
for p in removed:
print "D " + p.get_nevra()
for p in added:
print "A " + p.get_nevra()
for o, n in zip(modold, modnew):
print "M {0} {1} -> {2}".format(o.get_name(), o.get_evr(), n.get_evr())
#!/usr/bin/env python
import sys
from gi.repository import Gio, OSTree, RpmOstree
repopath, ref = sys.argv[1:3]
r = OSTree.Repo.new(Gio.File.new_for_path(repopath))
r.open(None)
qr = RpmOstree.db_query_all(r, ref, None)
print "Package list: "
for p in qr:
print p.get_nevra()
_,removed,added,modold,modnew = RpmOstree.db_diff(r, ref + '^', ref, None)
for p in removed:
print "D " + p.get_nevra()
for p in added:
print "A " + p.get_nevra()
for o,n in zip(modold, modnew):
print "M {0} {1} -> {2}".format(o.get_name(), o.get_evr(), n.get_evr())
def oscontainer_build(containers_storage,
src,
ref,
image_name_and_tag,
base_image,
push=False,
tls_verify=True,
cert_dir="",
authfile="",
inspect_out=None):
r = OSTree.Repo.new(Gio.File.new_for_path(src))
r.open(None)
[_, rev] = r.resolve_rev(ref, True)
if ref != rev:
print("Resolved {} = {}".format(ref, rev))
[_, ostree_commit, _] = r.load_commit(rev)
ostree_commitmeta = ostree_commit.get_child_value(0)
versionv = ostree_commitmeta.lookup_value("version",
GLib.VariantType.new("s"))
if versionv:
ostree_version = versionv.get_string()
else:
ostree_version = None
rootarg = '--root=' + containers_storage
bid = run_get_string(['buildah', rootarg, 'from', base_image])
mnt = run_get_string(['buildah', rootarg, 'mount', bid])
try:
dest_repo = os.path.join(mnt, 'srv/repo')
subprocess.check_call(['mkdir', '-p', dest_repo])
subprocess.check_call(
["ostree", "--repo=" + dest_repo, "init", "--mode=archive"])
# Note that oscontainers don't have refs
print("Copying ostree commit into container: {} ...".format(rev))
run_verbose(["ostree", "--repo=" + dest_repo, "pull-local", src, rev])
# Generate pkglist.txt in to the oscontainer at /
pkg_list_dest = os.path.join(mnt, 'pkglist.txt')
pkgs = RpmOstree.db_query_all(r, rev, None)
# should already be sorted, but just re-sort to be sure
nevras = sorted([pkg.get_nevra() for pkg in pkgs])
with open(pkg_list_dest, 'w') as f:
for nevra in nevras:
f.write(nevra)
f.write('\n')
# We use /noentry to trick `podman create` into not erroring out
# on a container with no cmd/entrypoint. It won't actually be run.
config = [
'--entrypoint', '["/noentry"]', '-l',
OSCONTAINER_COMMIT_LABEL + '=' + rev
]
if ostree_version is not None:
config += ['-l', 'version=' + ostree_version]
run_verbose(['buildah', rootarg, 'config'] + config + [bid])
print("Committing container...")
iid = run_get_string(
['buildah', rootarg, 'commit', bid, image_name_and_tag])
print("{} {}".format(image_name_and_tag, iid))
finally:
subprocess.call(['buildah', rootarg, 'umount', bid],
stdout=subprocess.DEVNULL)
subprocess.call(['buildah', rootarg, 'rm', bid],
stdout=subprocess.DEVNULL)
if push:
print("Pushing container")
podCmd = ['podman', rootarg, 'push']
if not tls_verify:
tls_arg = '--tls-verify=false'
else:
tls_arg = '--tls-verify'
podCmd.append(tls_arg)
if authfile != "":
podCmd.append("--authfile={}".format(authfile))
if cert_dir != "":
podCmd.append("--cert-dir={}".format(cert_dir))
podCmd.append(image_name_and_tag)
run_verbose(podCmd)
skopeoCmd = ['skopeo', 'inspect']
if authfile != "":
skopeoCmd.append("--authfile={}".format(authfile))
skopeoCmd.append("docker://" + image_name_and_tag)
inspect = run_get_json_retry(skopeoCmd)
else:
inspect = run_get_json(
['podman', rootarg, 'inspect', image_name_and_tag])[0]
if inspect_out is not None:
with open(inspect_out, 'w') as f:
json.dump(inspect, f)
#!/usr/bin/env python
import sys
from gi.repository import Gio, OSTree, RpmOstree
repopath, ref = sys.argv[1:3]
r = OSTree.Repo.new(Gio.File.new_for_path(repopath))
r.open(None)
q = RpmOstree.db_query(r, ref, None, None)
print "Package list: "
for p in q.get_packages():
print p
