def import_to_repo(self):
logging.info("Trying to use %s extractor...", self.__class__.__name__)
### Mmap the flatpak file and create a GLib.Variant from it
mapped_file = GLib.MappedFile.new(self._src_path, False)
delta = GLib.Variant.new_from_bytes(
GLib.VariantType(OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT),
mapped_file.get_bytes(), False)
### Parse flatpak metadata
# Use get_child_value instead of array index to avoid
# slowdown (constructing the whole array?)
checksum_variant = delta.get_child_value(3)
OSTree.validate_structureof_csum_v(checksum_variant)
metadata_variant = delta.get_child_value(0)
logging.debug("Metadata keys: %s", metadata_variant.keys())
commit = OSTree.checksum_from_bytes_v(checksum_variant)
self._metadata = {}
logging.debug("===== Start Metadata =====")
for key in FlatpakImporter.METADATA_KEYS:
try:
self._metadata[key] = metadata_variant[key]
except KeyError:
self._metadata[key] = ''
logging.debug(" %s: %s", key, self._metadata[key])
logging.debug("===== End Metadata =====")
self._apply_commit_to_repo(commit, self._metadata['ref'])
def validate_ostree_file(csum):
_, inputfile, file_info, xattrs = repo.load_file(csum)
# images are imported from layer tarballs, without any xattr. Don't use xattr to compute
# the OSTree object checksum.
xattrs = GLib.Variant("a(ayay)", [])
_, checksum_v = OSTree.checksum_file_from_input(file_info, xattrs, inputfile, OSTree.ObjectType.FILE)
return OSTree.checksum_from_bytes(checksum_v)
def _import_layers_into_ostree(repo, imagebranch, manifest, layers):
repo.prepare_transaction()
for layer, tar in layers.items():
mtree = OSTree.MutableTree()
repo.write_archive_to_mtree(Gio.File.new_for_path(tar), mtree, None, True)
root = repo.write_mtree(mtree)[1]
metav = GLib.Variant("a{sv}", {'docker.layer': GLib.Variant('s', layer)})
csum = repo.write_commit(None, "", None, metav, root)[1]
repo.transaction_set_ref(None, "%s%s" % (OSTREE_OCIIMAGE_PREFIX, layer), csum)
# create a $OSTREE_OCIIMAGE_PREFIX$image-$tag branch
if not isinstance(manifest, str):
manifest = json.dumps(manifest)
metadata = GLib.Variant("a{sv}", {'docker.manifest': GLib.Variant('s', manifest)})
mtree = OSTree.MutableTree()
file_info = Gio.FileInfo()
file_info.set_attribute_uint32("unix::uid", 0)
file_info.set_attribute_uint32("unix::gid", 0)
file_info.set_attribute_uint32("unix::mode", 0o755 | stat.S_IFDIR)
dirmeta = OSTree.create_directory_metadata(file_info, None)
csum_dirmeta = repo.write_metadata(OSTree.ObjectType.DIR_META, None, dirmeta)[1]
mtree.set_metadata_checksum(OSTree.checksum_from_bytes(csum_dirmeta))
root = repo.write_mtree(mtree)[1]
csum = repo.write_commit(None, "", None, metadata, root)[1]
repo.transaction_set_ref(None, imagebranch, csum)
repo.commit_transaction(None)
def validate_ostree_file(csum):
_, inputfile, file_info, xattrs = repo.load_file(csum)
# images are imported from layer tarballs, without any xattr. Don't use xattr to compute
# the OSTree object checksum.
xattrs = GLib.Variant("a(ayay)", [])
_, checksum_v = OSTree.checksum_file_from_input(file_info, xattrs, inputfile, OSTree.ObjectType.FILE)
return OSTree.checksum_from_bytes(checksum_v)
def get_system_image(rev):
commit_rev = repo.resolve_rev(rev, False)[1]
commit = repo.load_commit(commit_rev)[1]
tag = ":".join(rev.replace("ociimage/", "").rsplit('-', 1))
timestamp = OSTree.commit_get_timestamp(commit)
return {'Id' : commit_rev, 'RepoTags' : [tag], 'Names' : [], 'Created': timestamp }
def create_disks(self):
[res, rev] = self.repo.resolve_rev(self.ref, False)
[res, commit] = self.repo.load_variant(OSTree.ObjectType.COMMIT, rev)
commitdate = GLib.DateTime.new_from_unix_utc(
OSTree.commit_get_timestamp(commit)).format("%c")
print commitdate
# XXX - Define this somewhere?
imageoutputdir = os.path.join(self.outputdir, 'images')
imagedir = os.path.join(imageoutputdir, rev[:8])
if not os.path.exists(imagedir):
os.makedirs(imagedir)
imagestmpdir = os.path.join(self.workdir, 'images')
os.mkdir(imagestmpdir)
generated = []
imgtargetinstaller = os.path.join(imagestmpdir, 'install',
'%s-installer.iso' % self.os_name)
self.create_installer_image(self.workdir, imgtargetinstaller)
generated.append(imgtargetinstaller)
for f in generated:
destpath = os.path.join(imagedir, os.path.basename(f))
print "Created: " + destpath
shutil.move(f, destpath)
def prune_ostree_images(self):
repo = self._get_ostree_repo()
if not repo:
return
refs = {}
app_refs = []
for i in repo.list_refs()[1]:
if i.startswith(OSTREE_OCIIMAGE_PREFIX):
if len(i) == len(OSTREE_OCIIMAGE_PREFIX) + 64:
refs[i] = False
else:
app_refs.append(i)
def visit(rev):
manifest = self._image_manifest(repo,
repo.resolve_rev(rev, True)[1])
if not manifest:
return
for layer in SystemContainers.get_layers_from_manifest(
json.loads(manifest)):
refs[OSTREE_OCIIMAGE_PREFIX +
layer.replace("sha256:", "")] = True
for app in app_refs:
visit(app)
for k, v in refs.items():
if not v:
ref = OSTree.parse_refspec(k)
util.write_out("Deleting %s" % k)
repo.set_ref_immediate(ref[1], ref[2], None)
def run(self):
"""Pull a remote and delete it.
All pulls in our code follow the pattern pull + delete.
:raise: PayloadInstallationError if the pull fails
"""
# pull requires this for some reason
mainctx = create_new_context()
mainctx.push_thread_default()
cancellable = None
# Variable substitute the ref: https://pagure.io/atomic-wg/issue/299
ref = RpmOstree.varsubst_basearch(self._data.ref)
self.report_progress(
_("Starting pull of {branch_name} from {source}").format(
branch_name=ref, source=self._data.remote))
progress = OSTree.AsyncProgress.new()
progress.connect('changed', self._pull_progress_cb)
pull_opts = {'refs': Variant('as', [ref])}
# If we're doing a kickstart, we can at least use the content as a reference:
# See <https://github.com/rhinstaller/anaconda/issues/1117>
# The first path here is used by <https://pagure.io/fedora-lorax-templates>
# and the second by <https://github.com/projectatomic/rpm-ostree-toolbox/>
# FIXME extend tests to cover this part of code
if OSTree.check_version(2017, 8):
for path in ['/ostree/repo', '/install/ostree/repo']:
if os.path.isdir(path + '/objects'):
pull_opts['localcache-repos'] = Variant('as', [path])
break
sysroot_file = Gio.File.new_for_path(conf.target.physical_root)
sysroot = OSTree.Sysroot.new(sysroot_file)
sysroot.load(cancellable)
repo = sysroot.get_repo(None)[1]
# We don't support resuming from interrupted installs
repo.set_disable_fsync(True)
try:
repo.pull_with_options(self._data.remote,
Variant('a{sv}', pull_opts), progress,
cancellable)
except GError as e:
raise PayloadInstallationError(
"Failed to pull from repository: %s" % e) from e
log.info("ostree pull: %s", progress.get_status() or "")
self.report_progress(_("Preparing deployment of {}").format(ref))
# Now that we have the data pulled, delete the remote for now. This will allow a remote
# configuration defined in the tree (if any) to override what's in the kickstart.
# Otherwise, we'll re-add it in post. Ideally, ostree would support a pull without adding
# a remote, but that would get quite complex.
repo.remote_delete(self._data.remote, None)
mainctx.pop_thread_default()
def create_disks(self):
[res,rev] = self.repo.resolve_rev(self.ref, False)
[res,commit] = self.repo.load_variant(OSTree.ObjectType.COMMIT, rev)
commitdate = GLib.DateTime.new_from_unix_utc(OSTree.commit_get_timestamp(commit)).format("%c")
print commitdate
# XXX - Define this somewhere?
imageoutputdir=os.path.join(self.outputdir, 'images')
imagedir = os.path.join(imageoutputdir, rev[:8])
if not os.path.exists(imagedir):
os.makedirs(imagedir)
imagestmpdir = os.path.join(self.workdir, 'images')
if not os.path.exists(imagestmpdir):
os.mkdir(imagestmpdir)
generated = []
imgtargetcloud=os.path.join(imagestmpdir, self._name, '%s.qcow2' % self.os_name)
self.create_cloud_image(self.workdir, imgtargetcloud, self._kickstart)
generated.append(imgtargetcloud)
for f in generated:
destpath = os.path.join(imagedir, os.path.basename(f))
print "Created: " + destpath
shutil.move(f, destpath)
def _inspect_system_branch(self, repo, imagebranch):
commit_rev = repo.resolve_rev(imagebranch, False)[1]
commit = repo.load_commit(commit_rev)[1]
branch_id = imagebranch.replace(OSTREE_OCIIMAGE_PREFIX, "")
tag = ":".join(branch_id.rsplit('-', 1))
timestamp = OSTree.commit_get_timestamp(commit)
labels = {}
manifest = self._image_manifest(repo, commit_rev)
if len(branch_id) == 64:
image_id = branch_id
tag = "<none>"
else:
image_id = commit_rev
if manifest:
manifest = json.loads(manifest)
if 'Labels' in manifest:
labels = manifest['Labels']
if 'Digest' in manifest:
image_id = manifest['Digest'].replace("sha256:", "")
if self.user:
image_type = "user"
else:
image_type = "system"
return {'Id' : image_id, 'ImageId' : image_id, 'RepoTags' : [tag], 'Names' : [], 'Created': timestamp,
'ImageType' : image_type, 'Labels' : labels, 'OSTree-rev' : commit_rev}
def _checkout_layer(self, repo, rootfs_fd, rootfs, rev):
# ostree 2016.8 has a glib introspection safe API for checkout, use it
# when available.
if hasattr(repo, "checkout_at"):
options = OSTree.RepoCheckoutAtOptions() # pylint: disable=no-member
options.overwrite_mode = OSTree.RepoCheckoutOverwriteMode.UNION_FILES
options.process_whiteouts = True
options.disable_fsync = True
if self.user:
options.mode = OSTree.RepoCheckoutMode.USER
repo.checkout_at(options, rootfs_fd, rootfs, rev)
else:
if self.user:
user = ["--user-mode"]
else:
user = []
util.check_call(["ostree", "--repo=%s" % self.get_ostree_repo_location(),
"checkout",
"--union"] +
user +
["--whiteouts",
"--fsync=no",
rev,
rootfs],
stdin=DEVNULL,
stdout=DEVNULL,
stderr=DEVNULL)
def prune_ostree_images(self):
repo = self._get_ostree_repo()
if not repo:
return
refs = {}
app_refs = []
for i in repo.list_refs()[1]:
if i.startswith(OSTREE_OCIIMAGE_PREFIX):
if len(i) == len(OSTREE_OCIIMAGE_PREFIX) + 64:
refs[i] = False
else:
app_refs.append(i)
def visit(rev):
commit = repo.resolve_rev(rev, False)[1]
manifest = SystemContainers._get_commit_metadata(repo, commit, "docker.manifest")
if not manifest:
return
for layer in SystemContainers._get_layers_from_manifest(json.loads(manifest)):
refs[OSTREE_OCIIMAGE_PREFIX + layer.replace("sha256:", "")] = True
for app in app_refs:
visit(app)
for k, v in refs.items():
if not v:
ref = OSTree.parse_refspec(k)
self.write_out("Deleting %s" % k)
repo.set_ref_immediate(ref[1], ref[2], None)
return
def _checkout_layer(self, repo, rootfs_fd, rootfs, rev):
OSTREE_SAFE_GLIB_REPO_CHECKOUT_OPTIONS = False
# There is an issue in the way the RepoCheckoutOptions is mapped by glib, as the C
# struct is using bit fields that are not supported by the introspection.
# Accessing .disable_fsync and .process_whiteouts thus results in a segfault in
# libostree. Re-enable this once it gets fixed.
if OSTREE_SAFE_GLIB_REPO_CHECKOUT_OPTIONS:
options = OSTree.RepoCheckoutOptions() # pylint: disable=no-member
options.overwrite_mode = OSTree.RepoCheckoutOverwriteMode.UNION_FILES
options.process_whiteouts = True
options.disable_fsync = True
if self.user:
options.mode = OSTree.RepoCheckoutMode.USER
repo.checkout_tree_at(options, rootfs_fd, rootfs, rev)
else:
if self.user:
user = ["--user-mode"]
else:
user = []
util.check_call([
"ostree",
"--repo=%s" % self._get_ostree_repo_location(), "checkout",
"--union"
] + user + ["--whiteouts", "--fsync=no", rev, rootfs],
stdin=DEVNULL,
stdout=DEVNULL,
stderr=DEVNULL)
def create(self, outputdir, post=None):
[res,rev] = self.repo.resolve_rev(self.ref, False)
[res,commit] = self.repo.load_variant(OSTree.ObjectType.COMMIT, rev)
commitdate = GLib.DateTime.new_from_unix_utc(OSTree.commit_get_timestamp(commit)).format("%c")
print commitdate
lorax_opts = []
if self.local_overrides:
lorax_opts.extend([ '-s', self.local_overrides ])
if self.lorax_additional_repos:
for repourl in self.lorax_additional_repos.split(','):
lorax_opts.extend(['-s', repourl.strip()])
http_proxy = os.environ.get('http_proxy')
if http_proxy:
lorax_opts.extend([ '--proxy', http_proxy ])
template_src = self.pkgdatadir + '/lorax-embed-repo.tmpl'
template_dest = self.workdir + '/lorax-embed-repo.tmpl'
shutil.copy(template_src, template_dest)
if post is not None:
# Yeah, this is pretty awful.
post_str = '%r' % ('%post --erroronfail\n' + open(post).read() + '\n%end\n', )
with open(template_dest, 'a') as f:
f.write('\nappend usr/share/anaconda/interactive-defaults.ks %s\n' % (post_str, ))
lorax_workdir = os.path.join(self.workdir, 'lorax')
os.makedirs(lorax_workdir)
run_sync(['lorax', '--nomacboot',
'--add-template=%s' % template_dest,
'--add-template-var=ostree_osname=%s' % self.os_name,
'--add-template-var=ostree_repo=%s' % self.ostree_repo,
'--add-template-var=ostree_ref=%s' % self.ref,
'-p', self.os_pretty_name, '-v', self.release,
'-r', self.release, '-s', self.yum_baseurl,
'-e', 'systemd-container',
] + lorax_opts + ['output'],
cwd=lorax_workdir)
# We injected data into boot.iso, so it's now installer.iso
lorax_output = lorax_workdir + '/output'
lorax_images = lorax_output + '/images'
os.rename(lorax_images + '/boot.iso', lorax_images + '/installer.iso')
treeinfo = lorax_output + '/.treeinfo'
treeinfo_tmp = treeinfo + '.tmp'
with open(treeinfo) as treein:
with open(treeinfo_tmp, 'w') as treeout:
for line in treein:
if line.startswith('boot.iso'):
treeout.write(line.replace('boot.iso', 'installer.iso'))
else:
treeout.write(line)
os.rename(treeinfo_tmp, treeinfo)
for p in os.listdir(lorax_output):
print "Generated: " + p
shutil.move(os.path.join(lorax_output, p),
os.path.join(outputdir, p))
def _import_layers_into_ostree(repo, imagebranch, manifest, layers):
repo.prepare_transaction()
for layer, tar in layers.items():
mtree = OSTree.MutableTree()
def filter_func(*args):
info = args[2]
if info.get_file_type() == Gio.FileType.DIRECTORY:
info.set_attribute_uint32(
"unix::mode",
info.get_attribute_uint32("unix::mode") | stat.S_IWUSR)
return OSTree.RepoCommitFilterResult.ALLOW
modifier = OSTree.RepoCommitModifier.new(0, filter_func, None)
repo.write_archive_to_mtree(Gio.File.new_for_path(tar), mtree,
modifier, True)
root = repo.write_mtree(mtree)[1]
metav = GLib.Variant("a{sv}",
{'docker.layer': GLib.Variant('s', layer)})
csum = repo.write_commit(None, "", None, metav, root)[1]
repo.transaction_set_ref(None,
"%s%s" % (OSTREE_OCIIMAGE_PREFIX, layer),
csum)
# create a $OSTREE_OCIIMAGE_PREFIX$image-$tag branch
if not isinstance(manifest, str):
manifest = json.dumps(manifest)
metadata = GLib.Variant(
"a{sv}", {'docker.manifest': GLib.Variant('s', manifest)})
mtree = OSTree.MutableTree()
file_info = Gio.FileInfo()
file_info.set_attribute_uint32("unix::uid", 0)
file_info.set_attribute_uint32("unix::gid", 0)
file_info.set_attribute_uint32("unix::mode", 0o755 | stat.S_IFDIR)
dirmeta = OSTree.create_directory_metadata(file_info, None)
csum_dirmeta = repo.write_metadata(OSTree.ObjectType.DIR_META, None,
dirmeta)[1]
mtree.set_metadata_checksum(OSTree.checksum_from_bytes(csum_dirmeta))
root = repo.write_mtree(mtree)[1]
csum = repo.write_commit(None, "", None, metadata, root)[1]
repo.transaction_set_ref(None, imagebranch, csum)
repo.commit_transaction(None)
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.repo = OSTree.Repo.new(Gio.File.new_for_path(self.tmp.name))
self.repo.set_disable_fsync(True)
self.repo.create(OSTree.RepoMode.BARE_USER_ONLY)
self.mtree = OSTree.MutableTree()
def delete_image(self, image):
repo = self._get_ostree_repo()
if not repo:
return
imagebranch, commit_rev = self._resolve_image(repo, image)
if not commit_rev:
return
ref = OSTree.parse_refspec(imagebranch)
repo.set_ref_immediate(ref[1], ref[2], None)
def delete_image(self, image):
repo = self._get_ostree_repo()
if not repo:
return
imagebranch = SystemContainers._get_ostree_image_branch(image)
commit_rev = repo.resolve_rev(imagebranch, True)
if not commit_rev[1]:
return
ref = OSTree.parse_refspec(imagebranch)
repo.set_ref_immediate(ref[1], ref[2], None)
def delete_image(self, image):
repo = self._get_ostree_repo()
if not repo:
return
imagebranch = SystemContainers._get_ostree_image_branch(image)
commit_rev = repo.resolve_rev(imagebranch, True)
if not commit_rev[1]:
return
ref = OSTree.parse_refspec(imagebranch)
repo.set_ref_immediate(ref[1], ref[2], None)
def repo(self):
if not os.path.exists(self.ostree_repo):
# Remove the cache, if the repo. is gone ... or rpm-ostree is very
# confused.
shutil.rmtree(self.rpmostree_cache_dir)
os.makedirs(self.ostree_repo)
subprocess.check_call(
['ostree', 'init', "--repo=" + self.ostree_repo])
if self._repo is None:
self._repo = OSTree.Repo(
path=Gio.File.new_for_path(self.ostree_repo))
self._repo.open(None)
return self._repo
def checkout_container(self, container_name, rev_number):
"""
This method checks out a container into its corresponding folder, to a given
commit revision.
Before that, it stops the container using systemd, if found.
Parameters:
container_name (str): the name of the container
rev_number (str): the commit revision
"""
service = self.systemd.ListUnitsByNames([container_name + '.service'])
if service[0][2] != 'not-found':
self.logger.info("Stop the container {}".format(container_name))
self.stop_unit(container_name)
res = True
rootfs_fd = None
try:
options = OSTree.RepoCheckoutAtOptions()
options.overwrite_mode = OSTree.RepoCheckoutOverwriteMode.UNION_IDENTICAL
options.process_whiteouts = True
options.bareuseronly_dirs = True
options.no_copy_fallback = True
options.mode = OSTree.RepoCheckoutMode.USER
self.logger.info("Getting rev from repo:{}".format(container_name + ':' + container_name))
if rev_number is None:
rev = self.repo_containers.resolve_rev(container_name + ':' + container_name, False)[1]
else:
rev = rev_number
self.logger.info("Rev value:{}".format(rev))
if os.path.isdir(PATH_APPS + '/' + container_name):
shutil.rmtree(PATH_APPS + '/' + container_name)
os.mkdir(PATH_APPS + '/' + container_name)
self.logger.info("Create directory {}/{}".format(PATH_APPS, container_name))
rootfs_fd = os.open(PATH_APPS + '/' + container_name, os.O_DIRECTORY)
res = self.repo_containers.checkout_at(options, rootfs_fd, PATH_APPS + '/' + container_name, rev)
open(PATH_APPS + '/' + container_name + '/' + VALIDATE_CHECKOUT, 'a').close()
except GLib.Error as e:
self.logger.error("Checking out {} failed ({})".format(container_name, str(e)))
raise
if rootfs_fd is not None:
os.close(rootfs_fd)
if not res:
raise Exception("Checking out {} failed (returned False)")
def open_repository(repo_path):
# Open repository
repo_file = Gio.File.new_for_path(repo_path)
repo = OSTree.Repo(path=repo_file)
if os.path.exists(os.path.join(repo_path, 'config')):
logging.info('Opening repo at %s', repo_path)
repo.open()
else:
logging.info('Creating archive-z2 repo at %s', repo_path)
try:
os.makedirs(repo_path)
except OSError as err:
if err.errno != errno.EEXIST:
raise
repo.create(OSTree.RepoMode.ARCHIVE_Z2)
return repo
def repo(self):
if not os.path.exists(self.ostree_repo):
# Remove the cache, if the repo. is gone ... or rpm-ostree is very
# confused.
if (self.rpmostree_cache_dir is not None and
os.path.exists(self.rpmostree_cache_dir)):
shutil.rmtree(self.rpmostree_cache_dir)
os.makedirs(self.ostree_repo)
subprocess.check_call(['ostree', 'init',
"--repo="+self.ostree_repo, '--mode=archive-z2'])
if self._repo is None:
self._repo = OSTree.Repo(path=Gio.File.new_for_path(self.ostree_repo))
try:
self._repo.open(None)
except:
fail_msg("The repo location {0} has not been initialized. Use 'ostree --repo={0} init --mode=archive-z2' to initialize and re-run rpm-ostree-toolbox".format(self.ostree_repo))
return self._repo
def validate_layer(self, layer):
ret = []
layer = layer.replace("sha256:", "")
repo = self._get_ostree_repo()
if not repo:
return ret
def validate_ostree_file(csum):
_, inputfile, file_info, xattrs = repo.load_file(csum)
# images are imported from layer tarballs, without any xattr. Don't use xattr to compute
# the OSTree object checksum.
xattrs = GLib.Variant("a(ayay)", [])
_, checksum_v = OSTree.checksum_file_from_input(file_info, xattrs, inputfile, OSTree.ObjectType.FILE)
return OSTree.checksum_from_bytes(checksum_v)
def traverse(it):
def get_out_content_checksum(obj): return obj.out_content_checksum if hasattr(obj, 'out_content_checksum') else obj[1]
def get_out_checksum(obj): return obj.out_checksum if hasattr(obj, 'out_checksum') else obj[1]
while True:
res = it.next() # pylint: disable=next-method-called
if res == OSTree.RepoCommitIterResult.DIR:
dir_checksum = get_out_content_checksum(it.get_dir())
dir_it = OSTree.RepoCommitTraverseIter()
dirtree = repo.load_variant(OSTree.ObjectType.DIR_TREE, dir_checksum)
dir_it.init_dirtree(repo, dirtree[1], OSTree.RepoCommitTraverseFlags.REPO_COMMIT_TRAVERSE_FLAG_NONE)
traverse(dir_it)
elif res == OSTree.RepoCommitIterResult.FILE:
new_checksum = validate_ostree_file(get_out_checksum(it.get_file()))
if new_checksum != get_out_checksum(it.get_file()):
ret.append({"name" : it.get_file().out_name,
"old-checksum" : it.get_file().out_checksum,
"new-checksum" : new_checksum})
elif res == OSTree.RepoCommitIterResult.ERROR:
raise ValueError("Internal error while validating the layer")
elif res == OSTree.RepoCommitIterResult.END:
break
current_rev = repo.resolve_rev("%s%s" % (OSTREE_OCIIMAGE_PREFIX, layer), False)[1]
it = OSTree.RepoCommitTraverseIter()
it.init_commit(repo, repo.load_commit(current_rev)[1], OSTree.RepoCommitTraverseFlags.REPO_COMMIT_TRAVERSE_FLAG_NONE)
traverse(it)
return ret
def create_disks(self, outputdir):
[res,rev] = self.repo.resolve_rev(self.ref, False)
[res,commit] = self.repo.load_variant(OSTree.ObjectType.COMMIT, rev)
commitdate = GLib.DateTime.new_from_unix_utc(OSTree.commit_get_timestamp(commit)).format("%c")
print commitdate
imagestmpdir = os.path.join(self.workdir, 'images')
os.mkdir(imagestmpdir)
generated = []
imgtargetinstaller=os.path.join(imagestmpdir, 'install', '%s-installer.iso' % self.os_name)
self.create_installer_image(self.workdir, imgtargetinstaller)
generated.append(imgtargetinstaller)
for f in generated:
destpath = os.path.join(outputdir, os.path.basename(f))
print "Created: " + destpath
shutil.move(f, destpath)
def _look_up_ostree_ref_for_app_id(repo_path, app_id, remote_filter=None):
ostree_repo = OSTree.Repo.new(Gio.File.new_for_path(repo_path))
ostree_repo.open()
target_ref = None
_, refs = ostree_repo.list_refs()
logging.info("Searching for {} ref in {}".format(app_id, str(refs)))
for refspec in refs.keys():
_, remote, ref = OSTree.parse_refspec(refspec)
if remote_filter and remote != remote_filter:
logging.info("{} != {}, skipping...".format(remote, remote_filter))
continue
if str.startswith(ref, 'app/{}/'.format(app_id)):
target_ref = ref
break
return target_ref
def traverse(it):
def get_out_content_checksum(obj): return obj.out_content_checksum if hasattr(obj, 'out_content_checksum') else obj[1]
def get_out_checksum(obj): return obj.out_checksum if hasattr(obj, 'out_checksum') else obj[1]
while True:
res = it.next() # pylint: disable=next-method-called
if res == OSTree.RepoCommitIterResult.DIR:
dir_checksum = get_out_content_checksum(it.get_dir())
dir_it = OSTree.RepoCommitTraverseIter()
dirtree = repo.load_variant(OSTree.ObjectType.DIR_TREE, dir_checksum)
dir_it.init_dirtree(repo, dirtree[1], OSTree.RepoCommitTraverseFlags.REPO_COMMIT_TRAVERSE_FLAG_NONE)
traverse(dir_it)
elif res == OSTree.RepoCommitIterResult.FILE:
new_checksum = validate_ostree_file(get_out_checksum(it.get_file()))
if new_checksum != get_out_checksum(it.get_file()):
ret.append({"name" : it.get_file().out_name,
"old-checksum" : it.get_file().out_checksum,
"new-checksum" : new_checksum})
elif res == OSTree.RepoCommitIterResult.ERROR:
raise ValueError("Internal error while validating the layer")
elif res == OSTree.RepoCommitIterResult.END:
break
def checkout_container(self, container_name, rev_number):
res = True
rootfs_fd = None
try:
options = OSTree.RepoCheckoutAtOptions()
options.overwrite_mode = OSTree.RepoCheckoutOverwriteMode.UNION_IDENTICAL
options.process_whiteouts = True
options.bareuseronly_dirs = True
options.no_copy_fallback = True
options.mode = OSTree.RepoCheckoutMode.USER
self.logger.info(
"Getting rev from repo:{}".format(container_name + ':' +
container_name))
if rev_number == None:
rev = self.repo_containers.resolve_rev(
container_name + ':' + container_name, False)[1]
else:
rev = rev_number
self.logger.info("Rev value:{}".format(rev))
if os.path.isdir(PATH_APPS + '/' + container_name):
shutil.rmtree(PATH_APPS + '/' + container_name)
os.mkdir(PATH_APPS + '/' + container_name)
self.logger.info("Create directory {}/{}".format(
PATH_APPS, container_name))
rootfs_fd = os.open(PATH_APPS + '/' + container_name,
os.O_DIRECTORY)
res = self.repo_containers.checkout_at(
options, rootfs_fd, PATH_APPS + '/' + container_name, rev)
open(PATH_APPS + '/' + container_name + '/' + VALIDATE_CHECKOUT,
'a').close()
except GLib.Error as e:
self.logger.error("Checking out {} failed ({})".format(
container_name, str(e)))
res = False
if rootfs_fd != None:
os.close(rootfs_fd)
return res
def _inspect_system_branch(self, repo, imagebranch):
commit_rev = repo.resolve_rev(imagebranch, False)[1]
commit = repo.load_commit(commit_rev)[1]
branch_id = SystemContainers._decode_from_ostree_ref(imagebranch.replace(OSTREE_OCIIMAGE_PREFIX, ""))
tag = ":".join(branch_id.rsplit(":", 1))
timestamp = OSTree.commit_get_timestamp(commit)
labels = {}
manifest = self._image_manifest(repo, commit_rev)
if len(branch_id) == 64:
image_id = branch_id
tag = "<none>"
else:
image_id = commit_rev
if manifest:
manifest = json.loads(manifest)
if "Labels" in manifest:
labels = manifest["Labels"]
if "Digest" in manifest:
image_id = manifest["Digest"].replace("sha256:", "")
if self.user:
image_type = "user"
else:
image_type = "system"
return {
"Id": image_id,
"ImageId": image_id,
"RepoTags": [tag],
"Names": [],
"Created": timestamp,
"ImageType": image_type,
"Labels": labels,
"OSTree-rev": commit_rev,
}
def _inspect_system_branch(self, repo, imagebranch):
commit_rev = repo.resolve_rev(imagebranch, False)[1]
commit = repo.load_commit(commit_rev)[1]
branch_id = imagebranch.replace(OSTREE_OCIIMAGE_PREFIX, "")
tag = ":".join(branch_id.rsplit('-', 1))
timestamp = OSTree.commit_get_timestamp(commit)
labels = {}
manifest = self._image_manifest(repo, commit_rev)
if len(branch_id) == 64:
image_id = branch_id
tag = "<none>"
else:
image_id = commit_rev
if manifest:
manifest = json.loads(manifest)
if 'Labels' in manifest:
labels = manifest['Labels']
if 'Digest' in manifest:
image_id = manifest['Digest'].replace("sha256:", "")
if self.user:
image_type = "User"
else:
image_type = "System"
return {
'Id': image_id,
'RepoTags': [tag],
'Names': [],
'Created': timestamp,
'ImageType': image_type,
'Labels': labels,
'OSTree-rev': commit_rev
}
def prune_ostree_images(self):
repo = self._get_ostree_repo()
if not repo:
return
refs = {}
app_refs = []
for i in repo.list_refs()[1]:
if i.startswith(OSTREE_OCIIMAGE_PREFIX):
if len(i) == len(OSTREE_OCIIMAGE_PREFIX) + 64:
refs[i] = False
else:
invalid_encoding = False
for c in i.replace(OSTREE_OCIIMAGE_PREFIX, ""):
if not str.isalnum(str(c)) and c not in ".-_":
invalid_encoding = True
break
if invalid_encoding:
refs[i] = False
else:
app_refs.append(i)
def visit(rev):
manifest = self._image_manifest(repo, repo.resolve_rev(rev, True)[1])
if not manifest:
return
for layer in SystemContainers.get_layers_from_manifest(json.loads(manifest)):
refs[OSTREE_OCIIMAGE_PREFIX + layer.replace("sha256:", "")] = True
for app in app_refs:
visit(app)
for k, v in refs.items():
if not v:
ref = OSTree.parse_refspec(k)
util.write_out("Deleting %s" % k)
repo.set_ref_immediate(ref[1], ref[2], None)
def _inspect_system_branch(self, repo, imagebranch):
if imagebranch.startswith(OSTREE_OCIIMAGE_PREFIX):
commit_rev = repo.resolve_rev(imagebranch, False)[1]
else:
_, commit_rev = self._resolve_image(repo, imagebranch)
if commit_rev is None:
raise ValueError("Image %s not found" % imagebranch)
commit = repo.load_commit(commit_rev)[1]
branch_id = SystemContainers._decode_from_ostree_ref(imagebranch.replace(OSTREE_OCIIMAGE_PREFIX, ""))
tag = ":".join(branch_id.rsplit(':', 1))
timestamp = OSTree.commit_get_timestamp(commit)
labels = {}
manifest = self._image_manifest(repo, commit_rev)
if len(branch_id) == 64:
image_id = branch_id
tag = "<none>"
else:
image_id = commit_rev
if manifest:
manifest = json.loads(manifest)
if 'Labels' in manifest:
labels = manifest['Labels']
if 'Digest' in manifest:
image_id = manifest['Digest'].replace("sha256:", "")
if self.user:
image_type = "user"
else:
image_type = "system"
return {'Id' : image_id, 'Version' : tag, 'ImageId' : image_id, 'RepoTags' : [tag], 'Names' : [],
'Created': timestamp, 'ImageType' : image_type, 'Labels' : labels, 'OSTree-rev' : commit_rev}
def _import_layers_into_ostree(repo, imagebranch, manifest, layers):
repo.prepare_transaction()
for layer, tar in layers.items():
mtree = OSTree.MutableTree()
def filter_func(*args):
info = args[2]
if info.get_file_type() == Gio.FileType.DIRECTORY:
info.set_attribute_uint32("unix::mode", info.get_attribute_uint32("unix::mode") | stat.S_IWUSR)
return OSTree.RepoCommitFilterResult.ALLOW
modifier = OSTree.RepoCommitModifier.new(0, filter_func, None)
metav = GLib.Variant("a{sv}", {"docker.layer": GLib.Variant("s", layer)})
imported = False
try:
repo.write_archive_to_mtree(Gio.File.new_for_path(tar), mtree, modifier, True)
root = repo.write_mtree(mtree)[1]
csum = repo.write_commit(None, "", None, metav, root)[1]
imported = True
except GLib.GError as e: # pylint: disable=catching-non-exception
# libarchive which is used internally by OSTree to import a tarball doesn't support correctly
# files with xattrs. If that happens, extract the tarball and import the directory.
if e.domain != "g-io-error-quark": # pylint: disable=no-member
raise e # pylint: disable=raising-non-exception
if not imported:
try:
temp_dir = tempfile.mkdtemp()
with tarfile.open(tar, "r") as t:
t.extractall(temp_dir)
repo.write_directory_to_mtree(Gio.File.new_for_path(temp_dir), mtree, modifier)
root = repo.write_mtree(mtree)[1]
csum = repo.write_commit(None, "", None, metav, root)[1]
finally:
shutil.rmtree(temp_dir)
root = repo.write_mtree(mtree)[1]
csum = repo.write_commit(None, "", None, metav, root)[1]
repo.transaction_set_ref(None, "%s%s" % (OSTREE_OCIIMAGE_PREFIX, layer), csum)
# create a $OSTREE_OCIIMAGE_PREFIX$image-$tag branch
if not isinstance(manifest, str):
manifest = json.dumps(manifest)
metadata = GLib.Variant("a{sv}", {"docker.manifest": GLib.Variant("s", manifest)})
mtree = OSTree.MutableTree()
file_info = Gio.FileInfo()
file_info.set_attribute_uint32("unix::uid", 0)
file_info.set_attribute_uint32("unix::gid", 0)
file_info.set_attribute_uint32("unix::mode", 0o755 | stat.S_IFDIR)
dirmeta = OSTree.create_directory_metadata(file_info, None)
csum_dirmeta = repo.write_metadata(OSTree.ObjectType.DIR_META, None, dirmeta)[1]
mtree.set_metadata_checksum(OSTree.checksum_from_bytes(csum_dirmeta))
root = repo.write_mtree(mtree)[1]
csum = repo.write_commit(None, "", None, metadata, root)[1]
repo.transaction_set_ref(None, imagebranch, csum)
repo.commit_transaction(None)
def create(self, outputdir, name, ksfile, tdl, imageouttypes):
self._name = name
self._tdl = tdl
self._kickstart = ksfile
imgfunc = ImageFunctions()
[res, rev] = self.repo.resolve_rev(self.ref, False)
[res, commit] = self.repo.load_variant(OSTree.ObjectType.COMMIT, rev)
commitdate = GLib.DateTime.new_from_unix_utc(OSTree.commit_get_timestamp(commit)).format("%c")
print commitdate
port_file_path = self.workdir + '/repo-port'
# Start trivial-httpd
trivhttp = TrivialHTTP()
trivhttp.start(self.ostree_repo)
httpd_port = str(trivhttp.http_port)
print "trivial httpd port=%s, pid=%s" % (httpd_port, trivhttp.http_pid)
ks_basename = os.path.basename(ksfile)
flattened_ks = os.path.join(self.workdir, ks_basename)
# FIXME - eventually stop hardcoding this via some mapping
if ks_basename.find('fedora') >= 0:
kickstart_version = 'F21'
else:
kickstart_version = 'RHEL7'
run_sync(['ksflatten', '--version', kickstart_version,
'-c', ksfile, '-o', flattened_ks])
# TODO: Pull kickstart from separate git repo
ksdata = open(flattened_ks).read()
substitutions = { 'OSTREE_PORT': httpd_port,
'OSTREE_REF': self.ref,
'OSTREE_OSNAME': self.os_name}
if '@OSTREE_HOST_IP@' in ksdata:
host_ip = getDefaultIP(hostnet=self.virtnetwork)
substitutions['OSTREE_HOST_IP'] = host_ip
for subname, subval in substitutions.iteritems():
print subname, subval
ksdata = ksdata.replace('@%s@' % (subname, ), subval)
imgfunc.checkoz()
parameters = { "install_script": ksdata,
"generate_icicle": False,
"oz_overrides": json.dumps(imgfunc.ozoverrides)
}
print "Starting build"
image = self.builder.build(template=open(self._tdl).read(), parameters=parameters)
# For debug, you can comment out the above and enable the code below
# to skip the initial image creation. Just point myuuid at the proper
# image uuid
# self.builder.download()
# myuuid = "fd301dce-fba3-421d-a2e8-182cf2cefaf8"
# pim = PersistentImageManager.default_manager()
# image = pim.image_with_id(myuuid)
imageoutputdir = os.path.join(outputdir, "images")
if not os.path.exists(imageoutputdir):
os.mkdir(imageoutputdir)
# Copy the qcow2 file to the outputdir
outputname = os.path.join(imageoutputdir, '%s.qcow2' % (self.os_nr))
shutil.copyfile(image.data, outputname)
print "Created: {0}".format(outputname)
if 'raw' in imageouttypes:
print "Processing image from qcow2 to raw"
print image.data
outputname = os.path.join(imageoutputdir, '%s.raw' % (self.os_nr))
print outputname
qemucmd = ['qemu-img', 'convert', '-f', 'qcow2', '-O', 'raw', image.data, outputname]
subprocess.check_call(qemucmd)
imageouttypes.pop(imageouttypes.index("raw"))
print "Created: {0}".format(outputname)
for imagetype in imageouttypes:
if imagetype in ['vsphere', 'rhevm']:
# Imgfac will ensure proper qemu type is used
print "Creating {0} image".format(imagetype)
target_image = self.builder.buildimagetype(imagetype, image.identifier)
infile = target_image.data
outfile = os.path.join(imageoutputdir, '%s-%s.ova' % (self._name, imagetype))
shutil.copyfile(infile, outfile)
print "Created: {0}".format(outfile)
trivhttp.stop()
def install(self):
mainctx = GLib.MainContext.new()
mainctx.push_thread_default()
cancellable = None
gi.require_version("OSTree", "1.0")
gi.require_version("RpmOstree", "1.0")
from gi.repository import OSTree, RpmOstree
ostreesetup = self.data.ostreesetup
log.info("executing ostreesetup=%r", ostreesetup)
# Initialize the filesystem - this will create the repo as well
self._safeExecWithRedirect("ostree",
["admin", "--sysroot=" + iutil.getTargetPhysicalRoot(),
"init-fs", iutil.getTargetPhysicalRoot()])
# Here, we use the physical root as sysroot, because we haven't
# yet made a deployment.
sysroot_file = Gio.File.new_for_path(iutil.getTargetPhysicalRoot())
sysroot = OSTree.Sysroot.new(sysroot_file)
sysroot.load(cancellable)
repo = sysroot.get_repo(None)[1]
# We don't support resuming from interrupted installs
repo.set_disable_fsync(True)
self._remoteOptions = {}
if hasattr(ostreesetup, 'nogpg') and ostreesetup.nogpg:
self._remoteOptions['gpg-verify'] = GLib.Variant('b', False)
if flags.noverifyssl:
self._remoteOptions['tls-permissive'] = GLib.Variant('b', True)
repo.remote_change(None, OSTree.RepoRemoteChange.ADD_IF_NOT_EXISTS,
ostreesetup.remote, ostreesetup.url,
GLib.Variant('a{sv}', self._remoteOptions),
cancellable)
# Variable substitute the ref: https://pagure.io/atomic-wg/issue/299
ref = RpmOstree.varsubst_basearch(ostreesetup.ref)
progressQ.send_message(_("Starting pull of %(branchName)s from %(source)s") % \
{"branchName": ref, "source": ostreesetup.remote})
progress = OSTree.AsyncProgress.new()
progress.connect('changed', self._pullProgressCb)
pull_opts = {'refs': GLib.Variant('as', [ref])}
# If we're doing a kickstart, we can at least use the content as a reference:
# See <https://github.com/rhinstaller/anaconda/issues/1117>
# The first path here is used by <https://pagure.io/fedora-lorax-templates>
# and the second by <https://github.com/projectatomic/rpm-ostree-toolbox/>
if OSTree.check_version(2017, 8):
for path in ['/ostree/repo', '/install/ostree/repo']:
if os.path.isdir(path + '/objects'):
pull_opts['localcache-repos'] = GLib.Variant('as', [path])
break
try:
repo.pull_with_options(ostreesetup.remote,
GLib.Variant('a{sv}', pull_opts),
progress, cancellable)
except GLib.GError as e:
exn = PayloadInstallError("Failed to pull from repository: %s" % e)
log.error(str(exn))
if errors.errorHandler.cb(exn) == errors.ERROR_RAISE:
progressQ.send_quit(1)
iutil.ipmi_abort(scripts=self.data.scripts)
sys.exit(1)
log.info("ostree pull: " + (progress.get_status() or ""))
progressQ.send_message(_("Preparing deployment of %s") % (ref, ))
# Now that we have the data pulled, delete the remote for now.
# This will allow a remote configuration defined in the tree
# (if any) to override what's in the kickstart. Otherwise,
# we'll re-add it in post. Ideally, ostree would support a
# pull without adding a remote, but that would get quite
# complex.
repo.remote_delete(self.data.ostreesetup.remote, None)
self._safeExecWithRedirect("ostree",
["admin", "--sysroot=" + iutil.getTargetPhysicalRoot(),
"os-init", ostreesetup.osname])
admin_deploy_args = ["admin", "--sysroot=" + iutil.getTargetPhysicalRoot(),
"deploy", "--os=" + ostreesetup.osname]
admin_deploy_args.append(ostreesetup.remote + ':' + ref)
log.info("ostree admin deploy starting")
progressQ.send_message(_("Deployment starting: %s") % (ref, ))
self._safeExecWithRedirect("ostree", admin_deploy_args)
log.info("ostree admin deploy complete")
progressQ.send_message(_("Deployment complete: %s") % (ref, ))
# Reload now that we've deployed, find the path to the new deployment
sysroot.load(None)
deployments = sysroot.get_deployments()
assert len(deployments) > 0
deployment = deployments[0]
deployment_path = sysroot.get_deployment_directory(deployment)
iutil.setSysroot(deployment_path.get_path())
try:
self._copyBootloaderData()
except (OSError, RuntimeError) as e:
exn = PayloadInstallError("Failed to copy bootloader data: %s" % e)
log.error(str(exn))
if errors.errorHandler.cb(exn) == errors.ERROR_RAISE:
progressQ.send_quit(1)
iutil.ipmi_abort(scripts=self.data.scripts)
sys.exit(1)
mainctx.pop_thread_default()
def _do_checkout(self, repo, name, img, upgrade, values, destination, unitfileout, tmpfilesout, extract_only, remote):
if not values:
values = {}
remote_path = self._resolve_remote_path(remote)
_, rev = self._resolve_image(repo, img)
if rev is None:
raise ValueError("Image %s not found" % img)
if remote_path:
remote_rootfs = os.path.join(remote_path, "rootfs")
if os.path.exists(remote_rootfs):
util.write_out("The remote rootfs for this container is set to be %s" % remote_rootfs)
elif os.path.exists(os.path.join(remote, "usr")): # Assume that the user directly gave the location of the rootfs
remote_rootfs = remote
remote_path = os.path.dirname(remote_path) # Use the parent directory as the "container location"
else:
raise ValueError("--remote was specified but the given location does not contain a rootfs")
exports = os.path.join(remote_path, "rootfs/exports")
else:
exports = os.path.join(destination, "rootfs/exports")
unitfile = os.path.join(exports, "service.template")
tmpfiles = os.path.join(exports, "tmpfiles.template")
util.write_out("Extracting to %s" % destination)
# upgrade will not restart the service if it was not already running
was_service_active = self._is_service_active(name)
if self.display:
return
if self.user:
rootfs = os.path.join(destination, "rootfs")
elif extract_only:
rootfs = destination
elif remote_path:
rootfs = os.path.join(remote_path, "rootfs")
else:
# Under Atomic, get the real deployment location if we're using the
# system repo. It is needed to create the hard links.
if self.get_ostree_repo_location() == '/ostree/repo':
try:
sysroot = OSTree.Sysroot()
sysroot.load()
osname = sysroot.get_booted_deployment().get_osname()
destination = os.path.join("/ostree/deploy/", osname, os.path.relpath(destination, "/"))
destination = os.path.realpath(destination)
except: #pylint: disable=bare-except
pass
rootfs = os.path.join(destination, "rootfs")
if os.path.exists(destination):
shutil.rmtree(destination)
if remote_path:
os.makedirs(destination)
else:
os.makedirs(rootfs)
manifest = self._image_manifest(repo, rev)
if not remote_path:
rootfs_fd = None
try:
rootfs_fd = os.open(rootfs, os.O_DIRECTORY)
if manifest is None:
self._checkout_layer(repo, rootfs_fd, rootfs, rev)
else:
layers = SystemContainers.get_layers_from_manifest(json.loads(manifest))
for layer in layers:
rev_layer = repo.resolve_rev("%s%s" % (OSTREE_OCIIMAGE_PREFIX, layer.replace("sha256:", "")), True)[1]
if not rev_layer:
raise ValueError("Layer not found: %s. Please pull again the image" % layer.replace("sha256:", ""))
self._checkout_layer(repo, rootfs_fd, rootfs, rev_layer)
self._do_syncfs(rootfs, rootfs_fd)
finally:
if rootfs_fd:
os.close(rootfs_fd)
if extract_only:
return
if self.user:
values["RUN_DIRECTORY"] = os.environ.get("XDG_RUNTIME_DIR", "/run/user/%s" % (os.getuid()))
values["STATE_DIRECTORY"] = "%s/.data" % HOME
else:
values["RUN_DIRECTORY"] = "/run"
values["STATE_DIRECTORY"] = "/var/lib"
# When installing a new system container, set values in this order:
#
# 1) What comes from manifest.json, if present, as default value.
# 2) What the user sets explictly as --set
# 3) Values for DESTDIR and NAME
manifest_file = os.path.join(exports, "manifest.json")
if os.path.exists(manifest_file):
with open(manifest_file, "r") as f:
manifest = json.loads(f.read())
if "defaultValues" in manifest:
for key, val in manifest["defaultValues"].items():
if key not in values:
values[key] = val
if self.args.setvalues is not None:
for i in self.args.setvalues:
split = i.find("=")
if split < 0:
raise ValueError("Invalid value '%s'. Expected form NAME=VALUE" % i)
key, val = i[:split], i[split+1:]
values[key] = val
values["DESTDIR"] = destination
values["NAME"] = name
values["EXEC_START"], values["EXEC_STOP"] = self._generate_systemd_startstop_directives(name)
values["HOST_UID"] = os.getuid()
values["HOST_GID"] = os.getgid()
def _write_template(inputfilename, data, values, destination):
try:
os.makedirs(os.path.dirname(destination))
except OSError:
pass
with open(destination, "w") as outfile:
template = Template(data)
result = template.safe_substitute(values)
missing = {"".join(x) for x in template.pattern.findall(data) if "".join(x) not in values} # pylint: disable=no-member
if len(missing):
raise ValueError("The template file '%s' still contains unreplaced values for: %s" % \
(inputfilename, ", ".join(missing)))
outfile.write(result)
src = os.path.join(exports, "config.json")
destination_path = os.path.join(destination, "config.json")
if os.path.exists(src):
shutil.copyfile(src, destination_path)
elif os.path.exists(src + ".template"):
with open(src + ".template", 'r') as infile:
_write_template(src + ".template", infile.read(), values, destination_path)
else:
self._generate_default_oci_configuration(destination)
if remote_path:
with open(destination_path, 'r') as config_file:
config = json.loads(config_file.read())
config['root']['path'] = remote_rootfs
with open(destination_path, 'w') as config_file:
config_file.write(json.dumps(config, indent=4))
# When upgrading, stop the service and remove previously installed
# tmpfiles, before restarting the service.
if upgrade:
if was_service_active:
self._systemctl_command("stop", name)
if os.path.exists(tmpfilesout):
try:
self._systemd_tmpfiles("--remove", tmpfilesout)
except subprocess.CalledProcessError:
pass
missing_bind_paths = self._check_oci_configuration_file(destination_path, remote_path)
image_manifest = self._image_manifest(repo, rev)
image_id = rev
if image_manifest:
image_manifest = json.loads(image_manifest)
if 'Digest' in image_manifest:
image_id = image_manifest['Digest'].replace("sha256:", "")
with open(os.path.join(destination, "info"), 'w') as info_file:
info = {"image" : img,
"revision" : image_id,
"ostree-commit": rev,
'created' : calendar.timegm(time.gmtime()),
"values" : values,
"remote" : remote}
info_file.write(json.dumps(info, indent=4))
if os.path.exists(unitfile):
with open(unitfile, 'r') as infile:
systemd_template = infile.read()
else:
systemd_template = SYSTEMD_UNIT_FILE_DEFAULT_TEMPLATE
if os.path.exists(tmpfiles):
with open(tmpfiles, 'r') as infile:
tmpfiles_template = infile.read()
else:
tmpfiles_template = SystemContainers._generate_tmpfiles_data(missing_bind_paths, values["STATE_DIRECTORY"])
_write_template(unitfile, systemd_template, values, unitfileout)
shutil.copyfile(unitfileout, os.path.join(destination, "%s.service" % name))
if (tmpfiles_template):
_write_template(unitfile, tmpfiles_template, values, tmpfilesout)
shutil.copyfile(unitfileout, os.path.join(destination, "tmpfiles-%s.conf" % name))
sym = "%s/%s" % (self._get_system_checkout_path(), name)
if os.path.exists(sym):
os.unlink(sym)
os.symlink(destination, sym)
self._systemctl_command("daemon-reload")
if (tmpfiles_template):
self._systemd_tmpfiles("--create", tmpfilesout)
if not upgrade:
self._systemctl_command("enable", name)
elif was_service_active:
self._systemctl_command("start", name)
def copy_commit(repo, src_rev, dest_ref):
"""Copy commit src_rev to dest_ref
This makes the new commit at dest_ref have the proper collection
binding for this repo. The caller is expected to manage the
ostree transaction.
This is like "flatpak build-commit-from", but we need more
control over the transaction.
"""
logging.info('Copying commit %s to %s', src_rev, dest_ref)
_, src_root, _ = repo.read_commit(src_rev)
_, src_variant, src_state = repo.load_commit(src_rev)
# Only copy normal commits
if src_state != 0:
raise Exception('Cannot copy irregular commit {}'.format(src_rev))
# If the dest ref exists, use the current commit as the new
# commit's parent
_, dest_parent = repo.resolve_rev_ext(
dest_ref,
allow_noent=True,
flags=OSTree.RepoResolveRevExtFlags.REPO_RESOLVE_REV_EXT_NONE)
if dest_parent is not None:
logging.info('Using %s as new commit parent', dest_parent)
# Make a copy of the commit metadata to update. Like flatpak
# build-commit-from, the detached metadata is not copied since
# the only known usage is for GPG signatures, which would become
# invalid.
commit_metadata = GLib.VariantDict.new(src_variant.get_child_value(0))
# Set the collection binding if the repo has a collection ID,
# otherwise remove it
collection_id = get_collection_id(repo)
if collection_id is not None:
commit_metadata.insert_value(OSTree.COMMIT_META_KEY_COLLECTION_BINDING,
GLib.Variant('s', collection_id))
else:
commit_metadata.remove(OSTree.COMMIT_META_KEY_COLLECTION_BINDING)
# Include the destination ref in the ref bindings
ref_bindings = commit_metadata.lookup_value(
OSTree.COMMIT_META_KEY_REF_BINDING, GLib.VariantType('as'))
if ref_bindings is None:
ref_bindings = []
ref_bindings = set(ref_bindings)
ref_bindings.add(dest_ref)
commit_metadata.insert_value(OSTree.COMMIT_META_KEY_REF_BINDING,
GLib.Variant('as', sorted(ref_bindings)))
# Add flatpak specific metadata. xa.ref is deprecated, but some
# flatpak clients might expect it. xa.from_commit will be used
# by the app verifier to make sure the commit it sent actually
# got there
commit_metadata.insert_value('xa.ref', GLib.Variant('s', dest_ref))
commit_metadata.insert_value('xa.from_commit', GLib.Variant('s', src_rev))
# Convert from GVariantDict to GVariant vardict
commit_metadata = commit_metadata.end()
# Copy other commit data from source commit
commit_subject = src_variant[COMMIT_SUBJECT_INDEX]
commit_body = src_variant[COMMIT_BODY_INDEX]
commit_time = OSTree.commit_get_timestamp(src_variant)
# Make the new commit assuming the caller started a transaction
mtree = OSTree.MutableTree.new()
repo.write_directory_to_mtree(src_root, mtree, None)
_, dest_root = repo.write_mtree(mtree)
_, dest_checksum = repo.write_commit_with_time(dest_parent, commit_subject,
commit_body,
commit_metadata, dest_root,
commit_time)
logging.info('Created new commit %s', dest_checksum)
return dest_checksum
def _import_layers_into_ostree(repo, imagebranch, manifest, layers):
repo.prepare_transaction()
for layer, tar in layers.items():
mtree = OSTree.MutableTree()
def filter_func(*args):
info = args[2]
if info.get_file_type() == Gio.FileType.DIRECTORY:
info.set_attribute_uint32("unix::mode", info.get_attribute_uint32("unix::mode") | stat.S_IWUSR)
return OSTree.RepoCommitFilterResult.ALLOW
modifier = OSTree.RepoCommitModifier.new(0, filter_func, None)
metav = GLib.Variant("a{sv}", {'docker.layer': GLib.Variant('s', layer)})
imported = False
try:
repo.write_archive_to_mtree(Gio.File.new_for_path(tar), mtree, modifier, True)
root = repo.write_mtree(mtree)[1]
csum = repo.write_commit(None, "", None, metav, root)[1]
imported = True
except GLib.GError as e: #pylint: disable=catching-non-exception
# libarchive which is used internally by OSTree to import a tarball doesn't support correctly
# files with xattrs. If that happens, extract the tarball and import the directory.
if e.domain != "g-io-error-quark": # pylint: disable=no-member
raise e #pylint: disable=raising-non-exception
if not imported:
try:
temp_dir = tempfile.mkdtemp()
with tarfile.open(tar, 'r') as t:
t.extractall(temp_dir)
repo.write_directory_to_mtree(Gio.File.new_for_path(temp_dir), mtree, modifier)
root = repo.write_mtree(mtree)[1]
csum = repo.write_commit(None, "", None, metav, root)[1]
finally:
shutil.rmtree(temp_dir)
root = repo.write_mtree(mtree)[1]
csum = repo.write_commit(None, "", None, metav, root)[1]
repo.transaction_set_ref(None, "%s%s" % (OSTREE_OCIIMAGE_PREFIX, layer), csum)
# create a $OSTREE_OCIIMAGE_PREFIX$image-$tag branch
if not isinstance(manifest, str):
manifest = json.dumps(manifest)
metadata = GLib.Variant("a{sv}", {'docker.manifest': GLib.Variant('s', manifest)})
mtree = OSTree.MutableTree()
file_info = Gio.FileInfo()
file_info.set_attribute_uint32("unix::uid", 0)
file_info.set_attribute_uint32("unix::gid", 0)
file_info.set_attribute_uint32("unix::mode", 0o755 | stat.S_IFDIR)
dirmeta = OSTree.create_directory_metadata(file_info, None)
csum_dirmeta = repo.write_metadata(OSTree.ObjectType.DIR_META, None, dirmeta)[1]
mtree.set_metadata_checksum(OSTree.checksum_from_bytes(csum_dirmeta))
root = repo.write_mtree(mtree)[1]
csum = repo.write_commit(None, "", None, metadata, root)[1]
repo.transaction_set_ref(None, imagebranch, csum)
repo.commit_transaction(None)
def _checkout_system_container(self,
repo,
name,
img,
deployment,
upgrade,
values=None,
destination=None,
extract_only=False):
if not values:
values = {}
imagebranch = SystemContainers._get_ostree_image_branch(img)
destination = destination or "%s/%s.%d" % (
self._get_system_checkout_path(), name, deployment)
exports = os.path.join(destination, "rootfs/exports")
unitfile = os.path.join(exports, "service.template")
if self.user:
home = os.path.expanduser("~")
unitfileout = os.path.join(SYSTEMD_UNIT_FILES_DEST_USER % home,
"%s.service" % name)
else:
unitfileout = os.path.join(SYSTEMD_UNIT_FILES_DEST,
"%s.service" % name)
if not upgrade and os.path.exists(unitfileout):
raise ValueError("The file %s already exists." % unitfileout)
util.write_out("Extracting to %s" % destination)
if hasattr(self.args, 'display') and self.args.display:
return
if self.user:
rootfs = os.path.join(destination, "rootfs")
elif extract_only:
rootfs = destination
else:
# Under Atomic, get the real deployment location. It is needed to create the hard links.
try:
sysroot = OSTree.Sysroot()
sysroot.load()
osname = sysroot.get_booted_deployment().get_osname()
destination = os.path.join("/ostree/deploy/", osname,
os.path.relpath(destination, "/"))
destination = os.path.realpath(destination)
except: #pylint: disable=bare-except
pass
rootfs = os.path.join(destination, "rootfs")
if os.path.exists(destination):
shutil.rmtree(destination)
os.makedirs(rootfs)
rev = repo.resolve_rev(imagebranch, False)[1]
manifest = self._image_manifest(repo, rev)
rootfs_fd = None
try:
rootfs_fd = os.open(rootfs, os.O_DIRECTORY)
if manifest is None:
self._checkout_layer(repo, rootfs_fd, rootfs, rev)
else:
layers = SystemContainers.get_layers_from_manifest(
json.loads(manifest))
for layer in layers:
rev_layer = repo.resolve_rev(
"%s%s" %
(OSTREE_OCIIMAGE_PREFIX, layer.replace("sha256:", "")),
False)[1]
self._checkout_layer(repo, rootfs_fd, rootfs, rev_layer)
self._do_syncfs(rootfs, rootfs_fd)
finally:
if rootfs_fd:
os.close(rootfs_fd)
if extract_only:
return
# When installing a new system container, set values in this order:
#
# 1) What comes from manifest.json, if present, as default value.
# 2) What the user sets explictly as --set
# 3) Values for DESTDIR and NAME
manifest_file = os.path.join(exports, "manifest.json")
if os.path.exists(manifest_file):
with open(manifest_file, "r") as f:
manifest = json.loads(f.read())
if "defaultValues" in manifest:
for key, val in manifest["defaultValues"].items():
if key not in values:
values[key] = val
if self.args.setvalues is not None:
for i in self.args.setvalues:
split = i.find("=")
if split < 0:
raise ValueError(
"Invalid value '%s'. Expected form NAME=VALUE" % i)
key, val = i[:split], i[split + 1:]
values[key] = val
values["DESTDIR"] = destination
values["NAME"] = name
values["EXEC_START"], values[
"EXEC_STOP"] = self._generate_systemd_startstop_directives(name)
def _write_template(inputfilename, data, values, outfile):
template = Template(data)
result = template.safe_substitute(values)
if '$' in result.replace("$$", ""):
missing = {
x[1]
for x in template.pattern.findall(data, template.flags)
if len(x[1]) > 0 and x[1] not in values
} # pylint: disable=no-member
raise ValueError("The template file %s still contains unreplaced values for: %s" % \
(inputfilename, ", ".join(missing)))
outfile.write(result)
src = os.path.join(exports, "config.json")
destination_path = os.path.join(destination, "config.json")
if os.path.exists(src):
shutil.copyfile(src, destination_path)
elif os.path.exists(src + ".template"):
with open(src + ".template",
'r') as infile, open(destination_path, "w") as outfile:
_write_template(src + ".template", infile.read(), values,
outfile)
else:
self._generate_default_oci_configuration(destination)
self._check_oci_configuration_file(destination_path)
image_manifest = self._image_manifest(repo, rev)
image_id = rev
if image_manifest:
image_manifest = json.loads(image_manifest)
if 'Digest' in image_manifest:
image_id = image_manifest['Digest'].replace("sha256:", "")
with open(os.path.join(destination, "info"), 'w') as info_file:
info = {
"image": img,
"revision": image_id,
"ostree-commit": rev,
'created': calendar.timegm(time.gmtime()),
"values": values
}
info_file.write(json.dumps(info, indent=4))
if os.path.exists(unitfile):
with open(unitfile, 'r') as infile:
systemd_template = infile.read()
else:
systemd_template = SYSTEMD_UNIT_FILE_DEFAULT_TEMPLATE
try:
os.makedirs(os.path.dirname(unitfileout))
except OSError:
pass
with open(unitfileout, "w") as outfile:
_write_template(unitfile, systemd_template, values, outfile)
sym = "%s/%s" % (self._get_system_checkout_path(), name)
if os.path.exists(sym):
os.unlink(sym)
os.symlink(destination, sym)
self._systemctl_command("enable", name)
if upgrade:
self._systemctl_command("restart", name)
else:
self._systemctl_command("start", name)
return
def install(self):
mainctx = create_new_context()
mainctx.push_thread_default()
cancellable = None
gi.require_version("OSTree", "1.0")
gi.require_version("RpmOstree", "1.0")
from gi.repository import OSTree, RpmOstree
ostreesetup = self.data.ostreesetup
log.info("executing ostreesetup=%r", ostreesetup)
# Initialize the filesystem - this will create the repo as well
self._safe_exec_with_redirect("ostree", [
"admin", "--sysroot=" + conf.target.physical_root, "init-fs",
conf.target.physical_root
])
# Here, we use the physical root as sysroot, because we haven't
# yet made a deployment.
sysroot_file = Gio.File.new_for_path(conf.target.physical_root)
sysroot = OSTree.Sysroot.new(sysroot_file)
sysroot.load(cancellable)
repo = sysroot.get_repo(None)[1]
# We don't support resuming from interrupted installs
repo.set_disable_fsync(True)
self._remoteOptions = {}
if hasattr(ostreesetup, 'nogpg') and ostreesetup.nogpg:
self._remoteOptions['gpg-verify'] = Variant('b', False)
if not conf.payload.verify_ssl:
self._remoteOptions['tls-permissive'] = Variant('b', True)
repo.remote_change(None, OSTree.RepoRemoteChange.ADD_IF_NOT_EXISTS,
ostreesetup.remote, ostreesetup.url,
Variant('a{sv}', self._remoteOptions), cancellable)
# Variable substitute the ref: https://pagure.io/atomic-wg/issue/299
ref = RpmOstree.varsubst_basearch(ostreesetup.ref)
progressQ.send_message(
_("Starting pull of %(branchName)s from %(source)s") % {
"branchName": ref,
"source": ostreesetup.remote
})
progress = OSTree.AsyncProgress.new()
progress.connect('changed', self._pull_progress_cb)
pull_opts = {'refs': Variant('as', [ref])}
# If we're doing a kickstart, we can at least use the content as a reference:
# See <https://github.com/rhinstaller/anaconda/issues/1117>
# The first path here is used by <https://pagure.io/fedora-lorax-templates>
# and the second by <https://github.com/projectatomic/rpm-ostree-toolbox/>
if OSTree.check_version(2017, 8):
for path in ['/ostree/repo', '/install/ostree/repo']:
if os.path.isdir(path + '/objects'):
pull_opts['localcache-repos'] = Variant('as', [path])
break
try:
repo.pull_with_options(ostreesetup.remote,
Variant('a{sv}', pull_opts), progress,
cancellable)
except GError as e:
exn = PayloadInstallError("Failed to pull from repository: %s" % e)
log.error(str(exn))
if errors.errorHandler.cb(exn) == errors.ERROR_RAISE:
progressQ.send_quit(1)
util.ipmi_abort(scripts=self.data.scripts)
sys.exit(1)
log.info("ostree pull: %s", progress.get_status() or "")
progressQ.send_message(_("Preparing deployment of %s") % (ref, ))
# Now that we have the data pulled, delete the remote for now.
# This will allow a remote configuration defined in the tree
# (if any) to override what's in the kickstart. Otherwise,
# we'll re-add it in post. Ideally, ostree would support a
# pull without adding a remote, but that would get quite
# complex.
repo.remote_delete(self.data.ostreesetup.remote, None)
self._safe_exec_with_redirect("ostree", [
"admin", "--sysroot=" + conf.target.physical_root, "os-init",
ostreesetup.osname
])
admin_deploy_args = [
"admin", "--sysroot=" + conf.target.physical_root, "deploy",
"--os=" + ostreesetup.osname
]
admin_deploy_args.append(ostreesetup.remote + ':' + ref)
log.info("ostree admin deploy starting")
progressQ.send_message(_("Deployment starting: %s") % (ref, ))
self._safe_exec_with_redirect("ostree", admin_deploy_args)
log.info("ostree admin deploy complete")
progressQ.send_message(_("Deployment complete: %s") % (ref, ))
# Reload now that we've deployed, find the path to the new deployment
sysroot.load(None)
deployments = sysroot.get_deployments()
assert len(deployments) > 0
deployment = deployments[0]
deployment_path = sysroot.get_deployment_directory(deployment)
util.set_system_root(deployment_path.get_path())
try:
self._copy_bootloader_data()
except (OSError, RuntimeError) as e:
exn = PayloadInstallError("Failed to copy bootloader data: %s" % e)
log.error(str(exn))
if errors.errorHandler.cb(exn) == errors.ERROR_RAISE:
progressQ.send_quit(1)
util.ipmi_abort(scripts=self.data.scripts)
sys.exit(1)
mainctx.pop_thread_default()
def create(self, outputdir, name, ksfile, tdl, imageouttypes):
self._name = name
self._tdl = tdl
self._kickstart = ksfile
[res, rev] = self.repo.resolve_rev(self.ref, False)
[res, commit] = self.repo.load_variant(OSTree.ObjectType.COMMIT, rev)
commitdate = GLib.DateTime.new_from_unix_utc(OSTree.commit_get_timestamp(commit)).format("%c")
print commitdate
target = os.path.join(outputdir, '%s.raw' % (self._name))
port_file_path = self.workdir + '/repo-port'
subprocess.check_call(['ostree',
'trivial-httpd', '--autoexit', '--daemonize',
'--port-file', port_file_path],
cwd=self.ostree_repo)
httpd_port = open(port_file_path).read().strip()
print "trivial httpd port=%s" % (httpd_port, )
ks_basename = os.path.basename(ksfile)
flattened_ks = os.path.join(self.workdir, ks_basename)
# FIXME - eventually stop hardcoding this via some mapping
if ks_basename.find('fedora') >= 0:
kickstart_version = 'F21'
else:
kickstart_version = 'RHEL7'
run_sync(['ksflatten', '--version', kickstart_version,
'-c', ksfile, '-o', flattened_ks])
# TODO: Pull kickstart from separate git repo
ksdata = open(flattened_ks).read()
substitutions = { 'OSTREE_PORT': httpd_port,
'OSTREE_REF': self.ref,
'OSTREE_OSNAME': self.os_name }
for subname, subval in substitutions.iteritems():
ksdata = ksdata.replace('@%s@' % (subname, ), subval)
parameters = { "install_script": ksdata,
"generate_icicle": False,
}
defaultimagetype = checkoz()
print "Starting build"
image = self.builder.build(template=open(self._tdl).read(), parameters=parameters)
# For debug, you can comment out the above and enable the code below
# to skip the initial image creation. Just point myuuid at the proper
# image uuid
# self.builder.download()
# myuuid = "32a2d0b8-da84-415c-aec6-90bb5a7f8e8b"
# pim = PersistentImageManager.default_manager()
# image = pim.image_with_id(myuuid)
# This should probably be broken out to a sep. function
if defaultimagetype == "raw":
# Always create a qcow2
print "Processing image from raw to qcow2"
print image.data
outputname = os.path.join(outputdir, '%s.qcow2' % (self._name))
print outputname
qemucmd = ['qemu-img', 'convert', '-f', 'raw', '-O', 'qcow2', image.data, outputname]
imageouttypes.pop(imageouttypes.index("kvm"))
subprocess.check_call(qemucmd)
if 'kvm' in imageouttypes:
print "Processing image from qcow2 to raw"
print image.data
outputname = os.path.join(outputdir, '%s.raw' % (self._name))
print outputname
# We use compat=0.10 to ensure we run on RHEL6 era QEMU
qemucmd = ['qemu-img', 'convert', '-f', 'raw', '-O', 'qcow2', '-o', 'compat=0.10', image.data, outputname]
imageouttypes.pop(imageouttypes.index("raw"))
subprocess.check_call(qemucmd)
shutil.copyfile(image.data, target)
print "Created: " + target
for imagetype in imageouttypes:
print "Creating {0} image".format(imagetype)
target_image = self.builder.buildimagetype(imagetype, image.identifier)
infile = target_image.data
outfile = os.path.join(outputdir, '%s-%s.ova' % (self._name, imagetype))
shutil.copyfile(infile, outfile)
