def savefs_before_chroot(chrootdir, saveto = None):
""" backup chrootdir to another directory before chrooting in """
if configmgr.chroot['saveto']:
savefs = True
saveto = configmgr.chroot['saveto']
wrnmsg = "Can't save chroot fs for dir %s exists" % saveto
if saveto == chrootdir:
savefs = False
wrnmsg = "Dir %s is being used to chroot" % saveto
elif os.path.exists(saveto):
if msger.ask("Dir %s already exists, cleanup and continue?" %
saveto):
shutil.rmtree(saveto, ignore_errors = True)
savefs = True
else:
savefs = False
if savefs:
msger.info("Saving image to directory %s" % saveto)
fs_related.makedirs(os.path.dirname(os.path.abspath(saveto)))
runner.quiet("cp -af %s %s" % (chrootdir, saveto))
devs = ['dev/fd',
'dev/stdin',
'dev/stdout',
'dev/stderr',
'etc/mtab']
ignlst = [os.path.join(saveto, x) for x in devs]
map(os.unlink, filter(os.path.exists, ignlst))
else:
msger.warning(wrnmsg)
def print_outimage_info(self):
"""
Print the image(s) and artifacts used, for the user.
"""
msg = "The new image(s) can be found here:\n"
parts = self._get_parts()
for disk_name, disk in self.__instimage.disks.items():
full_path = self._full_path(self.__imgdir, disk_name, "direct")
msg += ' %s\n\n' % full_path
msg += 'The following build artifacts were used to create the image(s):\n'
for p in parts:
if p.get_rootfs() is None:
continue
if p.mountpoint == '/':
str = ':'
else:
str = '["%s"]:' % p.label
msg += ' ROOTFS_DIR%s%s\n' % (str.ljust(20), p.get_rootfs())
msg += ' BOOTIMG_DIR: %s\n' % self.bootimg_dir
msg += ' KERNEL_DIR: %s\n' % self.kernel_dir
msg += ' NATIVE_SYSROOT: %s\n' % self.native_sysroot
msger.info(msg)
def __create_iso(self, isodir):
iso = self._outdir + "/" + self.name + ".iso"
genisoimage = fs_related.find_binary_path("genisoimage")
args = [genisoimage,
"-J", "-r",
"-hide-rr-moved", "-hide-joliet-trans-tbl",
"-V", self.fslabel,
"-o", iso]
args.extend(self._get_mkisofs_options(isodir))
args.append(isodir)
if runner.show(args) != 0:
raise CreatorError("ISO creation failed!")
""" It should be ok still even if you haven't isohybrid """
isohybrid = None
if self._isohybrid:
try:
isohybrid = fs_related.find_binary_path("isohybrid")
msger.info("isohybrid found")
except:
msger.warning("isohybrid NOT found")
if isohybrid:
args = [isohybrid, "-partok", iso ]
if runner.show(args) != 0:
raise CreatorError("Hybrid ISO creation failed!")
else:
msger.info("Hybrid ISO created successfully")
self.__implant_md5sum(iso)
def configure(self, repodata=None):
"""Configure the system image according to the kickstart.
This method applies the (e.g. keyboard or network) configuration
specified in the kickstart and executes the kickstart %post scripts.
If necessary, it also prepares the image to be bootable by e.g.
creating an initrd and bootloader configuration.
"""
ksh = self.ks.handler
msger.info("Applying configurations ...")
try:
kickstart.LanguageConfig(self._instroot).apply(ksh.lang)
kickstart.KeyboardConfig(self._instroot).apply(ksh.keyboard)
kickstart.TimezoneConfig(self._instroot).apply(ksh.timezone)
# kickstart.AuthConfig(self._instroot).apply(ksh.authconfig)
kickstart.FirewallConfig(self._instroot).apply(ksh.firewall)
kickstart.RootPasswordConfig(self._instroot).apply(ksh.rootpw)
kickstart.UserConfig(self._instroot).apply(ksh.user)
kickstart.ServicesConfig(self._instroot).apply(ksh.services)
kickstart.XConfig(self._instroot).apply(ksh.xconfig)
kickstart.NetworkConfig(self._instroot).apply(ksh.network)
kickstart.RPMMacroConfig(self._instroot).apply(self.ks)
kickstart.DesktopConfig(self._instroot).apply(ksh.desktop)
self.__save_repo_keys(repodata)
kickstart.MoblinRepoConfig(self._instroot).apply(ksh.repo, repodata)
except:
msger.warning("Failed to apply configuration to image")
raise
self._create_bootconfig()
self.__run_post_scripts()
def create_manifest(self):
def get_pack_suffix():
return '.' + self.pack_to.split('.', 1)[1]
if not os.path.exists(self.destdir):
os.makedirs(self.destdir)
now = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
manifest_dict = {'version': VERSION,
'created': now}
if self.img_format:
manifest_dict.update({'format': self.img_format})
if hasattr(self, 'logfile') and self.logfile:
manifest_dict.update({'log_file': self.logfile})
if self.image_files:
if self.pack_to:
self.image_files.update({'pack': get_pack_suffix()})
manifest_dict.update({self.img_format: self.image_files})
msger.info('Creating manifest file...')
manifest_file_path = os.path.join(self.destdir, 'manifest.json')
with open(manifest_file_path, 'w') as fest_file:
json.dump(manifest_dict, fest_file, indent=4)
self.outimage.append(manifest_file_path)
def generate_bmap(self):
""" Generate block map file for the image. The idea is that while disk
images we generate may be large (e.g., 4GiB), they may actually contain
only little real data, e.g., 512MiB. This data are files, directories,
file-system meta-data, partition table, etc. In other words, when
flashing the image to the target device, you do not have to copy all the
4GiB of data, you can copy only 512MiB of it, which is 4 times faster.
This function generates the block map file for an arbitrary image that
mic has generated. The block map file is basically an XML file which
contains a list of blocks which have to be copied to the target device.
The other blocks are not used and there is no need to copy them. """
if self.bmap_needed is None:
return
from mic.utils import BmapCreate
msger.info("Generating the map file(s)")
for name in self.__disks.keys():
image = self._full_path(self.__imgdir, name, self.__disk_format)
bmap_file = self._full_path(self._outdir, name, "bmap")
msger.debug("Generating block map file '%s'" % bmap_file)
try:
creator = BmapCreate.BmapCreate(image, bmap_file)
creator.generate()
del creator
except BmapCreate.Error as err:
raise CreatorError("Failed to create bmap file: %s" % str(err))
def savefs_before_chroot(chrootdir, saveto=None):
""" backup chrootdir to another directory before chrooting in """
if configmgr.chroot['saveto']:
savefs = True
saveto = configmgr.chroot['saveto']
wrnmsg = "Can't save chroot fs for dir %s exists" % saveto
if saveto == chrootdir:
savefs = False
wrnmsg = "Dir %s is being used to chroot" % saveto
elif os.path.exists(saveto):
if msger.ask("Dir %s already exists, cleanup and continue?" %
saveto):
shutil.rmtree(saveto, ignore_errors=True)
savefs = True
else:
savefs = False
if savefs:
msger.info("Saving image to directory %s" % saveto)
fs_related.makedirs(os.path.dirname(os.path.abspath(saveto)))
runner.quiet("cp -af %s %s" % (chrootdir, saveto))
devs = [
'dev/fd', 'dev/stdin', 'dev/stdout', 'dev/stderr', 'etc/mtab'
]
ignlst = [os.path.join(saveto, x) for x in devs]
map(os.unlink, filter(os.path.exists, ignlst))
else:
msger.warning(wrnmsg)
def installLocal(self, pkg, po=None, updateonly=False):
if not self.ts:
self.__initialize_transaction()
solvfile = "%s/.solv" % (self.creator.cachedir)
rc, out = runner.runtool([fs_related.find_binary_path("rpms2solv"), pkg])
if rc == 0:
f = open(solvfile, "w+")
f.write(out)
f.close()
warnmsg = self.repo_manager.loadSolvFile(solvfile , os.path.basename(pkg))
if warnmsg:
msger.warning(warnmsg)
os.unlink(solvfile)
else:
msger.warning('Can not get %s solv data.' % pkg)
hdr = rpmmisc.readRpmHeader(self.ts, pkg)
arch = zypp.Arch(hdr['arch'])
if self.creator.target_arch == None:
# TODO, get the default_arch from conf or detected from global settings
sysarch = zypp.Arch('i686')
else:
sysarch = zypp.Arch(self.creator.target_arch)
if arch.compatible_with (sysarch):
pkgname = hdr['name']
self.localpkgs[pkgname] = pkg
self.selectPackage(pkgname)
msger.info("Marking %s to be installed" % (pkg))
else:
msger.warning ("Cannot add package %s to transaction. Not a compatible architecture: %s" % (pkg, hdr['arch']))
def _make_zipfile(archive_name, target_name):
""" Create a zip file from all the files under 'target_name' or itself.
@archive_name: the name of the archived file
@target_name: the directory or the file name to archive
@retval: indicate the archiving result
"""
import zipfile
msger.info("Zipping files to %s using zipfile module" % archive_name)
arv = zipfile.ZipFile(archive_name, 'w', compression=zipfile.ZIP_DEFLATED)
if os.path.isdir(target_name):
for dirpath, dirname, filenames in os.walk(target_name):
for filename in filenames:
filepath = os.path.normpath(os.path.join(dirpath, filename))
arcname = os.path.relpath(filepath, target_name)
if os.path.isfile(filepath):
arv.write(filepath, arcname)
else:
arv.write(target_name, os.path.basename(target_name))
arv.close()
return os.path.exists(archive_name)
def _stage_final_image(self):
if self.taring_to:
import tarfile
curdir = os.getcwd()
os.chdir(self.__imgdir)
self._resparse(0)
tarfile_name = self.taring_to
if not tarfile_name.endswith('.tar'):
tarfile_name += ".tar"
msger.info("Tar all loop images together to %s" % tarfile_name)
tar = tarfile.open(os.path.join(self._outdir, tarfile_name), 'w')
for item in self._instloops:
if item['fstype'] == "ext4":
runner.show('/sbin/tune2fs -O ^huge_file,extents,uninit_bg ' + item['name'])
tar.add(item['name'])
tar.close()
os.chdir(curdir)
else:
self._resparse()
for item in self._instloops:
shutil.move(os.path.join(self.__imgdir, item['name']), os.path.join(self._outdir, item['name']))
def _get_metadata_from_repo(baseurl,
proxies,
cachedir,
reponame,
filename,
sumtype=None,
checksum=None):
url = os.path.join(baseurl, filename)
filename_tmp = str("%s/%s/%s" %
(cachedir, reponame, os.path.basename(filename)))
if os.path.splitext(filename_tmp)[1] in (".gz", ".bz2"):
filename = os.path.splitext(filename_tmp)[0]
else:
filename = filename_tmp
if sumtype and checksum and os.path.exists(filename):
if sumtype == "sha":
sumtype = "sha1"
sumcmd = "%ssum" % sumtype
file_checksum = runner.outs([sumcmd, filename]).split()[0]
if file_checksum == checksum:
msger.info("%s checksum %s matches cache" %
(filename, file_checksum))
return filename
else:
msger.info("%s checksum %s does not match cache, removing it" %
(filename, file_checksum))
os.unlink(filename)
return _get_uncompressed_data_from_url(url, filename_tmp, proxies)
def package(self, destdir = "."):
"""Prepares the created image for final delivery.
In its simplest form, this method merely copies the install root to the
supplied destination directory; other subclasses may choose to package
the image by e.g. creating a bootable ISO containing the image and
bootloader configuration.
destdir -- the directory into which the final image should be moved;
this defaults to the current directory.
"""
self._stage_final_image()
if not os.path.exists(destdir):
fs.makedirs(destdir)
if self._img_compression_method:
if not self._img_name:
raise CreatorError("Image name not set.")
rc = None
img_location = os.path.join(self._outdir,self._img_name)
if self._img_compression_method == "bz2":
bzip2 = fs.find_binary_path('bzip2')
msger.info("Compressing %s with bzip2. Please wait..." \
% img_location)
rc = runner.show([bzip2, "-f", img_location])
if rc:
raise CreatorError("Failed to compress image %s with %s." \
% (img_location, self._img_compression_method))
for bootimg in glob.glob(os.path.dirname(img_location) + \
"/*-boot.bin"):
msger.info("Compressing %s with bzip2. Please wait..." \
% bootimg)
rc = runner.show([bzip2, "-f", bootimg])
if rc:
raise CreatorError("Failed to compress image %s with "
"%s." \
% (bootimg,
self._img_compression_method))
if self._recording_pkgs:
self._save_recording_pkgs(destdir)
# For image formats with two or multiple image files, it will be
# better to put them under a directory
if self.image_format in ("raw", "vmdk", "vdi", "nand", "mrstnand"):
destdir = os.path.join(destdir, "%s-%s" \
% (self.name, self.image_format))
msger.debug("creating destination dir: %s" % destdir)
fs.makedirs(destdir)
# Ensure all data is flushed to _outdir
runner.quiet('sync')
for f in os.listdir(self._outdir):
shutil.move(os.path.join(self._outdir, f),
os.path.join(destdir, f))
self.outimage.append(os.path.join(destdir, f))
self.do_genchecksum(os.path.join(destdir, f))
def normalize_ksfile(ksconf, release, arch):
def _clrtempks():
try:
os.unlink(ksconf)
except:
pass
if not os.path.exists(ksconf):
return
if not release:
release = "latest"
if not arch or re.match(r'i.86', arch):
arch = "ia32"
with open(ksconf) as f:
ksc = f.read()
if "@ARCH@" in ksc or "@BUILD_ID@" in ksc:
msger.info("Substitute macro variable @BUILD_ID@/@ARCH in ks: %s" %
ksconf)
ksc = ksc.replace("@ARCH@", arch)
ksc = ksc.replace("@BUILD_ID@", release)
fd, ksconf = tempfile.mkstemp(prefix=os.path.basename(ksconf),
dir="/tmp/")
os.write(fd, ksc)
os.close(fd)
msger.debug('new ks path %s' % ksconf)
import atexit
atexit.register(_clrtempks)
return ksconf
def _make_zipfile(archive_name, target_name):
""" Create a zip file from all the files under 'target_name' or itself.
@archive_name: the name of the archived file
@target_name: the directory or the file name to archive
@retval: indicate the archiving result
"""
import zipfile
msger.info("Zipping files to %s using zipfile module" % archive_name)
arv = zipfile.ZipFile(archive_name, 'w', compression=zipfile.ZIP_DEFLATED)
if os.path.isdir(target_name):
for dirpath, dirname, filenames in os.walk(target_name):
for filename in filenames:
filepath = os.path.normpath(os.path.join(dirpath, filename))
arcname = os.path.relpath(filepath, target_name)
if os.path.isfile(filepath):
arv.write(filepath, arcname)
else:
arv.write(target_name, os.path.basename(target_name))
arv.close()
return os.path.exists(archive_name)
def normalize_ksfile(ksconf, release, arch):
def _clrtempks():
try:
os.unlink(ksconf)
except:
pass
if not os.path.exists(ksconf):
return
if not release:
release = "latest"
if not arch or re.match(r'i.86', arch):
arch = "ia32"
with open(ksconf) as f:
ksc = f.read()
if "@ARCH@" in ksc or "@BUILD_ID@" in ksc:
msger.info("Substitute macro variable @BUILD_ID@/@ARCH in ks: %s" % ksconf)
ksc = ksc.replace("@ARCH@", arch)
ksc = ksc.replace("@BUILD_ID@", release)
fd, ksconf = tempfile.mkstemp(prefix=os.path.basename(ksconf), dir="/tmp/")
os.write(fd, ksc)
os.close(fd)
msger.debug('new ks path %s' % ksconf)
import atexit
atexit.register(_clrtempks)
return ksconf
def __run_post_scripts(self):
msger.info("Running scripts ...")
if os.path.exists(self._instroot + "/tmp"):
shutil.rmtree(self._instroot + "/tmp")
os.mkdir(self._instroot + "/tmp", 0755)
for s in kickstart.get_post_scripts(self.ks):
(fd, path) = tempfile.mkstemp(prefix="ks-script-", dir=self._instroot + "/tmp")
s.script = s.script.replace("\r", "")
os.write(fd, s.script)
os.close(fd)
os.chmod(path, 0700)
env = self._get_post_scripts_env(s.inChroot)
if not s.inChroot:
env["INSTALL_ROOT"] = self._instroot
env["IMG_NAME"] = self._name
preexec = None
script = path
else:
preexec = self._chroot
script = "/tmp/" + os.path.basename(path)
try:
try:
subprocess.call(
[s.interp, script], preexec_fn=preexec, env=env, stdout=sys.stdout, stderr=sys.stderr
)
except OSError, (err, msg):
raise CreatorError("Failed to execute %%post script " "with '%s' : %s" % (s.interp, msg))
finally:
os.unlink(path)
def _stage_final_image(self):
"""Stage the final system image in _outdir.
write meta data
"""
self._resparse()
if self.compress_image:
for imgfile in os.listdir(self.__imgdir):
if imgfile.endswith('.raw') or imgfile.endswith('bin'):
imgpath = os.path.join(self.__imgdir, imgfile)
msger.info("Compressing image %s" % imgfile)
misc.compressing(imgpath, self.compress_image)
if self.pack_to:
dst = os.path.join(self._outdir, self.pack_to)
msger.info("Pack all raw images to %s" % dst)
misc.packing(dst, self.__imgdir)
else:
msger.debug("moving disks to stage location")
for imgfile in os.listdir(self.__imgdir):
src = os.path.join(self.__imgdir, imgfile)
dst = os.path.join(self._outdir, imgfile)
msger.debug("moving %s to %s" % (src,dst))
shutil.move(src,dst)
self._write_image_xml()
def generate_bmap(self):
""" Generate block map file for the image. The idea is that while disk
images we generate may be large (e.g., 4GiB), they may actually contain
only little real data, e.g., 512MiB. This data are files, directories,
file-system meta-data, partition table, etc. In other words, when
flashing the image to the target device, you do not have to copy all the
4GiB of data, you can copy only 512MiB of it, which is 4 times faster.
This function generates the block map file for an arbitrary image that
mic has generated. The block map file is basically an XML file which
contains a list of blocks which have to be copied to the target device.
The other blocks are not used and there is no need to copy them. """
if self.bmap_needed is None:
return
from mic.utils import BmapCreate
msger.info("Generating the map file(s)")
for name in self.__disks.keys():
image = self._full_path(self.__imgdir, name, self.__disk_format)
bmap_file = self._full_path(self._outdir, name, "bmap")
msger.debug("Generating block map file '%s'" % bmap_file)
try:
creator = BmapCreate.BmapCreate(image, bmap_file)
creator.generate()
del creator
except BmapCreate.Error as err:
raise CreatorError("Failed to create bmap file: %s" % str(err))
def _stage_final_image(self):
if self.compress_to:
self._resparse(0)
cfile_name = self.compress_to
mountfp_xml = os.path.splitext(cfile_name)[0] + ".xml"
for item in self._instloops:
imgfile = os.path.join(self.__imgdir, item['name'])
if item['fstype'] == "ext4":
runner.show('/sbin/tune2fs '
'-O ^huge_file,extents,uninit_bg %s ' \
% imgfile)
msger.info("Compress all loop images together to %s" % cfile_name)
dstfile = os.path.join(self._outdir, cfile_name)
if self.compress_imgdir_method == "tar":
misc.taring(dstfile, self.__imgdir)
elif self.compress_imgdir_method == "zip":
misc.ziping(dstfile, self.__imgdir)
else:
raise CreatorError("Unsupported compress type: %s" \
% self.compress_imgdir_method)
# save mount points mapping file to xml
save_mountpoints(os.path.join(self._outdir, mountfp_xml),
self._instloops,
self.target_arch)
else:
self._resparse()
for item in self._instloops:
shutil.move(os.path.join(self.__imgdir, item['name']),
os.path.join(self._outdir, item['name']))
def do_unpack(cls, srcimg):
srcimgsize = (misc.get_file_size(srcimg)) * 1024L * 1024L
srcmnt = misc.mkdtemp("srcmnt")
disk = fs_related.SparseLoopbackDisk(srcimg, srcimgsize)
srcloop = PartitionedMount(srcmnt, skipformat = True)
srcloop.add_disk('/dev/sdb', disk)
srcloop.add_partition(srcimgsize/1024/1024, "/dev/sdb", "/", "ext3", boot=False)
try:
srcloop.mount()
except errors.MountError:
srcloop.cleanup()
raise
image = os.path.join(tempfile.mkdtemp(dir = "/var/tmp", prefix = "tmp"), "target.img")
args = ['dd', "if=%s" % srcloop.partitions[0]['device'], "of=%s" % image]
msger.info("`dd` image ...")
rc = runner.show(args)
srcloop.cleanup()
shutil.rmtree(os.path.dirname(srcmnt), ignore_errors = True)
if rc != 0:
raise errors.CreatorError("Failed to dd")
else:
return image
def print_outimage_info(self):
msg = "The new image can be found here:\n"
self.outimage.sort()
for file in self.outimage:
msg += " %s\n" % os.path.abspath(file)
msger.info(msg)
def _stage_final_image(self):
if self.taring_to:
import tarfile
curdir = os.getcwd()
os.chdir(self.__imgdir)
self._resparse(0)
tarfile_name = self.taring_to
if not tarfile_name.endswith('.tar'):
tarfile_name += ".tar"
msger.info("Tar all loop images together to %s" % tarfile_name)
tar = tarfile.open(os.path.join(self._outdir, tarfile_name), 'w')
for item in self._instloops:
if item['fstype'] == "ext4":
runner.show(
'/sbin/tune2fs -O ^huge_file,extents,uninit_bg ' +
item['name'])
tar.add(item['name'])
tar.close()
os.chdir(curdir)
else:
self._resparse()
for item in self._instloops:
shutil.move(os.path.join(self.__imgdir, item['name']),
os.path.join(self._outdir, item['name']))
def _stage_final_image(self):
if self.pack_to or self.shrink_image:
self._resparse(0)
else:
self._resparse()
for item in self._instloops:
imgfile = os.path.join(self.__imgdir, item['name'])
if item['fstype'] == "ext4":
runner.show('/sbin/tune2fs -O ^huge_file,extents,uninit_bg %s '
% imgfile)
if self.compress_image:
misc.compressing(imgfile, self.compress_image)
if not self.pack_to:
for item in os.listdir(self.__imgdir):
shutil.move(os.path.join(self.__imgdir, item),
os.path.join(self._outdir, item))
else:
msger.info("Pack all loop images together to %s" % self.pack_to)
dstfile = os.path.join(self._outdir, self.pack_to)
misc.packing(dstfile, self.__imgdir)
if self.pack_to:
mountfp_xml = os.path.splitext(self.pack_to)[0]
mountfp_xml = misc.strip_end(mountfp_xml, '.tar') + ".xml"
else:
mountfp_xml = self.name + ".xml"
# save mount points mapping file to xml
save_mountpoints(os.path.join(self._outdir, mountfp_xml),
self._instloops,
self.target_arch)
def __build_repo_cache(self, name):
repo = self.repo_manager.getRepositoryInfo(name)
if self.repo_manager.isCached(repo) or not repo.enabled():
return
msger.info('Refreshing repository: %s ...' % name)
self.repo_manager.buildCache(repo, zypp.RepoManager.BuildIfNeeded)
def generate_bmap(self):
""" Generate block map file for the image. The idea is that while disk
images we generate may be large (e.g., 4GiB), they may actually contain
only little real data, e.g., 512MiB. This data are files, directories,
file-system meta-data, partition table, etc. In other words, when
flashing the image to the target device, you do not have to copy all the
4GiB of data, you can copy only 512MiB of it, which is 4 times faster.
This function generates the block map file for an arbitrary image that
mic has generated. The block map file is basically an XML file which
contains a list of blocks which have to be copied to the target device.
The other blocks are not used and there is no need to copy them. """
if self.bmap_needed is None:
return
msger.info("Generating the map file(s)")
for name in self.__disks.keys():
image = self._full_path(self.__imgdir, name, self.__disk_format)
bmap_file = self._full_path(self._outdir, name, "bmap")
self.image_files.setdefault(name, {}).update({'bmap': \
os.path.basename(bmap_file)})
msger.debug("Generating block map file '%s'" % bmap_file)
bmaptoolcmd = misc.find_binary_path('bmaptool')
rc = runner.show([bmaptoolcmd, 'create', image, '-o', bmap_file])
if rc != 0:
raise CreatorError("Failed to create bmap file: %s" % bmap_file)
def _stage_final_image(self):
if self.pack_to or self.shrink_image:
self._resparse(0)
else:
self._resparse()
for item in self._instloops:
imgfile = os.path.join(self._imgdir, item['name'])
if item['fstype'] == "ext4":
runner.show(
'/sbin/tune2fs -O ^huge_file,extents,uninit_bg %s ' %
imgfile)
if self.compress_image:
misc.compressing(imgfile, self.compress_image)
if not self.pack_to:
for item in os.listdir(self._imgdir):
shutil.move(os.path.join(self._imgdir, item),
os.path.join(self._outdir, item))
else:
msger.info("Pack all loop images together to %s" % self.pack_to)
dstfile = os.path.join(self._outdir, self.pack_to)
misc.packing(dstfile, self._imgdir)
if self.pack_to:
mountfp_xml = os.path.splitext(self.pack_to)[0]
mountfp_xml = misc.strip_end(mountfp_xml, '.tar') + ".xml"
else:
mountfp_xml = self.name + ".xml"
# save mount points mapping file to xml
save_mountpoints(os.path.join(self._outdir, mountfp_xml),
self._instloops, self.target_arch)
def do_unpack(cls, srcimg):
srcimgsize = (misc.get_file_size(srcimg)) * 1024L * 1024L
srcmnt = misc.mkdtemp("srcmnt")
disk = fs_related.SparseLoopbackDisk(srcimg, srcimgsize)
srcloop = PartitionedMount(srcmnt, skipformat = True)
srcloop.add_disk('/dev/sdb', disk)
srcloop.add_partition(srcimgsize/1024/1024, "/dev/sdb", "/", "ext3", boot=False)
try:
srcloop.mount()
except errors.MountError:
srcloop.cleanup()
raise
image = os.path.join(tempfile.mkdtemp(dir = "/var/tmp", prefix = "tmp"), "target.img")
args = ['dd', "if=%s" % srcloop.partitions[0]['device'], "of=%s" % image]
msger.info("`dd` image ...")
rc = runner.show(args)
srcloop.cleanup()
shutil.rmtree(os.path.dirname(srcmnt), ignore_errors = True)
if rc != 0:
raise errors.CreatorError("Failed to dd")
else:
return image
def package_output(self, image_format, destdir=".", package="none"):
if not package or package == "none":
return
destdir = os.path.abspath(os.path.expanduser(destdir))
(pkg, comp) = os.path.splitext(package)
if comp:
comp = comp.lstrip(".")
if pkg == "tar":
if comp:
dst = "%s/%s-%s.tar.%s" % (destdir, self.name, image_format, comp)
else:
dst = "%s/%s-%s.tar" % (destdir, self.name, image_format)
msger.info("creating %s" % dst)
tar = tarfile.open(dst, "w:" + comp)
for file in self.outimage:
msger.info("adding %s to %s" % (file, dst))
tar.add(file, arcname=os.path.join("%s-%s" % (self.name, image_format), os.path.basename(file)))
if os.path.isdir(file):
shutil.rmtree(file, ignore_errors=True)
else:
os.remove(file)
tar.close()
"""All the file in outimage has been packaged into tar.* file"""
self.outimage = [dst]
def chroot(chrootdir, bindmounts = None, execute = "/bin/bash"):
""" chroot the chrootdir and execute the command """
def mychroot():
""" pre-execute function """
os.chroot(chrootdir)
os.chdir("/")
arch = ELF_arch(chrootdir)
if arch == "arm":
qemu_emulator = misc.setup_qemu_emulator(chrootdir, "arm")
elif arch == "mipsel":
qemu_emulator = misc.setup_qemu_emulator(chrootdir, "mipsel")
else:
qemu_emulator = None
savefs_before_chroot(chrootdir, None)
globalmounts = None
try:
msger.info("Launching shell. Exit to continue.\n"
"----------------------------------")
globalmounts = setup_chrootenv(chrootdir, bindmounts)
subprocess.call(execute, preexec_fn = mychroot, shell=True)
except OSError, err:
raise errors.CreatorError("chroot err: %s" % str(err))
def __create_subvolume_snapshots(self):
if not self.snapshots or self.snapped:
return
""" Remount with subvolid=0 """
if self.fsopts:
mountopts = self.fsopts + ",subvolid=0"
else:
mountopts = "subvolid=0"
rc = runner.show([self.umountcmd, self.mountdir])
if rc != 0:
raise MountError("Failed to umount %s" % self.mountdir)
rc = runner.show(
[self.mountcmd, "-o", mountopts, self.disk.device, self.mountdir])
if rc != 0:
raise MountError("Failed to mount %s" % self.mountdir)
""" Create all the subvolume snapshots """
for snap in self.snapshots:
subvolpath = os.path.join(self.mountdir, snap["base"])
snapshotpath = os.path.join(self.mountdir, snap["name"])
msger.info("Creating snapshot %s based on %s..." %
(snap["name"], snap["base"]))
rc = runner.show([
self.btrfscmd, "subvolume", "snapshot", subvolpath,
snapshotpath
])
if rc != 0:
raise MountError(
"Failed to create subvolume snapshot '%s' for '%s', return code: %d."
% (snapshotpath, subvolpath, rc))
self.snapped = True
def start(self, filename, url, *args, **kwargs):
self.url = url
self.termwidth = terminal_width()
if self.total is None:
msger.info("Retrieving %s ..." % truncate_url(self.url, self.termwidth - 15))
else:
msger.info("Retrieving %s [%d/%d] ..." % (truncate_url(self.url, self.termwidth - 25), self.counter, self.total))
def __build_repo_cache(self, name):
repo = self.repo_manager.getRepositoryInfo(name)
if self.repo_manager.isCached(repo) or not repo.enabled():
return
msger.info('Refreshing repository: %s ...' % name)
self.repo_manager.buildCache(repo, zypp.RepoManager.BuildIfNeeded)
def SrcpkgsDownload(pkgs, repometadata, instroot, cachedir):
def get_source_repometadata(repometadata):
src_repometadata = []
for repo in repometadata:
if repo["name"].endswith("-source"):
src_repometadata.append(repo)
if src_repometadata:
return src_repometadata
return None
def get_src_name(srpm):
m = re.match("(.*)-(\d+.*)-(\d+\.\d+).src.rpm", srpm)
if m:
return m.group(1)
return None
src_repometadata = get_source_repometadata(repometadata)
if not src_repometadata:
msger.warning("No source repo found")
return None
src_pkgs = []
lpkgs_dict = {}
lpkgs_path = []
for repo in src_repometadata:
cachepath = "%s/%s/packages/*.src.rpm" % (cachedir, repo["name"])
lpkgs_path += glob.glob(cachepath)
for lpkg in lpkgs_path:
lpkg_name = get_src_name(os.path.basename(lpkg))
lpkgs_dict[lpkg_name] = lpkg
localpkgs = lpkgs_dict.keys()
cached_count = 0
destdir = instroot + '/usr/src/SRPMS'
if not os.path.exists(destdir):
os.makedirs(destdir)
srcpkgset = set()
for _pkg in pkgs:
srcpkg_name = get_source_name(_pkg, repometadata)
if not srcpkg_name:
continue
srcpkgset.add(srcpkg_name)
for pkg in list(srcpkgset):
if pkg in localpkgs:
cached_count += 1
shutil.copy(lpkgs_dict[pkg], destdir)
src_pkgs.append(os.path.basename(lpkgs_dict[pkg]))
else:
src_pkg = get_package(pkg, src_repometadata, 'src')
if src_pkg:
shutil.copy(src_pkg, destdir)
src_pkgs.append(src_pkg)
msger.info("%d source packages gotten from cache" % cached_count)
return src_pkgs
def SrcpkgsDownload(pkgs, repometadata, instroot, cachedir):
def get_source_repometadata(repometadata):
src_repometadata=[]
for repo in repometadata:
if repo["name"].endswith("-source"):
src_repometadata.append(repo)
if src_repometadata:
return src_repometadata
return None
def get_src_name(srpm):
m = SRPM_RE.match(srpm)
if m:
return m.group(1)
return None
src_repometadata = get_source_repometadata(repometadata)
if not src_repometadata:
msger.warning("No source repo found")
return None
src_pkgs = []
lpkgs_dict = {}
lpkgs_path = []
for repo in src_repometadata:
cachepath = "%s/%s/packages/*.src.rpm" %(cachedir, repo["name"])
lpkgs_path += glob.glob(cachepath)
for lpkg in lpkgs_path:
lpkg_name = get_src_name(os.path.basename(lpkg))
lpkgs_dict[lpkg_name] = lpkg
localpkgs = lpkgs_dict.keys()
cached_count = 0
destdir = instroot+'/usr/src/SRPMS'
if not os.path.exists(destdir):
os.makedirs(destdir)
srcpkgset = set()
for _pkg in pkgs:
srcpkg_name = get_source_name(_pkg, repometadata)
if not srcpkg_name:
continue
srcpkgset.add(srcpkg_name)
for pkg in list(srcpkgset):
if pkg in localpkgs:
cached_count += 1
shutil.copy(lpkgs_dict[pkg], destdir)
src_pkgs.append(os.path.basename(lpkgs_dict[pkg]))
else:
src_pkg = get_package(pkg, src_repometadata, 'src')
if src_pkg:
shutil.copy(src_pkg, destdir)
src_pkgs.append(src_pkg)
msger.info("%d source packages gotten from cache" % cached_count)
return src_pkgs
def setup_qemu_emulator(rootdir, arch):
# mount binfmt_misc if it doesn't exist
if not os.path.exists("/proc/sys/fs/binfmt_misc"):
modprobecmd = find_binary_path("modprobe")
runner.show([modprobecmd, "binfmt_misc"])
if not os.path.exists("/proc/sys/fs/binfmt_misc/register"):
mountcmd = find_binary_path("mount")
runner.show([mountcmd, "-t", "binfmt_misc", "none", "/proc/sys/fs/binfmt_misc"])
# qemu_emulator is a special case, we can't use find_binary_path
# qemu emulator should be a statically-linked executable file
if arch == "aarch64":
node = "/proc/sys/fs/binfmt_misc/aarch64"
if os.path.exists("/usr/bin/qemu-arm64") and is_statically_linked("/usr/bin/qemu-arm64"):
arm_binary = "qemu-arm64"
elif os.path.exists("/usr/bin/qemu-aarch64") and is_statically_linked("/usr/bin/qemu-aarch64"):
arm_binary = "qemu-aarch64"
elif os.path.exists("/usr/bin/qemu-arm64-static"):
arm_binary = "qemu-arm64-static"
elif os.path.exists("/usr/bin/qemu-aarch64-static"):
arm_binary = "qemu-aarch64-static"
else:
raise CreatorError("Please install a statically-linked %s" % arm_binary)
else:
node = "/proc/sys/fs/binfmt_misc/arm"
arm_binary = "qemu-arm"
if not os.path.exists("/usr/bin/qemu-arm") or not is_statically_linked("/usr/bin/qemu-arm"):
arm_binary = "qemu-arm-static"
if not os.path.exists("/usr/bin/%s" % arm_binary):
raise CreatorError("Please install a statically-linked %s" % arm_binary)
qemu_emulator = "/usr/bin/%s" % arm_binary
if not os.path.exists(rootdir + "/usr/bin"):
makedirs(rootdir + "/usr/bin")
shutil.copy(qemu_emulator, rootdir + qemu_emulator)
# disable selinux, selinux will block qemu emulator to run
if os.path.exists("/usr/sbin/setenforce"):
msger.info('Try to disable selinux')
runner.show(["/usr/sbin/setenforce", "0"])
# unregister it if it has been registered and is a dynamically-linked executable
if os.path.exists(node):
qemu_unregister_string = "-1\n"
with open(node, "w") as fd:
fd.write(qemu_unregister_string)
# register qemu emulator for interpreting other arch executable file
if not os.path.exists(node):
if arch == "aarch64":
qemu_arm_string = ":aarch64:M::\\x7fELF\\x02\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\xb7:\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xfe\\xff\\xff:%s:\n" % qemu_emulator
else:
qemu_arm_string = ":arm:M::\\x7fELF\\x01\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x28\\x00:\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xfa\\xff\\xff\\xff:%s:\n" % qemu_emulator
with open("/proc/sys/fs/binfmt_misc/register", "w") as fd:
fd.write(qemu_arm_string)
return qemu_emulator
def mount(self, options=None, init_expand=False):
self.__create()
if init_expand:
expand_size = long(self.disk.size * 1.5)
msger.info("Initial partition size expanded : %ld -> %ld" %
(self.disk.size, expand_size))
self.__resize_filesystem(expand_size)
self.disk.reread_size()
DiskMount.mount(self, options)
def package(self, destdir = "."):
ignores = ["/dev/fd",
"/dev/stdin",
"/dev/stdout",
"/dev/stderr",
"/etc/mtab"]
if not os.path.exists(destdir):
os.makedirs(destdir)
if self._recording_pkgs:
self._save_recording_pkgs(destdir)
if self._img_compression_method == None:
fsdir = os.path.join(destdir, self.name)
misc.check_space_pre_cp(self._instroot, destdir)
msger.info("Copying %s to %s ..." % (self._instroot, fsdir))
runner.show(['cp', "-af", self._instroot, fsdir])
for exclude in ignores:
if os.path.exists(fsdir + exclude):
os.unlink(fsdir + exclude)
self.outimage.append(fsdir)
elif self._img_compression_method == "tar.bz2":
dst = "%s/%s.tar.bz2" % (destdir, self.name)
msger.info("Creating %s (compressing %s with %s). Please wait..." \
% (dst, self._instroot, self._img_compression_method))
tar = find_binary_path('tar')
tar_cmdline = [tar, "--numeric-owner",
"--preserve-permissions",
"--preserve-order",
"--one-file-system",
"--directory",
self._instroot]
for ignore_entry in ignores:
if ignore_entry.startswith('/'):
ignore_entry = ignore_entry[1:]
tar_cmdline.append("--exclude=%s" % (ignore_entry))
tar_cmdline.extend(["-cjf", dst, "."])
rc = call(tar_cmdline)
if rc:
raise CreatorError("Failed compress image with tar.bz2. "
"Cmdline: %s" % (" ".join(tar_cmdline)))
self.outimage.append(dst)
else:
raise CreatorError("Compression method '%s' not supported for 'fs' "
"image format." % (self._img_compression_method))
def generate_bmap(self):
""" Generate block map file for an image. The idea is that while disk
images we generate may be large (e.g., 4GiB), they may actually contain
only little real data, e.g., 512MiB. This data are files, directories,
file-system meta-data, partition table, etc. In other words, when
flashing the image to the target device, you do not have to copy all the
4GiB of data, you can copy only 512MiB of it, which is 4 times faster.
This function generates the block map file for an arbitrary image that
mic has generated. The block map file is basically an XML file which
contains a list of blocks which have to be copied to the target device.
The other blocks are not used and there is no need to copy them.
This function assumes the image file was originally created as a sparse
file. To generate the block map we use the FIBMAP ioctl. """
if self.bmap_needed is None:
return
msger.info("Generating the map file(s)")
for name in self.__disks.keys():
image = self._full_path(self.__imgdir, name, self.__disk_format)
bmap_file = self._full_path(self.__imgdir, name, "bmap")
msger.debug("Generating block map file '%s'" % bmap_file)
image_size = os.path.getsize(image)
with open(bmap_file, "w") as f_bmap:
with open(image, "rb") as f_image:
# Get the block size of the host file-system for the image
# file by calling the FIGETBSZ ioctl (number 2).
block_size = unpack('I', ioctl(f_image, 2, pack('I', 0)))[0]
blocks_cnt = (image_size + block_size - 1) / block_size
# Write general information to the block map file, without
# block map itself, which will be written next.
xml = self._bmap_file_start(block_size, image_size,
blocks_cnt)
f_bmap.write(xml)
# Generate the block map and write it to the XML block map
# file as we go.
mapped_cnt = 0
for first, last in self._get_ranges(f_image, blocks_cnt):
mapped_cnt += last - first + 1
sha1 = misc.calc_hashes(image, ('sha1', ),
first * block_size,
(last + 1) * block_size)
f_bmap.write("\t\t<Range sha1=\"%s\"> %s-%s " \
"</Range>\n" % (sha1[0], first, last))
# Finish the block map file
xml = self._bmap_file_end(mapped_cnt, block_size,
blocks_cnt)
f_bmap.write(xml)
def setup_qemu_emulator(rootdir, arch):
# mount binfmt_misc if it doesn't exist
if not os.path.exists("/proc/sys/fs/binfmt_misc"):
modprobecmd = find_binary_path("modprobe")
runner.show([modprobecmd, "binfmt_misc"])
if not os.path.exists("/proc/sys/fs/binfmt_misc/register"):
mountcmd = find_binary_path("mount")
runner.show([mountcmd, "-t", "binfmt_misc", "none", "/proc/sys/fs/binfmt_misc"])
# qemu_emulator is a special case, we can't use find_binary_path
# qemu emulator should be a statically-linked executable file
qemu_emulator = "/usr/bin/qemu-arm"
if not os.path.exists(qemu_emulator) or not is_statically_linked(qemu_emulator):
qemu_emulator = "/usr/bin/qemu-arm-static"
if not os.path.exists(qemu_emulator):
raise CreatorError("Please install a statically-linked qemu-arm")
# qemu emulator version check
armv7_list = [arch for arch in rpmmisc.archPolicies.keys() if arch.startswith('armv7')]
if arch in armv7_list: # need qemu (>=0.13.0)
qemuout = runner.outs([qemu_emulator, "-h"])
m = re.search("version\s*([.\d]+)", qemuout)
if m:
qemu_version = m.group(1)
if qemu_version < "0.13":
raise CreatorError("Requires %s version >=0.13 for %s" % (qemu_emulator, arch))
else:
msger.warning("Can't get version info of %s, please make sure it's higher than 0.13.0" % qemu_emulator)
if not os.path.exists(rootdir + "/usr/bin"):
makedirs(rootdir + "/usr/bin")
shutil.copy(qemu_emulator, rootdir + qemu_emulator)
# disable selinux, selinux will block qemu emulator to run
if os.path.exists("/usr/sbin/setenforce"):
msger.info('Try to disable selinux')
runner.show(["/usr/sbin/setenforce", "0"])
node = "/proc/sys/fs/binfmt_misc/arm"
if is_statically_linked(qemu_emulator) and os.path.exists(node):
return qemu_emulator
# unregister it if it has been registered and is a dynamically-linked executable
if not is_statically_linked(qemu_emulator) and os.path.exists(node):
qemu_unregister_string = "-1\n"
fd = open("/proc/sys/fs/binfmt_misc/arm", "w")
fd.write(qemu_unregister_string)
fd.close()
# register qemu emulator for interpreting other arch executable file
if not os.path.exists(node):
qemu_arm_string = ":arm:M::\\x7fELF\\x01\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x28\\x00:\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xfa\\xff\\xff\\xff:%s:\n" % qemu_emulator
fd = open("/proc/sys/fs/binfmt_misc/register", "w")
fd.write(qemu_arm_string)
fd.close()
return qemu_emulator
def bootstrap_mic(argv=None):
def mychroot():
os.chroot(rootdir)
os.chdir(cwd)
# by default, sys.argv is used to run mic in bootstrap
if not argv:
argv = sys.argv
if argv[0] not in ('/usr/bin/mic', 'mic'):
argv[0] = '/usr/bin/mic'
cropts = configmgr.create
bsopts = configmgr.bootstrap
distro = bsopts['distro_name'].lower()
rootdir = bsopts['rootdir']
pkglist = bsopts['packages']
cwd = os.getcwd()
# create bootstrap and run mic in bootstrap
bsenv = bootstrap.Bootstrap(rootdir, distro, cropts['arch'])
bsenv.logfile = cropts['logfile']
# rootdir is regenerated as a temp dir
rootdir = bsenv.rootdir
if 'optional' in bsopts:
optlist = bsopts['optional']
else:
optlist = []
try:
msger.info("Creating %s bootstrap ..." % distro)
bsenv.create(cropts['repomd'], pkglist, optlist)
# bootstrap is relocated under "bootstrap"
if os.path.exists(os.path.join(rootdir, "bootstrap")):
rootdir = os.path.join(rootdir, "bootstrap")
bsenv.dirsetup(rootdir)
sync_mic(rootdir)
#FIXME: sync the ks file to bootstrap
if "/" == os.path.dirname(os.path.abspath(configmgr._ksconf)):
safecopy(configmgr._ksconf, rootdir)
msger.info("Start mic in bootstrap: %s\n" % rootdir)
bindmounts = get_bindmounts(cropts)
ret = bsenv.run(argv, cwd, rootdir, bindmounts)
except errors.BootstrapError, err:
msger.warning('\n%s' % err)
if msger.ask("Switch to native mode and continue?"):
return
else:
raise errors.BootstrapError("Failed to create bootstrap: %s" % err)
def load_module(module):
found = False
for line in open('/proc/modules').xreadlines():
if line.startswith("%s " % module):
found = True
break
if not found:
msger.info("Loading %s..." % module)
runner.quiet(['modprobe', module])
def setup_qemu_emulator(rootdir, arch):
# mount binfmt_misc if it doesn't exist
if not os.path.exists("/proc/sys/fs/binfmt_misc"):
modprobecmd = find_binary_path("modprobe")
runner.show([modprobecmd, "binfmt_misc"])
if not os.path.exists("/proc/sys/fs/binfmt_misc/register"):
mountcmd = find_binary_path("mount")
runner.show([mountcmd, "-t", "binfmt_misc", "none", "/proc/sys/fs/binfmt_misc"])
# qemu_emulator is a special case, we can't use find_binary_path
# qemu emulator should be a statically-linked executable file
qemu_emulator = "/usr/bin/qemu-arm"
if not os.path.exists(qemu_emulator) or not is_statically_linked(qemu_emulator):
qemu_emulator = "/usr/bin/qemu-arm-static"
if not os.path.exists(qemu_emulator):
raise CreatorError("Please install a statically-linked qemu-arm")
# qemu emulator version check
armv7_list = [arch for arch in rpmmisc.archPolicies.keys() if arch.startswith('armv7')]
if arch in armv7_list: # need qemu (>=0.13.0)
qemuout = runner.outs([qemu_emulator, "-h"])
m = re.search("version\s*([.\d]+)", qemuout)
if m:
qemu_version = m.group(1)
if qemu_version < "0.13":
raise CreatorError("Requires %s version >=0.13 for %s" % (qemu_emulator, arch))
else:
msger.warning("Can't get version info of %s, please make sure it's higher than 0.13.0" % qemu_emulator)
if not os.path.exists(rootdir + "/usr/bin"):
makedirs(rootdir + "/usr/bin")
shutil.copy(qemu_emulator, rootdir + qemu_emulator)
# disable selinux, selinux will block qemu emulator to run
if os.path.exists("/usr/sbin/setenforce"):
msger.info('Try to disable selinux')
runner.show(["/usr/sbin/setenforce", "0"])
node = "/proc/sys/fs/binfmt_misc/arm"
if is_statically_linked(qemu_emulator) and os.path.exists(node):
return qemu_emulator
# unregister it if it has been registered and is a dynamically-linked executable
if not is_statically_linked(qemu_emulator) and os.path.exists(node):
qemu_unregister_string = "-1\n"
fd = open("/proc/sys/fs/binfmt_misc/arm", "w")
fd.write(qemu_unregister_string)
fd.close()
# register qemu emulator for interpreting other arch executable file
if not os.path.exists(node):
qemu_arm_string = ":arm:M::\\x7fELF\\x01\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x28\\x00:\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xfa\\xff\\xff\\xff:%s:\n" % qemu_emulator
fd = open("/proc/sys/fs/binfmt_misc/register", "w")
fd.write(qemu_arm_string)
fd.close()
return qemu_emulator
def start(self, filename, url, *args, **kwargs):
self.url = url
self.termwidth = terminal_width()
if self.total is None:
msger.info("Retrieving %s ..." %
truncate_url(self.url, self.termwidth - 15))
else:
msger.info("Retrieving %s [%d/%d] ..." % (truncate_url(
self.url, self.termwidth - 25), self.counter, self.total))
def load_module(module):
found = False
for line in open('/proc/modules').xreadlines():
if line.startswith("%s " % module):
found = True
break
if not found:
msger.info("Loading %s..." % module)
runner.quiet(['modprobe', module])
def bootstrap_mic(argv=None):
def mychroot():
os.chroot(rootdir)
os.chdir(cwd)
# by default, sys.argv is used to run mic in bootstrap
if not argv:
argv = sys.argv
if argv[0] not in ('/usr/bin/mic', 'mic'):
argv[0] = '/usr/bin/mic'
cropts = configmgr.create
bsopts = configmgr.bootstrap
distro = bsopts['distro_name'].lower()
rootdir = bsopts['rootdir']
pkglist = bsopts['packages']
cwd = os.getcwd()
# create bootstrap and run mic in bootstrap
bsenv = bootstrap.Bootstrap(rootdir, distro, cropts['arch'])
bsenv.logfile = cropts['logfile']
# rootdir is regenerated as a temp dir
rootdir = bsenv.rootdir
if 'optional' in bsopts:
optlist = bsopts['optional']
else:
optlist = []
try:
msger.info("Creating %s bootstrap ..." % distro)
bsenv.create(cropts['repomd'], pkglist, optlist)
# bootstrap is relocated under "bootstrap"
if os.path.exists(os.path.join(rootdir, "bootstrap")):
rootdir = os.path.join(rootdir, "bootstrap")
bsenv.dirsetup(rootdir)
sync_mic(rootdir, plugin=cropts['plugin_dir'])
#FIXME: sync the ks file to bootstrap
if "/" == os.path.dirname(os.path.abspath(configmgr._ksconf)):
safecopy(configmgr._ksconf, rootdir)
msger.info("Start mic in bootstrap: %s\n" % rootdir)
bsarch = ELF_arch(rootdir)
if bsarch in personality_defs:
condPersonality(bsarch)
bindmounts = get_bindmounts(cropts)
ret = bsenv.run(argv, cwd, rootdir, bindmounts)
except errors.BootstrapError, err:
raise errors.CreatorError("Failed to download/install bootstrap package " \
"or the package is in bad format: %s" % err)
def bootstrap_mic(argv=None):
def mychroot():
os.chroot(rootdir)
os.chdir(cwd)
# by default, sys.argv is used to run mic in bootstrap
if not argv:
argv = sys.argv
if argv[0] not in ('/usr/bin/mic', 'mic'):
argv[0] = '/usr/bin/mic'
cropts = configmgr.create
bsopts = configmgr.bootstrap
distro = bsopts['distro_name'].lower()
rootdir = bsopts['rootdir']
pkglist = bsopts['packages']
cwd = os.getcwd()
# create bootstrap and run mic in bootstrap
bsenv = bootstrap.Bootstrap(rootdir, distro, cropts['arch'])
bsenv.logfile = cropts['logfile']
# rootdir is regenerated as a temp dir
rootdir = bsenv.rootdir
if 'optional' in bsopts:
optlist = bsopts['optional']
else:
optlist = []
try:
msger.info("Creating %s bootstrap ..." % distro)
bsenv.create(cropts['repomd'], pkglist, optlist)
# bootstrap is relocated under "bootstrap"
if os.path.exists(os.path.join(rootdir, "bootstrap")):
rootdir = os.path.join(rootdir, "bootstrap")
bsenv.dirsetup(rootdir)
sync_mic(rootdir)
#FIXME: sync the ks file to bootstrap
if "/" == os.path.dirname(os.path.abspath(configmgr._ksconf)):
safecopy(configmgr._ksconf, rootdir)
msger.info("Start mic in bootstrap: %s\n" % rootdir)
bindmounts = get_bindmounts(cropts)
ret = bsenv.run(argv, cwd, rootdir, bindmounts)
except errors.BootstrapError, err:
msger.warning('\n%s' % err)
if msger.ask("Switch to native mode and continue?"):
return
raise
def launch_shell(self, launch):
"""Launch a shell in the install root.
This method is launches a bash shell chroot()ed in the install root;
this can be useful for debugging.
"""
if launch:
msger.info("Launching shell. Exit to continue.")
subprocess.call(["/bin/bash"], preexec_fn=self._chroot)
def bootstrap_mic(argv=None):
def mychroot():
os.chroot(rootdir)
os.chdir(cwd)
# by default, sys.argv is used to run mic in bootstrap
if not argv:
argv = sys.argv
if argv[0] not in ('/usr/bin/mic', 'mic'):
argv[0] = '/usr/bin/mic'
cropts = configmgr.create
bsopts = configmgr.bootstrap
distro = bsopts['distro_name'].lower()
rootdir = bsopts['rootdir']
pkglist = bsopts['packages']
cwd = os.getcwd()
# create bootstrap and run mic in bootstrap
bsenv = bootstrap.Bootstrap(rootdir, distro, cropts['arch'])
bsenv.logfile = cropts['logfile']
# rootdir is regenerated as a temp dir
rootdir = bsenv.rootdir
if 'optional' in bsopts:
optlist = bsopts['optional']
else:
optlist = []
try:
msger.info("Creating %s bootstrap ..." % distro)
bsenv.create(cropts['repomd'], pkglist, optlist)
# bootstrap is relocated under "bootstrap"
if os.path.exists(os.path.join(rootdir, "bootstrap")):
rootdir = os.path.join(rootdir, "bootstrap")
bsenv.dirsetup(rootdir)
sync_mic(rootdir, plugin=cropts['plugin_dir'])
#FIXME: sync the ks file to bootstrap
if "/" == os.path.dirname(os.path.abspath(configmgr._ksconf)):
safecopy(configmgr._ksconf, rootdir)
msger.info("Start mic in bootstrap: %s\n" % rootdir)
bsarch = ELF_arch(rootdir)
if bsarch in personality_defs:
condPersonality(bsarch)
bindmounts = get_bindmounts(cropts)
ret = bsenv.run(argv, cwd, rootdir, bindmounts)
except errors.BootstrapError, err:
raise errors.CreatorError("Failed to download/install bootstrap package " \
"or the package is in bad format: %s" % err)
def move_post_umount_scripts(self):
scripts_dir = self._instroot + "/var/tmp/post_umount_scripts"
if not os.path.exists(scripts_dir):
return
self._umountdir = self._mkdtemp("umount")
msger.info("Moving post umount scripts...")
for item in os.listdir(scripts_dir):
spath = os.path.join(scripts_dir, item)
dpath = os.path.join(self._umountdir, item)
msger.verbose("Move post umount scripts %s to %s" % (spath, dpath))
shutil.move(spath, dpath)
shutil.rmtree(scripts_dir)
def copy_attachment(self):
if not hasattr(self, '_attachment') or not self._attachment:
return
self._check_imgdir()
msger.info("Copying attachment files...")
for item in self._attachment:
if not os.path.exists(item):
continue
dpath = os.path.join(self._imgdir, os.path.basename(item))
msger.verbose("Copy attachment %s to %s" % (item, dpath))
shutil.copy(item, dpath)
def _parse_kickstart(self, ksconf=None):
if not ksconf:
return
ksconf = misc.normalize_ksfile(ksconf, self.create['release'],
self.create['arch'])
ks = kickstart.read_kickstart(ksconf)
self.create['ks'] = ks
self.create['name'] = os.path.splitext(os.path.basename(ksconf))[0]
self.create['name'] = misc.build_name(ksconf, self.create['release'],
self.create['name_prefix'],
self.create['name_suffix'])
msger.info("Retrieving repo metadata:")
ksrepos = misc.get_repostrs_from_ks(ks)
if not ksrepos:
raise errors.KsError('no valid repos found in ks file')
for repo in ksrepos:
if 'baseurl' in repo and repo['baseurl'].startswith("file:"):
repourl = repo['baseurl'].replace('file:', '')
repourl = "/%s" % repourl.lstrip('/')
self.create['localrepos'].append(repourl)
self.create['repomd'] = misc.get_metadata_from_repos(
ksrepos, self.create['cachedir'])
msger.raw(" DONE")
target_archlist, archlist = misc.get_arch(self.create['repomd'])
if self.create['arch']:
if self.create['arch'] not in archlist:
raise errors.ConfigError("Invalid arch %s for repository. "
"Valid arches: %s" \
% (self.create['arch'], ', '.join(archlist)))
else:
if len(target_archlist) == 1:
self.create['arch'] = str(target_archlist[0])
msger.info("\nUse detected arch %s." % target_archlist[0])
else:
raise errors.ConfigError("Please specify a valid arch, "
"the choice can be: %s" \
% ', '.join(archlist))
kickstart.resolve_groups(self.create, self.create['repomd'])
# check selinux, it will block arm and btrfs image creation
misc.selinux_check(self.create['arch'],
[p.fstype for p in ks.handler.partition.partitions])
def package(self, destdir = "."):
"""Prepares the created image for final delivery.
In its simplest form, this method merely copies the install root to the
supplied destination directory; other subclasses may choose to package
the image by e.g. creating a bootable ISO containing the image and
bootloader configuration.
destdir -- the directory into which the final image should be moved;
this defaults to the current directory.
"""
self._stage_final_image()
if not os.path.exists(destdir):
fs.makedirs(destdir)
if self._img_compression_method:
if not self._img_name:
raise CreatorError("Image name not set.")
rc = None
img_location = os.path.join(self._outdir,self._img_name)
if self._img_compression_method == "bz2":
bzip2 = fs.find_binary_path('bzip2')
msger.info("Compressing %s with bzip2. Please wait..." % img_location)
rc = runner.show([bzip2, "-f", img_location])
if rc:
raise CreatorError("Failed to compress image %s with %s." % (img_location, self._img_compression_method))
for bootimg in glob.glob(os.path.dirname(img_location) + "/*-boot.bin"):
msger.info("Compressing %s with bzip2. Please wait..." % bootimg)
rc = runner.show([bzip2, "-f", bootimg])
if rc:
raise CreatorError("Failed to compress image %s with %s." % (bootimg, self._img_compression_method))
if self._recording_pkgs:
self._save_recording_pkgs(destdir)
""" For image formats with two or multiple image files, it will be better to put them under a directory """
if self.image_format in ("raw", "vmdk", "vdi", "nand", "mrstnand"):
destdir = os.path.join(destdir, "%s-%s" % (self.name, self.image_format))
msger.debug("creating destination dir: %s" % destdir)
fs.makedirs(destdir)
# Ensure all data is flushed to _outdir
runner.quiet('sync')
for f in os.listdir(self._outdir):
shutil.move(os.path.join(self._outdir, f),
os.path.join(destdir, f))
self.outimage.append(os.path.join(destdir, f))
self.do_genchecksum(os.path.join(destdir, f))
def _get_uncompressed_data_from_url(url, filename, proxies):
filename = myurlgrab(url, filename, proxies)
suffix = None
if filename.endswith(".gz"):
suffix = ".gz"
runner.quiet(['gunzip', "-f", filename])
msger.info("filename %s gunzipped" % filename)
elif filename.endswith(".bz2"):
suffix = ".bz2"
runner.quiet(['bunzip2', "-f", filename])
if suffix:
filename = filename.replace(suffix, "")
msger.info("filename is now %s" % filename)
return filename
def runmic_in_bootstrap(name, argv, opts, ksfile, repolist):
bootstrap_env = bootstrap.Bootstrap(homedir=opts['bootstrapdir'])
bootstrap_lst = bootstrap_env.bootstraps
setattr(bootstrap_env, 'rootdir', name)
if not bootstrap_lst or not name in bootstrap_lst:
msger.info("Creating bootstrap %s under %s" %
(name, bootstrap_env.homedir))
bootstrap_env.create(name, repolist)
msger.info("Use bootstrap: %s" % bootstrap_env.rootdir)
# copy mic
msger.info("Sync native mic to bootstrap")
copy_mic(bootstrap_env.rootdir)
# bind mounts , opts['cachedir'], opts['tmpdir']
cwd = os.getcwd()
lst = [cwd, opts['outdir']]
if ksfile:
ksfp = os.path.abspath(os.path.expanduser(ksfile))
lst.append(os.path.dirname(ksfp))
if opts['logfile']:
logfile = os.path.abspath(os.path.expanduser(opts['logfile']))
lst.append(os.path.dirname(logfile))
# TBD local package path
# TBD local repo
# make unique
lst = list(set(lst)) # FIXME: wo need the original order here
bindmounts = ';'.join(
map(lambda p: os.path.abspath(os.path.expanduser(p)), lst))
msger.info("Start mic command in bootstrap")
bootstrap_env.run(name, argv, cwd, bindmounts)
def chroot(chrootdir, bindmounts=None, execute="/bin/bash"):
def mychroot():
os.chroot(chrootdir)
os.chdir("/")
dev_null = os.open("/dev/null", os.O_WRONLY)
files_to_check = ["/bin/bash", "/sbin/init"]
architecture_found = False
""" Register statically-linked qemu-arm if it is an ARM fs """
qemu_emulator = None
for ftc in files_to_check:
ftc = "%s/%s" % (chrootdir, ftc)
# Return code of 'file' is "almost always" 0 based on some man pages
# so we need to check the file existance first.
if not os.path.exists(ftc):
continue
for line in runner.outs(['file', ftc]).splitlines():
if 'ARM' in line:
qemu_emulator = misc.setup_qemu_emulator(chrootdir, "arm")
architecture_found = True
break
if 'Intel' in line:
architecture_found = True
break
if architecture_found:
break
os.close(dev_null)
if not architecture_found:
raise errors.CreatorError("Failed to get architecture from any of the "
"following files %s from chroot." \
% files_to_check)
try:
msger.info("Launching shell. Exit to continue.\n"
"----------------------------------")
globalmounts = setup_chrootenv(chrootdir, bindmounts)
subprocess.call(execute, preexec_fn=mychroot, shell=True)
except OSError, err:
raise errors.CreatorError("chroot err: %s" % str(err))
def do_create(self, subcmd, opts, *args):
"""${cmd_name}: create livecd image
Usage:
${name} ${cmd_name} <ksfile> [OPTS]
${cmd_option_list}
"""
creatoropts = common.creatoropts(args)
if creatoropts['arch'] and creatoropts['arch'].startswith('arm'):
msger.warning('livecd cannot support arm images, Quit')
return
creator = livecd.LiveCDImageCreator(creatoropts,
creatoropts['pkgmgr_pcls'])
creator._recording_pkgs = creatoropts['record_pkgs']
self.check_image_exists(creator.destdir, creator.pack_to,
[creator.name + ".iso"],
creatoropts['release'])
try:
creator.check_depend_tools()
creator.mount(None, creatoropts["cachedir"])
creator.install()
creator.configure(creatoropts["repomd"])
creator.copy_kernel()
creator.unmount()
creator.package(creatoropts["outdir"])
if creatoropts['release'] is not None:
creator.release_output(ksconf, creatoropts['outdir'],
creatoropts['release'])
else:
creator.outimage.append(creatoropts['dst_ks'])
creator.print_outimage_info()
except errors.CreatorError:
raise
finally:
creator.cleanup()
msger.info("Finished.")
return 0
def __resize_filesystem(self, size=None):
msger.info("Resizing filesystem ...")
current_size = os.stat(self.disk.lofile)[stat.ST_SIZE]
if size is None:
size = self.disk.size
if size == current_size:
return
if size > current_size:
self.disk.expand(size)
self.__fsck()
resize2fs(self.disk.lofile, size)
return size
def _stage_final_image(self):
"""Stage the final system image in _outdir.
write meta data
"""
self._resparse()
self.image_files.update({'disks': self.__disks.keys()})
if not (self.compress_image or self.pack_to):
for imgfile in os.listdir(self.__imgdir):
if imgfile.endswith('.raw'):
for disk in self.__disks.keys():
if imgfile.find(disk) != -1:
self.image_files.setdefault(disk, {}).update(
{'image': imgfile})
self.image_files.setdefault('image_files',
[]).append(imgfile)
if self.compress_image:
for imgfile in os.listdir(self.__imgdir):
if imgfile.endswith('.raw') or imgfile.endswith('bin'):
imgpath = os.path.join(self.__imgdir, imgfile)
msger.info("Compressing image %s" % imgfile)
compressing(imgpath, self.compress_image)
if imgfile.endswith('.raw') and not self.pack_to:
for disk in self.__disks.keys():
if imgfile.find(disk) != -1:
imgname = '%s.%s' % (imgfile, self.compress_image)
self.image_files.setdefault(disk, {}).update(
{'image': imgname})
self.image_files.setdefault('image_files',
[]).append(imgname)
if self.pack_to:
dst = os.path.join(self._outdir, self.pack_to)
msger.info("Pack all raw images to %s" % dst)
packing(dst, self.__imgdir)
self.image_files.update({'image_files': self.pack_to})
else:
msger.debug("moving disks to stage location")
for imgfile in os.listdir(self.__imgdir):
src = os.path.join(self.__imgdir, imgfile)
dst = os.path.join(self._outdir, imgfile)
msger.debug("moving %s to %s" % (src,dst))
shutil.move(src,dst)
self._write_image_xml()
def print_outimage_info(self):
"""
Print the image(s) and artifacts used, for the user.
"""
msg = "The new image(s) can be found here:\n"
for disk_name, disk in self.__instimage.disks.items():
full_path = self._full_path(self.__imgdir, disk_name, "direct")
msg += ' %s\n\n' % full_path
msg += 'The following build artifacts were used to create the image(s):\n'
msg += ' ROOTFS_DIR: %s\n' % self.rootfs_dir
msg += ' BOOTIMG_DIR: %s\n' % self.bootimg_dir
msg += ' KERNEL_DIR: %s\n' % self.kernel_dir
msg += ' NATIVE_SYSROOT: %s\n' % self.native_sysroot
msger.info(msg)