def myurlgrab(url, filename, proxies, progress_obj=None):
g = grabber.URLGrabber()
if progress_obj is None:
progress_obj = TextProgress()
if url.startswith("file:/"):
filepath = "/%s" % url.replace("file:", "").lstrip('/')
if not os.path.exists(filepath):
raise CreatorError("URLGrabber error: can't find file %s" % url)
if url.endswith('.rpm'):
return filepath
else:
# untouch repometadata in source path
runner.show(['cp', '-f', filepath, filename])
else:
try:
filename = g.urlgrab(url=str(url),
filename=filename,
ssl_verify_host=False,
ssl_verify_peer=False,
proxies=proxies,
http_headers=(('Pragma', 'no-cache'), ),
quote=0,
progress_obj=progress_obj)
except grabber.URLGrabError, err:
msg = str(err)
if msg.find(url) < 0:
msg += ' on %s' % url
raise CreatorError(msg)
def get_disk_names(self):
""" Returns a list of physical target disk names (e.g., 'sdb') which
will be created. """
if self._disk_names:
return self._disk_names
#get partition info from ks handler
parts = self._get_parts()
for i in range(len(parts)):
if parts[i].disk:
disk_name = parts[i].disk
else:
raise CreatorError("Failed to create disks, no --ondisk "
"specified in partition line of ks file")
if parts[i].mountpoint and not parts[i].fstype:
raise CreatorError("Failed to create disks, no --fstype "
"specified for partition with mountpoint "
"'%s' in the ks file")
self._disk_names.append(disk_name)
return self._disk_names
def __create_iso(self, isodir):
iso = self._outdir + "/" + self.name + ".iso"
genisoimage = fs_related.find_binary_path("genisoimage")
args = [
genisoimage, "-J", "-r", "-hide-rr-moved",
"-hide-joliet-trans-tbl", "-V", self.fslabel, "-o", iso
]
args.extend(self._get_mkisofs_options(isodir))
args.append(isodir)
if runner.show(args) != 0:
raise CreatorError("ISO creation failed!")
""" It should be ok still even if you haven't isohybrid """
isohybrid = None
try:
isohybrid = fs_related.find_binary_path("isohybrid")
except:
pass
if isohybrid:
args = [isohybrid, "-partok", iso]
if runner.show(args) != 0:
raise CreatorError("Hybrid ISO creation failed!")
self.__implant_md5sum(iso)
def get_diskinfo(self):
if self._diskinfo:
return self._diskinfo
#get partition info from ks handler
parts = self._get_parts()
for i in range(len(parts)):
if parts[i].disk:
disk = parts[i].disk
else:
raise CreatorError("Failed to create disks, no --ondisk "
"specified in partition line of ks file")
if not parts[i].fstype:
raise CreatorError("Failed to create disks, no --fstype "
"specified in partition line of ks file")
size = parts[i].size * 1024L * 1024L
found = False
for j in range(len(self._diskinfo)):
if self._diskinfo[j]['name'] == disk:
self._diskinfo[j][
'size'] = self._diskinfo[j]['size'] + size
found = True
break
else:
found = False
if not found:
self._diskinfo.append({'name': disk, 'size': size})
return self._diskinfo
def __set_blocksize(self, val):
if self._instloops:
raise CreatorError("_blocksize must be set before calling mount()")
try:
self.__blocksize = int(val)
except ValueError:
raise CreatorError("'%s' is not a valid integer value "
"for _blocksize" % val)
def compressing(fpath, method):
comp_map = {"gz": "gzip", "bz2": "bzip2"}
if method not in comp_map:
raise CreatorError("Unsupport compress format: %s, valid values: %s" %
(method, ','.join(comp_map.keys())))
cmd = find_binary_path(comp_map[method])
rc = runner.show([cmd, "-f", fpath])
if rc:
raise CreatorError("Failed to %s file: %s" % (comp_map[method], fpath))
def __sanity_check(self):
"""Ensure that the config we've been given is sane."""
if not (kickstart.get_packages(self.ks)
or kickstart.get_groups(self.ks)):
raise CreatorError("No packages or groups specified")
kickstart.convert_method_to_repo(self.ks)
if not kickstart.get_repos(self.ks):
raise CreatorError("No repositories specified")
def setup_qemu_emulator(rootdir, arch):
# mount binfmt_misc if it doesn't exist
if not os.path.exists("/proc/sys/fs/binfmt_misc"):
modprobecmd = find_binary_path("modprobe")
runner.show([modprobecmd, "binfmt_misc"])
if not os.path.exists("/proc/sys/fs/binfmt_misc/register"):
mountcmd = find_binary_path("mount")
runner.show([mountcmd, "-t", "binfmt_misc", "none", "/proc/sys/fs/binfmt_misc"])
# qemu_emulator is a special case, we can't use find_binary_path
# qemu emulator should be a statically-linked executable file
qemu_emulator = "/usr/bin/qemu-arm"
if not os.path.exists(qemu_emulator) or not is_statically_linked(qemu_emulator):
qemu_emulator = "/usr/bin/qemu-arm-static"
if not os.path.exists(qemu_emulator):
raise CreatorError("Please install a statically-linked qemu-arm")
# qemu emulator version check
armv7_list = [arch for arch in rpmmisc.archPolicies.keys() if arch.startswith('armv7')]
if arch in armv7_list: # need qemu (>=0.13.0)
qemuout = runner.outs([qemu_emulator, "-h"])
m = re.search("version\s*([.\d]+)", qemuout)
if m:
qemu_version = m.group(1)
if qemu_version < "0.13":
raise CreatorError("Requires %s version >=0.13 for %s" % (qemu_emulator, arch))
else:
msger.warning("Can't get version info of %s, please make sure it's higher than 0.13.0" % qemu_emulator)
if not os.path.exists(rootdir + "/usr/bin"):
makedirs(rootdir + "/usr/bin")
shutil.copy(qemu_emulator, rootdir + "/usr/bin/qemu-arm-static")
qemu_emulator = "/usr/bin/qemu-arm-static"
# disable selinux, selinux will block qemu emulator to run
if os.path.exists("/usr/sbin/setenforce"):
msger.info('Try to disable selinux')
runner.show(["/usr/sbin/setenforce", "0"])
# unregister it if it has been registered and is a dynamically-linked executable
node = "/proc/sys/fs/binfmt_misc/arm"
if os.path.exists(node):
qemu_unregister_string = "-1\n"
fd = open("/proc/sys/fs/binfmt_misc/arm", "w")
fd.write(qemu_unregister_string)
fd.close()
# register qemu emulator for interpreting other arch executable file
if not os.path.exists(node):
qemu_arm_string = ":arm:M::\\x7fELF\\x01\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x28\\x00:\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xfa\\xff\\xff\\xff:%s:\n" % qemu_emulator
fd = open("/proc/sys/fs/binfmt_misc/register", "w")
fd.write(qemu_arm_string)
fd.close()
return qemu_emulator
def package(self, destdir = "."):
"""Prepares the created image for final delivery.
In its simplest form, this method merely copies the install root to the
supplied destination directory; other subclasses may choose to package
the image by e.g. creating a bootable ISO containing the image and
bootloader configuration.
destdir -- the directory into which the final image should be moved;
this defaults to the current directory.
"""
self._stage_final_image()
if not os.path.exists(destdir):
fs.makedirs(destdir)
if self._img_compression_method:
if not self._img_name:
raise CreatorError("Image name not set.")
rc = None
img_location = os.path.join(self._outdir,self._img_name)
if self._img_compression_method == "bz2":
bzip2 = fs.find_binary_path('bzip2')
msger.info("Compressing %s with bzip2. Please wait..." % img_location)
rc = runner.show([bzip2, "-f", img_location])
if rc:
raise CreatorError("Failed to compress image %s with %s." % (img_location, self._img_compression_method))
for bootimg in glob.glob(os.path.dirname(img_location) + "/*-boot.bin"):
msger.info("Compressing %s with bzip2. Please wait..." % bootimg)
rc = runner.show([bzip2, "-f", bootimg])
if rc:
raise CreatorError("Failed to compress image %s with %s." % (bootimg, self._img_compression_method))
if self._recording_pkgs:
self._save_recording_pkgs(destdir)
""" For image formats with two or multiple image files, it will be better to put them under a directory """
if self.image_format in ("raw", "vmdk", "vdi", "nand", "mrstnand"):
destdir = os.path.join(destdir, "%s-%s" % (self.name, self.image_format))
msger.debug("creating destination dir: %s" % destdir)
fs.makedirs(destdir)
# Ensure all data is flushed to _outdir
runner.quiet('sync')
for f in os.listdir(self._outdir):
shutil.move(os.path.join(self._outdir, f),
os.path.join(destdir, f))
self.outimage.append(os.path.join(destdir, f))
self.do_genchecksum(os.path.join(destdir, f))
def selectPackage(self, pkg):
"""Select a given package.
Can be specified with name.arch or name*
"""
try:
self.install(pattern=pkg)
return None
except yum.Errors.InstallError:
return "No package(s) available to install"
except yum.Errors.RepoError as e:
raise CreatorError("Unable to download from repo : %s" % (e, ))
except yum.Errors.YumBaseError as e:
raise CreatorError("Unable to install: %s" % (e, ))
def selinux_check(arch, fstypes):
try:
getenforce = find_binary_path('getenforce')
except CreatorError:
return
selinux_status = runner.outs([getenforce])
if arch and arch.startswith("arm") and selinux_status == "Enforcing":
raise CreatorError("Can't create arm image if selinux is enabled, "
"please run 'setenforce 0' to disable selinux")
use_btrfs = filter(lambda typ: typ == 'btrfs', fstypes)
if use_btrfs and selinux_status == "Enforcing":
raise CreatorError("Can't create btrfs image if selinux is enabled,"
" please run 'setenforce 0' to disable selinux")
def get_file_size(filename):
""" Return size in MB unit """
cmd = ['du', "-s", "-b", "-B", "1M", filename]
rc, duOutput = runner.runtool(cmd)
if rc != 0:
raise CreatorError("Failed to run: %s" % ' '.join(cmd))
size1 = int(duOutput.split()[0])
cmd = ['du', "-s", "-B", "1M", filename]
rc, duOutput = runner.runtool(cmd)
if rc != 0:
raise CreatorError("Failed to run: %s" % ' '.join(cmd))
size2 = int(duOutput.split()[0])
return max(size1, size2)
def generate_bmap(self):
""" Generate block map file for the image. The idea is that while disk
images we generate may be large (e.g., 4GiB), they may actually contain
only little real data, e.g., 512MiB. This data are files, directories,
file-system meta-data, partition table, etc. In other words, when
flashing the image to the target device, you do not have to copy all the
4GiB of data, you can copy only 512MiB of it, which is 4 times faster.
This function generates the block map file for an arbitrary image that
mic has generated. The block map file is basically an XML file which
contains a list of blocks which have to be copied to the target device.
The other blocks are not used and there is no need to copy them. """
if self.bmap_needed is None:
return
from mic.utils import BmapCreate
msger.info("Generating the map file(s)")
for name in self.__disks.keys():
image = self._full_path(self.__imgdir, name, self.__disk_format)
bmap_file = self._full_path(self._outdir, name, "bmap")
msger.debug("Generating block map file '%s'" % bmap_file)
try:
creator = BmapCreate.BmapCreate(image, bmap_file)
creator.generate()
del creator
except BmapCreate.Error as err:
raise CreatorError("Failed to create bmap file: %s" % str(err))
def selectGroup(self, grp, include = ksparser.GROUP_DEFAULT):
def compareGroup(pitem):
item = zypp.asKindPattern(pitem)
return item.repoInfo().priority()
if not self.Z:
self.__initialize_zypp()
found = False
q = zypp.PoolQuery()
q.addKind(zypp.ResKind.pattern)
for pitem in sorted(q.queryResults(self.Z.pool()), key=compareGroup):
item = zypp.asKindPattern(pitem)
summary = "%s" % item.summary()
name = "%s" % item.name()
if name == grp or summary == grp:
found = True
pitem.status().setToBeInstalled (zypp.ResStatus.USER)
break
if found:
if include == ksparser.GROUP_REQUIRED:
map(
lambda p: self.deselectPackage(p),
grp.default_packages.keys())
return None
else:
raise CreatorError("Unable to find pattern: %s" % (grp,))
def runInstall(self, checksize = 0):
os.environ["HOME"] = "/"
os.environ["LD_PRELOAD"] = ""
try:
(res, resmsg) = self.buildTransaction()
except yum.Errors.RepoError, e:
raise CreatorError("Unable to download from repo : %s" %(e,))
def selectGroup(self, grp, include = ksconstants.GROUP_DEFAULT):
if not self.Z:
self.__initialize_zypp()
found = False
q=zypp.PoolQuery()
q.addKind(zypp.ResKind.pattern)
msger.debug("looking for %s" % grp)
for item in sorted(
q.queryResults(self.Z.pool()),
key=cmp_to_key(lambda x,y: cmpPackage(self._castKind(x), self._castKind(y))),
reverse=True):
xitem = self._castKind(item)
summary = "%s" % xitem.summary()
name = "%s" % xitem.name()
msger.debug("checking %s version %s" % (name, xitem.edition()))
if name == grp or summary == grp:
found = True
msger.info("marking pattern %s %s to be installed" % ( name, xitem.edition() ))
item.status().setToBeInstalled (zypp.ResStatus.USER)
break
if found:
if include == ksconstants.GROUP_REQUIRED:
list(map(
lambda p: self.deselectPackage(p),
list(grp.default_packages.keys())))
return None
else:
raise CreatorError("Unable to find pattern: %s" % (grp,))
def generate_bmap(self):
""" Generate block map file for the image. The idea is that while disk
images we generate may be large (e.g., 4GiB), they may actually contain
only little real data, e.g., 512MiB. This data are files, directories,
file-system meta-data, partition table, etc. In other words, when
flashing the image to the target device, you do not have to copy all the
4GiB of data, you can copy only 512MiB of it, which is 4 times faster.
This function generates the block map file for an arbitrary image that
mic has generated. The block map file is basically an XML file which
contains a list of blocks which have to be copied to the target device.
The other blocks are not used and there is no need to copy them. """
if self.bmap_needed is None:
return
msger.info("Generating the map file(s)")
for name in self.__disks.keys():
image = self._full_path(self.__imgdir, name, self.__disk_format)
bmap_file = self._full_path(self._outdir, name, "bmap")
self.image_files.setdefault(name, {}).update({'bmap': \
os.path.basename(bmap_file)})
msger.debug("Generating block map file '%s'" % bmap_file)
bmaptoolcmd = misc.find_binary_path('bmaptool')
rc = runner.show([bmaptoolcmd, 'create', image, '-o', bmap_file])
if rc != 0:
raise CreatorError("Failed to create bmap file: %s" % bmap_file)
def __run_post_scripts(self):
msger.info("Running scripts ...")
if os.path.exists(self._instroot + "/tmp"):
shutil.rmtree(self._instroot + "/tmp")
os.mkdir (self._instroot + "/tmp", 0755)
for s in kickstart.get_post_scripts(self.ks):
(fd, path) = tempfile.mkstemp(prefix = "ks-script-",
dir = self._instroot + "/tmp")
s.script = s.script.replace("\r", "")
os.write(fd, s.script)
os.close(fd)
os.chmod(path, 0700)
env = self._get_post_scripts_env(s.inChroot)
if not s.inChroot:
env["INSTALL_ROOT"] = self._instroot
env["IMG_NAME"] = self._name
preexec = None
script = path
else:
preexec = self._chroot
script = "/tmp/" + os.path.basename(path)
try:
try:
subprocess.call([s.interp, script],
preexec_fn = preexec, env = env, stdout = sys.stdout, stderr = sys.stderr)
except OSError, (err, msg):
raise CreatorError("Failed to execute %%post script "
"with '%s' : %s" % (s.interp, msg))
finally:
os.unlink(path)
def __find_syslinux_menu(self):
for menu in ["vesamenu.c32", "menu.c32"]:
if os.path.isfile(self._instroot + "/usr/share/syslinux/" + menu):
return menu
raise CreatorError("syslinux not installed : "
"no suitable /usr/share/syslinux/*menu.c32 found")
def checkRepositoryEULA(name, repo):
""" This function is to check the EULA file if provided.
return True: no EULA or accepted
return False: user declined the EULA
"""
import tempfile
import shutil
import urlparse
import urllib2 as u2
import httplib
from mic.utils.errors import CreatorError
def _check_and_download_url(u2opener, url, savepath):
try:
if u2opener:
f = u2opener.open(url)
else:
f = u2.urlopen(url)
except u2.HTTPError, httperror:
if httperror.code in (404, 503):
return None
else:
raise CreatorError(httperror)
except OSError, oserr:
if oserr.errno == 2:
return None
else:
raise CreatorError(oserr)
def __copy_kernel_and_initramfs(self, isodir, version, index):
bootdir = self._instroot + "/boot"
if self._alt_initrd_name:
src_initrd_path = os.path.join(bootdir, self._alt_initrd_name)
else:
src_initrd_path = os.path.join(bootdir,
"initrd-" + version + ".img")
try:
shutil.copyfile(bootdir + "/vmlinuz-" + version,
isodir + "/isolinux/vmlinuz" + index)
shutil.copyfile(src_initrd_path,
isodir + "/isolinux/initrd" + index + ".img")
except:
raise CreatorError("Unable to copy valid kernels or initrds, "
"please check the repo.")
is_xen = False
if os.path.exists(bootdir + "/xen.gz-" + version[:-3]):
shutil.copyfile(bootdir + "/xen.gz-" + version[:-3],
isodir + "/isolinux/xen" + index + ".gz")
is_xen = True
return is_xen
def selectGroup(self, grp, include=ksparser.GROUP_DEFAULT):
try:
yum.YumBase.selectGroup(self, grp)
if include == ksparser.GROUP_REQUIRED:
for p in list(grp.default_packages.keys()):
self.deselectPackage(p)
elif include == ksparser.GROUP_ALL:
for p in list(grp.optional_packages.keys()):
self.selectPackage(p)
return None
except (yum.Errors.InstallError, yum.Errors.GroupsError) as e:
return e
except yum.Errors.RepoError as e:
raise CreatorError("Unable to download from repo : %s" % (e, ))
except yum.Errors.YumBaseError as e:
raise CreatorError("Unable to install: %s" % (e, ))
def get_image_type(path):
def _get_extension_name(path):
match = re.search("(?<=\.)\w+$", path)
if match:
return match.group(0)
else:
return None
if os.path.isdir(path):
_check_mic_chroot(path)
return "fs"
maptab = {
"tar": "loop",
"raw":"raw",
"vmdk":"vmdk",
"vdi":"vdi",
"iso":"livecd",
"usbimg":"liveusb",
}
extension = _get_extension_name(path)
if extension in maptab:
return maptab[extension]
fd = open(path, "rb")
file_header = fd.read(1024)
fd.close()
vdi_flag = "<<< Sun VirtualBox Disk Image >>>"
if file_header[0:len(vdi_flag)] == vdi_flag:
return maptab["vdi"]
output = runner.outs(['file', path])
isoptn = re.compile(r".*ISO 9660 CD-ROM filesystem.*(bootable).*")
usbimgptn = re.compile(r".*x86 boot sector.*active.*")
rawptn = re.compile(r".*x86 boot sector.*")
vmdkptn = re.compile(r".*VMware. disk image.*")
ext3fsimgptn = re.compile(r".*Linux.*ext3 filesystem data.*")
ext4fsimgptn = re.compile(r".*Linux.*ext4 filesystem data.*")
btrfsimgptn = re.compile(r".*BTRFS.*")
if isoptn.match(output):
return maptab["iso"]
elif usbimgptn.match(output):
return maptab["usbimg"]
elif rawptn.match(output):
return maptab["raw"]
elif vmdkptn.match(output):
return maptab["vmdk"]
elif ext3fsimgptn.match(output):
return "ext3fsimg"
elif ext4fsimgptn.match(output):
return "ext4fsimg"
elif btrfsimgptn.match(output):
return "btrfsimg"
else:
raise CreatorError("Cannot detect the type of image: %s" % path)
def _configure_bootloader(self, isodir):
"""Create the architecture specific booloader configuration.
This is the hook where subclasses must create the booloader
configuration in order to allow a bootable ISO to be built.
isodir -- the directory where the contents of the ISO are to be staged
"""
raise CreatorError("Bootloader configuration is arch-specific, "
"but not implemented for this arch!")
def convert_image(srcimg, srcfmt, dstimg, dstfmt):
#convert disk format
if dstfmt != "raw":
raise CreatorError("Invalid destination image format: %s" % dstfmt)
msger.debug("converting %s image to %s" % (srcimg, dstimg))
if srcfmt == "vmdk":
path = find_binary_path("qemu-img")
argv = [path, "convert", "-f", "vmdk", srcimg, "-O", dstfmt, dstimg]
elif srcfmt == "vdi":
path = find_binary_path("VBoxManage")
argv = [path, "internalcommands", "converttoraw", srcimg, dstimg]
else:
raise CreatorError("Invalid soure image format: %s" % srcfmt)
rc = runner.show(argv)
if rc == 0:
msger.debug("convert successful")
if rc != 0:
raise CreatorError("Unable to convert disk to %s" % dstfmt)
def __ensure_builddir(self):
if not self.__builddir is None:
return
try:
self.__builddir = tempfile.mkdtemp(dir = self.tmpdir,
prefix = "imgcreate-")
except OSError, (err, msg):
raise CreatorError("Failed create build directory in %s: %s" %
(self.tmpdir, msg))
def check_space_pre_cp(src, dst):
"""Check whether disk space is enough before 'cp' like
operations, else exception will be raised.
"""
srcsize = get_file_size(src) * 1024 * 1024
freesize = get_filesystem_avail(dst)
if srcsize > freesize:
raise CreatorError("space on %s(%s) is not enough for about %s files"
% (dst, human_size(freesize), human_size(srcsize)))
def packing(dstfile, target):
(base, ext) = os.path.splitext(dstfile)
if ext in (".gz", ".bz2") and base.endswith(".tar"):
ext = ".tar" + ext
if ext not in pack_formats:
raise CreatorError("Unsupport pack format: %s, valid values: %s"
% (ext, ','.join(pack_formats.keys())))
func = pack_formats[ext]
# func should be callable
func(dstfile, target)
def __init__(self, creatoropts=None, pkgmgr=None):
"""Initialise a LiveImageCreator instance.
This method takes the same arguments as ImageCreator.__init__().
"""
LoopImageCreator.__init__(self, creatoropts, pkgmgr)
#Controls whether to use squashfs to compress the image.
self.skip_compression = False
#Controls whether an image minimizing snapshot should be created.
#
#This snapshot can be used when copying the system image from the ISO in
#order to minimize the amount of data that needs to be copied; simply,
#it makes it possible to create a version of the image's filesystem with
#no spare space.
self.skip_minimize = False
#A flag which indicates i act as a convertor default false
self.actasconvertor = False
#The bootloader timeout from kickstart.
if self.ks:
self._timeout = kickstart.get_timeout(self.ks, 10)
else:
self._timeout = 10
#The default kernel type from kickstart.
if self.ks:
self._default_kernel = kickstart.get_default_kernel(
self.ks, "kernel")
else:
self._default_kernel = None
if self.ks:
parts = kickstart.get_partitions(self.ks)
if len(parts) > 1:
raise CreatorError("Can't support multi partitions in ks file "
"for this image type")
# FIXME: rename rootfs img to self.name,
# else can't find files when create iso
self._instloops[0]['name'] = self.name + ".img"
self.__isodir = None
self.__modules = [
"=ata", "sym53c8xx", "aic7xxx", "=usb", "=firewire", "=mmc",
"=pcmcia", "mptsas"
]
if self.ks:
self.__modules.extend(kickstart.get_modules(self.ks))
self._dep_checks.extend(
["isohybrid", "unsquashfs", "mksquashfs", "dd", "genisoimage"])
def _check_and_download_url(u2opener, url, savepath):
try:
if u2opener:
f = u2opener.open(url)
else:
f = u2.urlopen(url)
except u2.HTTPError, httperror:
if httperror.code in (404, 503):
return None
else:
raise CreatorError(httperror)
