def setup_qemu_emulator(rootdir, arch):
    # mount binfmt_misc if it doesn't exist
    if not os.path.exists("/proc/sys/fs/binfmt_misc"):
        modprobecmd = find_binary_path("modprobe")
        runner.show([modprobecmd, "binfmt_misc"])
    if not os.path.exists("/proc/sys/fs/binfmt_misc/register"):
        mountcmd = find_binary_path("mount")
        runner.show([mountcmd, "-t", "binfmt_misc", "none", "/proc/sys/fs/binfmt_misc"])

    # qemu_emulator is a special case, we can't use find_binary_path
    # qemu emulator should be a statically-linked executable file
    qemu_emulator = "/usr/bin/qemu-arm"
    if not os.path.exists(qemu_emulator) or not is_statically_linked(qemu_emulator):
        qemu_emulator = "/usr/bin/qemu-arm-static"
    if not os.path.exists(qemu_emulator):
        raise CreatorError("Please install a statically-linked qemu-arm")

    # qemu emulator version check
    armv7_list = [arch for arch in rpmmisc.archPolicies.keys() if arch.startswith("armv7")]
    if arch in armv7_list:  # need qemu (>=0.13.0)
        qemuout = runner.outs([qemu_emulator, "-h"])
        m = re.search("version\s*([.\d]+)", qemuout)
        if m:
            qemu_version = m.group(1)
            if qemu_version < "0.13":
                raise CreatorError("Requires %s version >=0.13 for %s" % (qemu_emulator, arch))
        else:
            msger.warning("Can't get version info of %s, please make sure it's higher than 0.13.0" % qemu_emulator)

    if not os.path.exists(rootdir + "/usr/bin"):
        makedirs(rootdir + "/usr/bin")
    shutil.copy(qemu_emulator, rootdir + qemu_emulator)

    # disable selinux, selinux will block qemu emulator to run
    if os.path.exists("/usr/sbin/setenforce"):
        msger.info("Try to disable selinux")
        runner.show(["/usr/sbin/setenforce", "0"])

    node = "/proc/sys/fs/binfmt_misc/arm"
    if is_statically_linked(qemu_emulator) and os.path.exists(node):
        return qemu_emulator

    # unregister it if it has been registered and is a dynamically-linked executable
    if not is_statically_linked(qemu_emulator) and os.path.exists(node):
        qemu_unregister_string = "-1\n"
        fd = open("/proc/sys/fs/binfmt_misc/arm", "w")
        fd.write(qemu_unregister_string)
        fd.close()

    # register qemu emulator for interpreting other arch executable file
    if not os.path.exists(node):
        qemu_arm_string = (
            ":arm:M::\\x7fELF\\x01\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x28\\x00:\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xfa\\xff\\xff\\xff:%s:\n"
            % qemu_emulator
        )
        fd = open("/proc/sys/fs/binfmt_misc/register", "w")
        fd.write(qemu_arm_string)
        fd.close()

    return qemu_emulator
Example #2
0
    def print_outimage_info(self):
        """
        Print the image(s) and artifacts used, for the user.
        """
        msg = "The new image(s) can be found here:\n"

        parts = self._get_parts()

        for disk_name, disk in self.__instimage.disks.items():
            full_path = self._full_path(self.__imgdir, disk_name, "direct")
            msg += "  %s\n\n" % full_path

        msg += "The following build artifacts were used to create the image(s):\n"
        for p in parts:
            if p.get_rootfs() is None:
                continue
            if p.mountpoint == "/":
                str = ":"
            else:
                str = '["%s"]:' % p.label
            msg += "  ROOTFS_DIR%s%s\n" % (str.ljust(20), p.get_rootfs())

        msg += "  BOOTIMG_DIR:                  %s\n" % self.bootimg_dir
        msg += "  KERNEL_DIR:                   %s\n" % self.kernel_dir
        msg += "  NATIVE_SYSROOT:               %s\n" % self.native_sysroot

        msger.info(msg)
Example #3
0
    def _stage_final_image(self):
        if self.taring_to:
            import tarfile

            curdir = os.getcwd()
            os.chdir(self.__imgdir)
            self._resparse(0)

            tarfile_name = self.taring_to
            if not tarfile_name.endswith(".tar"):
                tarfile_name += ".tar"

            msger.info("Tar all loop images together to %s" % tarfile_name)
            tar = tarfile.open(os.path.join(self._outdir, tarfile_name), "w")
            for item in self._instloops:
                if item["fstype"] == "ext4":
                    runner.show("/sbin/tune2fs -O ^huge_file,extents,uninit_bg " + item["name"])
                tar.add(item["name"])

            tar.close()
            os.chdir(curdir)

        else:
            self._resparse()
            for item in self._instloops:
                shutil.move(os.path.join(self.__imgdir, item["name"]), os.path.join(self._outdir, item["name"]))
Example #4
0
    def package_output(self, image_format, destdir=".", package="none"):
        if not package or package == "none":
            return

        destdir = os.path.abspath(os.path.expanduser(destdir))
        (pkg, comp) = os.path.splitext(package)
        if comp:
            comp = comp.lstrip(".")

        if pkg == "tar":
            if comp:
                dst = "%s/%s-%s.tar.%s" % (destdir, self.name, image_format, comp)
            else:
                dst = "%s/%s-%s.tar" % (destdir, self.name, image_format)

            msger.info("creating %s" % dst)
            tar = tarfile.open(dst, "w:" + comp)

            for file in self.outimage:
                msger.info("adding %s to %s" % (file, dst))
                tar.add(file, arcname=os.path.join("%s-%s" % (self.name, image_format), os.path.basename(file)))
                if os.path.isdir(file):
                    shutil.rmtree(file, ignore_errors=True)
                else:
                    os.remove(file)

            tar.close()

            """All the file in outimage has been packaged into tar.* file"""
            self.outimage = [dst]
Example #5
0
    def configure(self, repodata=None):
        """Configure the system image according to the kickstart.

        This method applies the (e.g. keyboard or network) configuration
        specified in the kickstart and executes the kickstart %post scripts.

        If necessary, it also prepares the image to be bootable by e.g.
        creating an initrd and bootloader configuration.

        """
        ksh = self.ks.handler

        msger.info("Applying configurations ...")
        try:
            kickstart.LanguageConfig(self._instroot).apply(ksh.lang)
            kickstart.KeyboardConfig(self._instroot).apply(ksh.keyboard)
            kickstart.TimezoneConfig(self._instroot).apply(ksh.timezone)
            # kickstart.AuthConfig(self._instroot).apply(ksh.authconfig)
            kickstart.FirewallConfig(self._instroot).apply(ksh.firewall)
            kickstart.RootPasswordConfig(self._instroot).apply(ksh.rootpw)
            kickstart.UserConfig(self._instroot).apply(ksh.user)
            kickstart.ServicesConfig(self._instroot).apply(ksh.services)
            kickstart.XConfig(self._instroot).apply(ksh.xconfig)
            kickstart.NetworkConfig(self._instroot).apply(ksh.network)
            kickstart.RPMMacroConfig(self._instroot).apply(self.ks)
            kickstart.DesktopConfig(self._instroot).apply(ksh.desktop)
            self.__save_repo_keys(repodata)
            kickstart.MoblinRepoConfig(self._instroot).apply(ksh.repo, repodata)
        except:
            msger.warning("Failed to apply configuration to image")
            raise

        self._create_bootconfig()
        self.__run_post_scripts()
Example #6
0
    def do_unpack(cls, srcimg):
        srcimgsize = (misc.get_file_size(srcimg)) * 1024L * 1024L
        srcmnt = misc.mkdtemp("srcmnt")
        disk = fs_related.SparseLoopbackDisk(srcimg, srcimgsize)
        srcloop = PartitionedMount({"/dev/sdb": disk}, srcmnt, skipformat=True)

        srcloop.add_partition(srcimgsize / 1024 / 1024, "/dev/sdb", "/", "ext3", boot=False)
        try:
            srcloop.mount()

        except errors.MountError:
            srcloop.cleanup()
            raise

        image = os.path.join(tempfile.mkdtemp(dir="/var/tmp", prefix="tmp"), "target.img")
        args = ["dd", "if=%s" % srcloop.partitions[0]["device"], "of=%s" % image]

        msger.info("`dd` image ...")
        rc = runner.show(args)
        srcloop.cleanup()
        shutil.rmtree(os.path.dirname(srcmnt), ignore_errors=True)

        if rc != 0:
            raise errors.CreatorError("Failed to dd")
        else:
            return image
Example #7
0
    def installLocal(self, pkg, po=None, updateonly=False):
        if not self.ts:
            self.__initialize_transaction()

        solvfile = "%s/.solv" % (self.cachedir)

        rc, out = runner.runtool([fs_related.find_binary_path("rpms2solv"), pkg])
        if rc == 0:
            f = open(solvfile, "w+")
            f.write(out)
            f.close()

            warnmsg = self.repo_manager.loadSolvFile(solvfile, os.path.basename(pkg))
            if warnmsg:
                msger.warning(warnmsg)

            os.unlink(solvfile)
        else:
            msger.warning("Can not get %s solv data." % pkg)

        hdr = rpmmisc.readRpmHeader(self.ts, pkg)
        arch = zypp.Arch(hdr["arch"])
        sysarch = zypp.Arch(self.target_arch)

        if arch.compatible_with(sysarch):
            pkgname = hdr["name"]
            self.localpkgs[pkgname] = pkg
            self.selectPackage(pkgname)
            msger.info("Marking %s to be installed" % (pkg))

        else:
            msger.warning(
                "Cannot add package %s to transaction. " "Not a compatible architecture: %s" % (pkg, hdr["arch"])
            )
Example #8
0
    def __run_post_scripts(self):
        msger.info("Running scripts ...")
        if os.path.exists(self._instroot + "/tmp"):
            shutil.rmtree(self._instroot + "/tmp")
        os.mkdir(self._instroot + "/tmp", 0755)
        for s in kickstart.get_post_scripts(self.ks):
            (fd, path) = tempfile.mkstemp(prefix="ks-script-", dir=self._instroot + "/tmp")

            s.script = s.script.replace("\r", "")
            os.write(fd, s.script)
            os.close(fd)
            os.chmod(path, 0700)

            env = self._get_post_scripts_env(s.inChroot)

            if not s.inChroot:
                env["INSTALL_ROOT"] = self._instroot
                env["IMG_NAME"] = self._name
                preexec = None
                script = path
            else:
                preexec = self._chroot
                script = "/tmp/" + os.path.basename(path)

            try:
                try:
                    subprocess.call(
                        [s.interp, script], preexec_fn=preexec, env=env, stdout=sys.stdout, stderr=sys.stderr
                    )
                except OSError, (err, msg):
                    raise CreatorError("Failed to execute %%post script " "with '%s' : %s" % (s.interp, msg))
            finally:
                os.unlink(path)
Example #9
0
    def __build_repo_cache(self, name):
        repo = self.repo_manager.getRepositoryInfo(name)
        if self.repo_manager.isCached(repo) or not repo.enabled():
            return

        msger.info("Refreshing repository: %s ..." % name)
        self.repo_manager.buildCache(repo, zypp.RepoManager.BuildIfNeeded)
Example #10
0
File: misc.py Project: lbt/mic
def normalize_ksfile(ksconf, release, arch):
    def _clrtempks():
        try:
            os.unlink(ksconf)
        except:
            pass

    if not os.path.exists(ksconf):
        return

    if not release:
        release = "latest"
    if not arch or re.match(r"i.86", arch):
        arch = "ia32"

    with open(ksconf) as f:
        ksc = f.read()

    if "@ARCH@" in ksc or "@BUILD_ID@" in ksc:
        msger.info("Substitute macro variable @BUILD_ID@/@ARCH in ks: %s" % ksconf)
        ksc = ksc.replace("@ARCH@", arch)
        ksc = ksc.replace("@BUILD_ID@", release)
        fd, ksconf = tempfile.mkstemp(prefix=os.path.basename(ksconf), dir="/tmp/")
        os.write(fd, ksc)
        os.close(fd)

        msger.debug("new ks path %s" % ksconf)

        import atexit

        atexit.register(_clrtempks)

    return ksconf
Example #11
0
    def _stage_final_image(self):
        if self.compress_to:

            self._resparse(0)

            cfile_name = self.compress_to
            mountfp_xml = os.path.splitext(cfile_name)[0] + ".xml"

            for item in self._instloops:
                imgfile = os.path.join(self.__imgdir, item["name"])
                if item["fstype"] == "ext4":
                    runner.show("/sbin/tune2fs " "-O ^huge_file,extents,uninit_bg %s " % imgfile)

            msger.info("Compress all loop images together to %s" % cfile_name)
            dstfile = os.path.join(self._outdir, cfile_name)
            if self.compress_imgdir_method == "tar":
                misc.taring(dstfile, self.__imgdir)
            elif self.compress_imgdir_method == "zip":
                misc.ziping(dstfile, self.__imgdir)
            else:
                raise CreatorError("Unsupported compress type: %s" % self.compress_imgdir_method)

            # save mount points mapping file to xml
            save_mountpoints(os.path.join(self._outdir, mountfp_xml), self._instloops, self.target_arch)

        else:
            self._resparse()
            for item in self._instloops:
                shutil.move(os.path.join(self.__imgdir, item["name"]), os.path.join(self._outdir, item["name"]))
Example #12
0
def _make_zipfile(archive_name, target_name):
    """ Create a zip file from all the files under 'target_name' or itself.

    @archive_name: the name of the archived file
    @target_name: the directory or the file name to archive
    @retval: indicate the archiving result
    """
    import zipfile

    msger.info("Zipping files to %s using zipfile module" % archive_name)
    arv = zipfile.ZipFile(archive_name, "w", compression=zipfile.ZIP_DEFLATED)

    if os.path.isdir(target_name):
        for dirpath, dirname, filenames in os.walk(target_name):
            for filename in filenames:
                filepath = os.path.normpath(os.path.join(dirpath, filename))
                arcname = os.path.relpath(filepath, target_name)
                if os.path.isfile(filepath):
                    arv.write(filepath, arcname)
    else:
        arv.write(target_name, os.path.basename(target_name))

    arv.close()

    return os.path.exists(archive_name)
    def package(self, destdir="."):
        """Prepares the created image for final delivery.

        In its simplest form, this method merely copies the install root to the
        supplied destination directory; other subclasses may choose to package
        the image by e.g. creating a bootable ISO containing the image and
        bootloader configuration.

        destdir -- the directory into which the final image should be moved;
                   this defaults to the current directory.

        """
        self._stage_final_image()

        if not os.path.exists(destdir):
            fs.makedirs(destdir)
        if self._img_compression_method:
            if not self._img_name:
                raise CreatorError("Image name not set.")

            rc = None
            img_location = os.path.join(self._outdir, self._img_name)
            zipcmd = self.zips[self._img_compression_method]

            # confirm the existing of zip command
            fs.find_binary_path(zipcmd)

            msger.info("Compressing %s with %s. Please wait ..." % (img_location, zipcmd))
            rc = runner.show([zipcmd, "-f", img_location])
            if rc:
                raise CreatorError(
                    "Failed to compress image %s with %s." % (img_location, self._img_compression_method)
                )

            for bootimg in glob.glob(os.path.dirname(img_location) + "/*-boot.bin"):
                msger.info("Compressing %s with %s. Please wait..." % (bootimg, zipcmd))
                rc = runner.show([zipcmd, "-f", bootimg])
                if rc:
                    raise CreatorError(
                        "Failed to compress image %s with " "%s." % (bootimg, self._img_compression_method)
                    )

        if self._recording_pkgs:
            self._save_recording_pkgs(destdir)

        # For image formats with two or multiple image files, it will be
        # better to put them under a directory
        if self.image_format in ("raw", "vmdk", "vdi", "nand", "mrstnand"):
            destdir = os.path.join(destdir, "%s-%s" % (self.name, self.image_format))
            msger.debug("creating destination dir: %s" % destdir)
            fs.makedirs(destdir)

        # Ensure all data is flushed to _outdir
        runner.quiet("sync")

        misc.check_space_pre_cp(self._outdir, destdir)
        for f in os.listdir(self._outdir):
            shutil.move(os.path.join(self._outdir, f), os.path.join(destdir, f))
            self.outimage.append(os.path.join(destdir, f))
            self.do_genchecksum(os.path.join(destdir, f))
Example #14
0
    def print_outimage_info(self):
        msg = "The new image can be found here:\n"
        self.outimage.sort()
        for file in self.outimage:
            msg += "  %s\n" % os.path.abspath(file)

        msger.info(msg)
Example #15
0
    def create_manifest(self):
        def get_pack_suffix():
            return "." + self.pack_to.split(".", 1)[1]

        if not os.path.exists(self.destdir):
            os.makedirs(self.destdir)

        now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        manifest_dict = {"version": VERSION, "created": now}
        if self.img_format:
            manifest_dict.update({"format": self.img_format})

        if hasattr(self, "logfile") and self.logfile:
            manifest_dict.update({"log_file": self.logfile})

        if self.image_files:
            if self.pack_to:
                self.image_files.update({"pack": get_pack_suffix()})
            manifest_dict.update({self.img_format: self.image_files})

        msger.info("Creating manifest file...")
        manifest_file_path = os.path.join(self.destdir, "manifest.json")
        with open(manifest_file_path, "w") as fest_file:
            json.dump(manifest_dict, fest_file, indent=4)
        self.outimage.append(manifest_file_path)
Example #16
0
    def _parse_kickstart(self, ksconf=None):
        if not ksconf:
            return

        ksconf = misc.normalize_ksfile(ksconf, self.create["release"], self.create["arch"])

        ks = kickstart.read_kickstart(ksconf)

        self.create["ks"] = ks
        self.create["name"] = os.path.splitext(os.path.basename(ksconf))[0]

        self.create["name"] = misc.build_name(
            ksconf, self.create["release"], self.create["name_prefix"], self.create["name_suffix"]
        )

        self.create["destdir"] = self.create["outdir"]
        if self.create["release"] is not None:
            self.create["destdir"] = "%s/%s/images/%s/" % (
                self.create["outdir"],
                self.create["release"],
                self.create["name"],
            )
            self.create["name"] = self.create["release"] + "_" + self.create["name"]

            if not self.create["logfile"]:
                self.create["logfile"] = os.path.join(self.create["outdir"], self.create["name"] + ".log")
                self.create["releaselog"] = True
                self.set_logfile()

        msger.info("Retrieving repo metadata:")
        ksrepos = kickstart.get_repos(ks, self.create["extrarepos"], self.create["ignore_ksrepo"])
        if not ksrepos:
            raise errors.KsError("no valid repos found in ks file")

        for repo in ksrepos:
            if hasattr(repo, "baseurl") and repo.baseurl.startswith("file:"):
                repourl = repo.baseurl.replace("file:", "")
                repourl = "/%s" % repourl.lstrip("/")
                self.create["localrepos"].append(repourl)

        self.create["repomd"] = misc.get_metadata_from_repos(ksrepos, self.create["cachedir"])
        msger.raw(" DONE")

        target_archlist, archlist = misc.get_arch(self.create["repomd"])
        if self.create["arch"]:
            if self.create["arch"] not in archlist:
                raise errors.ConfigError(
                    "Invalid arch %s for repository. " "Valid arches: %s" % (self.create["arch"], ", ".join(archlist))
                )
        else:
            if len(target_archlist) == 1:
                self.create["arch"] = str(target_archlist[0])
                msger.info("Use detected arch %s." % target_archlist[0])
            else:
                raise errors.ConfigError("Please specify a valid arch, " "the choice can be: %s" % ", ".join(archlist))

        kickstart.resolve_groups(self.create, self.create["repomd"])

        # check selinux, it will block arm and btrfs image creation
        misc.selinux_check(self.create["arch"], [p.fstype for p in ks.handler.partition.partitions])
Example #17
0
    def _stage_final_image(self):

        if self.pack_to or self.shrink_image:
            self._resparse(0)
        else:
            self._resparse()

        for item in self._instloops:
            imgfile = os.path.join(self._imgdir, item["name"])
            if item["fstype"] == "ext4":
                runner.show("/sbin/tune2fs -O ^huge_file,extents,uninit_bg %s " % imgfile)
            self.image_files.setdefault("partitions", {}).update({item["mountpoint"]: item["label"]})
            if self.compress_image:
                compressing(imgfile, self.compress_image)
                self.image_files.setdefault("image_files", []).append(".".join([item["name"], self.compress_image]))
            else:
                self.image_files.setdefault("image_files", []).append(item["name"])

        if not self.pack_to:
            for item in os.listdir(self._imgdir):
                shutil.move(os.path.join(self._imgdir, item), os.path.join(self._outdir, item))
        else:
            msger.info("Pack all loop images together to %s" % self.pack_to)
            dstfile = os.path.join(self._outdir, self.pack_to)
            packing(dstfile, self._imgdir)
            self.image_files["image_files"] = [self.pack_to]

        if self.pack_to:
            mountfp_xml = os.path.splitext(self.pack_to)[0]
            mountfp_xml = misc.strip_end(mountfp_xml, ".tar") + ".xml"
        else:
            mountfp_xml = self.name + ".xml"
        # save mount points mapping file to xml
        save_mountpoints(os.path.join(self._outdir, mountfp_xml), self._instloops, self.target_arch)
Example #18
0
    def _stage_final_image(self):

        if self.pack_to or self.shrink_image:
            self._resparse(0)
        else:
            self._resparse()

        for item in self._instloops:
            imgfile = os.path.join(self.__imgdir, item["name"])
            if item["fstype"] == "ext4":
                runner.show("/sbin/tune2fs -O ^huge_file,extents,uninit_bg %s " % imgfile)
            if self.compress_image:
                misc.compressing(imgfile, self.compress_image)

        if not self.pack_to:
            for item in os.listdir(self.__imgdir):
                shutil.move(os.path.join(self.__imgdir, item), os.path.join(self._outdir, item))
        else:
            msger.info("Pack all loop images together to %s" % self.pack_to)
            dstfile = os.path.join(self._outdir, self.pack_to)
            misc.packing(dstfile, self.__imgdir)

        if self.pack_to:
            mountfp_xml = os.path.splitext(self.pack_to)[0]
            mountfp_xml = misc.strip_end(mountfp_xml, ".tar") + ".xml"
        else:
            mountfp_xml = self.name + ".xml"
        # save mount points mapping file to xml
        save_mountpoints(os.path.join(self._outdir, mountfp_xml), self._instloops, self.target_arch)
Example #19
0
File: fs.py Project: again4you/mic
    def package(self, destdir="."):

        ignores = ["/dev/fd", "/dev/stdin", "/dev/stdout", "/dev/stderr", "/etc/mtab"]

        if not os.path.exists(destdir):
            os.makedirs(destdir)

        if self._recording_pkgs:
            self._save_recording_pkgs(destdir)

        if not self.pack_to:
            self.image_files = {"image_files": [self.name]}
            fsdir = os.path.join(destdir, self.name)

            misc.check_space_pre_cp(self._instroot, destdir)
            msger.info("Copying %s to %s ..." % (self._instroot, fsdir))
            runner.show(["cp", "-af", self._instroot, fsdir])

            for exclude in ignores:
                if os.path.exists(fsdir + exclude):
                    os.unlink(fsdir + exclude)

            self.outimage.append(fsdir)

        else:
            self.image_files = {"image_files": [self.pack_to]}
            (tar, comp) = os.path.splitext(self.pack_to)
            try:
                tarcreat = {".tar": "-cf", ".gz": "-czf", ".bz2": "-cjf", ".tgz": "-czf", ".tbz": "-cjf"}[comp]
            except KeyError:
                raise CreatorError(
                    "Unsupported comression for this image type:" " '%s', try '.tar', '.tar.gz', etc" % comp
                )

            dst = os.path.join(destdir, self.pack_to)
            msger.info("Pack rootfs to %s. Please wait..." % dst)

            tar = find_binary_path("tar")
            tar_cmdline = [
                tar,
                "--numeric-owner",
                "--preserve-permissions",
                "--one-file-system",
                "--directory",
                self._instroot,
            ]
            for ignore_entry in ignores:
                if ignore_entry.startswith("/"):
                    ignore_entry = ignore_entry[1:]

                tar_cmdline.append("--exclude=%s" % (ignore_entry))

            tar_cmdline.extend([tarcreat, dst, "."])

            rc = runner.show(tar_cmdline)
            if rc:
                raise CreatorError("Failed compress image with tar.bz2. " "Cmdline: %s" % (" ".join(tar_cmdline)))

            self.outimage.append(dst)
Example #20
0
File: misc.py Project: lbt/mic
def SrcpkgsDownload(pkgs, repometadata, instroot, cachedir):
    def get_source_repometadata(repometadata):
        src_repometadata = []
        for repo in repometadata:
            if repo["name"].endswith("-source"):
                src_repometadata.append(repo)
        if src_repometadata:
            return src_repometadata
        return None

    def get_src_name(srpm):
        m = SRPM_RE.match(srpm)
        if m:
            return m.group(1)
        return None

    src_repometadata = get_source_repometadata(repometadata)

    if not src_repometadata:
        msger.warning("No source repo found")
        return None

    src_pkgs = []
    lpkgs_dict = {}
    lpkgs_path = []
    for repo in src_repometadata:
        cachepath = "%s/%s/packages/*.src.rpm" % (cachedir, repo["name"])
        lpkgs_path += glob.glob(cachepath)

    for lpkg in lpkgs_path:
        lpkg_name = get_src_name(os.path.basename(lpkg))
        lpkgs_dict[lpkg_name] = lpkg
    localpkgs = lpkgs_dict.keys()

    cached_count = 0
    destdir = instroot + "/usr/src/SRPMS"
    if not os.path.exists(destdir):
        os.makedirs(destdir)

    srcpkgset = set()
    for _pkg in pkgs:
        srcpkg_name = get_source_name(_pkg, repometadata)
        if not srcpkg_name:
            continue
        srcpkgset.add(srcpkg_name)

    for pkg in list(srcpkgset):
        if pkg in localpkgs:
            cached_count += 1
            shutil.copy(lpkgs_dict[pkg], destdir)
            src_pkgs.append(os.path.basename(lpkgs_dict[pkg]))
        else:
            src_pkg = get_package(pkg, src_repometadata, "src")
            if src_pkg:
                shutil.copy(src_pkg, destdir)
                src_pkgs.append(src_pkg)
    msger.info("%d source packages gotten from cache" % cached_count)

    return src_pkgs
Example #21
0
File: fs.py Project: saukko/mic
    def package(self, destdir="."):

        ignores = ["/dev/fd", "/dev/stdin", "/dev/stdout", "/dev/stderr", "/etc/mtab"]

        if not os.path.exists(destdir):
            os.makedirs(destdir)

        if self._recording_pkgs:
            self._save_recording_pkgs(destdir)

        if self._img_compression_method == None:
            fsdir = os.path.join(destdir, self.name)

            misc.check_space_pre_cp(self._instroot, destdir)
            msger.info("Copying %s to %s ..." % (self._instroot, fsdir))
            runner.show(["cp", "-af", self._instroot, fsdir])

            for exclude in ignores:
                if os.path.exists(fsdir + exclude):
                    os.unlink(fsdir + exclude)

            self.outimage.append(fsdir)

        elif self._img_compression_method == "tar.bz2":
            dst = "%s/%s.tar.bz2" % (destdir, self.name)
            msger.info(
                "Creating %s (compressing %s with %s). Please wait..."
                % (dst, self._instroot, self._img_compression_method)
            )

            tar = find_binary_path("tar")
            tar_cmdline = [
                tar,
                "--numeric-owner",
                "--preserve-permissions",
                "--preserve-order",
                "--one-file-system",
                "--directory",
                self._instroot,
            ]
            for ignore_entry in ignores:
                if ignore_entry.startswith("/"):
                    ignore_entry = ignore_entry[1:]

                tar_cmdline.append("--exclude=%s" % (ignore_entry))

            tar_cmdline.extend(["-cjf", dst, "."])

            rc = call(tar_cmdline)
            if rc:
                raise CreatorError("Failed compress image with tar.bz2. " "Cmdline: %s" % (" ".join(tar_cmdline)))

            self.outimage.append(dst)

        else:
            raise CreatorError(
                "Compression method '%s' not supported for 'fs' " "image format." % (self._img_compression_method)
            )
Example #22
0
def load_module(module):
    found = False
    for line in open("/proc/modules").xreadlines():
        if line.startswith("%s " % module):
            found = True
            break
    if not found:
        msger.info("Loading %s..." % module)
        runner.quiet(["modprobe", module])
Example #23
0
 def start(self, filename, url, *args, **kwargs):
     self.url = url
     self.termwidth = terminal_width()
     if self.total is None:
         msger.info("Retrieving %s ..." % truncate_url(self.url, self.termwidth - 15))
     else:
         msger.info(
             "Retrieving %s [%d/%d] ..." % (truncate_url(self.url, self.termwidth - 25), self.counter, self.total)
         )
Example #24
0
def bootstrap_mic(argv=None):
    def mychroot():
        os.chroot(rootdir)
        os.chdir(cwd)

    # by default, sys.argv is used to run mic in bootstrap
    if not argv:
        argv = sys.argv
    if argv[0] not in ("/usr/bin/mic", "mic"):
        argv[0] = "/usr/bin/mic"

    cropts = configmgr.create
    bsopts = configmgr.bootstrap
    distro = bsopts["distro_name"].lower()

    rootdir = bsopts["rootdir"]
    pkglist = bsopts["packages"]
    cwd = os.getcwd()

    # create bootstrap and run mic in bootstrap
    bsenv = bootstrap.Bootstrap(rootdir, distro, cropts["arch"])
    bsenv.logfile = cropts["logfile"]
    # rootdir is regenerated as a temp dir
    rootdir = bsenv.rootdir

    if "optional" in bsopts:
        optlist = bsopts["optional"]
    else:
        optlist = []

    try:
        msger.info("Creating %s bootstrap ..." % distro)
        bsenv.create(cropts["repomd"], pkglist, optlist)

        # bootstrap is relocated under "bootstrap"
        if os.path.exists(os.path.join(rootdir, "bootstrap")):
            rootdir = os.path.join(rootdir, "bootstrap")

        bsenv.dirsetup(rootdir)
        sync_mic(rootdir, plugin=cropts["plugin_dir"])

        # FIXME: sync the ks file to bootstrap
        if "/" == os.path.dirname(os.path.abspath(configmgr._ksconf)):
            safecopy(configmgr._ksconf, rootdir)

        msger.info("Start mic in bootstrap: %s\n" % rootdir)
        bsarch = ELF_arch(rootdir)
        if bsarch in personality_defs:
            condPersonality(bsarch)
        bindmounts = get_bindmounts(cropts)
        ret = bsenv.run(argv, cwd, rootdir, bindmounts)

    except errors.BootstrapError, err:
        raise errors.CreatorError(
            "Failed to download/install bootstrap package " "or the package is in bad format: %s" % err
        )
Example #25
0
    def launch_shell(self, launch):
        """Launch a shell in the install root.

        This method is launches a bash shell chroot()ed in the install root;
        this can be useful for debugging.

        """
        if launch:
            msger.info("Launching shell. Exit to continue.")
            subprocess.call(["/bin/bash"], preexec_fn=self._chroot)
Example #26
0
def bootstrap_mic(argv=None):
    def mychroot():
        os.chroot(rootdir)
        os.chdir(cwd)

    # by default, sys.argv is used to run mic in bootstrap
    if not argv:
        argv = sys.argv
    if argv[0] not in ("/usr/bin/mic", "mic"):
        argv[0] = "/usr/bin/mic"

    cropts = configmgr.create
    bsopts = configmgr.bootstrap
    distro = bsopts["distro_name"].lower()

    rootdir = bsopts["rootdir"]
    pkglist = bsopts["packages"]
    cwd = os.getcwd()

    # create bootstrap and run mic in bootstrap
    bsenv = bootstrap.Bootstrap(rootdir, distro, cropts["arch"])
    bsenv.logfile = cropts["logfile"]
    # rootdir is regenerated as a temp dir
    rootdir = bsenv.rootdir

    if "optional" in bsopts:
        optlist = bsopts["optional"]
    else:
        optlist = []

    try:
        msger.info("Creating %s bootstrap ..." % distro)
        bsenv.create(cropts["repomd"], pkglist, optlist)

        # bootstrap is relocated under "bootstrap"
        if os.path.exists(os.path.join(rootdir, "bootstrap")):
            rootdir = os.path.join(rootdir, "bootstrap")

        bsenv.dirsetup(rootdir)
        sync_mic(rootdir)

        # FIXME: sync the ks file to bootstrap
        if "/" == os.path.dirname(os.path.abspath(configmgr._ksconf)):
            safecopy(configmgr._ksconf, rootdir)

        msger.info("Start mic in bootstrap: %s\n" % rootdir)
        bindmounts = get_bindmounts(cropts)
        ret = bsenv.run(argv, cwd, rootdir, bindmounts)

    except errors.BootstrapError, err:
        msger.warning("\n%s" % err)
        if msger.ask("Switch to native mode and continue?"):
            return
        else:
            raise errors.BootstrapError("Failed to create bootstrap: %s" % err)
Example #27
0
    def copy_attachment(self):
        if not hasattr(self, "_attachment") or not self._attachment:
            return

        self._check_imgdir()

        msger.info("Copying attachment files...")
        for item in self._attachment:
            dpath = os.path.join(self.__imgdir, os.path.basename(item))
            msger.verbose("Copy attachment %s to %s" % (item, dpath))
            shutil.copy(item, dpath)
Example #28
0
    def generate_bmap(self):
        """ Generate block map file for an image. The idea is that while disk
        images we generate may be large (e.g., 4GiB), they may actually contain
        only little real data, e.g., 512MiB. This data are files, directories,
        file-system meta-data, partition table, etc. In other words, when
        flashing the image to the target device, you do not have to copy all the
        4GiB of data, you can copy only 512MiB of it, which is 4 times faster.

        This function generates the block map file for an arbitrary image that
        mic has generated. The block map file is basically an XML file which
        contains a list of blocks which have to be copied to the target device.
        The other blocks are not used and there is no need to copy them.

        This function assumes the image file was originally created as a sparse
        file. To generate the block map we use the FIBMAP ioctl. """

        if self.bmap_needed is None:
            return

        msger.info("Generating the map file(s)")

        for name in self.__disks.keys():
            image = self._full_path(self.__imgdir, name, self.__disk_format)
            bmap_file = self._full_path(self.__imgdir, name, "bmap")

            msger.debug("Generating block map file '%s'" % bmap_file)

            image_size = os.path.getsize(image)

            with open(bmap_file, "w") as f_bmap:
                with open(image, "rb") as f_image:
                    # Get the block size of the host file-system for the image
                    # file by calling the FIGETBSZ ioctl (number 2).
                    block_size = unpack("I", ioctl(f_image, 2, pack("I", 0)))[0]
                    blocks_cnt = (image_size + block_size - 1) / block_size

                    # Write general information to the block map file, without
                    # block map itself, which will be written next.
                    xml = self._bmap_file_start(block_size, image_size, blocks_cnt)
                    f_bmap.write(xml)

                    # Generate the block map and write it to the XML block map
                    # file as we go.
                    mapped_cnt = 0
                    for first, last in self._get_ranges(f_image, blocks_cnt):
                        mapped_cnt += last - first + 1
                        sha1 = misc.calc_hashes(image, ("sha1",), first * block_size, (last + 1) * block_size)
                        f_bmap.write('\t\t<Range sha1="%s"> %s-%s ' "</Range>\n" % (sha1[0], first, last))

                    # Finish the block map file
                    xml = self._bmap_file_end(mapped_cnt, block_size, blocks_cnt)
                    f_bmap.write(xml)
Example #29
0
def runmic_in_bootstrap(name, argv, opts, ksfile, repolist):
    bootstrap_env = bootstrap.Bootstrap(homedir=opts["bootstrapdir"])
    bootstrap_lst = bootstrap_env.bootstraps
    setattr(bootstrap_env, "rootdir", name)
    if not bootstrap_lst or not name in bootstrap_lst:
        msger.info("Creating bootstrap %s under %s" % (name, bootstrap_env.homedir))
        bootstrap_env.create(name, repolist)

    msger.info("Use bootstrap: %s" % bootstrap_env.rootdir)
    # copy mic
    msger.info("Sync native mic to bootstrap")
    copy_mic(bootstrap_env.rootdir)

    # bind mounts , opts['cachedir'], opts['tmpdir']
    cwd = os.getcwd()
    lst = [cwd, opts["outdir"]]
    if ksfile:
        ksfp = os.path.abspath(os.path.expanduser(ksfile))
        lst.append(os.path.dirname(ksfp))
    if opts["logfile"]:
        logfile = os.path.abspath(os.path.expanduser(opts["logfile"]))
        lst.append(os.path.dirname(logfile))
    if opts["local_pkgs_path"]:
        lppdir = os.path.abspath(os.path.expanduser(opts["local_pkgs_path"]))
        lst.append(lppdir)

    # TBD local repo

    # make unique and remain the original order
    lst = sorted(set(lst), key=lst.index)

    bindmounts = ";".join(map(lambda p: os.path.abspath(os.path.expanduser(p)), lst))

    msger.info("Start mic command in bootstrap")
    bootstrap_env.run(name, argv, cwd, bindmounts)
Example #30
0
def _call_external(cmdln_or_args):
    """ Wapper for subprocess calls.

    @cmdln_or_args: command line to be joined before execution.
    @retval: a tuple (returncode, outdata, errdata).
    """
    if isinstance(cmdln_or_args, list):
        shell = False
    else:
        shell = True
    msger.info("Running command: " + " ".join(cmdln_or_args))

    proc = subprocess.Popen(cmdln_or_args, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    (outdata, errdata) = proc.communicate()

    return (proc.returncode, outdata, errdata)