def test_initvm(): try: global exe_path # Properly initialise directories plug mod_dir = os.path.dirname (__file__) main_dir, _ = os.path.split( mod_dir ) exe_path = os.path.join( main_dir, "elbe" ) init_directories (exe_path) tmpdir = os.getenv( "ELBE_TEST_DIR" ) assert tmpdir is not None dname = os.path.join(tmpdir, "initvm_test") from elbepack.directories import examples_dir xml_name = os.path.join (examples_dir, "elbe-init-with-ssh.xml") system ('%s initvm create --devel --directory "%s" %s' % (exe_path, dname, xml_name)) except: system ("%s control shutdown_initvm" % (exe_path), allow_fail=True) sleep (10) tmpdir = os.getenv( "ELBE_TEST_DIR" ) assert tmpdir is not None dname = os.path.join(tmpdir, "initvm_test") shutil.rmtree (dname) raise
def execute(self, _initvmdir, _opt, _args): if self.initvm_state() != libvirt.VIR_DOMAIN_RUNNING: print('Error: Initvm not running properly.') sys.exit(20) print('Attaching to initvm console.') system('virsh --connect qemu:///system console %s' % cfg['initvm_domain'])
def copy_kinitrd(prj, target_dir): suite = prj.text("suite") try: tmp = TmpdirFilesystem() if prj.has("mirror/cdrom"): system('7z x -o%s "%s" initrd-cdrom.gz vmlinuz' % (tmp.fname('/'), prj.text("mirror/cdrom"))) # initrd.gz needs to be cdrom version ! copyfile(tmp.fname("initrd-cdrom.gz"), os.path.join(target_dir, "initrd.gz")) else: mirror = get_primary_mirror(prj) download_kinitrd(tmp, suite, mirror) copyfile(tmp.fname("initrd.gz"), os.path.join(target_dir, "initrd.gz")) copyfile(tmp.fname("initrd-cdrom.gz"), os.path.join(target_dir, "initrd-cdrom.gz")) copyfile(tmp.fname("vmlinuz"), os.path.join(target_dir, "vmlinuz")) except IOError as e: raise NoKinitrdException('IoError %s' % e.message) except InvalidSignature as e: raise NoKinitrdException('InvalidSignature %s' % e.message) except HashValidationFailed as e: raise NoKinitrdException('HashValidationFailed %s' % e.message)
def execute(self, opt, args): tmp = TmpdirFilesystem () if not opt.project: print ('you need to specify --project option', file=sys.stderr) sys.exit(20) prjdir = opt.project print ("Creating pbuilder") try: system ('%s control build_pbuilder "%s"' % (elbe_exe, prjdir)) except CommandError: print ("elbe control build_pbuilder Failed", file=sys.stderr) print ("Giving up", file=sys.stderr) sys.exit(20) try: system ('%s control wait_busy "%s"' % (elbe_exe, prjdir) ) except CommandError: print ("elbe control wait_busy Failed", file=sys.stderr) print ("Giving up", file=sys.stderr) sys.exit(20) print ("") print ("Building Pbuilder finished !") print ("")
def setup_apt_keyring(gpg_home, keyring_fname): ring_path = os.path.join(gpg_home, keyring_fname) if not os.path.isdir("/etc/apt/trusted.gpg.d"): print("/etc/apt/trusted.gpg.d doesn't exist") print("apt-get install debian-archive-keyring may " "fix this problem") sys.exit(20) if os.path.exists("/etc/apt/trusted.gpg"): system('cp /etc/apt/trusted.gpg "%s"' % ring_path) gpg_options = '--keyring "%s" --no-auto-check-trustdb ' \ '--trust-model always --no-default-keyring ' \ '--batch ' \ '--homedir "%s"' % (ring_path, gpg_home) trustkeys = os.listdir("/etc/apt/trusted.gpg.d") for key in trustkeys: print("Import %s: " % key) try: system('gpg %s --import "%s"' % ( gpg_options, os.path.join("/etc/apt/trusted.gpg.d", key))) except CommandError: print('adding keyring "%s" to keyring "%s" failed' % (key, ring_path))
def execute(self, _initvmdir, opt, args): try: system('%s initvm ensure' % elbe_exe) except CommandError: print("Starting the initvm Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) # Init cdrom to None, if we detect it, we set it cdrom = None if len(args) == 1: if args[0].endswith('.xml'): # We have an xml file, use that for elbe init xmlfile = args[0] url_validation = '' elif args[0].endswith('.iso'): # We have an iso image, extract xml from there. tmp = extract_cdrom(args[0]) xmlfile = tmp.fname('source.xml') url_validation = '--skip-urlcheck' cdrom = args[0] else: print( "Unknown file ending (use either xml or iso)", file=sys.stderr) sys.exit(20) submit_and_dl_result(xmlfile, cdrom, opt)
def download_and_validate_file(self, upstream_fname, local_fname): url = self.base_url + upstream_fname try: system('wget -O "%s" "%s"' % (local_fname, url)) except CommandError: raise HashValidationFailed('Failed to download %s' % url) self.validate_file(upstream_fname, local_fname)
def execute(self, _initvmdir, _opt, _args): if self.initvm_state() == libvirt.VIR_DOMAIN_SHUTOFF: system('%s initvm start' % elbe_exe) elif self.initvm_state() == libvirt.VIR_DOMAIN_RUNNING: pass else: print("Elbe initvm in bad state.") sys.exit(20)
def teardown(): system ("%s control shutdown_initvm" % (exe_path), allow_fail=True) sleep (10) tmpdir = os.getenv( "ELBE_TEST_DIR" ) assert tmpdir is not None dname = os.path.join(tmpdir, "initvm_test") shutil.rmtree (dname)
def init_example (xml_name): tmpdir = os.getenv( "ELBE_TEST_DIR" ) assert tmpdir is not None dname = os.path.join(tmpdir, "init_test") try: system ('%s init --directory "%s" %s' % (exe_path, dname, xml_name)) finally: shutil.rmtree (dname)
def unlock_key(fingerprint): ctx = core.Context() ctx.set_engine_info(PROTOCOL_OpenPGP, None, '/var/cache/elbe/gnupg') key = ctx.get_key(fingerprint, secret=True) keygrip = key.subkeys[0].keygrip system("/usr/lib/gnupg2/gpg-preset-passphrase " "--preset -P requiredToAvoidUserInput %s" % str(keygrip), env_add={"GNUPGHOME": "/var/cache/elbe/gnupg"})
def download_pkg(prj, target_dir, defs, package, arch="default", incl_deps=False, log=None): # pylint: disable=too-many-arguments # pylint: disable=too-many-locals # pylint: disable=too-many-branches try: urilist = get_uri(prj, defs, arch, package, incl_deps) except KeyError: raise NoPackageException('no package %s available' % package) except SystemError: raise NoPackageException('a configured mirror is not reachable') except CommandError: raise NoPackageException("couldn't download package %s" % package) if not urilist: raise NoPackageException("couldn't download package %s" % package) for u in urilist: sha256 = u[2] uri = u[1] dest = os.path.join(target_dir, "%s.deb" % u[0]) try: if uri.startswith("file://"): system('cp "%s" "%s"' % (uri[len("file://"):], dest)) elif uri.startswith("http://") or uri.startswith("ftp://"): system('wget -O "%s" "%s"' % (dest, uri)) else: raise NoPackageException('could not retreive %s' % uri) except CommandError: raise NoPackageException("couldn't download package %s" % package) if sha256: try: validate_sha256(dest, sha256) except HashValidationFailed as e: raise NoPackageException('%s failed to verify: %s' % package, e.message) else: if log: log.printo("WARNING: Using untrusted %s package" % package) else: print("-----------------------------------------------------") print("WARNING:") print("Using untrusted %s package" % package) print("-----------------------------------------------------") return [y[0] for y in urilist]
def execute(self, initvmdir, opt, args): try: have_session = os.system( "tmux has-session -t ElbeInitVMSession >/dev/null 2>&1" ) except CommandError as e: print ("tmux execution failed, tmux version 1.9 or higher is required") sys.exit(20) if have_session == 0: if os.environ.has_key('TMUX'): system( 'tmux link-window -s ElbeInitVMSession:initvm' ) else: system( 'tmux attach -t ElbeInitVMSession' ) else: print ("ElbeInitVMSession does not exist in tmux.", file=sys.stderr) print ("Try 'elbe initvm start' to start the session.", file=sys.stderr) sys.exit(20)
def execute(self, initvmdir, opt, args): try: have_session = os.system( "tmux has-session -t ElbeInitVMSession >/dev/null 2>&1" ) except CommandError as e: print ("tmux execution failed, tmux version 1.9 or higher is required") sys.exit(20) if have_session != 256: # other session exists... Good. exit sys.exit(0) # No other session exists, sanity check initvmdir sanity_check_dir (initvmdir) # Sanity check passed. start initvm session system( 'TMUX= tmux new-session -d -c "%s" -s ElbeInitVMSession -n initvm "make run-con"' % initvmdir )
def setup_gpg(self): ring_path = self.projectpath + "/etc/apt/trusted.gpg.d" if not os.path.isdir("/etc/apt/trusted.gpg.d"): print("/etc/apt/trusted.gpg.d doesn't exist") print("apt-get install debian-archive-keyring may fix this problem") sys.exit(20) system('cp /etc/apt/trusted.gpg.d/* "%s"' % ring_path) gpg_options = ( '--keyring "%s" --no-auto-check-trustdb --trust-model always --no-default-keyring --homedir "%s"' % (os.path.join(ring_path, "linutronix-elbe.gpg"), self.projectpath) ) system('gpg %s --import "%s"' % (gpg_options, elbe_pubkey_fname))
def execute(self, initvmdir, opt, args): try: have_session = os.system( "tmux has-session -t ElbeInitVMSession >/dev/null 2>&1" ) except CommandError as e: print ("tmux execution failed, tmux version 1.9 or higher is required") sys.exit(20) if have_session != 256: print ("ElbeInitVMSession already exists in tmux.", file=sys.stderr) print ("Try 'elbe initvm attach' to attach to the session.", file=sys.stderr) sys.exit(20) # No other session exists, sanity check initvmdir sanity_check_dir (initvmdir) system( 'TMUX= tmux new-session -d -c "%s" -s ElbeInitVMSession -n initvm "make"' % initvmdir )
def execute(self, opt, _args): if not opt.project: print("you need to specify --project option", file=sys.stderr) sys.exit(20) prjdir = opt.project print("Updating pbuilder") try: system('%s control update_pbuilder "%s"' % (elbe_exe, prjdir)) except CommandError: print("elbe control update_pbuilder Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("") print("Updating Pbuilder finished !") print("")
def setup(): try: global exe_path # Properly initialise directories plug mod_dir = os.path.dirname (__file__) main_dir, _ = os.path.split( mod_dir ) exe_path = os.path.join( main_dir, "elbe" ) init_directories (exe_path) tmpdir = os.getenv( "ELBE_TEST_DIR" ) assert tmpdir is not None dname = os.path.join(tmpdir, "initvm_test") from elbepack.directories import examples_dir xml_name = os.path.join (examples_dir, "elbe-init-with-ssh.xml") system ('%s initvm create --devel --directory "%s"' % (exe_path, dname)) except: teardown () raise
def execute(self, initvmdir, opt, args): try: have_session = os.system( "tmux has-session -t ElbeInitVMSession >/dev/null 2>&1" ) except CommandError as e: print ("tmux execution failed, tmux version 1.9 or higher is required") sys.exit(20) if have_session != 256: print ("ElbeInitVMSession already exists in tmux.", file=sys.stderr) print ("Try 'elbe initvm attach' to attach to the session.", file=sys.stderr) sys.exit(20) # No other session exists, sanity check initvmdir sanity_check_dir (initvmdir) # Sanity check passed. start initvm session try: try: os.remove (os.path.join (initvmdir, "run.log")) except OSError: pass system( 'TMUX= tmux new-session -d -c "%s" -s ElbeInitVMSession -n initvm "make run-con 2> %s"' % ( initvmdir, str(os.path.join (initvmdir, "run.log")).replace (' ', '\ '))) for i in range (1, 5): sys.stdout.write ("*") sys.stdout.flush () time.sleep (1) print ("*") have_session = os.system( "tmux has-session -t ElbeInitVMSession >/dev/null 2>&1" ) if have_session: print ("Maybe starting elbe initvm failed:", file=sys.stderr) with open (os.path.join (initvmdir, "run.log"), 'r') as f: for l in f: print (l, file=sys.stderr) os.remove (os.path.join (initvmdir, "run.log")) except CommandError as e: print ("tmux execution failed, tmux version 1.9 or higher is required") sys.exit(20)
def setup_gpg(self): ring_path = os.path.join(self.projectpath, "etc/apt/trusted.gpg") if not os.path.isdir("/etc/apt/trusted.gpg.d"): print("/etc/apt/trusted.gpg.d doesn't exist") print("apt-get install debian-archive-keyring may " "fix this problem") sys.exit(20) if os.path.exists("/etc/apt/trusted.gpg"): system('cp /etc/apt/trusted.gpg "%s"' % ring_path) gpg_options = '--keyring "%s" --no-auto-check-trustdb ' \ '--trust-model always --no-default-keyring ' \ '--homedir "%s"' % (ring_path, self.projectpath) trustkeys = os.listdir("/etc/apt/trusted.gpg.d") for key in trustkeys: print("Import %s: " % key) try: system('gpg %s --import "%s"' % ( gpg_options, os.path.join("/etc/apt/trusted.gpg.d", key))) except CommandError: print("adding elbe-pubkey to keyring failed")
def test_submit (): tmpdir = os.getenv( "ELBE_TEST_DIR" ) assert tmpdir is not None dname = os.path.join(tmpdir, "initvm_output") system ('mkdir "%s"' % dname) system ('mkdir "%s02"' % dname) try: from elbepack.directories import examples_dir xml_name = os.path.join (examples_dir, "rescue.xml") system ('%s initvm submit --build-bin --output "%s" "%s"' % (exe_path, dname, xml_name)) # Now submit the iso image system ('%s initvm submit --output "%s02" "%s"' % (exe_path, dname, os.path.join (dname, "bin-cdrom.iso"))) finally: shutil.rmtree (dname) shutil.rmtree (dname+"02")
def add_pubkey_url(self, url): ring_path = self.projectpath + "/etc/apt/trusted.gpg.d" tmpkey_path = self.projectpath + "/tmpkey.gpg" gpg_options = ( '--keyring "%s" --no-auto-check-trustdb --trust-model always --no-default-keyring --homedir "%s"' % (os.path.join(ring_path, "linutronix-elbe.gpg"), self.projectpath) ) try: system('wget -O "%s" "%s"' % (tmpkey_path, url)) system('gpg %s --import "%s"' % (gpg_options, tmpkey_path)) finally: system('rm "%s"' % tmpkey_path, allow_fail=True)
def add_pubkey_url(self, url): ring_path = os.path.join(self.projectpath, "etc/apt/trusted.gpg") tmpkey_path = os.path.join(self.projectpath, "tmpkey.gpg") gpg_options = '--keyring "%s" --no-auto-check-trustdb ' \ '--trust-model always --no-default-keyring ' \ '--homedir "%s"' % (ring_path, self.projectpath) try: system('wget -O "%s" "%s"' % (tmpkey_path, url)) system('gpg %s --import "%s"' % ( gpg_options, tmpkey_path)) finally: system('rm "%s"' % tmpkey_path, allow_fail=True)
def gen_update_pkg(project, xml_filename, upd_filename, override_buildtype=None, skip_validate=False, debug=False, cmd_dir=None, cfg_dir=None): # pylint: disable=too-many-arguments # pylint: disable=too-many-locals # pylint: disable=too-many-statements # pylint: disable=too-many-branches if xml_filename: xml = ElbeXML(xml_filename, buildtype=override_buildtype, skip_validate=skip_validate) if not xml.has("fullpkgs"): raise MissingData("Xml does not have fullpkgs list") if not project.xml.has("fullpkgs"): raise MissingData("Source Xml does not have fullpkgs list") if not project.buildenv.rfs: raise MissingData("Target does not have a build environment") cache = project.get_rpcaptcache() instpkgs = cache.get_installed_pkgs() instindex = {} for p in instpkgs: instindex[p.name] = p xmlpkgs = xml.node("/fullpkgs") xmlindex = {} fnamelist = [] for p in xmlpkgs: name = p.et.text ver = p.et.get('version') md5 = p.et.get('md5') xmlindex[name] = p if name not in instindex: print("package removed: %s" % name) continue ipkg = instindex[name] comp = cache.compare_versions(ipkg.installed_version, ver) pfname = ipkg.installed_deb if comp == 0: print("package ok: %s-%s" % (name, ipkg.installed_version)) if debug: fnamelist.append(pfname) continue if comp > 0: print("package upgrade: %s" % pfname) fnamelist.append(pfname) else: print( "package downgrade: %s-%s" % (name, ipkg.installed_version)) for p in instpkgs: if p.name in xmlindex: continue print("package %s newly installed" % p.name) pfname = p.installed_deb fnamelist.append(pfname) update = os.path.join(project.builddir, "update") if os.path.exists(update): rmtree(update) system('mkdir -p %s' % update) if xml_filename: repodir = os.path.join(update, "repo") repo = UpdateRepo(xml, repodir, project.log) for fname in fnamelist: path = os.path.join( project.chrootpath, "var/cache/apt/archives", fname) repo.includedeb(path) repo.finalize() dump_fullpkgs(project.xml, project.buildenv.rfs, cache) project.xml.xml.write(os.path.join(update, "new.xml")) system("cp %s %s" % (xml_filename, os.path.join(update, "base.xml"))) else: system("cp source.xml update/new.xml") if project.presh_file: copyfile(project.presh_file, update + '/pre.sh') os.chmod(update + '/pre.sh', 0o755) if project.postsh_file: copyfile(project.postsh_file, update + '/post.sh') os.chmod(update + '/post.sh', 0o755) if cmd_dir: inlucdedir(update, 'cmd', cmd_dir, mode=0o755) if cfg_dir: inlucdedir(update, 'conf', cfg_dir) create_zip_archive(upd_filename, update, ".") if project.postbuild_file: project.log.h2("postbuild script") project.log.do(project.postbuild_file + ' "%s %s %s"' % ( upd_filename, project.xml.text("project/version"), project.xml.text("project/name")), allow_fail=True)
def _umount(self, path): if os.path.ismount(path): system("umount %s" % path)
def run_command(argv): # pylint: disable=too-many-locals # pylint: disable=too-many-statements # pylint: disable=too-many-branches oparser = OptionParser(usage="usage: %prog init [options] <filename>") oparser.add_option("--skip-validation", action="store_true", dest="skip_validation", default=False, help="Skip xml schema validation") oparser.add_option("--directory", dest="directory", default="./build", help="Working directory (default is build)", metavar="FILE") oparser.add_option( "--cdrom", dest="cdrom", help="Use FILE as cdrom iso, and use that to build the initvm", metavar="FILE") oparser.add_option("--proxy", dest="proxy", help="Override the http Proxy") oparser.add_option("--buildtype", dest="buildtype", help="Override the buildtype") oparser.add_option( "--debug", dest="debug", action="store_true", default=False, help="start qemu in graphical mode to enable console switch") oparser.add_option( "--devel", dest="devel", action="store_true", default=False, help="use devel mode, and install current builddir inside initvm") oparser.add_option( "--nesting", dest="nesting", action="store_true", default=False, help="allow initvm to support nested kvm. " "This makes /proc/cpuinfo inside initvm differ per host.") oparser.add_option( "--skip-build-bin", action="store_false", dest="build_bin", default=True, help="Skip building Binary Repository CDROM, for exact Reproduction") oparser.add_option( "--skip-build-sources", action="store_false", dest="build_sources", default=True, help="Skip building Source CDROM") (opt, args) = oparser.parse_args(argv) if not args: print("no filename specified") oparser.print_help() sys.exit(20) elif len(args) > 1: print("too many filenames specified") oparser.print_help() sys.exit(20) if opt.devel: if not os.path.isdir(os.path.join(elbe_dir, "elbepack")): print("Devel Mode only valid, when running from elbe checkout") sys.exit(20) if not opt.skip_validation: validation = validate_xml(args[0]) if validation: print("xml validation failed. Bailing out") for i in validation: print(i) sys.exit(20) xml = etree(args[0]) if not xml.has("initvm"): print("fatal error: xml missing mandatory section 'initvm'") sys.exit(20) if opt.buildtype: buildtype = opt.buildtype elif xml.has("initvm/buildtype"): buildtype = xml.text("/initvm/buildtype") else: buildtype = "nodefaults" defs = ElbeDefaults(buildtype) http_proxy = "" if os.getenv("http_proxy"): http_proxy = os.getenv("http_proxy") elif opt.proxy: http_proxy = opt.proxy elif xml.has("initvm/mirror/primary_proxy"): http_proxy = xml.text("initvm/mirror/primary_proxy") http_proxy = http_proxy.strip().replace("LOCALMACHINE", "localhost") if opt.cdrom: mirror = xml.node("initvm/mirror") mirror.clear() cdrom = mirror.ensure_child("cdrom") cdrom.set_text(os.path.abspath(opt.cdrom)) # this is a workaround for # http://lists.linutronix.de/pipermail/elbe-devel/2017-July/000541.html _, virt = command_out('test -x /usr/bin/systemd-detect-virt && /usr/bin/systemd-detect-virt'); _, dist = command_out('lsb_release -cs') if 'vmware' in virt and 'stretch' in dist: machine_type = 'pc-i440fx-2.6' else: machine_type = 'pc' try: os.makedirs(opt.directory) except OSError as e: print( "unable to create project directory: %s (%s)" % (opt.directory, e.strerror)) sys.exit(30) out_path = os.path.join(opt.directory, ".elbe-in") try: os.makedirs(out_path) except OSError as e: print( "unable to create subdirectory: %s (%s)" % (out_path, e.strerror)) sys.exit(30) d = {"elbe_version": elbe_version, "defs": defs, "opt": opt, "xml": xml, "prj": xml.node("/initvm"), "http_proxy": http_proxy, "pkgs": xml.node("/initvm/pkg-list") or [], "preseed": get_initvm_preseed(xml), "machine_type": machine_type, "cfg": cfg} if http_proxy != "": os.putenv("http_proxy", http_proxy) os.putenv("https_proxy", http_proxy) os.putenv("no_proxy", "localhost,127.0.0.1") try: copy_kinitrd(xml.node("/initvm"), out_path) except NoKinitrdException as e: print("Failure to download kernel/initrd debian Package:") print("") print(e.message) print("") print("Check Mirror configuration") if 'SHA256SUMS' in e.message: print("If you use debmirror please read https://github.com/Linutronix/elbe/issues/188 " "on how to work around the issue") sys.exit(20) templates = os.listdir(init_template_dir) make_executable = ["init-elbe.sh.mako", "preseed.cfg.mako"] for t in templates: o = t.replace(".mako", "") if t == "Makefile.mako" or t == "libvirt.xml.mako": write_template( os.path.join( opt.directory, o), os.path.join( init_template_dir, t), d, linebreak=True) else: write_template( os.path.join( out_path, o), os.path.join( init_template_dir, t), d, linebreak=False) if t in make_executable: os.chmod(os.path.join(out_path, o), 0o755) shutil.copyfile(args[0], os.path.join(out_path, "source.xml")) if opt.cdrom: system('7z x -o%s "%s" elbe-keyring.gpg' % (out_path, opt.cdrom)) else: keys = [] for key in xml.all(".//initvm/mirror/url-list/url/raw-key"): keys.append(key.et.text) import_keyring = os.path.join(out_path, "elbe-keyring") command_out('gpg --no-options --no-default-keyring --keyring %s --import' % import_keyring, stdin="".join(keys)) export_keyring = import_keyring + ".gpg" command_out('gpg --no-options --no-default-keyring --keyring %s --export --output %s' % (import_keyring, export_keyring)) if opt.devel: out_real = os.path.realpath(out_path) ignore = '' if out_real.startswith(elbe_dir + os.sep): ignore = '--exclude "%s"' % os.path.relpath(out_path, start=elbe_dir) system('tar cfj "%s" %s -C "%s" .' % (os.path.join(out_path, "elbe-devel.tar.bz2"), ignore, elbe_dir))
def execute(self, opt, _args): # pylint: disable=too-many-statements # pylint: disable=too-many-branches crossopt = "" if opt.cross: crossopt = "--cross" tmp = TmpdirFilesystem() if opt.xmlfile: ret, prjdir, err = command_out_stderr( '%s control create_project --retries 60 "%s"' % (elbe_exe, opt.xmlfile)) if ret != 0: print("elbe control create_project failed.", file=sys.stderr) print(err, file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) prjdir = prjdir.strip() try: system('%s control build_pbuilder "%s"' % (elbe_exe, prjdir)) except CommandError: print("elbe control build_pbuilder Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) try: system('%s control wait_busy "%s"' % (elbe_exe, prjdir)) except CommandError: print("elbe control wait_busy Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("") print("Building Pbuilder finished !") print("") elif opt.project: prjdir = opt.project system('%s control rm_log %s' % (elbe_exe, prjdir)) else: print("you need to specify --project or --xmlfile option", file=sys.stderr) sys.exit(20) print("") print("Packing Source into tmp archive") print("") try: system('tar cfz "%s" .' % (tmp.fname("pdebuild.tar.gz"))) except CommandError: print("tar Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) for of in opt.origfile: print("") print("Pushing orig file '%s' into pbuilder" % of) print("") try: system('%s control set_orig "%s" "%s"' % (elbe_exe, prjdir, of)) except CommandError: print("elbe control set_orig Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("") print("Pushing source into pbuilder") print("") try: system('%s control set_pdebuild --cpuset "%d" --profile "%s" %s ' '"%s" "%s"' % (elbe_exe, opt.cpuset, opt.profile, crossopt, prjdir, tmp.fname("pdebuild.tar.gz"))) except CommandError: print("elbe control set_pdebuild Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) try: system('%s control wait_busy "%s"' % (elbe_exe, prjdir)) except CommandError: print("elbe control wait_busy Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("") print("Pdebuild finished !") print("") if opt.skip_download: print("") print("Listing available files:") print("") try: system('%s control --pbuilder-only get_files "%s"' % (elbe_exe, prjdir)) except CommandError: print("elbe control get_files Failed", file=sys.stderr) print("", file=sys.stderr) print("dumping logfile", file=sys.stderr) try: system('%s control dump_file "%s" log.txt' % (elbe_exe, prjdir)) except CommandError: print("elbe control dump_file Failed", file=sys.stderr) print("", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("") print("Get Files with: 'elbe control get_file %s <filename>'" % prjdir) else: print("") print("Getting generated Files") print("") ensure_outdir(opt) try: system( '%s control --pbuilder-only get_files --output "%s" "%s"' % (elbe_exe, opt.outdir, prjdir)) except CommandError: print("elbe control get_files Failed", file=sys.stderr) print("", file=sys.stderr) print("dumping logfile", file=sys.stderr) try: system('%s control dump_file "%s" log.txt' % (elbe_exe, prjdir)) except CommandError: print("elbe control dump_file Failed", file=sys.stderr) print("", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20)
def _del_rfs_file(self, filename, rfs): if rfs.lexists(filename): flags = "-f" if rfs.isdir(filename): flags += "r" system("rm %s %s" % (flags, rfs.fname(filename)))
def build(self, build_bin=False, build_sources=False, cdrom_size=None, skip_pkglist=False, skip_pbuild=False): # pylint: disable=too-many-arguments # pylint: disable=too-many-locals # pylint: disable=too-many-statements # pylint: disable=too-many-branches # Write the log header self.write_log_header() # Validate Apt Sources if build_sources: m = ValidationMode.CHECK_ALL else: m = ValidationMode.CHECK_BINARIES self.xml.validate_apt_sources(m, self.arch) if self.xml.has('target/pbuilder') and not skip_pbuild: if not os.path.exists(os.path.join(self.builddir, "pbuilder")): self.create_pbuilder() for p in self.xml.node('target/pbuilder'): self.pbuild(p) # the package might be needed by a following pbuild, so update # the project repo that it can be installed in as # build-dependency self.repo.finalize() # To avoid update cache errors, the project repo needs to have # Release and Packages files, even if it's empty. So don't do this # in the if case above! self.repo.finalize() # Create the build environment, if it does not a valid one # self.buildenv might be set when we come here. # However, if its not a full_buildenv, we specify clean here, # so it gets rebuilt properly. if not self.has_full_buildenv(): self.log.do('mkdir -p "%s"' % self.chrootpath) self.buildenv = BuildEnv(self.xml, self.log, self.chrootpath, build_sources=build_sources, clean=True) skip_pkglist = False # Import keyring self.buildenv.import_keys() self.log.printo("Keys imported") # Install packages if not skip_pkglist: self.install_packages(self.buildenv) try: self.buildenv.rfs.dump_elbeversion(self.xml) except IOError: self.log.printo("dump elbeversion failed") # Extract target FS. We always create a new instance here with # clean=true, because we want a pristine directory. self.targetfs = TargetFs(self.targetpath, self.log, self.buildenv.xml, clean=True) os.chdir(self.buildenv.rfs.fname('')) extract_target(self.buildenv.rfs, self.xml, self.targetfs, self.log, self.get_rpcaptcache()) # The validation file is created using check_full_pkgs() and # elbe_report(), both opening the file in append mode. So if an # old validation file already exists, it must be deleted first. if os.path.isfile(self.validationpath): os.unlink(self.validationpath) # Package validation and package list if not skip_pkglist: pkgs = self.xml.xml.node("/target/pkg-list") if self.xml.has("fullpkgs"): check_full_pkgs(pkgs, self.xml.xml.node("/fullpkgs"), self.validationpath, self.get_rpcaptcache()) else: check_full_pkgs(pkgs, None, self.validationpath, self.get_rpcaptcache()) dump_fullpkgs(self.xml, self.buildenv.rfs, self.get_rpcaptcache()) self.xml.dump_elbe_version() self.targetfs.write_fstab(self.xml) # Dump ELBE version try: self.targetfs.dump_elbeversion(self.xml) except MemoryError: self.log.printo("dump elbeversion failed") # install packages for buildenv if not skip_pkglist: self.install_packages(self.buildenv, buildenv=True) # Write source.xml try: sourcexmlpath = os.path.join(self.builddir, "source.xml") self.xml.xml.write(sourcexmlpath) except MemoryError: self.log.printo("write source.xml failed (archive to huge?)") # Elbe report reportpath = os.path.join(self.builddir, "elbe-report.txt") elbe_report(self.xml, self.buildenv, self.get_rpcaptcache(), reportpath, self.validationpath, self.targetfs) # the current license code raises an exception that interrupts the hole # build if a licence can't be converted to utf-8. Exception handling # can be removed as soon as the licence code is more stable lic_err = False try: f = io.open(os.path.join(self.builddir, "licence.txt"), "w+", encoding='utf-8') self.buildenv.rfs.write_licenses( f, self.log, os.path.join(self.builddir, "licence.xml")) except Exception: self.log.printo("error during generating licence.txt/xml") self.log.printo(sys.exc_info()[0]) lic_err = True finally: f.close() if lic_err: os.remove(os.path.join(self.builddir, "licence.txt")) os.remove(os.path.join(self.builddir, "licence.xml")) # Use some handwaving to determine grub version # jessie and wheezy grubs are 2.0 but differ in behaviour # # We might also want support for legacy grub if (self.get_rpcaptcache().is_installed('grub-pc') and self.get_rpcaptcache().is_installed('grub-efi-amd64-bin')): grub_version = 202 grub_fw_type = "hybrid" elif self.get_rpcaptcache().is_installed('grub-pc'): if self.codename == "wheezy": grub_version = 199 else: grub_version = 202 grub_fw_type = "bios" elif self.get_rpcaptcache().is_installed('grub-efi-amd64'): grub_version = 202 grub_fw_type = "efi" elif self.get_rpcaptcache().is_installed('grub-legacy'): self.log.printo("package grub-legacy is installed, " "this is obsolete, skipping grub") grub_version = 0 grub_fw_type = "" else: self.log.printo("package grub-pc is not installed, skipping grub") # version 0 == skip_grub grub_version = 0 grub_fw_type = "" self.targetfs.part_target(self.builddir, grub_version, grub_fw_type) self.build_cdroms(build_bin, build_sources, cdrom_size) if self.postbuild_file: self.log.h2("postbuild script:") self.log.do(self.postbuild_file + ' "%s %s %s"' % (self.builddir, self.xml.text("project/version"), self.xml.text("project/name")), allow_fail=True) do_prj_finetuning(self.xml, self.log, self.buildenv, self.targetfs, self.builddir) self.targetfs.pack_images(self.builddir) system('cat "%s"' % self.validationpath)
def execute(self, opt, args): tmp = TmpdirFilesystem () if opt.xmlfile: ret, prjdir, err = command_out_stderr ('%s control create_project --retries 60 "%s"' % (elbe_exe, opt.xmlfile)) if ret != 0: print ("elbe control create_project failed.", file=sys.stderr) print (err, file=sys.stderr) print ("Giving up", file=sys.stderr) sys.exit(20) prjdir = prjdir.strip() try: system ('%s control build "%s"' % (elbe_exe, prjdir) ) except CommandError: print ("elbe control build Failed", file=sys.stderr) print ("Giving up", file=sys.stderr) sys.exit(20) print ("Build started, waiting till it finishes") try: system ('%s control wait_busy "%s"' % (elbe_exe, prjdir) ) except CommandError: print ("elbe control wait_busy Failed", file=sys.stderr) print ("Giving up", file=sys.stderr) sys.exit(20) print ("") print ("Build finished !") print ("") print ("Creating pbuilder") try: system ('%s control build_pbuilder "%s"' % (elbe_exe, prjdir)) except CommandError: print ("elbe control build_pbuilder Failed", file=sys.stderr) print ("Giving up", file=sys.stderr) sys.exit(20) try: system ('%s control wait_busy "%s"' % (elbe_exe, prjdir) ) except CommandError: print ("elbe control wait_busy Failed", file=sys.stderr) print ("Giving up", file=sys.stderr) sys.exit(20) print ("") print ("Building Pbuilder finished !") print ("") elif opt.project: prjdir = opt.project else: print ('you need to specify --project or --xmlfile option', file=sys.stderr) sys.exit(20) print ("") print ("Packing Source into tmp archive") print ("") try: system ('tar cvfz "%s" .' % (tmp.fname ("pdebuild.tar.gz"))) except CommandError: print ("tar Failed", file=sys.stderr) print ("Giving up", file=sys.stderr) sys.exit(20) print ("") print ("Pushing source into pbuilder") print ("") try: system ('%s control set_pdebuild "%s" "%s"' % (elbe_exe, prjdir, tmp.fname ("pdebuild.tar.gz"))) except CommandError: print ("elbe control set_pdebuild Failed", file=sys.stderr) print ("Giving up", file=sys.stderr) sys.exit(20) try: system ('%s control wait_busy "%s"' % (elbe_exe, prjdir) ) except CommandError: print ("elbe control wait_busy Failed", file=sys.stderr) print ("Giving up", file=sys.stderr) sys.exit(20) print ("") print ("Pdebuild finished !") print ("") if opt.skip_download: print ("") print ("Listing available files:") print ("") try: system ('%s control --pbuilder-only get_files "%s"' % (elbe_exe, prjdir) ) except CommandError: print ("elbe control get_files Failed", file=sys.stderr) print ("", file=sys.stderr) print ("dumping logfile", file=sys.stderr) try: system ('%s control dump_file "%s" log.txt' % ( elbe_exe, prjdir )) except CommandError: print ("elbe control dump_file Failed", file=sys.stderr) print ("", file=sys.stderr) print ("Giving up", file=sys.stderr) sys.exit(20) print ("") print ('Get Files with: elbe control get_file "%s" <filename>' % prjdir) else: print ("") print ("Getting generated Files") print ("") ensure_outdir (wdfs, opt) try: system ('%s control --pbuilder-only get_files --output "%s" "%s"' % ( elbe_exe, opt.outdir, prjdir )) except CommandError: print ("elbe control get_files Failed", file=sys.stderr) print ("", file=sys.stderr) print ("dumping logfile", file=sys.stderr) try: system ('%s control dump_file "%s" log.txt' % ( elbe_exe, prjdir )) except CommandError: print ("elbe control dump_file Failed", file=sys.stderr) print ("", file=sys.stderr) print ("Giving up", file=sys.stderr) sys.exit(20)
def copy_filelist(src, file_lst, dst): files = set() copied = set() # Make sure to copy parent directories # # For example, if file_lst = ['/usr/bin/bash'], # we might get {'/usr/bin', '/usr', '/usr/bin/bash'} for f in file_lst: parts = f.rstrip('\n') while parts != os.sep: files.add(parts) parts, _ = os.path.split(parts) # Start from closest to root first files = list(files) files.sort() files.reverse() while files: f = files.pop() copied.add(f) if src.islink(f): tgt = src.readlink(f) # If the target of the symlink is relative, we need the # absolute path of it if not os.path.isabs(tgt): tgt = os.path.join(os.path.dirname(f), tgt) # If the target is not yet in the destination RFS, we need # to defer the copy of the symlink after the target is # resolved. Thus, we put the symlink back on the stack # and we add the target to resolve on top of it. # # Not that this will result in an infinite loop for # circular symlinks if not dst.lexists(tgt): files.append(f) files.append(tgt) else: dst.symlink(tgt, f) elif src.isdir(f): if not dst.isdir(f): dst.mkdir(f) st = src.stat(f) dst.chown(f, st.st_uid, st.st_gid) else: system('cp -a --reflink=auto "%s" "%s"' % (src.realpath(f), dst.realpath(f))) # update utime which will change after a file has been copied into # the directory for f in copied: if src.isdir(f) and not src.islink(f): shutil.copystat(src.fname(f), dst.fname(f))
def execute(self, initvmdir, opt, args): try: have_session = os.system( "tmux has-session -t ElbeInitVMSession >/dev/null 2>&1") except CommandError as e: print( "tmux execution failed, tmux version 1.9 or higher is required" ) sys.exit(20) if have_session == 256: print("ElbeInitVMSession does not exist in tmux.", file=sys.stderr) print("Try 'elbe initvm start' to start the session.", file=sys.stderr) sys.exit(20) try: system('%s initvm ensure --directory "%s"' % (elbe_exe, initvmdir)) except CommandError: print("Starting the initvm Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) # Init cdrom to None, if we detect it, we set it cdrom = None if len(args) == 1: if args[0].endswith('.xml'): # We have an xml file, use that for elbe init xmlfile = args[0] elif args[0].endswith('.iso'): # We have an iso image, extract xml from there. tmp = TmpdirFilesystem() os.system('7z x -o%s "%s" source.xml' % (tmp.path, args[0])) print('', file=sys.stderr) if not tmp.isfile('source.xml'): print('Iso image does not contain a source.xml file', file=sys.stderr) print('This is not supported by "elbe initvm"', file=sys.stderr) print('', file=sys.stderr) print('Exiting !!!', file=sys.stderr) sys.exit(20) try: exml = ElbeXML(tmp.fname('source.xml'), skip_urlcheck=True) except ValidationError as e: print('Iso image does contain a source.xml file.', file=sys.stderr) print('But that xml does not validate correctly', file=sys.stderr) print('', file=sys.stderr) print('Exiting !!!', file=sys.stderr) sys.exit(20) print('Iso Image with valid source.xml detected !') print('Image was generated using Elbe Version %s' % exml.get_elbe_version()) xmlfile = tmp.fname('source.xml') cdrom = args[0] else: print('Unknown file ending (use either xml or iso)', file=sys.stderr) sys.exit(20) ret, prjdir, err = command_out_stderr( '%s control create_project --retries 60 "%s"' % (elbe_exe, xmlfile)) if ret != 0: print("elbe control create_project failed.", file=sys.stderr) print(err, file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) prjdir = prjdir.strip() if cdrom is not None: print("Uploading CDROM. This might take a while") try: system('%s control set_cdrom "%s" "%s"' % (elbe_exe, prjdir, cdrom)) except CommandError: print("elbe control set_cdrom Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("Upload finished") build_opts = '' if opt.build_bin: build_opts += '--build-bin ' if opt.build_sources: build_opts += '--build-sources ' try: system('%s control build "%s" %s' % (elbe_exe, prjdir, build_opts)) except CommandError: print("elbe control build Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("Build started, waiting till it finishes") try: system('%s control wait_busy "%s"' % (elbe_exe, prjdir)) except CommandError: print("elbe control wait_busy Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("") print("Build finished !") print("") try: system('%s control dump_file "%s" validation.txt' % (elbe_exe, prjdir)) except CommandError: print("Project failed to generate validation.txt", file=sys.stderr) print("Getting log.txt", file=sys.stderr) try: system('%s control dump_file "%s" log.txt' % (elbe_exe, prjdir)) except CommandError: print("Failed to dump log.txt", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) if opt.skip_download: print("") print("Listing available files:") print("") try: system('%s control get_files "%s"' % (elbe_exe, prjdir)) except CommandError: print("elbe control get_files Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("") print('Get Files with: elbe control get_file "%s" <filename>' % prjdir) else: print("") print("Getting generated Files") print("") ensure_outdir(wdfs, opt) try: system('%s control get_files --output "%s" "%s"' % (elbe_exe, opt.outdir, prjdir)) except CommandError: print("elbe control get_files Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) if not opt.keep_files: try: system('%s control del_project "%s"' % (elbe_exe, prjdir)) except CommandError: print("remove project from initvm failed", file=sys.stderr) sys.exit(20)
def download(url, local_fname): try: system('wget -O "%s" "%s"' % (local_fname, url)) except CommandError: raise NoKinitrdException('Failed to download %s' % url)
def export_key(fingerprint, outfile): system("/usr/bin/gpg -a -o %s --export %s" % (outfile, fingerprint), env_add={"GNUPGHOME": "/var/cache/elbe/gnupg"})
def execute(self, initvmdir, opt, args): # pylint: disable=too-many-branches # pylint: disable=too-many-statements if self.initvm is not None: print("Initvm is already defined for the libvirt domain '%s'.\n" % cfg['initvm_domain']) print("If you want to build in your old initvm, " "use `elbe initvm submit <xml>`.") print("If you want to remove your old initvm from libvirt " "run `virsh --connect qemu:///system undefine %s`.\n" % cfg['initvm_domain']) print( "You can specify another libvirt domain by setting the " "ELBE_INITVM_DOMAIN environment variable to an unused domain name.\n" ) print("Note:") print("\t1) You can reimport your old initvm via " "`virsh --connect qemu:///system define <file>`") print("\t where <file> is the corresponding libvirt.xml") print( "\t2) virsh --connect qemu:///system undefine does not delete the image " "of your old initvm.") sys.exit(20) # Upgrade from older versions which used tmux try: system("tmux has-session -t ElbeInitVMSession 2>/dev/null") print( "ElbeInitVMSession exists in tmux. " "It may belong to an old elbe version. " "Please stop it to prevent interfering with this version.", file=sys.stderr) sys.exit(20) except CommandError: pass # Init cdrom to None, if we detect it, we set it cdrom = None if len(args) == 1: if args[0].endswith('.xml'): # We have an xml file, use that for elbe init xmlfile = args[0] try: xml = etree(xmlfile) except ValidationError as e: print("XML file is invalid: %s" % str(e)) # Use default XML if no initvm was specified if not xml.has("initvm"): xmlfile = os.path.join(elbepack.__path__[0], "init/default-init.xml") elif args[0].endswith('.iso'): # We have an iso image, extract xml from there. tmp = extract_cdrom(args[0]) xmlfile = tmp.fname('source.xml') cdrom = args[0] else: print("Unknown file ending (use either xml or iso)", file=sys.stderr) sys.exit(20) else: # No xml File was specified, build the default elbe-init-with-ssh xmlfile = os.path.join(elbepack.__path__[0], "init/default-init.xml") try: init_opts = '' if opt.devel: init_opts += ' --devel' if opt.nesting: init_opts += ' --nesting' if not opt.build_bin: init_opts += ' --skip-build-bin' if not opt.build_sources: init_opts += ' --skip-build-source' with PreprocessWrapper(xmlfile, opt) as ppw: if cdrom: system( '%s init %s --directory "%s" --cdrom "%s" "%s"' % (elbe_exe, init_opts, initvmdir, cdrom, ppw.preproc)) else: system('%s init %s --directory "%s" "%s"' % (elbe_exe, init_opts, initvmdir, ppw.preproc)) except CommandError: print("'elbe init' Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) # Read xml file for libvirt with open(os.path.join(initvmdir, 'libvirt.xml')) as f: xml = f.read() # Register initvm in libvirt try: self.conn.defineXML(xml) except CommandError: print('Registering initvm in libvirt failed', file=sys.stderr) print( 'Try `virsh --connect qemu:///system undefine %s` to delete existing initvm' % cfg['initvm_domain'], file=sys.stderr) sys.exit(20) # Build initvm try: system('cd "%s"; make' % (initvmdir)) except CommandError: print("Building the initvm Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) try: system('%s initvm start' % elbe_exe) except CommandError: print("Starting the initvm Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) if len(args) == 1: # if provided xml file has no initvm section xmlfile is set to a # default initvm XML file. But we need the original file here if args[0].endswith('.xml'): # stop here if no project node was specified try: x = etree(args[0]) except ValidationError as e: print("XML file is invalid: %s" % str(e)) sys.exit(20) if not x.has('project'): print("elbe initvm ready: use 'elbe initvm submit " "myproject.xml' to build a project") sys.exit(0) xmlfile = args[0] elif cdrom is not None: xmlfile = tmp.fname('source.xml') submit_and_dl_result(xmlfile, cdrom, opt)
def shutdown_initvm (self, uid): system ("shutdown -h now")
def shutdown_initvm(self, uid): # pylint: disable=unused-argument system("systemctl --no-block poweroff")
def copy_filelist(src, file_lst, dst): files = set() copied = set() # Make sure to copy parent directories # # For example, if file_lst = ['/usr/bin/bash'], # we might get {'/usr/bin', '/usr', '/usr/bin/bash'} for f in file_lst: parts = f.rstrip('\n') while parts != os.sep: files.add(parts) parts, _ = os.path.split(parts) # Start from closest to root first files = list(files) files.sort() files.reverse() while files: f = files.pop() copied.add(f) if src.islink(f): tgt = src.readlink(f) if not src.lexists(tgt): dst.symlink(tgt, f, allow_exists=True) continue # If the target is not yet in the destination RFS, we need # to defer the copy of the symlink after the target is # resolved. Thus, we recusively call copy_filelist # # Not that this will result in an infinite loop for # circular symlinks if not dst.lexists(tgt): if not os.path.isabs(tgt): lst = [os.path.join(os.path.dirname(f), tgt)] else: lst = [tgt] copy_filelist(src, lst, dst) dst.symlink(tgt, f, allow_exists=True) elif src.isdir(f): if not dst.isdir(f): dst.mkdir(f) st = src.stat(f) dst.chown(f, st.st_uid, st.st_gid) else: try: system('cp -a --reflink=auto "%s" "%s"' % (src.realpath(f), dst.realpath(f))) except CommandError as E: logging.warning( "Error while copying from %s to %s of file %s - %s", src.path, dst.path, f, E) # update utime which will change after a file has been copied into # the directory for f in copied: if src.isdir(f) and not src.islink(f): shutil.copystat(src.fname(f), dst.fname(f))
def validate_example (xml_name): system ("%s validate %s" % (exe_path, xml_name))
def _umount(self, path): path = os.path.join(self.path, path) if os.path.ismount(path): system("umount %s" % path)
def copy_kinitrd(prj, target_dir, defs, arch="default"): try: sha1, uri = get_initrd_uri(prj, defs, arch) except KeyError: raise NoKinitrdException('no elbe-bootstrap package available') return except SystemError: raise NoKinitrdException('a configured mirror is not reachable') return except CommandError as e: raise NoKinitrdException("couldn't download elbe-bootstrap package") return try: tmpdir = mkdtemp() try: if uri.startswith("file://"): system('cp "%s" "%s"' % (uri[len("file://"):], os.path.join(tmpdir, "pkg.deb"))) elif uri.startswith("http://"): system('wget -O "%s" "%s"' % (os.path.join(tmpdir, "pkg.deb"), uri)) elif uri.startswith("ftp://"): system('wget -O "%s" "%s"' % (os.path.join(tmpdir, "pkg.deb"), uri)) else: raise NoKinitrdException('no elbe-bootstrap package available') except CommandError as e: raise NoKinitrdException( "couldn't download elbe-bootstrap package") return if len(sha1) > 0: m = hashlib.sha1() with open(os.path.join(tmpdir, "pkg.deb"), "rb") as f: buf = f.read(65536) while len(buf) > 0: m.update(buf) buf = f.read(65536) if m.hexdigest() != sha1: raise NoKinitrdException('elbe-bootstrap failed to verify !!!') else: print "-----------------------------------------------------" print "WARNING:" print "Using untrusted elbe-bootstrap" print "-----------------------------------------------------" try: system('dpkg -x "%s" "%s"' % (os.path.join(tmpdir, "pkg.deb"), tmpdir)) except CommandError: try: # dpkg did not work, try falling back to ar and tar system('ar p "%s" data.tar.gz | tar xz -C "%s"' % (os.path.join(tmpdir, "pkg.deb"), tmpdir)) except CommandError: system('ar p "%s" data.tar.xz | tar xJ -C "%s"' % (os.path.join(tmpdir, "pkg.deb"), tmpdir)) # copy is done twice, because paths in elbe-bootstarp_1.0 and 0.9 differ if prj.has("mirror/cdrom"): system('cp "%s" "%s"' % (os.path.join( tmpdir, 'var', 'lib', 'elbe', 'initrd', 'initrd-cdrom.gz'), os.path.join(target_dir, "initrd.gz"))) else: system('cp "%s" "%s"' % (os.path.join( tmpdir, 'var', 'lib', 'elbe', 'initrd', 'initrd.gz'), os.path.join(target_dir, "initrd.gz"))) system('cp "%s" "%s"' % (os.path.join(tmpdir, 'var', 'lib', 'elbe', 'initrd', 'vmlinuz'), os.path.join(target_dir, "vmlinuz"))) finally: system('rm -rf "%s"' % tmpdir)
def test_gen_sdk_scripts(): system("mkdir -p /tmp/test/sdk") gen_sdk_scripts('armhf-linux-gnueabihf', 'ARM', 'testproject', '08.15', '/tmp/test', '/tmp/test/sdk')
def execute(self, opt, _args): # pylint: disable=too-many-statements # pylint: disable=too-many-branches tmp = TmpdirFilesystem() if opt.xmlfile: ret, prjdir, err = command_out_stderr( '%s control create_project --retries 60 "%s"' % (elbe_exe, opt.xmlfile)) if ret != 0: print("elbe control create_project failed.", file=sys.stderr) print(err, file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) prjdir = prjdir.strip() try: system('%s control build_pbuilder "%s"' % (elbe_exe, prjdir)) except CommandError: print("elbe control build_pbuilder Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) try: system('%s control wait_busy "%s"' % (elbe_exe, prjdir)) except CommandError: print("elbe control wait_busy Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("") print("Building Pbuilder finished !") print("") elif opt.project: prjdir = opt.project system('%s control rm_log %s' % (elbe_exe, prjdir)) else: print( "you need to specify --project or --xmlfile option", file=sys.stderr) sys.exit(20) print("") print("Packing Source into tmp archive") print("") try: system('tar cfz "%s" .' % (tmp.fname("pdebuild.tar.gz"))) except CommandError: print("tar Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) for of in opt.origfile: print("") print("Pushing orig file '%s' into pbuilder" % of) print("") try: system( '%s control set_orig "%s" "%s"' % (elbe_exe, prjdir, of)) except CommandError: print("elbe control set_orig Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("") print("Pushing source into pbuilder") print("") try: system('%s control set_pdebuild --cpuset "%d" --profile "%s" ' '"%s" "%s"' % (elbe_exe, opt.cpuset, opt.profile, prjdir, tmp.fname("pdebuild.tar.gz"))) except CommandError: print("elbe control set_pdebuild Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) try: system('%s control wait_busy "%s"' % (elbe_exe, prjdir)) except CommandError: print("elbe control wait_busy Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("") print("Pdebuild finished !") print("") if opt.skip_download: print("") print("Listing available files:") print("") try: system( '%s control --pbuilder-only get_files "%s"' % (elbe_exe, prjdir)) except CommandError: print("elbe control get_files Failed", file=sys.stderr) print("", file=sys.stderr) print("dumping logfile", file=sys.stderr) try: system('%s control dump_file "%s" log.txt' % ( elbe_exe, prjdir)) except CommandError: print("elbe control dump_file Failed", file=sys.stderr) print("", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("") print( "Get Files with: 'elbe control get_file %s <filename>'" % prjdir) else: print("") print("Getting generated Files") print("") ensure_outdir(opt) try: system( '%s control --pbuilder-only get_files --output "%s" "%s"' % (elbe_exe, opt.outdir, prjdir)) except CommandError: print("elbe control get_files Failed", file=sys.stderr) print("", file=sys.stderr) print("dumping logfile", file=sys.stderr) try: system('%s control dump_file "%s" log.txt' % ( elbe_exe, prjdir)) except CommandError: print("elbe control dump_file Failed", file=sys.stderr) print("", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20)
def shutdown_initvm(self, uid): system("systemctl --no-block poweroff")
def execute(self, opt, _args): if opt.xmlfile: try: with PreprocessWrapper(opt.xmlfile, opt) as ppw: ret, prjdir, err = command_out_stderr( '%s control create_project' % (elbe_exe)) if ret != 0: print("elbe control create_project failed.", file=sys.stderr) print(err, file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) prjdir = prjdir.strip() ret, _, err = command_out_stderr( '%s control set_xml "%s" "%s"' % (elbe_exe, prjdir, ppw.preproc)) if ret != 0: print("elbe control set_xml failed.", file=sys.stderr) print(err, file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) except CommandError: # this is the failure from PreprocessWrapper # it already printed the error message from # elbe preprocess print("Giving up", file=sys.stderr) sys.exit(20) if opt.writeproject: wpf = open(opt.writeproject, "w") wpf.write(prjdir) wpf.close() elif opt.project: prjdir = opt.project else: print("you need to specify --project option", file=sys.stderr) sys.exit(20) print("Creating pbuilder") try: system('%s control build_pbuilder "%s"' % (elbe_exe, prjdir)) except CommandError: print("elbe control build_pbuilder Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) try: system('%s control wait_busy "%s"' % (elbe_exe, prjdir)) except CommandError: print("elbe control wait_busy Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("") print("Building Pbuilder finished !") print("")
def execute(self, opt, _args): crossopt = "" if opt.cross: crossopt = "--cross" if opt.noccache: ccacheopt = "--no-ccache" ccachesize = "" else: ccacheopt = "--ccache-size" ccachesize = opt.ccachesize if opt.xmlfile: try: with PreprocessWrapper(opt.xmlfile, opt) as ppw: ret, prjdir, err = command_out_stderr( '%s control create_project' % (elbe_exe)) if ret != 0: print("elbe control create_project failed.", file=sys.stderr) print(err, file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) prjdir = prjdir.strip() ret, _, err = command_out_stderr( '%s control set_xml "%s" "%s"' % (elbe_exe, prjdir, ppw.preproc)) if ret != 0: print("elbe control set_xml failed.", file=sys.stderr) print(err, file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) except CommandError: # this is the failure from PreprocessWrapper # it already printed the error message from # elbe preprocess print("Giving up", file=sys.stderr) sys.exit(20) if opt.writeproject: wpf = open(opt.writeproject, "w") wpf.write(prjdir) wpf.close() elif opt.project: prjdir = opt.project else: print("you need to specify --project option", file=sys.stderr) sys.exit(20) print("Creating pbuilder") try: system('%s control build_pbuilder "%s" %s %s %s' % (elbe_exe, prjdir, crossopt, ccacheopt, ccachesize)) except CommandError: print("elbe control build_pbuilder Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) try: system('%s control wait_busy "%s"' % (elbe_exe, prjdir)) except CommandError: print("elbe control wait_busy Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("") print("Building Pbuilder finished !") print("")
def run_command(argv): # pylint: disable=too-many-locals # pylint: disable=too-many-statements # pylint: disable=too-many-branches oparser = OptionParser(usage="usage: %prog init [options] <filename>") oparser.add_option("--skip-validation", action="store_true", dest="skip_validation", default=False, help="Skip xml schema validation") oparser.add_option("--directory", dest="directory", default="./build", help="Working directory (default is build)", metavar="FILE") oparser.add_option( "--cdrom", dest="cdrom", help="Use FILE as cdrom iso, and use that to build the initvm", metavar="FILE") oparser.add_option("--buildtype", dest="buildtype", help="Override the buildtype") oparser.add_option( "--debug", dest="debug", action="store_true", default=False, help="start qemu in graphical mode to enable console switch") oparser.add_option( "--devel", dest="devel", action="store_true", default=False, help="use devel mode, and install current builddir inside initvm") oparser.add_option( "--nesting", dest="nesting", action="store_true", default=False, help="allow initvm to support nested kvm. " "This makes /proc/cpuinfo inside initvm differ per host.") oparser.add_option( "--skip-build-bin", action="store_false", dest="build_bin", default=True, help="Skip building Binary Repository CDROM, for exact Reproduction") oparser.add_option("--skip-build-sources", action="store_false", dest="build_sources", default=True, help="Skip building Source CDROM") (opt, args) = oparser.parse_args(argv) if not args: print("no filename specified") oparser.print_help() sys.exit(20) elif len(args) > 1: print("too many filenames specified") oparser.print_help() sys.exit(20) with elbe_logging({"files": None}): if opt.devel: if not os.path.isdir(os.path.join(elbe_dir, "elbepack")): logging.error("Devel Mode only valid, " "when running from elbe checkout") sys.exit(20) if not opt.skip_validation: validation = validate_xml(args[0]) if validation: logging.error("xml validation failed. Bailing out") for i in validation: logging.error(i) sys.exit(20) xml = etree(args[0]) if not xml.has("initvm"): logging.error("fatal error: " "xml missing mandatory section 'initvm'") sys.exit(20) if opt.buildtype: buildtype = opt.buildtype elif xml.has("initvm/buildtype"): buildtype = xml.text("/initvm/buildtype") else: buildtype = "nodefaults" defs = ElbeDefaults(buildtype) http_proxy = xml.text("/initvm/mirror/primary_proxy", default="") http_proxy = http_proxy.strip().replace("LOCALMACHINE", "localhost") if opt.cdrom: mirror = xml.node("initvm/mirror") mirror.clear() cdrom = mirror.ensure_child("cdrom") cdrom.set_text(os.path.abspath(opt.cdrom)) # this is a workaround for # http://lists.linutronix.de/pipermail/elbe-devel/2017-July/000541.html _, virt = command_out('test -x /usr/bin/systemd-detect-virt && ' '/usr/bin/systemd-detect-virt') _, dist = command_out('lsb_release -cs') if 'vmware' in virt and 'stretch' in dist: machine_type = 'pc-i440fx-2.6' else: machine_type = 'pc' try: os.makedirs(opt.directory) except OSError as e: logging.error("unable to create project directory: %s (%s)", opt.directory, e.strerror) sys.exit(30) out_path = os.path.join(opt.directory, ".elbe-in") try: os.makedirs(out_path) except OSError as e: logging.error("unable to create subdirectory: %s (%s)", out_path, e.strerror) sys.exit(30) initvm_http_proxy = http_proxy.replace('http://localhost:', 'http://10.0.2.2:') d = { "elbe_version": elbe_version, "defs": defs, "opt": opt, "xml": xml, "prj": xml.node("/initvm"), "http_proxy": initvm_http_proxy, "pkgs": xml.node("/initvm/pkg-list") or [], "preseed": get_initvm_preseed(xml), "machine_type": machine_type, "cfg": cfg } if http_proxy != "": os.putenv("http_proxy", http_proxy) os.putenv("https_proxy", http_proxy) os.putenv("no_proxy", "localhost,127.0.0.1") try: copy_kinitrd(xml.node("/initvm"), out_path) except NoKinitrdException as e: msg = str(e) logging.error("Failure to download kernel/initrd debian Package:") logging.error("") logging.error(msg) logging.error("") logging.error("Check Mirror configuration") if 'SHA256SUMS' in msg: logging.error("If you use debmirror please read " "https://github.com/Linutronix/elbe/issues/188 " "on how to work around the issue") sys.exit(20) templates = os.listdir(init_template_dir) make_executable = ["init-elbe.sh.mako", "preseed.cfg.mako"] for t in templates: o = t.replace(".mako", "") if t in ("Makefile.mako", "libvirt.xml.mako"): write_template(os.path.join(opt.directory, o), os.path.join(init_template_dir, t), d, linebreak=True) else: write_template(os.path.join(out_path, o), os.path.join(init_template_dir, t), d, linebreak=False) if t in make_executable: os.chmod(os.path.join(out_path, o), 0o755) shutil.copyfile(args[0], os.path.join(out_path, "source.xml")) keys = [] for key in xml.all(".//initvm/mirror/url-list/url/raw-key"): keys.append(key.et.text) if opt.cdrom: cmd = '7z x -so "%s" repo.pub' % opt.cdrom keys.append(system_out(cmd)) import_keyring = os.path.join(out_path, "elbe-keyring") do('gpg --no-options \ --no-default-keyring \ --keyring %s --import' % import_keyring, stdin="".join(keys).encode('ascii'), allow_fail=True, env_add={'GNUPGHOME': out_path}) export_keyring = import_keyring + ".gpg" # No need to set GNUPGHOME because both input and output # keyring files are specified. do('gpg --no-options \ --no-default-keyring \ --keyring %s \ --export \ --output %s' % (import_keyring, export_keyring)) if opt.devel: out_real = os.path.realpath(out_path) opts = [] if out_real.startswith(elbe_dir + os.sep): opts.append('--exclude "%s"' % os.path.relpath(out_path, start=elbe_dir)) opts.append("--exclude-vcs") opts.append("--exclude-vcs-ignores") opts.append("--exclude='elbe-build*'") opts.append("--exclude='docs/*'") tar_fname = os.path.join(out_path, "elbe-devel.tar.bz2") system('tar cfj "%s" %s -C "%s" .' % (tar_fname, " ".join(opts), elbe_dir)) to_cpy = [("apt.conf", "etc/apt"), ("init-elbe.sh", ""), ("source.xml", ""), ("initrd-cdrom.gz", ""), ("vmlinuz", ""), ("preseed.cfg", "")] elbe_in = Filesystem(out_path) if opt.devel: to_cpy.append(("elbe-devel.tar.bz2", "")) # Convert relative rfs path to absolute in the system to_cpy = [(elbe_in.fname(src), elbe_in.fname(os.path.join("initrd-tree", dst))) for src, dst in to_cpy] # These are already absolute path! keyrings = elbe_in.fname( os.path.join("initrd-tree", "usr/share/keyrings")) for gpg in elbe_in.glob("*.gpg"): to_cpy.append((gpg, keyrings)) for src, dst in to_cpy: try: os.makedirs(dst) except FileExistsError: pass shutil.copy(src, dst)
def _undo_excursion(self, rfs): saved_to = self._saved_to() self._del_rfs_file(self.origin, rfs) if self.restore is True and rfs.lexists(saved_to): system('mv %s %s' % (rfs.fname(saved_to), rfs.fname(self.origin)))
def run_command(argv): # pylint: disable=too-many-locals # pylint: disable=too-many-statements # pylint: disable=too-many-branches oparser = OptionParser( usage="usage: %prog check_updates [options] <source-xmlfile>") oparser.add_option( "-s", "--script", dest="script", help="filename of script to run when an update is required") oparser.add_option("--skip-validation", action="store_true", dest="skip_validation", default=False, help="Skip xml schema validation") oparser.add_option("-c", "--changelogs", dest="changelogs", help="filename of changelog xml file") (opt, args) = oparser.parse_args(argv) if len(args) != 1: print("Wrong number of arguments") oparser.print_help() sys.exit(20) if not opt.skip_validation: validation = validate_xml(args[0]) if validation: print("xml validation failed. Bailing out") for i in validation: print(i) sys.exit(20) print("checking %s" % args[0]) xml = ElbeXML(args[0]) fullp = xml.node("fullpkgs") arch = xml.text("project/buildimage/arch", key="arch") v = virtapt.VirtApt(xml) for p in fullp: pname = p.et.text pauto = p.et.get('auto') if pauto != "true": v.mark_install(pname) errors = 0 required_updates = 0 update_packages = [] for p in fullp: xp = XMLPackage(p, arch) pname = p.et.text pauto = p.et.get('auto') if not v.has_pkg(xp.name): if not xp.is_auto_installed: print( "%s does not exist in cache but is specified in pkg-list" % xp.name) errors += 1 else: print("%s is no more required" % xp.name) required_updates += 1 continue if v.marked_install(xp.name): cver = v.get_candidate_ver(xp.name) if xp.installed_version != cver: print("%s: %s != %s" % (xp.name, xp.installed_version, cver)) required_updates += 1 if opt.changelogs: v.mark_pkg_download(xp.name) xp.candidate_version = cver update_packages.append(xp) sys.stdout.flush() sys.stderr.flush() if errors > 0: print("%d Errors occured, xml files needs fixing" % errors) if opt.script: system("%s ERRORS %s" % (opt.script, args[0]), allow_fail=True) elif required_updates > 0: print("%d updates required" % required_updates) if opt.changelogs: build_changelog_xml(v, opt, update_packages) if opt.script: system("%s UPDATE %s" % (opt.script, args[0]), allow_fail=True) else: print("No Updates available")
def execute(self, initvmdir, opt, args): try: have_session = os.system( "tmux has-session -t ElbeInitVMSession >/dev/null 2>&1") except CommandError as e: print( "tmux execution failed, tmux version 1.9 or higher is required" ) sys.exit(20) if have_session == 0: print("ElbeInitVMSession already exists in tmux.", file=sys.stderr) print("", file=sys.stderr) print( "There can only exist a single ElbeInitVMSession, and this session", file=sys.stderr) print("can also be used to make your build.", file=sys.stderr) print( "See 'elbe initvm submit', 'elbe initvm attach' and 'elbe control'", file=sys.stderr) sys.exit(20) # Init cdrom to None, if we detect it, we set it cdrom = None if len(args) == 1: if args[0].endswith('.xml'): # We have an xml file, use that for elbe init exampl = args[0] try: xml = etree(exampl) except ValidationError as e: print('XML file is inavlid: ' + str(e)) # Use default XML if no initvm was specified if not xml.has("initvm"): exampl = os.path.join(elbepack.__path__[0], "init/default-init.xml") elif args[0].endswith('.iso'): # We have an iso image, extract xml from there. tmp = TmpdirFilesystem() os.system('7z x -o%s "%s" source.xml' % (tmp.path, args[0])) if not tmp.isfile('source.xml'): print('Iso image does not contain a source.xml file', file=sys.stderr) print('This is not supported by "elbe initvm"', file=sys.stderr) print('', file=sys.stderr) print('Exiting !!!', file=sys.stderr) sys.exit(20) try: exml = ElbeXML(tmp.fname('source.xml'), skip_urlcheck=True) except ValidationError as e: print('Iso image does contain a source.xml file.', file=sys.stderr) print('But that xml does not validate correctly', file=sys.stderr) print('', file=sys.stderr) print('Exiting !!!', file=sys.stderr) sys.exit(20) print('Iso Image with valid source.xml detected !') print('Image was generated using Elbe Version %s' % exml.get_elbe_version()) exampl = tmp.fname('source.xml') cdrom = args[0] else: print('Unknown file ending (use either xml or iso)', file=sys.stderr) sys.exit(20) else: # No xml File was specified, build the default elbe-init-with-ssh exampl = os.path.join(elbepack.__path__[0], "init/default-init.xml") try: if opt.devel: devel = ' --devel' else: devel = '' if cdrom: system('%s init %s --directory "%s" --cdrom "%s" "%s"' % (elbe_exe, devel, initvmdir, cdrom, exampl)) else: system('%s init %s --directory "%s" "%s"' % (elbe_exe, devel, initvmdir, exampl)) except CommandError: print("'elbe init' Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) try: system('cd "%s"; make' % (initvmdir)) except CommandError: print("Building the initvm Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) try: system('%s initvm start --directory "%s"' % (elbe_exe, initvmdir)) except CommandError: print("Starting the initvm Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) if len(args) == 1: # if provided xml file has no initvm section exampl is set to a # default initvm XML file. But we need the original file here if args[0].endswith('.xml'): # stop here if no project node was specified try: x = ElbeXML(args[0]) except ValidationError as e: print('XML file is inavlid: ' + str(e)) sys.exit(20) if not x.has('project'): print( 'elbe initvm ready: use "elbe initvm submit myproject.xml" to build a project' ) sys.exit(0) ret, prjdir, err = command_out_stderr( '%s control create_project "%s"' % (elbe_exe, args[0])) else: ret, prjdir, err = command_out_stderr( '%s control create_project "%s"' % (elbe_exe, exampl)) if ret != 0: print("elbe control create_project failed.", file=sys.stderr) print(err, file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) prjdir = prjdir.strip() if cdrom is not None: print("Uploading CDROM. This might take a while") try: system('%s control set_cdrom "%s" "%s"' % (elbe_exe, prjdir, cdrom)) except CommandError: print("elbe control set_cdrom Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("Upload finished") build_opts = '' if opt.build_bin: build_opts += '--build-bin ' if opt.build_sources: build_opts += '--build-sources ' try: system('%s control build "%s" %s' % (elbe_exe, prjdir, build_opts)) except CommandError: print("elbe control build Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) try: system('%s control wait_busy "%s"' % (elbe_exe, prjdir)) except CommandError: print("elbe control wait_busy Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("") print("Build finished !") print("") try: system('%s control dump_file "%s" validation.txt' % (elbe_exe, prjdir)) except CommandError: print("Project failed to generate validation.txt", file=sys.stderr) print("Getting log.txt", file=sys.stderr) try: system('%s control dump_file "%s" log.txt' % (elbe_exe, prjdir)) except CommandError: print("Failed to dump log.txt", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) if opt.skip_download: print("") print("Listing available files:") print("") try: system('%s control get_files "%s"' % (elbe_exe, prjdir)) except CommandError: print("elbe control Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("") print('Get Files with: elbe control get_file "%s" <filename>' % prjdir) else: ensure_outdir(wdfs, opt) try: system('%s control get_files --output "%s" "%s"' % (elbe_exe, opt.outdir, prjdir)) except CommandError: print("elbe control get_files Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20)
def tuning(self, loopdev): if self.tune: system('tune2fs "%s" %s' % (self.tune, loopdev))
def submit_and_dl_result(xmlfile, cdrom, opt): # pylint: disable=too-many-statements # pylint: disable=too-many-branches try: with PreprocessWrapper(xmlfile, opt) as ppw: xmlfile = ppw.preproc ret, prjdir, err = command_out_stderr( '%s control create_project' % (elbe_exe)) if ret != 0: print("elbe control create_project failed.", file=sys.stderr) print(err, file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) prjdir = prjdir.strip() cmd = '%s control set_xml %s %s' % (elbe_exe, prjdir, xmlfile) ret, _, err = command_out_stderr(cmd) if ret != 0: print("elbe control set_xml failed2", file=sys.stderr) print(err, file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) except CommandError: # this is the failure from PreprocessWrapper # it already printed the error message from # elbe preprocess print("Giving up", file=sys.stderr) sys.exit(20) if opt.writeproject: with open(opt.writeproject, "w") as wpf: wpf.write(prjdir) if cdrom is not None: print("Uploading CDROM. This might take a while") try: system( '%s control set_cdrom "%s" "%s"' % (elbe_exe, prjdir, cdrom)) except CommandError: print("elbe control set_cdrom Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("Upload finished") build_opts = '' if opt.build_bin: build_opts += '--build-bin ' if opt.build_sources: build_opts += '--build-sources ' if cdrom: build_opts += '--skip-pbuilder ' try: system( '%s control build "%s" %s' % (elbe_exe, prjdir, build_opts)) except CommandError: print("elbe control build Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("Build started, waiting till it finishes") try: system('%s control wait_busy "%s"' % (elbe_exe, prjdir)) except CommandError: print("elbe control wait_busy Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("") print("Build finished !") print("") try: system( '%s control dump_file "%s" validation.txt' % (elbe_exe, prjdir)) except CommandError: print( "Project failed to generate validation.txt", file=sys.stderr) print("Getting log.txt", file=sys.stderr) try: system( '%s control dump_file "%s" log.txt' % (elbe_exe, prjdir)) except CommandError: print("Failed to dump log.txt", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) if opt.skip_download: print("") print("Listing available files:") print("") try: system('%s control get_files "%s"' % (elbe_exe, prjdir)) except CommandError: print("elbe control get_files Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("") print( 'Get Files with: elbe control get_file "%s" <filename>' % prjdir) else: print("") print("Getting generated Files") print("") ensure_outdir(opt) try: system('%s control get_files --output "%s" "%s"' % ( elbe_exe, opt.outdir, prjdir)) except CommandError: print("elbe control get_files Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) if not opt.keep_files: try: system('%s control del_project "%s"' % ( elbe_exe, prjdir)) except CommandError: print("remove project from initvm failed", file=sys.stderr) sys.exit(20)
def build(self, build_bin=False, build_sources=False, cdrom_size=None, skip_pkglist=False, skip_pbuild=False): # pylint: disable=too-many-arguments # pylint: disable=too-many-locals # pylint: disable=too-many-statements # pylint: disable=too-many-branches # Write the log header self.write_log_header() # Validate Apt Sources if build_sources: m = ValidationMode.CHECK_ALL else: m = ValidationMode.CHECK_BINARIES self.xml.validate_apt_sources(m, self.arch) if self.xml.has('target/pbuilder') and not skip_pbuild: if not os.path.exists(os.path.join(self.builddir, "pbuilder")): self.create_pbuilder(cross=False, noccache=False, ccachesize="10G") for p in self.xml.node('target/pbuilder'): self.pbuild(p) # the package might be needed by a following pbuild, so update # the project repo that it can be installed in as # build-dependency self.repo.finalize() # To avoid update cache errors, the project repo needs to have # Release and Packages files, even if it's empty. So don't do this # in the if case above! self.repo.finalize() # Create the build environment, if it does not a valid one # self.buildenv might be set when we come here. # However, if its not a full_buildenv, we specify clean here, # so it gets rebuilt properly. if not self.has_full_buildenv(): do('mkdir -p "%s"' % self.chrootpath) self.buildenv = BuildEnv(self.xml, self.chrootpath, build_sources=build_sources, clean=True) skip_pkglist = False # Import keyring self.buildenv.import_keys() logging.info("Keys imported") # Install packages if not skip_pkglist: self.install_packages(self.buildenv) try: self.buildenv.rfs.dump_elbeversion(self.xml) except IOError: logging.exception("Dump elbeversion failed") # Extract target FS. We always create a new instance here with # clean=true, because we want a pristine directory. self.targetfs = TargetFs(self.targetpath, self.buildenv.xml, clean=True) os.chdir(self.buildenv.rfs.fname('')) extract_target(self.buildenv.rfs, self.xml, self.targetfs, self.get_rpcaptcache()) # Package validation and package list if not skip_pkglist: pkgs = self.xml.xml.node("/target/pkg-list") if self.xml.has("fullpkgs"): check_full_pkgs(pkgs, self.xml.xml.node("/fullpkgs"), self.get_rpcaptcache()) else: check_full_pkgs(pkgs, None, self.get_rpcaptcache()) dump_fullpkgs(self.xml, self.buildenv.rfs, self.get_rpcaptcache()) self.xml.dump_elbe_version() self.targetfs.write_fstab(self.xml) # Dump ELBE version try: self.targetfs.dump_elbeversion(self.xml) except MemoryError: logging.exception("Dump elbeversion failed") # install packages for buildenv if not skip_pkglist: self.install_packages(self.buildenv, buildenv=True) # Write source.xml try: sourcexmlpath = os.path.join(self.builddir, "source.xml") self.xml.xml.write(sourcexmlpath) except MemoryError: logging.exception("Write source.xml failed (archive to huge?)") # Elbe report cache = self.get_rpcaptcache() tgt_pkgs = elbe_report(self.xml, self.buildenv, cache, self.targetfs) # chroot' licenses self.gen_licenses("chroot", self.buildenv, [p.name for p in cache.get_installed_pkgs()]) self.gen_licenses("target", self.buildenv, tgt_pkgs) # Use some handwaving to determine grub version grub_arch = "ia32" if self.arch == "i386" else self.arch grub_fw_type = [] grub_version = 0 if self.get_rpcaptcache().is_installed('grub-pc'): grub_version = 202 grub_fw_type.append("bios") if self.get_rpcaptcache().is_installed('grub-efi-%s-bin' % grub_arch): grub_version = 202 grub_tgt = "x86_64" if self.arch == "amd64" else self.arch grub_fw_type.extend(["efi", grub_tgt + "-efi"]) if (self.get_rpcaptcache().is_installed('shim-signed') and self.get_rpcaptcache().is_installed( 'grub-efi-%s-signed' % grub_arch)): grub_version = 202 grub_fw_type.append("shimfix") if self.get_rpcaptcache().is_installed('grub-legacy'): logging.warning("package grub-legacy is installed, " "this is obsolete.") grub_version = 97 grub_fw_type.append("bios") elif not grub_fw_type: logging.warning( "neither package grub-pc nor grub-efi-%s-bin " "are installed, skipping grub", grub_arch) self.targetfs.part_target(self.builddir, grub_version, grub_fw_type) self.build_cdroms(build_bin, build_sources, cdrom_size, tgt_pkg_lst=tgt_pkgs) if self.postbuild_file: logging.info("Postbuild script") cmd = ' "%s %s %s"' % (self.builddir, self.xml.text("project/version"), self.xml.text("project/name")) do(self.postbuild_file + cmd, allow_fail=True) do_prj_finetuning(self.xml, self.buildenv, self.targetfs, self.builddir) self.targetfs.pack_images(self.builddir) system('cat "%s"' % self.validationpath)
def submit_and_dl_result(xmlfile, cdrom, opt): # pylint: disable=too-many-statements # pylint: disable=too-many-branches try: with PreprocessWrapper(xmlfile, opt) as ppw: xmlfile = ppw.preproc ret, prjdir, err = command_out_stderr('%s control create_project' % (elbe_exe)) if ret != 0: print("elbe control create_project failed.", file=sys.stderr) print(err, file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) prjdir = prjdir.strip() cmd = '%s control set_xml %s %s' % (elbe_exe, prjdir, xmlfile) ret, _, err = command_out_stderr(cmd) if ret != 0: print("elbe control set_xml failed2", file=sys.stderr) print(err, file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) except CommandError: # this is the failure from PreprocessWrapper # it already printed the error message from # elbe preprocess print("Giving up", file=sys.stderr) sys.exit(20) if opt.writeproject: with open(opt.writeproject, "w") as wpf: wpf.write(prjdir) if cdrom is not None: print("Uploading CDROM. This might take a while") try: system('%s control set_cdrom "%s" "%s"' % (elbe_exe, prjdir, cdrom)) except CommandError: print("elbe control set_cdrom Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("Upload finished") build_opts = '' if opt.build_bin: build_opts += '--build-bin ' if opt.build_sources: build_opts += '--build-sources ' if cdrom: build_opts += '--skip-pbuilder ' try: system('%s control build "%s" %s' % (elbe_exe, prjdir, build_opts)) except CommandError: print("elbe control build Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("Build started, waiting till it finishes") try: system('%s control wait_busy "%s"' % (elbe_exe, prjdir)) except CommandError: print("elbe control wait_busy Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("") print("Build finished !") print("") if opt.build_sdk: try: system('%s control build_sdk "%s" %s' % (elbe_exe, prjdir, build_opts)) except CommandError: print("elbe control build_sdk Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("SDK Build started, waiting till it finishes") try: system('%s control wait_busy "%s"' % (elbe_exe, prjdir)) except CommandError: print("elbe control wait_busy Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("") print("SDK Build finished !") print("") try: system('%s control dump_file "%s" validation.txt' % (elbe_exe, prjdir)) except CommandError: print("Project failed to generate validation.txt", file=sys.stderr) print("Getting log.txt", file=sys.stderr) try: system('%s control dump_file "%s" log.txt' % (elbe_exe, prjdir)) except CommandError: print("Failed to dump log.txt", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) if opt.skip_download: print("") print("Listing available files:") print("") try: system('%s control get_files "%s"' % (elbe_exe, prjdir)) except CommandError: print("elbe control get_files Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("") print('Get Files with: elbe control get_file "%s" <filename>' % prjdir) else: print("") print("Getting generated Files") print("") ensure_outdir(opt) try: system('%s control get_files --output "%s" "%s"' % (elbe_exe, opt.outdir, prjdir)) except CommandError: print("elbe control get_files Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) if not opt.keep_files: try: system('%s control del_project "%s"' % (elbe_exe, prjdir)) except CommandError: print("remove project from initvm failed", file=sys.stderr) sys.exit(20)
def execute(self, initvmdir, opt, args): # pylint: disable=too-many-branches # pylint: disable=too-many-statements if self.initvm is not None: print("Initvm is already defined for the libvirt domain '%s'.\n" % cfg['initvm_domain']) print("If you want to build in your old initvm, " "use `elbe initvm submit <xml>`.") print("If you want to remove your old initvm from libvirt " "run `virsh --connect qemu:///system undefine %s`.\n" % cfg['initvm_domain']) print("You can specify another libvirt domain by setting the " "ELBE_INITVM_DOMAIN environment variable to an unused domain name.\n") print("Note:") print("\t1) You can reimport your old initvm via " "`virsh --connect qemu:///system define <file>`") print("\t where <file> is the corresponding libvirt.xml") print("\t2) virsh --connect qemu:///system undefine does not delete the image " "of your old initvm.") sys.exit(20) # Upgrade from older versions which used tmux try: system("tmux has-session -t ElbeInitVMSession 2>/dev/null") print ("ElbeInitVMSession exists in tmux. " "It may belong to an old elbe version. " "Please stop it to prevent interfering with this version.", file=sys.stderr) sys.exit(20) except CommandError: pass # Init cdrom to None, if we detect it, we set it cdrom = None if len(args) == 1: if args[0].endswith('.xml'): # We have an xml file, use that for elbe init xmlfile = args[0] try: xml = etree(xmlfile) except ValidationError as e: print("XML file is invalid: %s" % str(e)) # Use default XML if no initvm was specified if not xml.has("initvm"): xmlfile = os.path.join( elbepack.__path__[0], "init/default-init.xml") elif args[0].endswith('.iso'): # We have an iso image, extract xml from there. tmp = extract_cdrom(args[0]) xmlfile = tmp.fname('source.xml') cdrom = args[0] else: print( "Unknown file ending (use either xml or iso)", file=sys.stderr) sys.exit(20) else: # No xml File was specified, build the default elbe-init-with-ssh xmlfile = os.path.join( elbepack.__path__[0], "init/default-init.xml") try: init_opts = '' if opt.devel: init_opts += ' --devel' if opt.nesting: init_opts += ' --nesting' if not opt.build_bin: init_opts += ' --skip-build-bin' if not opt.build_sources: init_opts += ' --skip-build-source' if cdrom: system('%s init %s --directory "%s" --cdrom "%s" "%s"' % (elbe_exe, init_opts, initvmdir, cdrom, xmlfile)) else: system( '%s init %s --directory "%s" "%s"' % (elbe_exe, init_opts, initvmdir, xmlfile)) except CommandError: print("'elbe init' Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) # Read xml file for libvirt with open(os.path.join(initvmdir, 'libvirt.xml')) as f: xml = f.read() # Register initvm in libvirt try: self.conn.defineXML(xml) except CommandError: print('Registering initvm in libvirt failed', file=sys.stderr) print('Try `virsh --connect qemu:///system undefine %s` to delete existing initvm' % cfg['initvm_domain'], file=sys.stderr) sys.exit(20) # Build initvm try: system('cd "%s"; make' % (initvmdir)) except CommandError: print("Building the initvm Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) try: system('%s initvm start' % elbe_exe) except CommandError: print("Starting the initvm Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) if len(args) == 1: # if provided xml file has no initvm section xmlfile is set to a # default initvm XML file. But we need the original file here if args[0].endswith('.xml'): # stop here if no project node was specified try: x = etree(args[0]) except ValidationError as e: print("XML file is invalid: %s" % str(e)) sys.exit(20) if not x.has('project'): print("elbe initvm ready: use 'elbe initvm submit " "myproject.xml' to build a project") sys.exit(0) xmlfile = args[0] elif cdrom is not None: xmlfile = tmp.fname('source.xml') submit_and_dl_result(xmlfile, cdrom, opt)
def shutdown_initvm(self): # pylint: disable=no-self-use system("systemctl --no-block poweroff")
def run_command(argv): oparser = OptionParser( usage="usage: %prog chroot [options] <builddir> [cmd]") oparser.add_option("--skip-validation", action="store_true", dest="skip_validation", default=False, help="Skip xml schema validation") oparser.add_option("--target", action="store_true", dest="target", help="chroot into target instead of buildenv", default=False) oparser.add_option("--buildtype", dest="buildtype", help="Override the buildtype") (opt, args) = oparser.parse_args(argv) if len(args) < 1: print("wrong number of arguments") oparser.print_help() sys.exit(20) with elbe_logging({"streams": sys.stdout}): try: project = ElbeProject(args[0], override_buildtype=opt.buildtype, skip_validate=opt.skip_validation, url_validation=ValidationMode.NO_CHECK) except ValidationError: logging.exception("XML validation failed. Bailing out") sys.exit(20) os.environ["LANG"] = "C" os.environ["LANGUAGE"] = "C" os.environ["LC_ALL"] = "C" # TODO: howto set env in chroot? os.environ["PS1"] = project.xml.text('project/name') + r': \w\$' cmd = "/bin/bash" if len(args) > 1: cmd = "" cmd2 = args[1:] for c in cmd2: cmd += (c + " ") if opt.target: try: with project.targetfs: system("/usr/sbin/chroot %s %s" % (project.targetpath, cmd)) except CommandError as e: print(repr(e)) else: try: with project.buildenv: system("/usr/sbin/chroot %s %s" % (project.chrootpath, cmd)) except CommandError as e: print(repr(e))