def test_initvm():
try:
global exe_path
# Properly initialise directories plug
mod_dir = os.path.dirname (__file__)
main_dir, _ = os.path.split( mod_dir )
exe_path = os.path.join( main_dir, "elbe" )
init_directories (exe_path)
tmpdir = os.getenv( "ELBE_TEST_DIR" )
assert tmpdir is not None
dname = os.path.join(tmpdir, "initvm_test")
from elbepack.directories import examples_dir
xml_name = os.path.join (examples_dir, "elbe-init-with-ssh.xml")
system ('%s initvm create --devel --directory "%s" %s' % (exe_path, dname, xml_name))
except:
system ("%s control shutdown_initvm" % (exe_path), allow_fail=True)
sleep (10)
tmpdir = os.getenv( "ELBE_TEST_DIR" )
assert tmpdir is not None
dname = os.path.join(tmpdir, "initvm_test")
shutil.rmtree (dname)
raise
def execute(self, _initvmdir, _opt, _args):
if self.initvm_state() != libvirt.VIR_DOMAIN_RUNNING:
print('Error: Initvm not running properly.')
sys.exit(20)
print('Attaching to initvm console.')
system('virsh --connect qemu:///system console %s' % cfg['initvm_domain'])
def copy_kinitrd(prj, target_dir):
suite = prj.text("suite")
try:
tmp = TmpdirFilesystem()
if prj.has("mirror/cdrom"):
system('7z x -o%s "%s" initrd-cdrom.gz vmlinuz' %
(tmp.fname('/'), prj.text("mirror/cdrom")))
# initrd.gz needs to be cdrom version !
copyfile(tmp.fname("initrd-cdrom.gz"),
os.path.join(target_dir, "initrd.gz"))
else:
mirror = get_primary_mirror(prj)
download_kinitrd(tmp, suite, mirror)
copyfile(tmp.fname("initrd.gz"),
os.path.join(target_dir, "initrd.gz"))
copyfile(tmp.fname("initrd-cdrom.gz"),
os.path.join(target_dir, "initrd-cdrom.gz"))
copyfile(tmp.fname("vmlinuz"),
os.path.join(target_dir, "vmlinuz"))
except IOError as e:
raise NoKinitrdException('IoError %s' % e.message)
except InvalidSignature as e:
raise NoKinitrdException('InvalidSignature %s' % e.message)
except HashValidationFailed as e:
raise NoKinitrdException('HashValidationFailed %s' % e.message)
def execute(self, opt, args):
tmp = TmpdirFilesystem ()
if not opt.project:
print ('you need to specify --project option', file=sys.stderr)
sys.exit(20)
prjdir = opt.project
print ("Creating pbuilder")
try:
system ('%s control build_pbuilder "%s"' % (elbe_exe, prjdir))
except CommandError:
print ("elbe control build_pbuilder Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
try:
system ('%s control wait_busy "%s"' % (elbe_exe, prjdir) )
except CommandError:
print ("elbe control wait_busy Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
print ("")
print ("Building Pbuilder finished !")
print ("")
def setup_apt_keyring(gpg_home, keyring_fname):
ring_path = os.path.join(gpg_home, keyring_fname)
if not os.path.isdir("/etc/apt/trusted.gpg.d"):
print("/etc/apt/trusted.gpg.d doesn't exist")
print("apt-get install debian-archive-keyring may "
"fix this problem")
sys.exit(20)
if os.path.exists("/etc/apt/trusted.gpg"):
system('cp /etc/apt/trusted.gpg "%s"' % ring_path)
gpg_options = '--keyring "%s" --no-auto-check-trustdb ' \
'--trust-model always --no-default-keyring ' \
'--batch ' \
'--homedir "%s"' % (ring_path, gpg_home)
trustkeys = os.listdir("/etc/apt/trusted.gpg.d")
for key in trustkeys:
print("Import %s: " % key)
try:
system('gpg %s --import "%s"' % (
gpg_options,
os.path.join("/etc/apt/trusted.gpg.d", key)))
except CommandError:
print('adding keyring "%s" to keyring "%s" failed' % (key,
ring_path))
def execute(self, _initvmdir, opt, args):
try:
system('%s initvm ensure' % elbe_exe)
except CommandError:
print("Starting the initvm Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
# Init cdrom to None, if we detect it, we set it
cdrom = None
if len(args) == 1:
if args[0].endswith('.xml'):
# We have an xml file, use that for elbe init
xmlfile = args[0]
url_validation = ''
elif args[0].endswith('.iso'):
# We have an iso image, extract xml from there.
tmp = extract_cdrom(args[0])
xmlfile = tmp.fname('source.xml')
url_validation = '--skip-urlcheck'
cdrom = args[0]
else:
print(
"Unknown file ending (use either xml or iso)",
file=sys.stderr)
sys.exit(20)
submit_and_dl_result(xmlfile, cdrom, opt)
def download_and_validate_file(self, upstream_fname, local_fname):
url = self.base_url + upstream_fname
try:
system('wget -O "%s" "%s"' % (local_fname, url))
except CommandError:
raise HashValidationFailed('Failed to download %s' % url)
self.validate_file(upstream_fname, local_fname)
def execute(self, _initvmdir, _opt, _args):
if self.initvm_state() == libvirt.VIR_DOMAIN_SHUTOFF:
system('%s initvm start' % elbe_exe)
elif self.initvm_state() == libvirt.VIR_DOMAIN_RUNNING:
pass
else:
print("Elbe initvm in bad state.")
sys.exit(20)
def teardown():
system ("%s control shutdown_initvm" % (exe_path), allow_fail=True)
sleep (10)
tmpdir = os.getenv( "ELBE_TEST_DIR" )
assert tmpdir is not None
dname = os.path.join(tmpdir, "initvm_test")
shutil.rmtree (dname)
def init_example (xml_name):
tmpdir = os.getenv( "ELBE_TEST_DIR" )
assert tmpdir is not None
dname = os.path.join(tmpdir, "init_test")
try:
system ('%s init --directory "%s" %s' % (exe_path, dname, xml_name))
finally:
shutil.rmtree (dname)
def unlock_key(fingerprint):
ctx = core.Context()
ctx.set_engine_info(PROTOCOL_OpenPGP,
None,
'/var/cache/elbe/gnupg')
key = ctx.get_key(fingerprint, secret=True)
keygrip = key.subkeys[0].keygrip
system("/usr/lib/gnupg2/gpg-preset-passphrase "
"--preset -P requiredToAvoidUserInput %s" % str(keygrip),
env_add={"GNUPGHOME": "/var/cache/elbe/gnupg"})
def download_pkg(prj,
target_dir,
defs,
package,
arch="default",
incl_deps=False,
log=None):
# pylint: disable=too-many-arguments
# pylint: disable=too-many-locals
# pylint: disable=too-many-branches
try:
urilist = get_uri(prj, defs, arch, package, incl_deps)
except KeyError:
raise NoPackageException('no package %s available' % package)
except SystemError:
raise NoPackageException('a configured mirror is not reachable')
except CommandError:
raise NoPackageException("couldn't download package %s" % package)
if not urilist:
raise NoPackageException("couldn't download package %s" % package)
for u in urilist:
sha256 = u[2]
uri = u[1]
dest = os.path.join(target_dir, "%s.deb" % u[0])
try:
if uri.startswith("file://"):
system('cp "%s" "%s"' % (uri[len("file://"):], dest))
elif uri.startswith("http://") or uri.startswith("ftp://"):
system('wget -O "%s" "%s"' % (dest, uri))
else:
raise NoPackageException('could not retreive %s' % uri)
except CommandError:
raise NoPackageException("couldn't download package %s" % package)
if sha256:
try:
validate_sha256(dest, sha256)
except HashValidationFailed as e:
raise NoPackageException('%s failed to verify: %s' % package,
e.message)
else:
if log:
log.printo("WARNING: Using untrusted %s package" % package)
else:
print("-----------------------------------------------------")
print("WARNING:")
print("Using untrusted %s package" % package)
print("-----------------------------------------------------")
return [y[0] for y in urilist]
def execute(self, initvmdir, opt, args):
try:
have_session = os.system( "tmux has-session -t ElbeInitVMSession >/dev/null 2>&1" )
except CommandError as e:
print ("tmux execution failed, tmux version 1.9 or higher is required")
sys.exit(20)
if have_session == 0:
if os.environ.has_key('TMUX'):
system( 'tmux link-window -s ElbeInitVMSession:initvm' )
else:
system( 'tmux attach -t ElbeInitVMSession' )
else:
print ("ElbeInitVMSession does not exist in tmux.", file=sys.stderr)
print ("Try 'elbe initvm start' to start the session.", file=sys.stderr)
sys.exit(20)
def execute(self, initvmdir, opt, args):
try:
have_session = os.system( "tmux has-session -t ElbeInitVMSession >/dev/null 2>&1" )
except CommandError as e:
print ("tmux execution failed, tmux version 1.9 or higher is required")
sys.exit(20)
if have_session != 256:
# other session exists... Good. exit
sys.exit(0)
# No other session exists, sanity check initvmdir
sanity_check_dir (initvmdir)
# Sanity check passed. start initvm session
system( 'TMUX= tmux new-session -d -c "%s" -s ElbeInitVMSession -n initvm "make run-con"' % initvmdir )
def setup_gpg(self):
ring_path = self.projectpath + "/etc/apt/trusted.gpg.d"
if not os.path.isdir("/etc/apt/trusted.gpg.d"):
print("/etc/apt/trusted.gpg.d doesn't exist")
print("apt-get install debian-archive-keyring may fix this problem")
sys.exit(20)
system('cp /etc/apt/trusted.gpg.d/* "%s"' % ring_path)
gpg_options = (
'--keyring "%s" --no-auto-check-trustdb --trust-model always --no-default-keyring --homedir "%s"'
% (os.path.join(ring_path, "linutronix-elbe.gpg"), self.projectpath)
)
system('gpg %s --import "%s"' % (gpg_options, elbe_pubkey_fname))
def execute(self, initvmdir, opt, args):
try:
have_session = os.system( "tmux has-session -t ElbeInitVMSession >/dev/null 2>&1" )
except CommandError as e:
print ("tmux execution failed, tmux version 1.9 or higher is required")
sys.exit(20)
if have_session != 256:
print ("ElbeInitVMSession already exists in tmux.", file=sys.stderr)
print ("Try 'elbe initvm attach' to attach to the session.", file=sys.stderr)
sys.exit(20)
# No other session exists, sanity check initvmdir
sanity_check_dir (initvmdir)
system( 'TMUX= tmux new-session -d -c "%s" -s ElbeInitVMSession -n initvm "make"' % initvmdir )
def execute(self, opt, _args):
if not opt.project:
print("you need to specify --project option", file=sys.stderr)
sys.exit(20)
prjdir = opt.project
print("Updating pbuilder")
try:
system('%s control update_pbuilder "%s"' % (elbe_exe, prjdir))
except CommandError:
print("elbe control update_pbuilder Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print("Updating Pbuilder finished !")
print("")
def setup():
try:
global exe_path
# Properly initialise directories plug
mod_dir = os.path.dirname (__file__)
main_dir, _ = os.path.split( mod_dir )
exe_path = os.path.join( main_dir, "elbe" )
init_directories (exe_path)
tmpdir = os.getenv( "ELBE_TEST_DIR" )
assert tmpdir is not None
dname = os.path.join(tmpdir, "initvm_test")
from elbepack.directories import examples_dir
xml_name = os.path.join (examples_dir, "elbe-init-with-ssh.xml")
system ('%s initvm create --devel --directory "%s"' % (exe_path, dname))
except:
teardown ()
raise
def execute(self, initvmdir, opt, args):
try:
have_session = os.system( "tmux has-session -t ElbeInitVMSession >/dev/null 2>&1" )
except CommandError as e:
print ("tmux execution failed, tmux version 1.9 or higher is required")
sys.exit(20)
if have_session != 256:
print ("ElbeInitVMSession already exists in tmux.", file=sys.stderr)
print ("Try 'elbe initvm attach' to attach to the session.", file=sys.stderr)
sys.exit(20)
# No other session exists, sanity check initvmdir
sanity_check_dir (initvmdir)
# Sanity check passed. start initvm session
try:
try:
os.remove (os.path.join (initvmdir, "run.log"))
except OSError:
pass
system( 'TMUX= tmux new-session -d -c "%s" -s ElbeInitVMSession -n initvm "make run-con 2> %s"' % (
initvmdir, str(os.path.join (initvmdir, "run.log")).replace (' ', '\ ')))
for i in range (1, 5):
sys.stdout.write ("*")
sys.stdout.flush ()
time.sleep (1)
print ("*")
have_session = os.system( "tmux has-session -t ElbeInitVMSession >/dev/null 2>&1" )
if have_session:
print ("Maybe starting elbe initvm failed:", file=sys.stderr)
with open (os.path.join (initvmdir, "run.log"), 'r') as f:
for l in f:
print (l, file=sys.stderr)
os.remove (os.path.join (initvmdir, "run.log"))
except CommandError as e:
print ("tmux execution failed, tmux version 1.9 or higher is required")
sys.exit(20)
def setup_gpg(self):
ring_path = os.path.join(self.projectpath, "etc/apt/trusted.gpg")
if not os.path.isdir("/etc/apt/trusted.gpg.d"):
print("/etc/apt/trusted.gpg.d doesn't exist")
print("apt-get install debian-archive-keyring may "
"fix this problem")
sys.exit(20)
if os.path.exists("/etc/apt/trusted.gpg"):
system('cp /etc/apt/trusted.gpg "%s"' % ring_path)
gpg_options = '--keyring "%s" --no-auto-check-trustdb ' \
'--trust-model always --no-default-keyring ' \
'--homedir "%s"' % (ring_path, self.projectpath)
trustkeys = os.listdir("/etc/apt/trusted.gpg.d")
for key in trustkeys:
print("Import %s: " % key)
try:
system('gpg %s --import "%s"' % (
gpg_options,
os.path.join("/etc/apt/trusted.gpg.d", key)))
except CommandError:
print("adding elbe-pubkey to keyring failed")
def test_submit ():
tmpdir = os.getenv( "ELBE_TEST_DIR" )
assert tmpdir is not None
dname = os.path.join(tmpdir, "initvm_output")
system ('mkdir "%s"' % dname)
system ('mkdir "%s02"' % dname)
try:
from elbepack.directories import examples_dir
xml_name = os.path.join (examples_dir, "rescue.xml")
system ('%s initvm submit --build-bin --output "%s" "%s"' % (exe_path, dname, xml_name))
# Now submit the iso image
system ('%s initvm submit --output "%s02" "%s"' % (exe_path, dname, os.path.join (dname, "bin-cdrom.iso")))
finally:
shutil.rmtree (dname)
shutil.rmtree (dname+"02")
def add_pubkey_url(self, url):
ring_path = self.projectpath + "/etc/apt/trusted.gpg.d"
tmpkey_path = self.projectpath + "/tmpkey.gpg"
gpg_options = (
'--keyring "%s" --no-auto-check-trustdb --trust-model always --no-default-keyring --homedir "%s"'
% (os.path.join(ring_path, "linutronix-elbe.gpg"), self.projectpath)
)
try:
system('wget -O "%s" "%s"' % (tmpkey_path, url))
system('gpg %s --import "%s"' % (gpg_options, tmpkey_path))
finally:
system('rm "%s"' % tmpkey_path, allow_fail=True)
def add_pubkey_url(self, url):
ring_path = os.path.join(self.projectpath, "etc/apt/trusted.gpg")
tmpkey_path = os.path.join(self.projectpath, "tmpkey.gpg")
gpg_options = '--keyring "%s" --no-auto-check-trustdb ' \
'--trust-model always --no-default-keyring ' \
'--homedir "%s"' % (ring_path, self.projectpath)
try:
system('wget -O "%s" "%s"' % (tmpkey_path, url))
system('gpg %s --import "%s"' % (
gpg_options,
tmpkey_path))
finally:
system('rm "%s"' % tmpkey_path, allow_fail=True)
def gen_update_pkg(project, xml_filename, upd_filename,
override_buildtype=None, skip_validate=False, debug=False,
cmd_dir=None, cfg_dir=None):
# pylint: disable=too-many-arguments
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
if xml_filename:
xml = ElbeXML(xml_filename, buildtype=override_buildtype,
skip_validate=skip_validate)
if not xml.has("fullpkgs"):
raise MissingData("Xml does not have fullpkgs list")
if not project.xml.has("fullpkgs"):
raise MissingData("Source Xml does not have fullpkgs list")
if not project.buildenv.rfs:
raise MissingData("Target does not have a build environment")
cache = project.get_rpcaptcache()
instpkgs = cache.get_installed_pkgs()
instindex = {}
for p in instpkgs:
instindex[p.name] = p
xmlpkgs = xml.node("/fullpkgs")
xmlindex = {}
fnamelist = []
for p in xmlpkgs:
name = p.et.text
ver = p.et.get('version')
md5 = p.et.get('md5')
xmlindex[name] = p
if name not in instindex:
print("package removed: %s" % name)
continue
ipkg = instindex[name]
comp = cache.compare_versions(ipkg.installed_version, ver)
pfname = ipkg.installed_deb
if comp == 0:
print("package ok: %s-%s" % (name, ipkg.installed_version))
if debug:
fnamelist.append(pfname)
continue
if comp > 0:
print("package upgrade: %s" % pfname)
fnamelist.append(pfname)
else:
print(
"package downgrade: %s-%s" %
(name, ipkg.installed_version))
for p in instpkgs:
if p.name in xmlindex:
continue
print("package %s newly installed" % p.name)
pfname = p.installed_deb
fnamelist.append(pfname)
update = os.path.join(project.builddir, "update")
if os.path.exists(update):
rmtree(update)
system('mkdir -p %s' % update)
if xml_filename:
repodir = os.path.join(update, "repo")
repo = UpdateRepo(xml, repodir, project.log)
for fname in fnamelist:
path = os.path.join(
project.chrootpath,
"var/cache/apt/archives",
fname)
repo.includedeb(path)
repo.finalize()
dump_fullpkgs(project.xml, project.buildenv.rfs, cache)
project.xml.xml.write(os.path.join(update, "new.xml"))
system("cp %s %s" % (xml_filename,
os.path.join(update, "base.xml")))
else:
system("cp source.xml update/new.xml")
if project.presh_file:
copyfile(project.presh_file, update + '/pre.sh')
os.chmod(update + '/pre.sh', 0o755)
if project.postsh_file:
copyfile(project.postsh_file, update + '/post.sh')
os.chmod(update + '/post.sh', 0o755)
if cmd_dir:
inlucdedir(update, 'cmd', cmd_dir, mode=0o755)
if cfg_dir:
inlucdedir(update, 'conf', cfg_dir)
create_zip_archive(upd_filename, update, ".")
if project.postbuild_file:
project.log.h2("postbuild script")
project.log.do(project.postbuild_file + ' "%s %s %s"' % (
upd_filename,
project.xml.text("project/version"),
project.xml.text("project/name")),
allow_fail=True)
def _umount(self, path):
if os.path.ismount(path):
system("umount %s" % path)
def run_command(argv):
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
oparser = OptionParser(usage="usage: %prog init [options] <filename>")
oparser.add_option("--skip-validation", action="store_true",
dest="skip_validation", default=False,
help="Skip xml schema validation")
oparser.add_option("--directory", dest="directory", default="./build",
help="Working directory (default is build)",
metavar="FILE")
oparser.add_option(
"--cdrom",
dest="cdrom",
help="Use FILE as cdrom iso, and use that to build the initvm",
metavar="FILE")
oparser.add_option("--proxy", dest="proxy",
help="Override the http Proxy")
oparser.add_option("--buildtype", dest="buildtype",
help="Override the buildtype")
oparser.add_option(
"--debug",
dest="debug",
action="store_true",
default=False,
help="start qemu in graphical mode to enable console switch")
oparser.add_option(
"--devel",
dest="devel",
action="store_true",
default=False,
help="use devel mode, and install current builddir inside initvm")
oparser.add_option(
"--nesting",
dest="nesting",
action="store_true",
default=False,
help="allow initvm to support nested kvm. "
"This makes /proc/cpuinfo inside initvm differ per host.")
oparser.add_option(
"--skip-build-bin",
action="store_false",
dest="build_bin",
default=True,
help="Skip building Binary Repository CDROM, for exact Reproduction")
oparser.add_option(
"--skip-build-sources",
action="store_false",
dest="build_sources",
default=True,
help="Skip building Source CDROM")
(opt, args) = oparser.parse_args(argv)
if not args:
print("no filename specified")
oparser.print_help()
sys.exit(20)
elif len(args) > 1:
print("too many filenames specified")
oparser.print_help()
sys.exit(20)
if opt.devel:
if not os.path.isdir(os.path.join(elbe_dir, "elbepack")):
print("Devel Mode only valid, when running from elbe checkout")
sys.exit(20)
if not opt.skip_validation:
validation = validate_xml(args[0])
if validation:
print("xml validation failed. Bailing out")
for i in validation:
print(i)
sys.exit(20)
xml = etree(args[0])
if not xml.has("initvm"):
print("fatal error: xml missing mandatory section 'initvm'")
sys.exit(20)
if opt.buildtype:
buildtype = opt.buildtype
elif xml.has("initvm/buildtype"):
buildtype = xml.text("/initvm/buildtype")
else:
buildtype = "nodefaults"
defs = ElbeDefaults(buildtype)
http_proxy = ""
if os.getenv("http_proxy"):
http_proxy = os.getenv("http_proxy")
elif opt.proxy:
http_proxy = opt.proxy
elif xml.has("initvm/mirror/primary_proxy"):
http_proxy = xml.text("initvm/mirror/primary_proxy")
http_proxy = http_proxy.strip().replace("LOCALMACHINE", "localhost")
if opt.cdrom:
mirror = xml.node("initvm/mirror")
mirror.clear()
cdrom = mirror.ensure_child("cdrom")
cdrom.set_text(os.path.abspath(opt.cdrom))
# this is a workaround for
# http://lists.linutronix.de/pipermail/elbe-devel/2017-July/000541.html
_, virt = command_out('test -x /usr/bin/systemd-detect-virt && /usr/bin/systemd-detect-virt');
_, dist = command_out('lsb_release -cs')
if 'vmware' in virt and 'stretch' in dist:
machine_type = 'pc-i440fx-2.6'
else:
machine_type = 'pc'
try:
os.makedirs(opt.directory)
except OSError as e:
print(
"unable to create project directory: %s (%s)" %
(opt.directory, e.strerror))
sys.exit(30)
out_path = os.path.join(opt.directory, ".elbe-in")
try:
os.makedirs(out_path)
except OSError as e:
print(
"unable to create subdirectory: %s (%s)" %
(out_path, e.strerror))
sys.exit(30)
d = {"elbe_version": elbe_version,
"defs": defs,
"opt": opt,
"xml": xml,
"prj": xml.node("/initvm"),
"http_proxy": http_proxy,
"pkgs": xml.node("/initvm/pkg-list") or [],
"preseed": get_initvm_preseed(xml),
"machine_type": machine_type,
"cfg": cfg}
if http_proxy != "":
os.putenv("http_proxy", http_proxy)
os.putenv("https_proxy", http_proxy)
os.putenv("no_proxy", "localhost,127.0.0.1")
try:
copy_kinitrd(xml.node("/initvm"), out_path)
except NoKinitrdException as e:
print("Failure to download kernel/initrd debian Package:")
print("")
print(e.message)
print("")
print("Check Mirror configuration")
if 'SHA256SUMS' in e.message:
print("If you use debmirror please read https://github.com/Linutronix/elbe/issues/188 "
"on how to work around the issue")
sys.exit(20)
templates = os.listdir(init_template_dir)
make_executable = ["init-elbe.sh.mako",
"preseed.cfg.mako"]
for t in templates:
o = t.replace(".mako", "")
if t == "Makefile.mako" or t == "libvirt.xml.mako":
write_template(
os.path.join(
opt.directory, o), os.path.join(
init_template_dir, t), d, linebreak=True)
else:
write_template(
os.path.join(
out_path, o), os.path.join(
init_template_dir, t), d, linebreak=False)
if t in make_executable:
os.chmod(os.path.join(out_path, o), 0o755)
shutil.copyfile(args[0],
os.path.join(out_path, "source.xml"))
if opt.cdrom:
system('7z x -o%s "%s" elbe-keyring.gpg' % (out_path, opt.cdrom))
else:
keys = []
for key in xml.all(".//initvm/mirror/url-list/url/raw-key"):
keys.append(key.et.text)
import_keyring = os.path.join(out_path, "elbe-keyring")
command_out('gpg --no-options --no-default-keyring --keyring %s --import' % import_keyring,
stdin="".join(keys))
export_keyring = import_keyring + ".gpg"
command_out('gpg --no-options --no-default-keyring --keyring %s --export --output %s' % (import_keyring,
export_keyring))
if opt.devel:
out_real = os.path.realpath(out_path)
ignore = ''
if out_real.startswith(elbe_dir + os.sep):
ignore = '--exclude "%s"' % os.path.relpath(out_path,
start=elbe_dir)
system('tar cfj "%s" %s -C "%s" .' % (os.path.join(out_path, "elbe-devel.tar.bz2"),
ignore,
elbe_dir))
def execute(self, opt, _args):
# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
crossopt = ""
if opt.cross:
crossopt = "--cross"
tmp = TmpdirFilesystem()
if opt.xmlfile:
ret, prjdir, err = command_out_stderr(
'%s control create_project --retries 60 "%s"' %
(elbe_exe, opt.xmlfile))
if ret != 0:
print("elbe control create_project failed.", file=sys.stderr)
print(err, file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
prjdir = prjdir.strip()
try:
system('%s control build_pbuilder "%s"' % (elbe_exe, prjdir))
except CommandError:
print("elbe control build_pbuilder Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
try:
system('%s control wait_busy "%s"' % (elbe_exe, prjdir))
except CommandError:
print("elbe control wait_busy Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print("Building Pbuilder finished !")
print("")
elif opt.project:
prjdir = opt.project
system('%s control rm_log %s' % (elbe_exe, prjdir))
else:
print("you need to specify --project or --xmlfile option",
file=sys.stderr)
sys.exit(20)
print("")
print("Packing Source into tmp archive")
print("")
try:
system('tar cfz "%s" .' % (tmp.fname("pdebuild.tar.gz")))
except CommandError:
print("tar Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
for of in opt.origfile:
print("")
print("Pushing orig file '%s' into pbuilder" % of)
print("")
try:
system('%s control set_orig "%s" "%s"' %
(elbe_exe, prjdir, of))
except CommandError:
print("elbe control set_orig Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print("Pushing source into pbuilder")
print("")
try:
system('%s control set_pdebuild --cpuset "%d" --profile "%s" %s '
'"%s" "%s"' % (elbe_exe, opt.cpuset, opt.profile, crossopt,
prjdir, tmp.fname("pdebuild.tar.gz")))
except CommandError:
print("elbe control set_pdebuild Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
try:
system('%s control wait_busy "%s"' % (elbe_exe, prjdir))
except CommandError:
print("elbe control wait_busy Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print("Pdebuild finished !")
print("")
if opt.skip_download:
print("")
print("Listing available files:")
print("")
try:
system('%s control --pbuilder-only get_files "%s"' %
(elbe_exe, prjdir))
except CommandError:
print("elbe control get_files Failed", file=sys.stderr)
print("", file=sys.stderr)
print("dumping logfile", file=sys.stderr)
try:
system('%s control dump_file "%s" log.txt' %
(elbe_exe, prjdir))
except CommandError:
print("elbe control dump_file Failed", file=sys.stderr)
print("", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print("Get Files with: 'elbe control get_file %s <filename>'" %
prjdir)
else:
print("")
print("Getting generated Files")
print("")
ensure_outdir(opt)
try:
system(
'%s control --pbuilder-only get_files --output "%s" "%s"' %
(elbe_exe, opt.outdir, prjdir))
except CommandError:
print("elbe control get_files Failed", file=sys.stderr)
print("", file=sys.stderr)
print("dumping logfile", file=sys.stderr)
try:
system('%s control dump_file "%s" log.txt' %
(elbe_exe, prjdir))
except CommandError:
print("elbe control dump_file Failed", file=sys.stderr)
print("", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
def _del_rfs_file(self, filename, rfs):
if rfs.lexists(filename):
flags = "-f"
if rfs.isdir(filename):
flags += "r"
system("rm %s %s" % (flags, rfs.fname(filename)))
def build(self,
build_bin=False,
build_sources=False,
cdrom_size=None,
skip_pkglist=False,
skip_pbuild=False):
# pylint: disable=too-many-arguments
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
# Write the log header
self.write_log_header()
# Validate Apt Sources
if build_sources:
m = ValidationMode.CHECK_ALL
else:
m = ValidationMode.CHECK_BINARIES
self.xml.validate_apt_sources(m, self.arch)
if self.xml.has('target/pbuilder') and not skip_pbuild:
if not os.path.exists(os.path.join(self.builddir, "pbuilder")):
self.create_pbuilder()
for p in self.xml.node('target/pbuilder'):
self.pbuild(p)
# the package might be needed by a following pbuild, so update
# the project repo that it can be installed in as
# build-dependency
self.repo.finalize()
# To avoid update cache errors, the project repo needs to have
# Release and Packages files, even if it's empty. So don't do this
# in the if case above!
self.repo.finalize()
# Create the build environment, if it does not a valid one
# self.buildenv might be set when we come here.
# However, if its not a full_buildenv, we specify clean here,
# so it gets rebuilt properly.
if not self.has_full_buildenv():
self.log.do('mkdir -p "%s"' % self.chrootpath)
self.buildenv = BuildEnv(self.xml,
self.log,
self.chrootpath,
build_sources=build_sources,
clean=True)
skip_pkglist = False
# Import keyring
self.buildenv.import_keys()
self.log.printo("Keys imported")
# Install packages
if not skip_pkglist:
self.install_packages(self.buildenv)
try:
self.buildenv.rfs.dump_elbeversion(self.xml)
except IOError:
self.log.printo("dump elbeversion failed")
# Extract target FS. We always create a new instance here with
# clean=true, because we want a pristine directory.
self.targetfs = TargetFs(self.targetpath,
self.log,
self.buildenv.xml,
clean=True)
os.chdir(self.buildenv.rfs.fname(''))
extract_target(self.buildenv.rfs, self.xml, self.targetfs, self.log,
self.get_rpcaptcache())
# The validation file is created using check_full_pkgs() and
# elbe_report(), both opening the file in append mode. So if an
# old validation file already exists, it must be deleted first.
if os.path.isfile(self.validationpath):
os.unlink(self.validationpath)
# Package validation and package list
if not skip_pkglist:
pkgs = self.xml.xml.node("/target/pkg-list")
if self.xml.has("fullpkgs"):
check_full_pkgs(pkgs, self.xml.xml.node("/fullpkgs"),
self.validationpath, self.get_rpcaptcache())
else:
check_full_pkgs(pkgs, None, self.validationpath,
self.get_rpcaptcache())
dump_fullpkgs(self.xml, self.buildenv.rfs, self.get_rpcaptcache())
self.xml.dump_elbe_version()
self.targetfs.write_fstab(self.xml)
# Dump ELBE version
try:
self.targetfs.dump_elbeversion(self.xml)
except MemoryError:
self.log.printo("dump elbeversion failed")
# install packages for buildenv
if not skip_pkglist:
self.install_packages(self.buildenv, buildenv=True)
# Write source.xml
try:
sourcexmlpath = os.path.join(self.builddir, "source.xml")
self.xml.xml.write(sourcexmlpath)
except MemoryError:
self.log.printo("write source.xml failed (archive to huge?)")
# Elbe report
reportpath = os.path.join(self.builddir, "elbe-report.txt")
elbe_report(self.xml, self.buildenv, self.get_rpcaptcache(),
reportpath, self.validationpath, self.targetfs)
# the current license code raises an exception that interrupts the hole
# build if a licence can't be converted to utf-8. Exception handling
# can be removed as soon as the licence code is more stable
lic_err = False
try:
f = io.open(os.path.join(self.builddir, "licence.txt"),
"w+",
encoding='utf-8')
self.buildenv.rfs.write_licenses(
f, self.log, os.path.join(self.builddir, "licence.xml"))
except Exception:
self.log.printo("error during generating licence.txt/xml")
self.log.printo(sys.exc_info()[0])
lic_err = True
finally:
f.close()
if lic_err:
os.remove(os.path.join(self.builddir, "licence.txt"))
os.remove(os.path.join(self.builddir, "licence.xml"))
# Use some handwaving to determine grub version
# jessie and wheezy grubs are 2.0 but differ in behaviour
#
# We might also want support for legacy grub
if (self.get_rpcaptcache().is_installed('grub-pc')
and self.get_rpcaptcache().is_installed('grub-efi-amd64-bin')):
grub_version = 202
grub_fw_type = "hybrid"
elif self.get_rpcaptcache().is_installed('grub-pc'):
if self.codename == "wheezy":
grub_version = 199
else:
grub_version = 202
grub_fw_type = "bios"
elif self.get_rpcaptcache().is_installed('grub-efi-amd64'):
grub_version = 202
grub_fw_type = "efi"
elif self.get_rpcaptcache().is_installed('grub-legacy'):
self.log.printo("package grub-legacy is installed, "
"this is obsolete, skipping grub")
grub_version = 0
grub_fw_type = ""
else:
self.log.printo("package grub-pc is not installed, skipping grub")
# version 0 == skip_grub
grub_version = 0
grub_fw_type = ""
self.targetfs.part_target(self.builddir, grub_version, grub_fw_type)
self.build_cdroms(build_bin, build_sources, cdrom_size)
if self.postbuild_file:
self.log.h2("postbuild script:")
self.log.do(self.postbuild_file + ' "%s %s %s"' %
(self.builddir, self.xml.text("project/version"),
self.xml.text("project/name")),
allow_fail=True)
do_prj_finetuning(self.xml, self.log, self.buildenv, self.targetfs,
self.builddir)
self.targetfs.pack_images(self.builddir)
system('cat "%s"' % self.validationpath)
def execute(self, opt, args):
tmp = TmpdirFilesystem ()
if opt.xmlfile:
ret, prjdir, err = command_out_stderr ('%s control create_project --retries 60 "%s"' % (elbe_exe, opt.xmlfile))
if ret != 0:
print ("elbe control create_project failed.", file=sys.stderr)
print (err, file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
prjdir = prjdir.strip()
try:
system ('%s control build "%s"' % (elbe_exe, prjdir) )
except CommandError:
print ("elbe control build Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
print ("Build started, waiting till it finishes")
try:
system ('%s control wait_busy "%s"' % (elbe_exe, prjdir) )
except CommandError:
print ("elbe control wait_busy Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
print ("")
print ("Build finished !")
print ("")
print ("Creating pbuilder")
try:
system ('%s control build_pbuilder "%s"' % (elbe_exe, prjdir))
except CommandError:
print ("elbe control build_pbuilder Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
try:
system ('%s control wait_busy "%s"' % (elbe_exe, prjdir) )
except CommandError:
print ("elbe control wait_busy Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
print ("")
print ("Building Pbuilder finished !")
print ("")
elif opt.project:
prjdir = opt.project
else:
print ('you need to specify --project or --xmlfile option', file=sys.stderr)
sys.exit(20)
print ("")
print ("Packing Source into tmp archive")
print ("")
try:
system ('tar cvfz "%s" .' % (tmp.fname ("pdebuild.tar.gz")))
except CommandError:
print ("tar Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
print ("")
print ("Pushing source into pbuilder")
print ("")
try:
system ('%s control set_pdebuild "%s" "%s"' % (elbe_exe, prjdir, tmp.fname ("pdebuild.tar.gz")))
except CommandError:
print ("elbe control set_pdebuild Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
try:
system ('%s control wait_busy "%s"' % (elbe_exe, prjdir) )
except CommandError:
print ("elbe control wait_busy Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
print ("")
print ("Pdebuild finished !")
print ("")
if opt.skip_download:
print ("")
print ("Listing available files:")
print ("")
try:
system ('%s control --pbuilder-only get_files "%s"' % (elbe_exe, prjdir) )
except CommandError:
print ("elbe control get_files Failed", file=sys.stderr)
print ("", file=sys.stderr)
print ("dumping logfile", file=sys.stderr)
try:
system ('%s control dump_file "%s" log.txt' % (
elbe_exe, prjdir ))
except CommandError:
print ("elbe control dump_file Failed", file=sys.stderr)
print ("", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
print ("")
print ('Get Files with: elbe control get_file "%s" <filename>' % prjdir)
else:
print ("")
print ("Getting generated Files")
print ("")
ensure_outdir (wdfs, opt)
try:
system ('%s control --pbuilder-only get_files --output "%s" "%s"' % (
elbe_exe, opt.outdir, prjdir ))
except CommandError:
print ("elbe control get_files Failed", file=sys.stderr)
print ("", file=sys.stderr)
print ("dumping logfile", file=sys.stderr)
try:
system ('%s control dump_file "%s" log.txt' % (
elbe_exe, prjdir ))
except CommandError:
print ("elbe control dump_file Failed", file=sys.stderr)
print ("", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
def copy_filelist(src, file_lst, dst):
files = set()
copied = set()
# Make sure to copy parent directories
#
# For example, if file_lst = ['/usr/bin/bash'],
# we might get {'/usr/bin', '/usr', '/usr/bin/bash'}
for f in file_lst:
parts = f.rstrip('\n')
while parts != os.sep:
files.add(parts)
parts, _ = os.path.split(parts)
# Start from closest to root first
files = list(files)
files.sort()
files.reverse()
while files:
f = files.pop()
copied.add(f)
if src.islink(f):
tgt = src.readlink(f)
# If the target of the symlink is relative, we need the
# absolute path of it
if not os.path.isabs(tgt):
tgt = os.path.join(os.path.dirname(f), tgt)
# If the target is not yet in the destination RFS, we need
# to defer the copy of the symlink after the target is
# resolved. Thus, we put the symlink back on the stack
# and we add the target to resolve on top of it.
#
# Not that this will result in an infinite loop for
# circular symlinks
if not dst.lexists(tgt):
files.append(f)
files.append(tgt)
else:
dst.symlink(tgt, f)
elif src.isdir(f):
if not dst.isdir(f):
dst.mkdir(f)
st = src.stat(f)
dst.chown(f, st.st_uid, st.st_gid)
else:
system('cp -a --reflink=auto "%s" "%s"' %
(src.realpath(f), dst.realpath(f)))
# update utime which will change after a file has been copied into
# the directory
for f in copied:
if src.isdir(f) and not src.islink(f):
shutil.copystat(src.fname(f), dst.fname(f))
def execute(self, initvmdir, opt, args):
try:
have_session = os.system(
"tmux has-session -t ElbeInitVMSession >/dev/null 2>&1")
except CommandError as e:
print(
"tmux execution failed, tmux version 1.9 or higher is required"
)
sys.exit(20)
if have_session == 256:
print("ElbeInitVMSession does not exist in tmux.", file=sys.stderr)
print("Try 'elbe initvm start' to start the session.",
file=sys.stderr)
sys.exit(20)
try:
system('%s initvm ensure --directory "%s"' % (elbe_exe, initvmdir))
except CommandError:
print("Starting the initvm Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
# Init cdrom to None, if we detect it, we set it
cdrom = None
if len(args) == 1:
if args[0].endswith('.xml'):
# We have an xml file, use that for elbe init
xmlfile = args[0]
elif args[0].endswith('.iso'):
# We have an iso image, extract xml from there.
tmp = TmpdirFilesystem()
os.system('7z x -o%s "%s" source.xml' % (tmp.path, args[0]))
print('', file=sys.stderr)
if not tmp.isfile('source.xml'):
print('Iso image does not contain a source.xml file',
file=sys.stderr)
print('This is not supported by "elbe initvm"',
file=sys.stderr)
print('', file=sys.stderr)
print('Exiting !!!', file=sys.stderr)
sys.exit(20)
try:
exml = ElbeXML(tmp.fname('source.xml'), skip_urlcheck=True)
except ValidationError as e:
print('Iso image does contain a source.xml file.',
file=sys.stderr)
print('But that xml does not validate correctly',
file=sys.stderr)
print('', file=sys.stderr)
print('Exiting !!!', file=sys.stderr)
sys.exit(20)
print('Iso Image with valid source.xml detected !')
print('Image was generated using Elbe Version %s' %
exml.get_elbe_version())
xmlfile = tmp.fname('source.xml')
cdrom = args[0]
else:
print('Unknown file ending (use either xml or iso)',
file=sys.stderr)
sys.exit(20)
ret, prjdir, err = command_out_stderr(
'%s control create_project --retries 60 "%s"' %
(elbe_exe, xmlfile))
if ret != 0:
print("elbe control create_project failed.", file=sys.stderr)
print(err, file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
prjdir = prjdir.strip()
if cdrom is not None:
print("Uploading CDROM. This might take a while")
try:
system('%s control set_cdrom "%s" "%s"' %
(elbe_exe, prjdir, cdrom))
except CommandError:
print("elbe control set_cdrom Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("Upload finished")
build_opts = ''
if opt.build_bin:
build_opts += '--build-bin '
if opt.build_sources:
build_opts += '--build-sources '
try:
system('%s control build "%s" %s' %
(elbe_exe, prjdir, build_opts))
except CommandError:
print("elbe control build Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("Build started, waiting till it finishes")
try:
system('%s control wait_busy "%s"' % (elbe_exe, prjdir))
except CommandError:
print("elbe control wait_busy Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print("Build finished !")
print("")
try:
system('%s control dump_file "%s" validation.txt' %
(elbe_exe, prjdir))
except CommandError:
print("Project failed to generate validation.txt",
file=sys.stderr)
print("Getting log.txt", file=sys.stderr)
try:
system('%s control dump_file "%s" log.txt' %
(elbe_exe, prjdir))
except CommandError:
print("Failed to dump log.txt", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
if opt.skip_download:
print("")
print("Listing available files:")
print("")
try:
system('%s control get_files "%s"' % (elbe_exe, prjdir))
except CommandError:
print("elbe control get_files Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print('Get Files with: elbe control get_file "%s" <filename>' %
prjdir)
else:
print("")
print("Getting generated Files")
print("")
ensure_outdir(wdfs, opt)
try:
system('%s control get_files --output "%s" "%s"' %
(elbe_exe, opt.outdir, prjdir))
except CommandError:
print("elbe control get_files Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
if not opt.keep_files:
try:
system('%s control del_project "%s"' %
(elbe_exe, prjdir))
except CommandError:
print("remove project from initvm failed",
file=sys.stderr)
sys.exit(20)
def download(url, local_fname):
try:
system('wget -O "%s" "%s"' % (local_fname, url))
except CommandError:
raise NoKinitrdException('Failed to download %s' % url)
def export_key(fingerprint, outfile):
system("/usr/bin/gpg -a -o %s --export %s" % (outfile, fingerprint),
env_add={"GNUPGHOME": "/var/cache/elbe/gnupg"})
def execute(self, initvmdir, opt, args):
# pylint: disable=too-many-branches
# pylint: disable=too-many-statements
if self.initvm is not None:
print("Initvm is already defined for the libvirt domain '%s'.\n" %
cfg['initvm_domain'])
print("If you want to build in your old initvm, "
"use `elbe initvm submit <xml>`.")
print("If you want to remove your old initvm from libvirt "
"run `virsh --connect qemu:///system undefine %s`.\n" %
cfg['initvm_domain'])
print(
"You can specify another libvirt domain by setting the "
"ELBE_INITVM_DOMAIN environment variable to an unused domain name.\n"
)
print("Note:")
print("\t1) You can reimport your old initvm via "
"`virsh --connect qemu:///system define <file>`")
print("\t where <file> is the corresponding libvirt.xml")
print(
"\t2) virsh --connect qemu:///system undefine does not delete the image "
"of your old initvm.")
sys.exit(20)
# Upgrade from older versions which used tmux
try:
system("tmux has-session -t ElbeInitVMSession 2>/dev/null")
print(
"ElbeInitVMSession exists in tmux. "
"It may belong to an old elbe version. "
"Please stop it to prevent interfering with this version.",
file=sys.stderr)
sys.exit(20)
except CommandError:
pass
# Init cdrom to None, if we detect it, we set it
cdrom = None
if len(args) == 1:
if args[0].endswith('.xml'):
# We have an xml file, use that for elbe init
xmlfile = args[0]
try:
xml = etree(xmlfile)
except ValidationError as e:
print("XML file is invalid: %s" % str(e))
# Use default XML if no initvm was specified
if not xml.has("initvm"):
xmlfile = os.path.join(elbepack.__path__[0],
"init/default-init.xml")
elif args[0].endswith('.iso'):
# We have an iso image, extract xml from there.
tmp = extract_cdrom(args[0])
xmlfile = tmp.fname('source.xml')
cdrom = args[0]
else:
print("Unknown file ending (use either xml or iso)",
file=sys.stderr)
sys.exit(20)
else:
# No xml File was specified, build the default elbe-init-with-ssh
xmlfile = os.path.join(elbepack.__path__[0],
"init/default-init.xml")
try:
init_opts = ''
if opt.devel:
init_opts += ' --devel'
if opt.nesting:
init_opts += ' --nesting'
if not opt.build_bin:
init_opts += ' --skip-build-bin'
if not opt.build_sources:
init_opts += ' --skip-build-source'
with PreprocessWrapper(xmlfile, opt) as ppw:
if cdrom:
system(
'%s init %s --directory "%s" --cdrom "%s" "%s"' %
(elbe_exe, init_opts, initvmdir, cdrom, ppw.preproc))
else:
system('%s init %s --directory "%s" "%s"' %
(elbe_exe, init_opts, initvmdir, ppw.preproc))
except CommandError:
print("'elbe init' Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
# Read xml file for libvirt
with open(os.path.join(initvmdir, 'libvirt.xml')) as f:
xml = f.read()
# Register initvm in libvirt
try:
self.conn.defineXML(xml)
except CommandError:
print('Registering initvm in libvirt failed', file=sys.stderr)
print(
'Try `virsh --connect qemu:///system undefine %s` to delete existing initvm'
% cfg['initvm_domain'],
file=sys.stderr)
sys.exit(20)
# Build initvm
try:
system('cd "%s"; make' % (initvmdir))
except CommandError:
print("Building the initvm Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
try:
system('%s initvm start' % elbe_exe)
except CommandError:
print("Starting the initvm Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
if len(args) == 1:
# if provided xml file has no initvm section xmlfile is set to a
# default initvm XML file. But we need the original file here
if args[0].endswith('.xml'):
# stop here if no project node was specified
try:
x = etree(args[0])
except ValidationError as e:
print("XML file is invalid: %s" % str(e))
sys.exit(20)
if not x.has('project'):
print("elbe initvm ready: use 'elbe initvm submit "
"myproject.xml' to build a project")
sys.exit(0)
xmlfile = args[0]
elif cdrom is not None:
xmlfile = tmp.fname('source.xml')
submit_and_dl_result(xmlfile, cdrom, opt)
def shutdown_initvm (self, uid):
system ("shutdown -h now")
def shutdown_initvm(self, uid):
# pylint: disable=unused-argument
system("systemctl --no-block poweroff")
def copy_filelist(src, file_lst, dst):
files = set()
copied = set()
# Make sure to copy parent directories
#
# For example, if file_lst = ['/usr/bin/bash'],
# we might get {'/usr/bin', '/usr', '/usr/bin/bash'}
for f in file_lst:
parts = f.rstrip('\n')
while parts != os.sep:
files.add(parts)
parts, _ = os.path.split(parts)
# Start from closest to root first
files = list(files)
files.sort()
files.reverse()
while files:
f = files.pop()
copied.add(f)
if src.islink(f):
tgt = src.readlink(f)
if not src.lexists(tgt):
dst.symlink(tgt, f, allow_exists=True)
continue
# If the target is not yet in the destination RFS, we need
# to defer the copy of the symlink after the target is
# resolved. Thus, we recusively call copy_filelist
#
# Not that this will result in an infinite loop for
# circular symlinks
if not dst.lexists(tgt):
if not os.path.isabs(tgt):
lst = [os.path.join(os.path.dirname(f), tgt)]
else:
lst = [tgt]
copy_filelist(src, lst, dst)
dst.symlink(tgt, f, allow_exists=True)
elif src.isdir(f):
if not dst.isdir(f):
dst.mkdir(f)
st = src.stat(f)
dst.chown(f, st.st_uid, st.st_gid)
else:
try:
system('cp -a --reflink=auto "%s" "%s"' %
(src.realpath(f), dst.realpath(f)))
except CommandError as E:
logging.warning(
"Error while copying from %s to %s of file %s - %s",
src.path, dst.path, f, E)
# update utime which will change after a file has been copied into
# the directory
for f in copied:
if src.isdir(f) and not src.islink(f):
shutil.copystat(src.fname(f), dst.fname(f))
def validate_example (xml_name):
system ("%s validate %s" % (exe_path, xml_name))
def _umount(self, path):
path = os.path.join(self.path, path)
if os.path.ismount(path):
system("umount %s" % path)
def copy_kinitrd(prj, target_dir, defs, arch="default"):
try:
sha1, uri = get_initrd_uri(prj, defs, arch)
except KeyError:
raise NoKinitrdException('no elbe-bootstrap package available')
return
except SystemError:
raise NoKinitrdException('a configured mirror is not reachable')
return
except CommandError as e:
raise NoKinitrdException("couldn't download elbe-bootstrap package")
return
try:
tmpdir = mkdtemp()
try:
if uri.startswith("file://"):
system('cp "%s" "%s"' %
(uri[len("file://"):], os.path.join(tmpdir, "pkg.deb")))
elif uri.startswith("http://"):
system('wget -O "%s" "%s"' %
(os.path.join(tmpdir, "pkg.deb"), uri))
elif uri.startswith("ftp://"):
system('wget -O "%s" "%s"' %
(os.path.join(tmpdir, "pkg.deb"), uri))
else:
raise NoKinitrdException('no elbe-bootstrap package available')
except CommandError as e:
raise NoKinitrdException(
"couldn't download elbe-bootstrap package")
return
if len(sha1) > 0:
m = hashlib.sha1()
with open(os.path.join(tmpdir, "pkg.deb"), "rb") as f:
buf = f.read(65536)
while len(buf) > 0:
m.update(buf)
buf = f.read(65536)
if m.hexdigest() != sha1:
raise NoKinitrdException('elbe-bootstrap failed to verify !!!')
else:
print "-----------------------------------------------------"
print "WARNING:"
print "Using untrusted elbe-bootstrap"
print "-----------------------------------------------------"
try:
system('dpkg -x "%s" "%s"' %
(os.path.join(tmpdir, "pkg.deb"), tmpdir))
except CommandError:
try:
# dpkg did not work, try falling back to ar and tar
system('ar p "%s" data.tar.gz | tar xz -C "%s"' %
(os.path.join(tmpdir, "pkg.deb"), tmpdir))
except CommandError:
system('ar p "%s" data.tar.xz | tar xJ -C "%s"' %
(os.path.join(tmpdir, "pkg.deb"), tmpdir))
# copy is done twice, because paths in elbe-bootstarp_1.0 and 0.9 differ
if prj.has("mirror/cdrom"):
system('cp "%s" "%s"' % (os.path.join(
tmpdir, 'var', 'lib', 'elbe', 'initrd',
'initrd-cdrom.gz'), os.path.join(target_dir, "initrd.gz")))
else:
system('cp "%s" "%s"' % (os.path.join(
tmpdir, 'var', 'lib', 'elbe', 'initrd',
'initrd.gz'), os.path.join(target_dir, "initrd.gz")))
system('cp "%s" "%s"' %
(os.path.join(tmpdir, 'var', 'lib', 'elbe', 'initrd',
'vmlinuz'), os.path.join(target_dir, "vmlinuz")))
finally:
system('rm -rf "%s"' % tmpdir)
def test_gen_sdk_scripts():
system("mkdir -p /tmp/test/sdk")
gen_sdk_scripts('armhf-linux-gnueabihf', 'ARM', 'testproject', '08.15',
'/tmp/test', '/tmp/test/sdk')
def execute(self, opt, _args):
# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
tmp = TmpdirFilesystem()
if opt.xmlfile:
ret, prjdir, err = command_out_stderr(
'%s control create_project --retries 60 "%s"' %
(elbe_exe, opt.xmlfile))
if ret != 0:
print("elbe control create_project failed.", file=sys.stderr)
print(err, file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
prjdir = prjdir.strip()
try:
system('%s control build_pbuilder "%s"' % (elbe_exe, prjdir))
except CommandError:
print("elbe control build_pbuilder Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
try:
system('%s control wait_busy "%s"' % (elbe_exe, prjdir))
except CommandError:
print("elbe control wait_busy Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print("Building Pbuilder finished !")
print("")
elif opt.project:
prjdir = opt.project
system('%s control rm_log %s' % (elbe_exe, prjdir))
else:
print(
"you need to specify --project or --xmlfile option",
file=sys.stderr)
sys.exit(20)
print("")
print("Packing Source into tmp archive")
print("")
try:
system('tar cfz "%s" .' % (tmp.fname("pdebuild.tar.gz")))
except CommandError:
print("tar Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
for of in opt.origfile:
print("")
print("Pushing orig file '%s' into pbuilder" % of)
print("")
try:
system(
'%s control set_orig "%s" "%s"' %
(elbe_exe, prjdir, of))
except CommandError:
print("elbe control set_orig Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print("Pushing source into pbuilder")
print("")
try:
system('%s control set_pdebuild --cpuset "%d" --profile "%s" '
'"%s" "%s"' %
(elbe_exe, opt.cpuset, opt.profile,
prjdir, tmp.fname("pdebuild.tar.gz")))
except CommandError:
print("elbe control set_pdebuild Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
try:
system('%s control wait_busy "%s"' % (elbe_exe, prjdir))
except CommandError:
print("elbe control wait_busy Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print("Pdebuild finished !")
print("")
if opt.skip_download:
print("")
print("Listing available files:")
print("")
try:
system(
'%s control --pbuilder-only get_files "%s"' %
(elbe_exe, prjdir))
except CommandError:
print("elbe control get_files Failed", file=sys.stderr)
print("", file=sys.stderr)
print("dumping logfile", file=sys.stderr)
try:
system('%s control dump_file "%s" log.txt' % (
elbe_exe, prjdir))
except CommandError:
print("elbe control dump_file Failed", file=sys.stderr)
print("", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print(
"Get Files with: 'elbe control get_file %s <filename>'" %
prjdir)
else:
print("")
print("Getting generated Files")
print("")
ensure_outdir(opt)
try:
system(
'%s control --pbuilder-only get_files --output "%s" "%s"' %
(elbe_exe, opt.outdir, prjdir))
except CommandError:
print("elbe control get_files Failed", file=sys.stderr)
print("", file=sys.stderr)
print("dumping logfile", file=sys.stderr)
try:
system('%s control dump_file "%s" log.txt' % (
elbe_exe, prjdir))
except CommandError:
print("elbe control dump_file Failed", file=sys.stderr)
print("", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
def shutdown_initvm(self, uid):
system("systemctl --no-block poweroff")
def execute(self, opt, _args):
if opt.xmlfile:
try:
with PreprocessWrapper(opt.xmlfile, opt) as ppw:
ret, prjdir, err = command_out_stderr(
'%s control create_project' % (elbe_exe))
if ret != 0:
print("elbe control create_project failed.",
file=sys.stderr)
print(err, file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
prjdir = prjdir.strip()
ret, _, err = command_out_stderr(
'%s control set_xml "%s" "%s"' %
(elbe_exe, prjdir, ppw.preproc))
if ret != 0:
print("elbe control set_xml failed.", file=sys.stderr)
print(err, file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
except CommandError:
# this is the failure from PreprocessWrapper
# it already printed the error message from
# elbe preprocess
print("Giving up", file=sys.stderr)
sys.exit(20)
if opt.writeproject:
wpf = open(opt.writeproject, "w")
wpf.write(prjdir)
wpf.close()
elif opt.project:
prjdir = opt.project
else:
print("you need to specify --project option", file=sys.stderr)
sys.exit(20)
print("Creating pbuilder")
try:
system('%s control build_pbuilder "%s"' % (elbe_exe, prjdir))
except CommandError:
print("elbe control build_pbuilder Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
try:
system('%s control wait_busy "%s"' % (elbe_exe, prjdir))
except CommandError:
print("elbe control wait_busy Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print("Building Pbuilder finished !")
print("")
def execute(self, opt, _args):
crossopt = ""
if opt.cross:
crossopt = "--cross"
if opt.noccache:
ccacheopt = "--no-ccache"
ccachesize = ""
else:
ccacheopt = "--ccache-size"
ccachesize = opt.ccachesize
if opt.xmlfile:
try:
with PreprocessWrapper(opt.xmlfile, opt) as ppw:
ret, prjdir, err = command_out_stderr(
'%s control create_project' % (elbe_exe))
if ret != 0:
print("elbe control create_project failed.",
file=sys.stderr)
print(err, file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
prjdir = prjdir.strip()
ret, _, err = command_out_stderr(
'%s control set_xml "%s" "%s"' %
(elbe_exe, prjdir, ppw.preproc))
if ret != 0:
print("elbe control set_xml failed.", file=sys.stderr)
print(err, file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
except CommandError:
# this is the failure from PreprocessWrapper
# it already printed the error message from
# elbe preprocess
print("Giving up", file=sys.stderr)
sys.exit(20)
if opt.writeproject:
wpf = open(opt.writeproject, "w")
wpf.write(prjdir)
wpf.close()
elif opt.project:
prjdir = opt.project
else:
print("you need to specify --project option", file=sys.stderr)
sys.exit(20)
print("Creating pbuilder")
try:
system('%s control build_pbuilder "%s" %s %s %s' %
(elbe_exe, prjdir, crossopt, ccacheopt, ccachesize))
except CommandError:
print("elbe control build_pbuilder Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
try:
system('%s control wait_busy "%s"' % (elbe_exe, prjdir))
except CommandError:
print("elbe control wait_busy Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print("Building Pbuilder finished !")
print("")
def run_command(argv):
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
oparser = OptionParser(usage="usage: %prog init [options] <filename>")
oparser.add_option("--skip-validation",
action="store_true",
dest="skip_validation",
default=False,
help="Skip xml schema validation")
oparser.add_option("--directory",
dest="directory",
default="./build",
help="Working directory (default is build)",
metavar="FILE")
oparser.add_option(
"--cdrom",
dest="cdrom",
help="Use FILE as cdrom iso, and use that to build the initvm",
metavar="FILE")
oparser.add_option("--buildtype",
dest="buildtype",
help="Override the buildtype")
oparser.add_option(
"--debug",
dest="debug",
action="store_true",
default=False,
help="start qemu in graphical mode to enable console switch")
oparser.add_option(
"--devel",
dest="devel",
action="store_true",
default=False,
help="use devel mode, and install current builddir inside initvm")
oparser.add_option(
"--nesting",
dest="nesting",
action="store_true",
default=False,
help="allow initvm to support nested kvm. "
"This makes /proc/cpuinfo inside initvm differ per host.")
oparser.add_option(
"--skip-build-bin",
action="store_false",
dest="build_bin",
default=True,
help="Skip building Binary Repository CDROM, for exact Reproduction")
oparser.add_option("--skip-build-sources",
action="store_false",
dest="build_sources",
default=True,
help="Skip building Source CDROM")
(opt, args) = oparser.parse_args(argv)
if not args:
print("no filename specified")
oparser.print_help()
sys.exit(20)
elif len(args) > 1:
print("too many filenames specified")
oparser.print_help()
sys.exit(20)
with elbe_logging({"files": None}):
if opt.devel:
if not os.path.isdir(os.path.join(elbe_dir, "elbepack")):
logging.error("Devel Mode only valid, "
"when running from elbe checkout")
sys.exit(20)
if not opt.skip_validation:
validation = validate_xml(args[0])
if validation:
logging.error("xml validation failed. Bailing out")
for i in validation:
logging.error(i)
sys.exit(20)
xml = etree(args[0])
if not xml.has("initvm"):
logging.error("fatal error: "
"xml missing mandatory section 'initvm'")
sys.exit(20)
if opt.buildtype:
buildtype = opt.buildtype
elif xml.has("initvm/buildtype"):
buildtype = xml.text("/initvm/buildtype")
else:
buildtype = "nodefaults"
defs = ElbeDefaults(buildtype)
http_proxy = xml.text("/initvm/mirror/primary_proxy", default="")
http_proxy = http_proxy.strip().replace("LOCALMACHINE", "localhost")
if opt.cdrom:
mirror = xml.node("initvm/mirror")
mirror.clear()
cdrom = mirror.ensure_child("cdrom")
cdrom.set_text(os.path.abspath(opt.cdrom))
# this is a workaround for
# http://lists.linutronix.de/pipermail/elbe-devel/2017-July/000541.html
_, virt = command_out('test -x /usr/bin/systemd-detect-virt && '
'/usr/bin/systemd-detect-virt')
_, dist = command_out('lsb_release -cs')
if 'vmware' in virt and 'stretch' in dist:
machine_type = 'pc-i440fx-2.6'
else:
machine_type = 'pc'
try:
os.makedirs(opt.directory)
except OSError as e:
logging.error("unable to create project directory: %s (%s)",
opt.directory, e.strerror)
sys.exit(30)
out_path = os.path.join(opt.directory, ".elbe-in")
try:
os.makedirs(out_path)
except OSError as e:
logging.error("unable to create subdirectory: %s (%s)", out_path,
e.strerror)
sys.exit(30)
initvm_http_proxy = http_proxy.replace('http://localhost:',
'http://10.0.2.2:')
d = {
"elbe_version": elbe_version,
"defs": defs,
"opt": opt,
"xml": xml,
"prj": xml.node("/initvm"),
"http_proxy": initvm_http_proxy,
"pkgs": xml.node("/initvm/pkg-list") or [],
"preseed": get_initvm_preseed(xml),
"machine_type": machine_type,
"cfg": cfg
}
if http_proxy != "":
os.putenv("http_proxy", http_proxy)
os.putenv("https_proxy", http_proxy)
os.putenv("no_proxy", "localhost,127.0.0.1")
try:
copy_kinitrd(xml.node("/initvm"), out_path)
except NoKinitrdException as e:
msg = str(e)
logging.error("Failure to download kernel/initrd debian Package:")
logging.error("")
logging.error(msg)
logging.error("")
logging.error("Check Mirror configuration")
if 'SHA256SUMS' in msg:
logging.error("If you use debmirror please read "
"https://github.com/Linutronix/elbe/issues/188 "
"on how to work around the issue")
sys.exit(20)
templates = os.listdir(init_template_dir)
make_executable = ["init-elbe.sh.mako", "preseed.cfg.mako"]
for t in templates:
o = t.replace(".mako", "")
if t in ("Makefile.mako", "libvirt.xml.mako"):
write_template(os.path.join(opt.directory, o),
os.path.join(init_template_dir, t),
d,
linebreak=True)
else:
write_template(os.path.join(out_path, o),
os.path.join(init_template_dir, t),
d,
linebreak=False)
if t in make_executable:
os.chmod(os.path.join(out_path, o), 0o755)
shutil.copyfile(args[0], os.path.join(out_path, "source.xml"))
keys = []
for key in xml.all(".//initvm/mirror/url-list/url/raw-key"):
keys.append(key.et.text)
if opt.cdrom:
cmd = '7z x -so "%s" repo.pub' % opt.cdrom
keys.append(system_out(cmd))
import_keyring = os.path.join(out_path, "elbe-keyring")
do('gpg --no-options \
--no-default-keyring \
--keyring %s --import' % import_keyring,
stdin="".join(keys).encode('ascii'),
allow_fail=True,
env_add={'GNUPGHOME': out_path})
export_keyring = import_keyring + ".gpg"
# No need to set GNUPGHOME because both input and output
# keyring files are specified.
do('gpg --no-options \
--no-default-keyring \
--keyring %s \
--export \
--output %s' % (import_keyring, export_keyring))
if opt.devel:
out_real = os.path.realpath(out_path)
opts = []
if out_real.startswith(elbe_dir + os.sep):
opts.append('--exclude "%s"' %
os.path.relpath(out_path, start=elbe_dir))
opts.append("--exclude-vcs")
opts.append("--exclude-vcs-ignores")
opts.append("--exclude='elbe-build*'")
opts.append("--exclude='docs/*'")
tar_fname = os.path.join(out_path, "elbe-devel.tar.bz2")
system('tar cfj "%s" %s -C "%s" .' %
(tar_fname, " ".join(opts), elbe_dir))
to_cpy = [("apt.conf", "etc/apt"), ("init-elbe.sh", ""),
("source.xml", ""), ("initrd-cdrom.gz", ""), ("vmlinuz", ""),
("preseed.cfg", "")]
elbe_in = Filesystem(out_path)
if opt.devel:
to_cpy.append(("elbe-devel.tar.bz2", ""))
# Convert relative rfs path to absolute in the system
to_cpy = [(elbe_in.fname(src),
elbe_in.fname(os.path.join("initrd-tree", dst)))
for src, dst in to_cpy]
# These are already absolute path!
keyrings = elbe_in.fname(
os.path.join("initrd-tree", "usr/share/keyrings"))
for gpg in elbe_in.glob("*.gpg"):
to_cpy.append((gpg, keyrings))
for src, dst in to_cpy:
try:
os.makedirs(dst)
except FileExistsError:
pass
shutil.copy(src, dst)
def _undo_excursion(self, rfs):
saved_to = self._saved_to()
self._del_rfs_file(self.origin, rfs)
if self.restore is True and rfs.lexists(saved_to):
system('mv %s %s' % (rfs.fname(saved_to), rfs.fname(self.origin)))
def run_command(argv):
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
oparser = OptionParser(
usage="usage: %prog check_updates [options] <source-xmlfile>")
oparser.add_option(
"-s",
"--script",
dest="script",
help="filename of script to run when an update is required")
oparser.add_option("--skip-validation",
action="store_true",
dest="skip_validation",
default=False,
help="Skip xml schema validation")
oparser.add_option("-c",
"--changelogs",
dest="changelogs",
help="filename of changelog xml file")
(opt, args) = oparser.parse_args(argv)
if len(args) != 1:
print("Wrong number of arguments")
oparser.print_help()
sys.exit(20)
if not opt.skip_validation:
validation = validate_xml(args[0])
if validation:
print("xml validation failed. Bailing out")
for i in validation:
print(i)
sys.exit(20)
print("checking %s" % args[0])
xml = ElbeXML(args[0])
fullp = xml.node("fullpkgs")
arch = xml.text("project/buildimage/arch", key="arch")
v = virtapt.VirtApt(xml)
for p in fullp:
pname = p.et.text
pauto = p.et.get('auto')
if pauto != "true":
v.mark_install(pname)
errors = 0
required_updates = 0
update_packages = []
for p in fullp:
xp = XMLPackage(p, arch)
pname = p.et.text
pauto = p.et.get('auto')
if not v.has_pkg(xp.name):
if not xp.is_auto_installed:
print(
"%s does not exist in cache but is specified in pkg-list" %
xp.name)
errors += 1
else:
print("%s is no more required" % xp.name)
required_updates += 1
continue
if v.marked_install(xp.name):
cver = v.get_candidate_ver(xp.name)
if xp.installed_version != cver:
print("%s: %s != %s" % (xp.name, xp.installed_version, cver))
required_updates += 1
if opt.changelogs:
v.mark_pkg_download(xp.name)
xp.candidate_version = cver
update_packages.append(xp)
sys.stdout.flush()
sys.stderr.flush()
if errors > 0:
print("%d Errors occured, xml files needs fixing" % errors)
if opt.script:
system("%s ERRORS %s" % (opt.script, args[0]), allow_fail=True)
elif required_updates > 0:
print("%d updates required" % required_updates)
if opt.changelogs:
build_changelog_xml(v, opt, update_packages)
if opt.script:
system("%s UPDATE %s" % (opt.script, args[0]), allow_fail=True)
else:
print("No Updates available")
def execute(self, initvmdir, opt, args):
try:
have_session = os.system(
"tmux has-session -t ElbeInitVMSession >/dev/null 2>&1")
except CommandError as e:
print(
"tmux execution failed, tmux version 1.9 or higher is required"
)
sys.exit(20)
if have_session == 0:
print("ElbeInitVMSession already exists in tmux.", file=sys.stderr)
print("", file=sys.stderr)
print(
"There can only exist a single ElbeInitVMSession, and this session",
file=sys.stderr)
print("can also be used to make your build.", file=sys.stderr)
print(
"See 'elbe initvm submit', 'elbe initvm attach' and 'elbe control'",
file=sys.stderr)
sys.exit(20)
# Init cdrom to None, if we detect it, we set it
cdrom = None
if len(args) == 1:
if args[0].endswith('.xml'):
# We have an xml file, use that for elbe init
exampl = args[0]
try:
xml = etree(exampl)
except ValidationError as e:
print('XML file is inavlid: ' + str(e))
# Use default XML if no initvm was specified
if not xml.has("initvm"):
exampl = os.path.join(elbepack.__path__[0],
"init/default-init.xml")
elif args[0].endswith('.iso'):
# We have an iso image, extract xml from there.
tmp = TmpdirFilesystem()
os.system('7z x -o%s "%s" source.xml' % (tmp.path, args[0]))
if not tmp.isfile('source.xml'):
print('Iso image does not contain a source.xml file',
file=sys.stderr)
print('This is not supported by "elbe initvm"',
file=sys.stderr)
print('', file=sys.stderr)
print('Exiting !!!', file=sys.stderr)
sys.exit(20)
try:
exml = ElbeXML(tmp.fname('source.xml'), skip_urlcheck=True)
except ValidationError as e:
print('Iso image does contain a source.xml file.',
file=sys.stderr)
print('But that xml does not validate correctly',
file=sys.stderr)
print('', file=sys.stderr)
print('Exiting !!!', file=sys.stderr)
sys.exit(20)
print('Iso Image with valid source.xml detected !')
print('Image was generated using Elbe Version %s' %
exml.get_elbe_version())
exampl = tmp.fname('source.xml')
cdrom = args[0]
else:
print('Unknown file ending (use either xml or iso)',
file=sys.stderr)
sys.exit(20)
else:
# No xml File was specified, build the default elbe-init-with-ssh
exampl = os.path.join(elbepack.__path__[0],
"init/default-init.xml")
try:
if opt.devel:
devel = ' --devel'
else:
devel = ''
if cdrom:
system('%s init %s --directory "%s" --cdrom "%s" "%s"' %
(elbe_exe, devel, initvmdir, cdrom, exampl))
else:
system('%s init %s --directory "%s" "%s"' %
(elbe_exe, devel, initvmdir, exampl))
except CommandError:
print("'elbe init' Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
try:
system('cd "%s"; make' % (initvmdir))
except CommandError:
print("Building the initvm Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
try:
system('%s initvm start --directory "%s"' % (elbe_exe, initvmdir))
except CommandError:
print("Starting the initvm Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
if len(args) == 1:
# if provided xml file has no initvm section exampl is set to a
# default initvm XML file. But we need the original file here
if args[0].endswith('.xml'):
# stop here if no project node was specified
try:
x = ElbeXML(args[0])
except ValidationError as e:
print('XML file is inavlid: ' + str(e))
sys.exit(20)
if not x.has('project'):
print(
'elbe initvm ready: use "elbe initvm submit myproject.xml" to build a project'
)
sys.exit(0)
ret, prjdir, err = command_out_stderr(
'%s control create_project "%s"' % (elbe_exe, args[0]))
else:
ret, prjdir, err = command_out_stderr(
'%s control create_project "%s"' % (elbe_exe, exampl))
if ret != 0:
print("elbe control create_project failed.", file=sys.stderr)
print(err, file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
prjdir = prjdir.strip()
if cdrom is not None:
print("Uploading CDROM. This might take a while")
try:
system('%s control set_cdrom "%s" "%s"' %
(elbe_exe, prjdir, cdrom))
except CommandError:
print("elbe control set_cdrom Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("Upload finished")
build_opts = ''
if opt.build_bin:
build_opts += '--build-bin '
if opt.build_sources:
build_opts += '--build-sources '
try:
system('%s control build "%s" %s' %
(elbe_exe, prjdir, build_opts))
except CommandError:
print("elbe control build Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
try:
system('%s control wait_busy "%s"' % (elbe_exe, prjdir))
except CommandError:
print("elbe control wait_busy Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print("Build finished !")
print("")
try:
system('%s control dump_file "%s" validation.txt' %
(elbe_exe, prjdir))
except CommandError:
print("Project failed to generate validation.txt",
file=sys.stderr)
print("Getting log.txt", file=sys.stderr)
try:
system('%s control dump_file "%s" log.txt' %
(elbe_exe, prjdir))
except CommandError:
print("Failed to dump log.txt", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
if opt.skip_download:
print("")
print("Listing available files:")
print("")
try:
system('%s control get_files "%s"' % (elbe_exe, prjdir))
except CommandError:
print("elbe control Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print('Get Files with: elbe control get_file "%s" <filename>' %
prjdir)
else:
ensure_outdir(wdfs, opt)
try:
system('%s control get_files --output "%s" "%s"' %
(elbe_exe, opt.outdir, prjdir))
except CommandError:
print("elbe control get_files Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
def tuning(self, loopdev):
if self.tune:
system('tune2fs "%s" %s' % (self.tune, loopdev))
def submit_and_dl_result(xmlfile, cdrom, opt):
# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
try:
with PreprocessWrapper(xmlfile, opt) as ppw:
xmlfile = ppw.preproc
ret, prjdir, err = command_out_stderr(
'%s control create_project' % (elbe_exe))
if ret != 0:
print("elbe control create_project failed.", file=sys.stderr)
print(err, file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
prjdir = prjdir.strip()
cmd = '%s control set_xml %s %s' % (elbe_exe, prjdir, xmlfile)
ret, _, err = command_out_stderr(cmd)
if ret != 0:
print("elbe control set_xml failed2", file=sys.stderr)
print(err, file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
except CommandError:
# this is the failure from PreprocessWrapper
# it already printed the error message from
# elbe preprocess
print("Giving up", file=sys.stderr)
sys.exit(20)
if opt.writeproject:
with open(opt.writeproject, "w") as wpf:
wpf.write(prjdir)
if cdrom is not None:
print("Uploading CDROM. This might take a while")
try:
system(
'%s control set_cdrom "%s" "%s"' %
(elbe_exe, prjdir, cdrom))
except CommandError:
print("elbe control set_cdrom Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("Upload finished")
build_opts = ''
if opt.build_bin:
build_opts += '--build-bin '
if opt.build_sources:
build_opts += '--build-sources '
if cdrom:
build_opts += '--skip-pbuilder '
try:
system(
'%s control build "%s" %s' %
(elbe_exe, prjdir, build_opts))
except CommandError:
print("elbe control build Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("Build started, waiting till it finishes")
try:
system('%s control wait_busy "%s"' % (elbe_exe, prjdir))
except CommandError:
print("elbe control wait_busy Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print("Build finished !")
print("")
try:
system(
'%s control dump_file "%s" validation.txt' %
(elbe_exe, prjdir))
except CommandError:
print(
"Project failed to generate validation.txt",
file=sys.stderr)
print("Getting log.txt", file=sys.stderr)
try:
system(
'%s control dump_file "%s" log.txt' %
(elbe_exe, prjdir))
except CommandError:
print("Failed to dump log.txt", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
if opt.skip_download:
print("")
print("Listing available files:")
print("")
try:
system('%s control get_files "%s"' % (elbe_exe, prjdir))
except CommandError:
print("elbe control get_files Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print(
'Get Files with: elbe control get_file "%s" <filename>' %
prjdir)
else:
print("")
print("Getting generated Files")
print("")
ensure_outdir(opt)
try:
system('%s control get_files --output "%s" "%s"' % (
elbe_exe, opt.outdir, prjdir))
except CommandError:
print("elbe control get_files Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
if not opt.keep_files:
try:
system('%s control del_project "%s"' % (
elbe_exe, prjdir))
except CommandError:
print("remove project from initvm failed",
file=sys.stderr)
sys.exit(20)
def build(self,
build_bin=False,
build_sources=False,
cdrom_size=None,
skip_pkglist=False,
skip_pbuild=False):
# pylint: disable=too-many-arguments
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
# Write the log header
self.write_log_header()
# Validate Apt Sources
if build_sources:
m = ValidationMode.CHECK_ALL
else:
m = ValidationMode.CHECK_BINARIES
self.xml.validate_apt_sources(m, self.arch)
if self.xml.has('target/pbuilder') and not skip_pbuild:
if not os.path.exists(os.path.join(self.builddir, "pbuilder")):
self.create_pbuilder(cross=False,
noccache=False,
ccachesize="10G")
for p in self.xml.node('target/pbuilder'):
self.pbuild(p)
# the package might be needed by a following pbuild, so update
# the project repo that it can be installed in as
# build-dependency
self.repo.finalize()
# To avoid update cache errors, the project repo needs to have
# Release and Packages files, even if it's empty. So don't do this
# in the if case above!
self.repo.finalize()
# Create the build environment, if it does not a valid one
# self.buildenv might be set when we come here.
# However, if its not a full_buildenv, we specify clean here,
# so it gets rebuilt properly.
if not self.has_full_buildenv():
do('mkdir -p "%s"' % self.chrootpath)
self.buildenv = BuildEnv(self.xml,
self.chrootpath,
build_sources=build_sources,
clean=True)
skip_pkglist = False
# Import keyring
self.buildenv.import_keys()
logging.info("Keys imported")
# Install packages
if not skip_pkglist:
self.install_packages(self.buildenv)
try:
self.buildenv.rfs.dump_elbeversion(self.xml)
except IOError:
logging.exception("Dump elbeversion failed")
# Extract target FS. We always create a new instance here with
# clean=true, because we want a pristine directory.
self.targetfs = TargetFs(self.targetpath,
self.buildenv.xml,
clean=True)
os.chdir(self.buildenv.rfs.fname(''))
extract_target(self.buildenv.rfs, self.xml, self.targetfs,
self.get_rpcaptcache())
# Package validation and package list
if not skip_pkglist:
pkgs = self.xml.xml.node("/target/pkg-list")
if self.xml.has("fullpkgs"):
check_full_pkgs(pkgs, self.xml.xml.node("/fullpkgs"),
self.get_rpcaptcache())
else:
check_full_pkgs(pkgs, None, self.get_rpcaptcache())
dump_fullpkgs(self.xml, self.buildenv.rfs, self.get_rpcaptcache())
self.xml.dump_elbe_version()
self.targetfs.write_fstab(self.xml)
# Dump ELBE version
try:
self.targetfs.dump_elbeversion(self.xml)
except MemoryError:
logging.exception("Dump elbeversion failed")
# install packages for buildenv
if not skip_pkglist:
self.install_packages(self.buildenv, buildenv=True)
# Write source.xml
try:
sourcexmlpath = os.path.join(self.builddir, "source.xml")
self.xml.xml.write(sourcexmlpath)
except MemoryError:
logging.exception("Write source.xml failed (archive to huge?)")
# Elbe report
cache = self.get_rpcaptcache()
tgt_pkgs = elbe_report(self.xml, self.buildenv, cache, self.targetfs)
# chroot' licenses
self.gen_licenses("chroot", self.buildenv,
[p.name for p in cache.get_installed_pkgs()])
self.gen_licenses("target", self.buildenv, tgt_pkgs)
# Use some handwaving to determine grub version
grub_arch = "ia32" if self.arch == "i386" else self.arch
grub_fw_type = []
grub_version = 0
if self.get_rpcaptcache().is_installed('grub-pc'):
grub_version = 202
grub_fw_type.append("bios")
if self.get_rpcaptcache().is_installed('grub-efi-%s-bin' % grub_arch):
grub_version = 202
grub_tgt = "x86_64" if self.arch == "amd64" else self.arch
grub_fw_type.extend(["efi", grub_tgt + "-efi"])
if (self.get_rpcaptcache().is_installed('shim-signed')
and self.get_rpcaptcache().is_installed(
'grub-efi-%s-signed' % grub_arch)):
grub_version = 202
grub_fw_type.append("shimfix")
if self.get_rpcaptcache().is_installed('grub-legacy'):
logging.warning("package grub-legacy is installed, "
"this is obsolete.")
grub_version = 97
grub_fw_type.append("bios")
elif not grub_fw_type:
logging.warning(
"neither package grub-pc nor grub-efi-%s-bin "
"are installed, skipping grub", grub_arch)
self.targetfs.part_target(self.builddir, grub_version, grub_fw_type)
self.build_cdroms(build_bin,
build_sources,
cdrom_size,
tgt_pkg_lst=tgt_pkgs)
if self.postbuild_file:
logging.info("Postbuild script")
cmd = ' "%s %s %s"' % (self.builddir,
self.xml.text("project/version"),
self.xml.text("project/name"))
do(self.postbuild_file + cmd, allow_fail=True)
do_prj_finetuning(self.xml, self.buildenv, self.targetfs,
self.builddir)
self.targetfs.pack_images(self.builddir)
system('cat "%s"' % self.validationpath)
def submit_and_dl_result(xmlfile, cdrom, opt):
# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
try:
with PreprocessWrapper(xmlfile, opt) as ppw:
xmlfile = ppw.preproc
ret, prjdir, err = command_out_stderr('%s control create_project' %
(elbe_exe))
if ret != 0:
print("elbe control create_project failed.", file=sys.stderr)
print(err, file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
prjdir = prjdir.strip()
cmd = '%s control set_xml %s %s' % (elbe_exe, prjdir, xmlfile)
ret, _, err = command_out_stderr(cmd)
if ret != 0:
print("elbe control set_xml failed2", file=sys.stderr)
print(err, file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
except CommandError:
# this is the failure from PreprocessWrapper
# it already printed the error message from
# elbe preprocess
print("Giving up", file=sys.stderr)
sys.exit(20)
if opt.writeproject:
with open(opt.writeproject, "w") as wpf:
wpf.write(prjdir)
if cdrom is not None:
print("Uploading CDROM. This might take a while")
try:
system('%s control set_cdrom "%s" "%s"' %
(elbe_exe, prjdir, cdrom))
except CommandError:
print("elbe control set_cdrom Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("Upload finished")
build_opts = ''
if opt.build_bin:
build_opts += '--build-bin '
if opt.build_sources:
build_opts += '--build-sources '
if cdrom:
build_opts += '--skip-pbuilder '
try:
system('%s control build "%s" %s' % (elbe_exe, prjdir, build_opts))
except CommandError:
print("elbe control build Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("Build started, waiting till it finishes")
try:
system('%s control wait_busy "%s"' % (elbe_exe, prjdir))
except CommandError:
print("elbe control wait_busy Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print("Build finished !")
print("")
if opt.build_sdk:
try:
system('%s control build_sdk "%s" %s' %
(elbe_exe, prjdir, build_opts))
except CommandError:
print("elbe control build_sdk Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("SDK Build started, waiting till it finishes")
try:
system('%s control wait_busy "%s"' % (elbe_exe, prjdir))
except CommandError:
print("elbe control wait_busy Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print("SDK Build finished !")
print("")
try:
system('%s control dump_file "%s" validation.txt' % (elbe_exe, prjdir))
except CommandError:
print("Project failed to generate validation.txt", file=sys.stderr)
print("Getting log.txt", file=sys.stderr)
try:
system('%s control dump_file "%s" log.txt' % (elbe_exe, prjdir))
except CommandError:
print("Failed to dump log.txt", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
if opt.skip_download:
print("")
print("Listing available files:")
print("")
try:
system('%s control get_files "%s"' % (elbe_exe, prjdir))
except CommandError:
print("elbe control get_files Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print('Get Files with: elbe control get_file "%s" <filename>' % prjdir)
else:
print("")
print("Getting generated Files")
print("")
ensure_outdir(opt)
try:
system('%s control get_files --output "%s" "%s"' %
(elbe_exe, opt.outdir, prjdir))
except CommandError:
print("elbe control get_files Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
if not opt.keep_files:
try:
system('%s control del_project "%s"' % (elbe_exe, prjdir))
except CommandError:
print("remove project from initvm failed", file=sys.stderr)
sys.exit(20)
def execute(self, initvmdir, opt, args):
# pylint: disable=too-many-branches
# pylint: disable=too-many-statements
if self.initvm is not None:
print("Initvm is already defined for the libvirt domain '%s'.\n" % cfg['initvm_domain'])
print("If you want to build in your old initvm, "
"use `elbe initvm submit <xml>`.")
print("If you want to remove your old initvm from libvirt "
"run `virsh --connect qemu:///system undefine %s`.\n" % cfg['initvm_domain'])
print("You can specify another libvirt domain by setting the "
"ELBE_INITVM_DOMAIN environment variable to an unused domain name.\n")
print("Note:")
print("\t1) You can reimport your old initvm via "
"`virsh --connect qemu:///system define <file>`")
print("\t where <file> is the corresponding libvirt.xml")
print("\t2) virsh --connect qemu:///system undefine does not delete the image "
"of your old initvm.")
sys.exit(20)
# Upgrade from older versions which used tmux
try:
system("tmux has-session -t ElbeInitVMSession 2>/dev/null")
print ("ElbeInitVMSession exists in tmux. "
"It may belong to an old elbe version. "
"Please stop it to prevent interfering with this version.", file=sys.stderr)
sys.exit(20)
except CommandError:
pass
# Init cdrom to None, if we detect it, we set it
cdrom = None
if len(args) == 1:
if args[0].endswith('.xml'):
# We have an xml file, use that for elbe init
xmlfile = args[0]
try:
xml = etree(xmlfile)
except ValidationError as e:
print("XML file is invalid: %s" % str(e))
# Use default XML if no initvm was specified
if not xml.has("initvm"):
xmlfile = os.path.join(
elbepack.__path__[0], "init/default-init.xml")
elif args[0].endswith('.iso'):
# We have an iso image, extract xml from there.
tmp = extract_cdrom(args[0])
xmlfile = tmp.fname('source.xml')
cdrom = args[0]
else:
print(
"Unknown file ending (use either xml or iso)",
file=sys.stderr)
sys.exit(20)
else:
# No xml File was specified, build the default elbe-init-with-ssh
xmlfile = os.path.join(
elbepack.__path__[0],
"init/default-init.xml")
try:
init_opts = ''
if opt.devel:
init_opts += ' --devel'
if opt.nesting:
init_opts += ' --nesting'
if not opt.build_bin:
init_opts += ' --skip-build-bin'
if not opt.build_sources:
init_opts += ' --skip-build-source'
if cdrom:
system('%s init %s --directory "%s" --cdrom "%s" "%s"' %
(elbe_exe, init_opts, initvmdir, cdrom, xmlfile))
else:
system(
'%s init %s --directory "%s" "%s"' %
(elbe_exe, init_opts, initvmdir, xmlfile))
except CommandError:
print("'elbe init' Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
# Read xml file for libvirt
with open(os.path.join(initvmdir, 'libvirt.xml')) as f:
xml = f.read()
# Register initvm in libvirt
try:
self.conn.defineXML(xml)
except CommandError:
print('Registering initvm in libvirt failed', file=sys.stderr)
print('Try `virsh --connect qemu:///system undefine %s` to delete existing initvm' % cfg['initvm_domain'],
file=sys.stderr)
sys.exit(20)
# Build initvm
try:
system('cd "%s"; make' % (initvmdir))
except CommandError:
print("Building the initvm Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
try:
system('%s initvm start' % elbe_exe)
except CommandError:
print("Starting the initvm Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
if len(args) == 1:
# if provided xml file has no initvm section xmlfile is set to a
# default initvm XML file. But we need the original file here
if args[0].endswith('.xml'):
# stop here if no project node was specified
try:
x = etree(args[0])
except ValidationError as e:
print("XML file is invalid: %s" % str(e))
sys.exit(20)
if not x.has('project'):
print("elbe initvm ready: use 'elbe initvm submit "
"myproject.xml' to build a project")
sys.exit(0)
xmlfile = args[0]
elif cdrom is not None:
xmlfile = tmp.fname('source.xml')
submit_and_dl_result(xmlfile, cdrom, opt)
def shutdown_initvm(self):
# pylint: disable=no-self-use
system("systemctl --no-block poweroff")
def run_command(argv):
oparser = OptionParser(
usage="usage: %prog chroot [options] <builddir> [cmd]")
oparser.add_option("--skip-validation",
action="store_true",
dest="skip_validation",
default=False,
help="Skip xml schema validation")
oparser.add_option("--target",
action="store_true",
dest="target",
help="chroot into target instead of buildenv",
default=False)
oparser.add_option("--buildtype",
dest="buildtype",
help="Override the buildtype")
(opt, args) = oparser.parse_args(argv)
if len(args) < 1:
print("wrong number of arguments")
oparser.print_help()
sys.exit(20)
with elbe_logging({"streams": sys.stdout}):
try:
project = ElbeProject(args[0],
override_buildtype=opt.buildtype,
skip_validate=opt.skip_validation,
url_validation=ValidationMode.NO_CHECK)
except ValidationError:
logging.exception("XML validation failed. Bailing out")
sys.exit(20)
os.environ["LANG"] = "C"
os.environ["LANGUAGE"] = "C"
os.environ["LC_ALL"] = "C"
# TODO: howto set env in chroot?
os.environ["PS1"] = project.xml.text('project/name') + r': \w\$'
cmd = "/bin/bash"
if len(args) > 1:
cmd = ""
cmd2 = args[1:]
for c in cmd2:
cmd += (c + " ")
if opt.target:
try:
with project.targetfs:
system("/usr/sbin/chroot %s %s" %
(project.targetpath, cmd))
except CommandError as e:
print(repr(e))
else:
try:
with project.buildenv:
system("/usr/sbin/chroot %s %s" %
(project.chrootpath, cmd))
except CommandError as e:
print(repr(e))
