def __init__(self, xml_fname=None):
dict.__init__(self)
self.perpackage_mapping = {}
self.perpackage_override = {}
if xml_fname is None:
return
xml = etree(xml_fname)
if xml.root.has('global'):
for mapping in xml.root.node('global'):
self[mapping.et.attrib['name']] = mapping.et.text
if xml.root.has('perpackage'):
for pkg in xml.root.node('perpackage'):
pname = pkg.et.attrib['name']
self.perpackage_mapping[pname] = {}
self.perpackage_override[pname] = []
for pp in pkg:
if pp.tag == 'mapping':
self.perpackage_mapping[pname][pp.et.attrib['name']
] = pp.et.text
if pp.tag == 'license':
self.perpackage_override[pname].append(pp.et.text)
def __init__(
self,
fname,
buildtype=None,
skip_validate=False,
url_validation=ValidationMode.NO_CHECK):
if not skip_validate:
validation = validate_xml(fname)
if validation:
raise ValidationError(validation)
self.xml = etree(fname)
self.prj = self.xml.node("/project")
self.tgt = self.xml.node("/target")
if buildtype:
pass
elif self.xml.has("project/buildtype"):
buildtype = self.xml.text("/project/buildtype")
else:
buildtype = "nodefaults"
self.defs = ElbeDefaults(buildtype)
if not skip_validate and url_validation != ValidationMode.NO_CHECK:
self.validate_apt_sources(url_validation, buildtype)
def run_command( argv ):
oparser = OptionParser( usage="usage: %prog get_archive <xmlfile> <archive>")
(opt,args) = oparser.parse_args(argv)
if len(args) != 2:
print "Wrong number of arguments"
oparser.print_help()
sys.exit(20)
if os.path.exists( args[1] ):
print "archive already exists, bailing out"
sys.exit(20)
try:
xml = etree( args[0] )
except:
print "Error reading xml file!"
sys.exit(20)
if xml.has("archive"):
try:
unbase( xml.text("/archive"), args[1] )
except:
print "Error writing archive"
sys.exit(20)
else:
print "no archive in this xml file."
sys.exit(20)
def __init__(self, xml_fname=None):
dict.__init__(self)
self.perpackage_mapping = {}
self.perpackage_override = {}
if xml_fname is None:
return
xml = etree(xml_fname)
if xml.root.has('global'):
for mapping in xml.root.node('global'):
self[mapping.et.attrib['name']] = mapping.et.text
if xml.root.has('perpackage'):
for pkg in xml.root.node('perpackage'):
pname = pkg.et.attrib['name']
self.perpackage_mapping[pname] = {}
self.perpackage_override[pname] = []
for pp in pkg:
if pp.tag == 'mapping':
self.perpackage_mapping[pname][
pp.et.attrib['name']] = pp.et.text
if pp.tag == 'license':
self.perpackage_override[pname].append(pp.et.text)
def run_command( argv ):
oparser = OptionParser( usage="usage: %prog setcdrom <xmlfile> <cdrom>")
(opt,args) = oparser.parse_args(argv)
if len(args) != 2:
print "Wrong number of arguments"
oparser.print_help()
sys.exit(20)
try:
xml = etree( args[0] )
except:
print "Error reading xml file!"
sys.exit(20)
mirror = xml.node("project/mirror")
mirror.clear()
cdrom = mirror.ensure_child("cdrom")
cdrom.set_text( args[1] )
try:
xml.write( args[0] )
except:
print "Unable to write new xml file"
sys.exit(20)
def run_command( argv ):
pack_dir = elbepack.__path__[0]
template_dir = os.path.join( pack_dir, "mako" )
oparser = OptionParser(usage="usage: %prog xsdtoasciidoc [options] <xsdfile>")
oparser.add_option( "--output", dest="out",
help="specify output filename",
metavar="FILE" )
(opt,args) = oparser.parse_args(argv)
if len(args) != 1:
print "Wrong number of arguments"
oparser.print_help()
sys.exit(20)
xml = etree( args[0] )
if not opt.out:
print 'output is mandatory'
sys.exit(20)
d = {"opt": opt,
"xml": xml }
write_template(opt.out, os.path.join(pack_dir, "xsdtoasciidoc.mako"), d )
def run_command(argv):
oparser = OptionParser(usage="usage: %prog adjustpkgs [options] <xmlfile>")
oparser.add_option("-o", "--output", dest="output", help="name of logfile")
oparser.add_option("-n",
"--name",
dest="name",
help="name of the project (included in the report)")
(opt, args) = oparser.parse_args(argv)
if len(args) != 1:
print("Wrong number of arguments")
oparser.print_help()
sys.exit(20)
if not opt.output:
return 0
xml = etree(args[0])
xml_pkglist = xml.node("/target/pkg-list")
xml_pkgs = [p.et.text for p in xml_pkglist]
# TODO: install buildimage packages after target image generation
# and remove theme before target image generation
# we need to introduce additional arguments for this
# in default copy mode chroot to the target and remove elbe-daemon
# and its dependencies (if it is not in target/pkg-list.
buildenv_pkgs = ["python3-elbe-buildenv"]
if xml.has("./project/buildimage/pkg-list"):
buildenv_pkgs.extend(
[p.et.text for p in xml.node("project/buildimage/pkg-list")])
with elbe_logging({"files": opt.output}):
logging.info("ELBE Report for Project %s", opt.name)
return set_pkgs(xml_pkgs + buildenv_pkgs)
def run_command(argv):
oparser = OptionParser(
usage="usage: %prog xsdtoasciidoc [options] <xsdfile>")
oparser.add_option("--output",
dest="out",
help="specify output filename",
metavar="FILE")
(opt, args) = oparser.parse_args(argv)
if len(args) != 1:
print "Wrong number of arguments"
oparser.print_help()
sys.exit(20)
xml = etree(args[0])
if not opt.out:
print 'output is mandatory'
sys.exit(20)
d = {"opt": opt, "xml": xml}
write_template(opt.out, xsdtoasciidoc_mako_fname, d)
def run_command( argv ):
oparser = OptionParser( usage="usage: %prog chg_archive <xmlfile> <archive>")
(opt,args) = oparser.parse_args(argv)
if len(args) != 2:
print "Wrong number of arguments"
oparser.print_help()
sys.exit(20)
try:
xml = etree( args[0] )
except:
print "Error reading xml file!"
sys.exit(20)
try:
arch = xml.ensure_child( "archive" )
arch.set_text( enbase( args[1] ) )
except:
print "Error reading archive"
sys.exit(20)
try:
xml.write( args[0] )
except:
print "Unable to write new xml file"
sys.exit(20)
def run_command(argv):
oparser = OptionParser(usage="usage: %prog setcdrom <xmlfile> <cdrom>")
(opt, args) = oparser.parse_args(argv)
if len(args) != 2:
print "Wrong number of arguments"
oparser.print_help()
sys.exit(20)
try:
xml = etree(args[0])
except:
print "Error reading xml file!"
sys.exit(20)
mirror = xml.node("project/mirror")
mirror.clear()
cdrom = mirror.ensure_child("cdrom")
cdrom.set_text(args[1])
try:
xml.write(args[0])
except:
print "Unable to write new xml file"
sys.exit(20)
def run_command(argv):
oparser = OptionParser(
usage="usage: %prog xsdtoasciidoc [options] <xsdfile>")
oparser.add_option("--output", dest="out",
help="specify output filename",
metavar="FILE")
(opt, args) = oparser.parse_args(argv)
if len(args) != 1:
print("Wrong number of arguments")
oparser.print_help()
sys.exit(20)
xml = etree(args[0])
if not opt.out:
print("--output is mandatory")
sys.exit(20)
d = {"opt": opt,
"xml": xml}
write_template(opt.out, xsdtoasciidoc_mako_fname, d)
def run_command(argv):
oparser = OptionParser(
usage="usage: %prog get_archive <xmlfile> <archive>")
(opt, args) = oparser.parse_args(argv)
if len(args) != 2:
print "Wrong number of arguments"
oparser.print_help()
sys.exit(20)
if os.path.exists(args[1]):
print "archive already exists, bailing out"
sys.exit(20)
try:
xml = etree(args[0])
except:
print "Error reading xml file!"
sys.exit(20)
if xml.has("archive"):
try:
unbase(xml.text("/archive"), args[1])
except:
print "Error writing archive"
sys.exit(20)
else:
print "no archive in this xml file."
sys.exit(20)
def __init__(
self,
fname,
buildtype=None,
skip_validate=False,
url_validation=ValidationMode.NO_CHECK):
if not skip_validate:
validation = validate_xml(fname)
if validation:
raise ValidationError(validation)
self.xml = etree(fname)
self.prj = self.xml.node("/project")
self.tgt = self.xml.node("/target")
if buildtype:
pass
elif self.xml.has("project/buildtype"):
buildtype = self.xml.text("/project/buildtype")
else:
buildtype = "nodefaults"
self.defs = ElbeDefaults(buildtype)
if not skip_validate and url_validation != ValidationMode.NO_CHECK:
self.validate_apt_sources(url_validation, buildtype)
def run_command(argv):
oparser = OptionParser(
usage="usage: %prog pin_versions [options] <xmlfile>")
oparser.add_option("--skip-validation",
action="store_true",
dest="skip_validation",
default=False,
help="Skip xml schema validation")
(opt, args) = oparser.parse_args(argv)
if len(args) != 1:
print("Wrong number of arguments")
oparser.print_help()
sys.exit(20)
if not opt.skip_validation:
validation = validate_xml(args[0])
if validation:
print("xml validation failed. Bailing out")
for i in validation:
print(i)
sys.exit(20)
try:
xml = etree(args[0])
except BaseException:
print("Error reading xml file!")
sys.exit(20)
if not xml.has("fullpkgs"):
print("xml file does not have fullpkgs node")
sys.exit(20)
plist = xml.ensure_child("/target/pkg-list")
plist.clear()
fullp = xml.node("fullpkgs")
for p in fullp:
pname = p.et.text
pver = p.et.get('version')
pak = plist.append('pkg')
pak.set_text(pname)
pak.et.tail = '\n'
pak.et.set('version', pver)
try:
xml.write(args[0])
except BaseException:
print("Unable to write new xml file")
sys.exit(20)
def run_command(argv):
oparser = OptionParser(
usage="usage: %prog pin_versions [options] <xmlfile>")
oparser.add_option("--skip-validation", action="store_true",
dest="skip_validation", default=False,
help="Skip xml schema validation")
(opt, args) = oparser.parse_args(argv)
if len(args) != 1:
print("Wrong number of arguments")
oparser.print_help()
sys.exit(20)
if not opt.skip_validation:
validation = validate_xml(args[0])
if validation:
print("xml validation failed. Bailing out")
for i in validation:
print(i)
sys.exit(20)
try:
xml = etree(args[0])
except BaseException:
print("Error reading xml file!")
sys.exit(20)
if not xml.has("fullpkgs"):
print("xml file does not have fullpkgs node")
sys.exit(20)
plist = xml.ensure_child("/target/pkg-list")
plist.clear()
fullp = xml.node("fullpkgs")
for p in fullp:
pname = p.et.text
pver = p.et.get('version')
pak = plist.append('pkg')
pak.set_text(pname)
pak.et.tail = '\n'
pak.et.set('version', pver)
try:
xml.write(args[0])
except BaseException:
print("Unable to write new xml file")
sys.exit(20)
def run_command(_argv):
try:
xml = etree("/etc/elbe_base.xml")
except IOError:
print("/etc/elbe_base.xml removed by user")
return -1
bootup_check(xml)
try:
bootup_info()
except IOError:
print("/etc/elbe_version removed by user")
return -1
def run_command (argv):
try:
xml = etree ("/etc/elbe_base.xml")
except IOError:
print '/etc/elbe_base.xml removed by user'
return -1
bootup_check (xml)
try:
bootup_info ()
except IOError:
print '/etc/elbe_version removed by user'
return -1
def run(self):
# pylint: disable=attribute-defined-outside-init
self.xml = etree("source.xml")
fail_cnt = 0
total_cnt = 0
# For all image
for tag in self.xml.all(".//check-image-list/check"):
fail_cnt += self.do_img(tag)
total_cnt += 1
logging.info("Succesfully validate %d images out of %d",
total_cnt - fail_cnt, total_cnt)
return fail_cnt
def run_command( argv ):
oparser = OptionParser( usage="usage: %prog chg_archive [options] <xmlfile> [<archive>|<directory>]")
oparser.add_option ("--keep-attributes", action="store_true",
help="keep file owners and groups, if not specified all files will belong to root:root",
dest="keep_attributes", default=False)
(opt,args) = oparser.parse_args(argv)
if len(args) != 2:
print "Wrong number of arguments"
oparser.print_help()
sys.exit(20)
try:
xml = etree( args[0] )
except:
print "Error reading xml file!"
sys.exit(20)
if os.path.isdir (args[1]):
archive = '.archive.tbz'
if opt.keep_attributes:
cmd = 'tar cfj .archive.tbz -C '
else:
cmd = 'tar cjf .archive.tbz --owner=root --group=root -C '
cmd += args[1] + ' .'
os.system (cmd)
else:
archive = args[1]
try:
arch = xml.ensure_child( "archive" )
arch.set_text( enbase( archive ) )
except:
print "Error reading archive"
sys.exit(20)
try:
xml.write( args[0] )
except:
print "Unable to write new xml file"
sys.exit(20)
if os.path.isdir (args[1]):
os.remove (archive)
def __init__(self, fname, buildtype=None, skip_validate=False, skip_urlcheck=False):
if not skip_validate:
validation = validate_xml (fname)
if len (validation) != 0:
raise ValidationError (validation)
self.xml = etree( fname )
self.prj = self.xml.node("/project")
self.tgt = self.xml.node("/target")
if buildtype:
pass
elif self.xml.has( "project/buildtype" ):
buildtype = self.xml.text( "/project/buildtype" )
else:
buildtype = "nodefaults"
self.defs = ElbeDefaults(buildtype)
if not skip_validate and not skip_urlcheck:
self.validate_apt_sources ()
def get_initvm_preseed(xml):
def_xml = etree(default_preseed_fname)
preseed = {}
for c in def_xml.node("/preseed"):
k = (c.et.attrib["owner"], c.et.attrib["key"])
v = (c.et.attrib["type"], c.et.attrib["value"])
preseed[k] = v
if not xml.has("./initvm/preseed"):
return preseed
for c in xml.node("/initvm/preseed"):
k = (c.et.attrib["owner"], c.et.attrib["key"])
v = (c.et.attrib["type"], c.et.attrib["value"])
preseed[k] = v
return preseed
def get_initvm_preseed(xml):
def_xml = etree(default_preseed_fname)
preseed = {}
for c in def_xml.node("/preseed"):
k = (c.et.attrib["owner"], c.et.attrib["key"])
v = (c.et.attrib["type"], c.et.attrib["value"])
preseed[k] = v
if not xml.has("./initvm/preseed"):
return preseed
for c in xml.node("/initvm/preseed"):
k = (c.et.attrib["owner"], c.et.attrib["key"])
v = (c.et.attrib["type"], c.et.attrib["value"])
preseed[k] = v
return preseed
def get_preseed(xml):
pack_dir = elbepack.__path__[0]
def_xml = etree(os.path.join(pack_dir, "default-preseed.xml"))
preseed = {}
for c in def_xml.node("/preseed"):
k = (c.et.attrib["owner"], c.et.attrib["key"])
v = (c.et.attrib["type"], c.et.attrib["value"])
preseed[k] = v
if not xml.has("./project/preseed"):
return preseed
for c in xml.node("/project/preseed"):
k = (c.et.attrib["owner"], c.et.attrib["key"])
v = (c.et.attrib["type"], c.et.attrib["value"])
preseed[k] = v
return preseed
def get_preseed( xml ):
pack_dir = elbepack.__path__[0]
def_xml = etree( os.path.join( pack_dir, "default-preseed.xml" ) )
preseed = {}
for c in def_xml.node("/preseed"):
k = (c.et.attrib["owner"], c.et.attrib["key"])
v = (c.et.attrib["type"], c.et.attrib["value"])
preseed[k] = v
if not xml.has("./project/preseed"):
return preseed
for c in xml.node("/project/preseed"):
k = (c.et.attrib["owner"], c.et.attrib["key"])
v = (c.et.attrib["type"], c.et.attrib["value"])
preseed[k] = v
return preseed
def run_command(argv):
oparser = OptionParser(usage="usage: %prog adjustpkgs [options] <xmlfile>")
oparser.add_option("-o", "--output", dest="output", help="name of logfile")
oparser.add_option("-n",
"--name",
dest="name",
help="name of the project (included in the report)")
(opt, args) = oparser.parse_args(argv)
if len(args) != 1:
print "Wrong number of arguments"
oparser.print_help()
sys.exit(20)
if not opt.output:
return 0
xml = etree(args[0])
xml_pkglist = xml.node("/target/pkg-list")
xml_pkgs = [p.et.text for p in xml_pkglist]
mandatory_pkgs = ["elbe-buildenv"]
if xml.has("target/images/msdoshd/grub-install"):
mandatory_pkgs = ["elbe-buildenv", "grub-pc"]
# TODO: install buildimage packages after target image generation
# and remove theme before target image generation
# we need to introduce additional arguments for this
# in default copy mode chroot to the target and remove elbe-daemon
# and its dependencies (if it is not in target/pkg-list.
buildenv_pkgs = []
if xml.has("./project/buildimage/pkg-list"):
buildenv_pkgs = [
p.et.text for p in xml.node("project/buildimage/pkg-list")
]
adj = adjpkg(opt.output, opt.name)
return adj.set_pkgs(xml_pkgs + mandatory_pkgs + buildenv_pkgs)
def _combinearchivedir(xml, xpath, use_volume):
elbexml = etree(None)
elbexml.et = xml
tmp = TmpdirFilesystem()
for archivedir in elbexml.all(xpath):
try:
archiveurl = urljoin(archivedir.et.base, archivedir.et.text)
keep = archivedir.bool_attr("keep-attributes")
parent = archivedir.get_parent()
if use_volume:
volume_attr = archivedir.et.get('volume', default='all')
fname_suffix = volume_attr
arch = parent.node("archive[@volume='%s']" % volume_attr)
if arch is None:
arch = parent.append("archive")
arch.et.set("volume", volume_attr)
else:
arch = parent.ensure_child("archive")
fname_suffix = ''
get_and_append = get_and_append_method(archiveurl)
archname = tmp.fname('archive%s.tar.bz2' % fname_suffix)
get_and_append(archiveurl, archname, keep)
arch.set_text(enbase(archname, True))
parent.remove_child(archivedir)
except (CalledProcessError, OSError):
msg = "Failure while processing \"" + archivedir.text + "\":\n"
msg += str(sys.exc_info()[1])
raise ArchivedirError(msg)
def _combinearchivedir(xml):
elbexml = etree(None)
elbexml.et = xml
archive = '.combinedarchive.tar'
for archivedir in xml.iterfind("archivedir"):
try:
archiveurl = urljoin(archivedir.base, archivedir.text)
keep = elbexml.check_boolean(archivedir, "keep-attributes")
get_and_append = get_and_append_method(archiveurl)
get_and_append(archiveurl, archive, keep)
archivedir.getparent().remove(archivedir)
except (CalledProcessError, OSError):
msg = "Failure while processing \"" + archivedir.text + "\":\n"
msg += str(sys.exc_info()[1])
raise ArchivedirError(msg)
arch = elbexml.ensure_child("archive")
arch.set_text(enbase(archive, True))
os.remove(archive)
return xml
def run_command(argv):
oparser = OptionParser(usage="usage: %prog setsel <xmlfile> <pkglist.txt>")
(_, args) = oparser.parse_args(argv)
if len(args) != 2:
print("Wrong number of arguments")
oparser.print_help()
sys.exit(20)
xml = etree(args[0])
pkg_list = xml.node("/pkg-list")
pkg_list.clear()
sels = parse_selections(args[1])
for s in sels:
new = pkg_list.append('pkg')
new.set_text(s)
xml.write(args[0])
def run_command(argv):
oparser = OptionParser(usage="usage: %prog setsel <xmlfile> <pkglist.txt>")
(_, args) = oparser.parse_args(argv)
if len(args) != 2:
print("Wrong number of arguments")
oparser.print_help()
sys.exit(20)
xml = etree(args[0])
pkg_list = xml.node("/pkg-list")
pkg_list.clear()
sels = parse_selections(args[1])
for s in sels:
new = pkg_list.append('pkg')
new.set_text(s)
xml.write(args[0])
def _combinearchivedir(xml):
elbexml = etree(None)
elbexml.et = xml
archive = '.combinedarchive.tar'
for archivedir in xml.iterfind("archivedir"):
try:
archiveurl = urljoin(archivedir.base, archivedir.text)
keep = elbexml.check_boolean(archivedir, "keep-attributes")
get_and_append = get_and_append_method(archiveurl)
get_and_append(archiveurl, archive, keep)
archivedir.getparent().remove(archivedir)
except (CalledProcessError, OSError):
msg = "Failure while processing \"" + archivedir.text + "\":\n"
msg += str(sys.exc_info()[1])
raise ArchivedirError(msg)
arch = elbexml.ensure_child("archive")
arch.set_text(enbase(archive, True))
os.remove(archive)
return xml
def run_command(argv):
oparser = OptionParser(usage="usage: %prog adjustpkgs [options] <xmlfile>")
oparser.add_option("-o", "--output", dest="output",
help="name of logfile")
oparser.add_option("-n", "--name", dest="name",
help="name of the project (included in the report)")
(opt, args) = oparser.parse_args(argv)
if len(args) != 1:
print("Wrong number of arguments")
oparser.print_help()
sys.exit(20)
if not opt.output:
return 0
xml = etree(args[0])
xml_pkglist = xml.node("/target/pkg-list")
xml_pkgs = [p.et.text for p in xml_pkglist]
mandatory_pkgs = ["python-elbe-buildenv"]
if xml.has("target/images/msdoshd/grub-install"):
mandatory_pkgs = ["python-elbe-buildenv", "grub-pc"]
# TODO: install buildimage packages after target image generation
# and remove theme before target image generation
# we need to introduce additional arguments for this
# in default copy mode chroot to the target and remove elbe-daemon
# and its dependencies (if it is not in target/pkg-list.
buildenv_pkgs = []
if xml.has("./project/buildimage/pkg-list"):
buildenv_pkgs = [p.et.text for p in xml.node(
"project/buildimage/pkg-list")]
adj = adjpkg(opt.output, opt.name)
return adj.set_pkgs(xml_pkgs + mandatory_pkgs + buildenv_pkgs)
def run_command(argv):
oparser = OptionParser(
usage="usage: %prog chg_archive [options] <xmlfile> "
"[<archive>|<directory>]")
oparser.add_option(
"--keep-attributes",
action="store_true",
help="keep file owners and groups, if not specified all files will "
"belong to root:root",
dest="keep_attributes",
default=False)
(opt, args) = oparser.parse_args(argv)
if len(args) != 2:
print("Wrong number of arguments")
oparser.print_help()
sys.exit(20)
try:
xml = etree(args[0])
except BaseException:
print("Error reading xml file!")
sys.exit(20)
try:
xml = chg_archive(xml, args[1], opt.keep_attributes)
except BaseException:
print("Error reading archive")
sys.exit(20)
try:
xml.write(args[0])
except BaseException:
print("Unable to write new xml file")
sys.exit(20)
def run_command(argv):
oparser = OptionParser(
usage="usage: %prog chg_archive [options] <xmlfile> "
"[<archive>|<directory>]")
oparser.add_option(
"--keep-attributes",
action="store_true",
help="keep file owners and groups, if not specified all files will "
"belong to root:root",
dest="keep_attributes",
default=False)
(opt, args) = oparser.parse_args(argv)
if len(args) != 2:
print("Wrong number of arguments")
oparser.print_help()
sys.exit(20)
try:
xml = etree(args[0])
except BaseException:
print("Error reading xml file!")
sys.exit(20)
try:
xml = chg_archive(xml, args[1], opt.keep_attributes)
except BaseException:
print("Error reading archive")
sys.exit(20)
try:
xml.write(args[0])
except BaseException:
print("Unable to write new xml file")
sys.exit(20)
def run_command(argv):
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
oparser = OptionParser(
usage="usage: %prog parselicence [options] <licencefile>")
oparser.add_option("--output", dest="output", help="outputfilename")
oparser.add_option("--mapping", dest="mapping", help="mapping filename")
oparser.add_option(
"--use-nomos",
action="store_true",
dest="use_nomos",
default=False,
help="Use the external nomos tool on the copyright text, "
"and record the ouput in out xml")
oparser.add_option("--errors-only",
action="store_true",
dest="only_errors",
default=False,
help="Only Output Packages with errors, "
"needing a fix in the mapping file")
oparser.add_option("--tvout",
dest="tagvalue",
help="tag value output filename")
(opt, args) = oparser.parse_args(argv)
if len(args) != 1:
print("wrong number of arguments")
oparser.print_help()
sys.exit(20)
tree = etree(args[0])
num_pkg = 0
mr = 0
hr = 0
err_pkg = 0
if not opt.mapping:
print("A mapping file is required")
oparser.print_help()
sys.exit(20)
mapping = license_dep5_to_spdx(opt.mapping)
# Dont use direct iterator, because we might want to delete
# elements, when --errors-only is active
for pkg in list(tree.root):
errors = []
pkg_name = pkg.et.attrib['name']
num_pkg += 1
if pkg.has('machinereadable'):
mr += 1
if pkg.has('heuristics'):
hr += 1
if not mapping.have_override(pkg_name):
errors.append('no override for heuristics based package "%s"' %
pkg_name)
if mapping.have_override(pkg_name):
pkg.append('have_override')
if pkg.has('debian_licenses'):
sp = pkg.ensure_child('spdx_licenses')
sp.clear()
sp.et.text = '\n'
lics = []
for l in pkg.node('debian_licenses'):
if l.et.text in lics:
continue
lics.append(l.et.text)
mapped_lics = mapping.map_lic(pkg_name, lics, errors)
for l in mapped_lics:
ll = sp.append('license')
ll.et.text = l
if not mapped_lics:
errors.append('empty mapped licenses in package "%s"' %
pkg_name)
else:
if not mapping.have_override(pkg_name):
errors.append(
'no debian_licenses and no override in package "%s"' %
pkg_name)
else:
sp = pkg.ensure_child('spdx_licenses')
sp.clear()
sp.et.text = '\n'
for l in mapping.get_override(pkg_name):
ll = sp.append('license')
ll.et.text = l
if opt.use_nomos:
nomos_l = scan_nomos(pkg.text('text'))
if nomos_l[0] != 'No_license_found':
nomos_node = pkg.append('nomos_licenses')
nomos_node.et.text = '\n'
for l in nomos_l:
ll = nomos_node.append('license')
ll.et.text = l
if errors:
for e in errors:
ee = pkg.append('error')
ee.et.text = e
err_pkg += 1
elif opt.only_errors:
# No Errors, and only_errors is active
# Remove package node
tree.root.remove_child(pkg)
if opt.tagvalue is not None:
with io.open(opt.tagvalue, "wt", encoding='utf-8') as fp:
fp.write(u'SPDXVersion: SPDX-1.2\n')
fp.write(u'DataLicense: CC0-1.0\n')
fp.write(u'\n')
fp.write(u'## Creation Information\n')
fp.write(u'Creator: Tool: elbe-%s\n' % elbe_version)
fp.write(u'Created: %s\n' % datetime.now().isoformat())
fp.write(u'\n')
fp.write(u'\n')
fp.write(u'## Package Information\n')
fp.write(u'\n')
for pkg in tree.root:
fp.write(u'## Package %s\n' % pkg.et.attrib['name'])
fp.write(u'PackageName: %s\n' % pkg.et.attrib['name'])
fp.write(u'PackageDownloadLocation: NOASSERTION\n')
if pkg.has('have_override'):
fp.write(u'PackageLicenseConcluded: %s\n' %
license_string(pkg))
fp.write(u'PackageLicenseDeclared: NOASSERTION\n')
else:
fp.write(u'PackageLicenseConcluded: NOASSERTION\n')
fp.write(u'PackageLicenseDeclared: %s\n' %
license_string(pkg))
fp.write(u'PackageLicenseInfoFromFiles: NOASSERTION\n')
fp.write(u'\n')
if opt.output is not None:
tree.write(opt.output)
print("statistics:")
print("num:%d mr:%d hr:%d err_pkg:%d" % (num_pkg, mr, hr, err_pkg))
def run_command(argv):
oparser = OptionParser(usage="usage: %prog check_updates [options] <source-xmlfile>")
oparser.add_option("-s", "--script", dest="script", help="filename of script to run when an update is required")
oparser.add_option(
"--skip-validation",
action="store_true",
dest="skip_validation",
default=False,
help="Skip xml schema validation",
)
(opt, args) = oparser.parse_args(argv)
if len(args) != 1:
print "Wrong number of arguments"
oparser.print_help()
sys.exit(20)
if not opt.skip_validation:
if not validate_xml(args[0]):
print "xml validation failed. Bailing out"
sys.exit(20)
print "checking %s" % args[0]
xml = etree(args[0])
arch = xml.text("project/buildimage/arch")
suite = xml.text("project/suite")
name = xml.text("project/name")
apt_sources = xml.text("sources_list")
apt_prefs = xml.text("apt_prefs")
fullp = xml.node("fullpkgs")
v = virtapt.VirtApt(name, arch, suite, apt_sources, apt_prefs)
d = virtapt.apt_pkg.DepCache(v.cache)
d.read_pinfile(v.projectpath + "/etc/apt/preferences")
for p in fullp:
pname = p.et.text
pver = p.et.get("version")
pauto = p.et.get("auto")
if pauto != "true":
d.mark_install(v.cache[pname])
errors = 0
required_updates = 0
for p in fullp:
pname = p.et.text
pver = p.et.get("version")
pauto = p.et.get("auto")
if not pname in v.cache:
if pauto == "false":
print pname, "does not exist in cache but is specified in pkg-list"
errors += 1
else:
print pname, "is no more required"
required_updates += 1
continue
centry = v.cache[pname]
if d.marked_install(centry):
cver = d.get_candidate_ver(v.cache[pname]).ver_str
if pver != cver:
print pname, "%s != %s" % (pver, cver)
required_updates += 1
if errors > 0:
print errors, "Errors occured, xml files needs fixing"
if opt.script:
os.system("%s ERRORS %s" % (opt.script, args[0]))
elif required_updates > 0:
print required_updates, "updates required"
if opt.script:
os.system("%s UPDATE %s" % (opt.script, args[0]))
else:
print "No Updates available"
def run_command(argv):
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
oparser = OptionParser(
usage="usage: %prog parselicence [options] <licencefile>")
oparser.add_option("--output", dest="output",
help="outputfilename")
oparser.add_option("--mapping", dest="mapping",
help="mapping filename")
oparser.add_option(
"--use-nomos",
action="store_true",
dest="use_nomos",
default=False,
help="Use the external nomos tool on the copyright text, "
"and record the ouput in out xml")
oparser.add_option(
"--errors-only",
action="store_true",
dest="only_errors",
default=False,
help="Only Output Packages with errors, "
"needing a fix in the mapping file")
oparser.add_option("--tvout", dest="tagvalue",
help="tag value output filename")
(opt, args) = oparser.parse_args(argv)
if len(args) != 1:
print("wrong number of arguments")
oparser.print_help()
sys.exit(20)
tree = etree(args[0])
num_pkg = 0
mr = 0
hr = 0
err_pkg = 0
if not opt.mapping:
print("A mapping file is required")
oparser.print_help()
sys.exit(20)
mapping = license_dep5_to_spdx(opt.mapping)
# Dont use direct iterator, because we might want to delete
# elements, when --errors-only is active
for pkg in list(tree.root):
errors = []
pkg_name = pkg.et.attrib['name']
num_pkg += 1
if pkg.has('machinereadable'):
mr += 1
if pkg.has('heuristics'):
hr += 1
if not mapping.have_override(pkg_name):
errors.append(
'no override for heuristics based package "%s"' %
pkg_name)
if mapping.have_override(pkg_name):
pkg.append('have_override')
if pkg.has('debian_licenses'):
sp = pkg.ensure_child('spdx_licenses')
sp.clear()
sp.et.text = '\n'
lics = []
for l in pkg.node('debian_licenses'):
if l.et.text in lics:
continue
lics.append(l.et.text)
mapped_lics = mapping.map_lic(pkg_name, lics, errors)
for l in mapped_lics:
ll = sp.append('license')
ll.et.text = l
if not mapped_lics:
errors.append(
'empty mapped licenses in package "%s"' %
pkg_name)
else:
if not mapping.have_override(pkg_name):
errors.append(
'no debian_licenses and no override in package "%s"' %
pkg_name)
else:
sp = pkg.ensure_child('spdx_licenses')
sp.clear()
sp.et.text = '\n'
for l in mapping.get_override(pkg_name):
ll = sp.append('license')
ll.et.text = l
if opt.use_nomos:
nomos_l = scan_nomos(pkg.text('text'))
if nomos_l[0] != 'No_license_found':
nomos_node = pkg.append('nomos_licenses')
nomos_node.et.text = '\n'
for l in nomos_l:
ll = nomos_node.append('license')
ll.et.text = l
if errors:
for e in errors:
ee = pkg.append('error')
ee.et.text = e
err_pkg += 1
elif opt.only_errors:
# No Errors, and only_errors is active
# Remove package node
tree.root.remove_child(pkg)
if opt.tagvalue is not None:
with io.open(opt.tagvalue, "wt", encoding='utf-8') as fp:
fp.write(u'SPDXVersion: SPDX-1.2\n')
fp.write(u'DataLicense: CC0-1.0\n')
fp.write(u'\n')
fp.write(u'## Creation Information\n')
fp.write(u'Creator: Tool: elbe-%s\n' % elbe_version)
fp.write(u'Created: %s\n' % datetime.now().isoformat())
fp.write(u'\n')
fp.write(u'\n')
fp.write(u'## Package Information\n')
fp.write(u'\n')
for pkg in tree.root:
fp.write(u'## Package %s\n' % pkg.et.attrib['name'])
fp.write(u'PackageName: %s\n' % pkg.et.attrib['name'])
fp.write(u'PackageDownloadLocation: NOASSERTION\n')
if pkg.has('have_override'):
fp.write(
u'PackageLicenseConcluded: %s\n' %
license_string(pkg))
fp.write(u'PackageLicenseDeclared: NOASSERTION\n')
else:
fp.write(u'PackageLicenseConcluded: NOASSERTION\n')
fp.write(
u'PackageLicenseDeclared: %s\n' %
license_string(pkg))
fp.write(u'PackageLicenseInfoFromFiles: NOASSERTION\n')
fp.write(u'\n')
if opt.output is not None:
tree.write(opt.output)
print("statistics:")
print("num:%d mr:%d hr:%d err_pkg:%d" % (num_pkg, mr, hr, err_pkg))
def run_command( argv ):
pack_dir = elbepack.__path__[0]
template_dir = os.path.join( pack_dir, "mako" )
oparser = OptionParser( usage="usage: %prog create [options] <filename>" )
oparser.add_option( "--oldkvm", action="store_true", dest="oldkvm",
default=False,
help="We are building for an old kvm version" )
oparser.add_option( "--debug", action="store_true", dest="debug",
default=False,
help="Enable various features to debug the build" )
oparser.add_option( "--skip-validation", action="store_true",
dest="skip_validation", default=False,
help="Skip xml schema validation" )
oparser.add_option( "--skip-cds", action="store_true", dest="skip_cds",
default=False,
help="Skip cd generation" )
oparser.add_option( "--directory", dest="dir",
help="Write Makefile into specified directory",
metavar="FILE" )
oparser.add_option( "--build-source", action="store_true",
dest="buildsources", default=False,
help="Build source cdrom" )
(opt,args) = oparser.parse_args(argv)
if len(args) == 0:
print "No Filename specified"
oparser.print_help()
sys.exit(20)
if len(args) > 1:
print "too many filenames specified"
oparser.print_help()
sys.exit(20)
try:
if not opt.skip_validation:
if not validate_xml( args[0] ):
print "xml validation failed. Bailing out"
sys.exit(20)
xml = etree( args[0] )
except:
print "Unable to open xml File. Bailing out"
sys.exit(20)
if not opt.dir:
path = "./build"
else:
path = opt.dir
try:
os.makedirs(path)
except:
print 'unable to create project directory: %s' % path
sys.exit(30)
d = {"opt": opt,
"xml": xml,
"prj": xml.node("/project"),
"tgt": xml.node("/target"),
"pkgs": xml.node("/target/pkg-list"),
"fine": xml.node("/finetuning"),
"preseed": get_preseed(xml) }
try:
copy_kinitrd(xml, path)
except:
print "Failure to download kernel/initrd debian Package"
print "Check your source URLs"
sys.exit(20)
if xml.has("archive"):
unbase( xml.text("/archive"), os.path.join(path,"archive.tar.bz2") )
templates = os.listdir( template_dir )
make_executable = [ "02pinning.mako",
"finetuning.sh.mako",
"changeroot-into-buildenv.sh.mako",
"cp-scipts-into-buildenv.sh.mako",
"create-target-rfs.sh.mako",
"part-target.sh.mako",
"post-inst.sh.mako",
"print_licence.sh.mako",
"purge.sh.mako",
"mkcdrom.sh.mako" ]
for t in templates:
print t
o = t.replace( ".mako", "" )
write_template(os.path.join(path,o), os.path.join(template_dir, t), d )
if t in make_executable:
os.chmod( os.path.join(path,o), 0755 )
shutil.copyfile( args[0],
os.path.join(path, "source.xml" ) )
shutil.copyfile( os.path.join( pack_dir, "treeutils.py" ),
os.path.join(path, "treeutils.py" ) )
shutil.copyfile( os.path.join( pack_dir, "dump.py" ),
os.path.join(path, "dump.py" ) )
def run_command(argv):
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
oparser = OptionParser(
usage="usage: %prog check_updates [options] <source-xmlfile>")
oparser.add_option(
"-s",
"--script",
dest="script",
help="filename of script to run when an update is required")
oparser.add_option("--skip-validation", action="store_true",
dest="skip_validation", default=False,
help="Skip xml schema validation")
(opt, args) = oparser.parse_args(argv)
if len(args) != 1:
print("Wrong number of arguments")
oparser.print_help()
sys.exit(20)
if not opt.skip_validation:
validation = validate_xml(args[0])
if validation:
print("xml validation failed. Bailing out")
for i in validation:
print(i)
sys.exit(20)
print("checking %s" % args[0])
xml = etree(args[0])
if xml.has("project/buildtype"):
buildtype = xml.text("/project/buildtype")
else:
buildtype = "nodefaults"
defs = ElbeDefaults(buildtype)
arch = xml.text("project/buildimage/arch", default=defs, key="arch")
suite = xml.text("project/suite")
apt_sources = xml.text("sources_list").replace("10.0.2.2", "localhost")
apt_prefs = xml.text("apt_prefs")
fullp = xml.node("fullpkgs")
v = virtapt.VirtApt(arch, suite, apt_sources, apt_prefs)
d = virtapt.apt_pkg.DepCache(v.cache)
d.read_pinfile(v.projectpath + "/etc/apt/preferences")
for p in fullp:
pname = p.et.text
pver = p.et.get('version')
pauto = p.et.get('auto')
if pauto != "true":
d.mark_install(v.cache[pname])
errors = 0
required_updates = 0
for p in fullp:
pname = p.et.text
pver = p.et.get('version')
pauto = p.et.get('auto')
if pname not in v.cache:
if pauto == 'false':
print(
"%s does not exist in cache but is specified in pkg-list" %
pname)
errors += 1
else:
print("%s is no more required" % pname)
required_updates += 1
continue
centry = v.cache[pname]
if d.marked_install(centry):
cver = d.get_candidate_ver(v.cache[pname]).ver_str
if pver != cver:
print("%s: %s != %s" % (pname, pver, cver))
required_updates += 1
sys.stdout.flush()
sys.stderr.flush()
if errors > 0:
print("%d Errors occured, xml files needs fixing" % errors)
if opt.script:
os.system("%s ERRORS %s" % (opt.script, args[0]))
elif required_updates > 0:
print("%d updates required" % required_updates)
if opt.script:
os.system("%s UPDATE %s" % (opt.script, args[0]))
else:
print("No Updates available")
def _apply_update(fname, status):
# pylint: disable=too-many-locals
try:
xml = etree(fname)
except BaseException:
raise Exception("reading %s failed " % fname)
fpl = xml.node("fullpkgs")
sources = apt_pkg.SourceList()
sources.read_main_list()
status.log("initialize apt")
apt_pkg.init()
cache = apt_pkg.Cache(progress=ElbeOpProgress(cb=status.log))
status.set_progress(1)
status.log("updating package cache")
cache.update(ElbeAcquireProgress(cb=status.log), sources)
# quote from python-apt api doc: "A call to this method does not affect the
# current Cache object, instead a new one should be created in order to use
# the changed index files."
cache = apt_pkg.Cache(progress=ElbeOpProgress(cb=status.log))
depcache = apt_pkg.DepCache(cache)
hl_cache = apt.cache.Cache(progress=ElbeOpProgress(cb=status.log))
hl_cache.update(fetch_progress=ElbeAcquireProgress(cb=status.log))
# go through package cache, if a package is in the fullpkg list of the XML
# mark the package for installation (with the specified version)
# if it is not mentioned in the fullpkg list purge the package out of the
# system.
status.set_progress(2)
status.log("calculating packages to install/remove")
count = len(hl_cache)
step = count / 10
i = 0
percent = 0
for p in hl_cache:
i = i + 1
if not i % step:
percent = percent + 10
status.log(str(percent) + "% - " + str(i) + "/" + str(count))
status.set_progress(2, str(percent) + "%")
pkg = cache[p.name]
marked = False
for fpi in fpl:
if pkg.name == fpi.et.text:
mark_install(depcache, pkg,
fpi.et.get('version'),
fpi.et.get('auto'),
status)
marked = True
if not marked:
depcache.mark_delete(pkg, True)
status.set_progress(3)
status.log("applying snapshot")
depcache.commit(ElbeAcquireProgress(cb=status.log),
ElbeInstallProgress(cb=status.log))
del depcache
del hl_cache
del cache
del sources
version_file = open("/etc/updated_version", "w")
version_file.write(xml.text("/project/version"))
version_file.close()
def get_base_version():
xml = etree("/etc/elbe_base.xml")
return xml.text("/project/version")
def get_target_version(fname):
xml = etree(fname)
return xml.text("/project/version")
def run_command( argv ):
pack_dir = elbepack.__path__[0]
template_dir = os.path.join( pack_dir, "mako" )
oparser = OptionParser( usage="usage: %prog create [options] <filename>" )
oparser.add_option( "--oldkvm", action="store_true", dest="oldkvm",
default=False,
help="We are building for an old kvm version" )
oparser.add_option( "--debug", action="store_true", dest="debug",
default=False,
help="Enable various features to debug the build" )
oparser.add_option( "--skip-validation", action="store_true",
dest="skip_validation", default=False,
help="Skip xml schema validation" )
oparser.add_option( "--skip-cds", action="store_true", dest="skip_cds",
default=False,
help="Skip cd generation" )
oparser.add_option( "--directory", dest="dir",
help="Write Makefile into specified directory",
metavar="FILE" )
oparser.add_option( "--buildtype", dest="buildtype",
help="Override the buildtype" )
oparser.add_option( "--build-source", action="store_true",
dest="buildsources", default=False,
help="Build source cdrom" )
oparser.add_option( "--proxy", dest="proxy",
help="Override the http Proxy" )
(opt,args) = oparser.parse_args(argv)
if len(args) == 0:
print "No Filename specified"
oparser.print_help()
sys.exit(20)
if len(args) > 1:
print "too many filenames specified"
oparser.print_help()
sys.exit(20)
try:
if not opt.skip_validation:
if not validate_xml( args[0] ):
print "xml validation failed. Bailing out"
sys.exit(20)
xml = etree( args[0] )
except:
print "Unable to open xml File. Bailing out"
sys.exit(20)
if opt.buildtype:
buildtype = opt.buildtype
elif xml.has( "project/buildtype" ):
buildtype = xml.text( "/project/buildtype" )
else:
buildtype = "nodefaults"
if opt.proxy:
http_proxy = opt.proxy
elif xml.has("project/mirror/primary_proxy"):
http_proxy = xml.text("project/mirror/primary_proxy")
else:
http_proxy = ""
defs = ElbeDefaults( buildtype )
if xml.node("/project/mirror/url-list"):
for n in xml.node("/project/mirror/url-list"):
if n.has("binary"):
if n.text("binary").find("localhost") != -1:
print "localhost is not allowed as url - use LOCALMACHINE to refer to your system."
sys.exit(40)
if n.has("source"):
if n.text("source").find("localhost") != -1:
print "localhost is not allowed as url - use LOCALMACHINE to refer to your system."
sys.exit(40)
if not opt.dir:
path = "./build"
else:
path = opt.dir
try:
os.makedirs(path)
except:
print 'unable to create project directory: %s' % path
sys.exit(30)
out_path = os.path.join(path,".elbe-in")
try:
os.makedirs(out_path)
except:
print 'unable to create subdirectory: %s' % out_path
sys.exit(30)
d = {"elbe_version": elbe_version,
"opt": opt,
"xml": xml,
"prj": xml.node("/project"),
"tgt": xml.node("/target"),
"pkgs": xml.node("/target/pkg-list"),
"fine": xml.node("/finetuning"),
"defs": defs,
"http_proxy": http_proxy,
"buildchroot": False,
"preseed": get_preseed(xml) }
try:
copy_kinitrd(xml, out_path, defs)
except:
print "Failure to download kernel/initrd debian Package"
print "Check your source URLs"
sys.exit(20)
if xml.has("archive"):
unbase( xml.text("/archive"), os.path.join(out_path,"archive.tar.bz2") )
templates = os.listdir( template_dir )
make_executable = [ "02pinning.mako",
"finetuning.sh.mako",
"changeroot-into-buildenv.sh.mako",
"cp-scipts-into-buildenv.sh.mako",
"create-target-rfs.sh.mako",
"part-target.sh.mako",
"post-inst.sh.mako",
"print_licence.sh.mako",
"mkcdrom.sh.mako" ]
for t in templates:
print t
o = t.replace( ".mako", "" )
if t == "Makefile.mako":
write_template(os.path.join(path,o), os.path.join(template_dir, t), d )
else:
write_template(os.path.join(out_path,o), os.path.join(template_dir, t), d )
if t in make_executable:
os.chmod( os.path.join(out_path,o), 0755 )
shutil.copyfile( args[0],
os.path.join(out_path, "source.xml" ) )
shutil.copyfile( os.path.join( pack_dir, "treeutils.py" ),
os.path.join(out_path, "treeutils.py" ) )
shutil.copyfile( os.path.join( pack_dir, "version.py" ),
os.path.join(out_path, "version.py" ) )
shutil.copyfile( os.path.join( pack_dir, "dump.py" ),
os.path.join(out_path, "dump.py" ) )
shutil.copyfile( os.path.join( pack_dir, "hdimg.py" ),
os.path.join(out_path, "hdimg.py" ) )
shutil.copyfile( os.path.join( pack_dir, "fstab.py" ),
os.path.join(out_path, "fstab.py" ) )
def action_select(upd_file, status):
status.log("updating: " + upd_file)
try:
upd_file_z = ZipFile(upd_file)
except BadZipfile:
status.log("update aborted (bad zip file: %s)" % upd_file)
return
if not "new.xml" in upd_file_z.namelist():
status.log("update invalid (new.xml missing)")
return
with rw_access("/tmp", status):
upd_file_z.extract("new.xml", "/tmp/")
xml = etree("/tmp/new.xml")
prefix = status.repo_dir + "/" + fname_replace(xml.text("/project/name"))
prefix += "_" + fname_replace(xml.text("/project/version")) + "/"
status.log("preparing update: " + prefix)
with rw_access(prefix, status):
for i in upd_file_z.namelist():
(dirname, filename) = os.path.split(i)
try:
zi = upd_file_z.getinfo(i)
upd_file_z.extract(zi, prefix)
os.chmod(prefix + '/' + i, zi.external_attr >> 16)
except OSError:
status.log("extraction failed: %s" % sys.exc_info()[1])
return
with rw_access("/var/cache/elbe", status):
if os.path.isfile(prefix + '/' + 'pre.sh'):
try:
copy(prefix + '/' + 'pre.sh', '/var/cache/elbe/' + 'pre.sh')
except OSError as e:
status.log('presh-copy failed: ' + str(e))
except IOError as e:
status.log('presh-copy failed: ' + str(e))
if os.path.isfile(prefix + '/' + 'post.sh'):
try:
copy(prefix + '/' + 'post.sh', '/var/cache/elbe/' + 'post.sh')
except OSError as e:
status.log('postsh-copy failed: ' + str(e))
except IOError as e:
status.log('postsh-copy failed: ' + str(e))
if os.path.isdir(prefix + "conf"):
status.log("copying config files:")
for path, pathname, filenames in os.walk(prefix + "conf"):
dst = path[len(prefix + "conf"):]
with rw_access(dst, status):
for f in filenames:
src = os.path.join(path, f)
status.log("cp " + src + " " + dst)
try:
mkdir_p(dst)
copyfile(src, dst + '/' + f)
except OSError as e:
status.log('failed: ' + str(e))
except IOError as e:
status.log('failed: ' + str(e))
with rw_access(prefix + "conf", status):
rmtree(prefix + "conf")
if os.path.isdir(prefix + "cmd"):
status.log("executing scripts:")
for path, pathname, filenames in os.walk(prefix + "cmd"):
for f in filenames:
cmd = os.path.join(path, f)
if os.path.isfile(cmd):
status.log('exec: ' + cmd)
try:
execute(cmd, status)
except OSError as e:
status.log('exec: ' + cmd + ' - ' + str(e))
with rw_access(prefix + "cmd", status):
rmtree(prefix + "cmd")
if os.path.isdir(prefix + "repo"):
try:
update_sourceslist(xml, prefix + "repo", status)
except Exception, err:
status.log(str(err))
status.set_finished('error')
status.log("update apt sources list failed: " + prefix)
return
try:
apply_update("/tmp/new.xml", status)
except Exception, err:
status.log(str(err))
status.set_finished('error')
status.log("apply update failed: " + prefix)
return
def __init__(self):
self.outxml = etree(None)
self.pkglist = self.outxml.setroot('pkgchangelogs')
def run_command(argv):
pack_dir = elbepack.__path__[0]
template_dir = os.path.join(pack_dir, "mako")
oparser = OptionParser(usage="usage: %prog create [options] <filename>")
oparser.add_option("--oldkvm",
action="store_true",
dest="oldkvm",
default=False,
help="We are building for an old kvm version")
oparser.add_option("--debug",
action="store_true",
dest="debug",
default=False,
help="Enable various features to debug the build")
oparser.add_option("--skip-validation",
action="store_true",
dest="skip_validation",
default=False,
help="Skip xml schema validation")
oparser.add_option("--skip-cds",
action="store_true",
dest="skip_cds",
default=False,
help="Skip cd generation")
oparser.add_option("--directory",
dest="dir",
help="Write Makefile into specified directory",
metavar="FILE")
oparser.add_option("--buildtype",
dest="buildtype",
help="Override the buildtype")
oparser.add_option("--build-source",
action="store_true",
dest="buildsources",
default=False,
help="Build source cdrom")
oparser.add_option("--proxy", dest="proxy", help="Override the http Proxy")
(opt, args) = oparser.parse_args(argv)
if len(args) == 0:
print "No Filename specified"
oparser.print_help()
sys.exit(20)
if len(args) > 1:
print "too many filenames specified"
oparser.print_help()
sys.exit(20)
try:
if not opt.skip_validation:
if not validate_xml(args[0]):
print "xml validation failed. Bailing out"
sys.exit(20)
xml = etree(args[0])
except:
print "Unable to open xml File. Bailing out"
sys.exit(20)
if opt.buildtype:
buildtype = opt.buildtype
elif xml.has("project/buildtype"):
buildtype = xml.text("/project/buildtype")
else:
buildtype = "nodefaults"
if opt.proxy:
http_proxy = opt.proxy
elif xml.has("project/mirror/primary_proxy"):
http_proxy = xml.text("project/mirror/primary_proxy")
else:
http_proxy = ""
defs = ElbeDefaults(buildtype)
if xml.node("/project/mirror/url-list"):
for n in xml.node("/project/mirror/url-list"):
if n.has("binary"):
if n.text("binary").find("localhost") != -1:
print "localhost is not allowed as url - use LOCALMACHINE to refer to your system."
sys.exit(40)
if n.has("source"):
if n.text("source").find("localhost") != -1:
print "localhost is not allowed as url - use LOCALMACHINE to refer to your system."
sys.exit(40)
if not opt.dir:
path = "./build"
else:
path = opt.dir
try:
os.makedirs(path)
except:
print 'unable to create project directory: %s' % path
sys.exit(30)
out_path = os.path.join(path, ".elbe-in")
try:
os.makedirs(out_path)
except:
print 'unable to create subdirectory: %s' % out_path
sys.exit(30)
d = {
"elbe_version": elbe_version,
"opt": opt,
"xml": xml,
"prj": xml.node("/project"),
"tgt": xml.node("/target"),
"pkgs": xml.node("/target/pkg-list"),
"fine": xml.node("/finetuning"),
"defs": defs,
"http_proxy": http_proxy,
"buildchroot": False,
"preseed": get_preseed(xml)
}
try:
copy_kinitrd(xml, out_path, defs)
except:
print "Failure to download kernel/initrd debian Package"
print "Check your source URLs"
sys.exit(20)
if xml.has("archive"):
unbase(xml.text("/archive"), os.path.join(out_path, "archive.tar.bz2"))
templates = os.listdir(template_dir)
make_executable = [
"02pinning.mako", "finetuning.sh.mako",
"changeroot-into-buildenv.sh.mako", "cp-scipts-into-buildenv.sh.mako",
"create-target-rfs.sh.mako", "part-target.sh.mako",
"post-inst.sh.mako", "print_licence.sh.mako", "mkcdrom.sh.mako"
]
for t in templates:
print t
o = t.replace(".mako", "")
if t == "Makefile.mako":
write_template(os.path.join(path, o),
os.path.join(template_dir, t), d)
else:
write_template(os.path.join(out_path, o),
os.path.join(template_dir, t), d)
if t in make_executable:
os.chmod(os.path.join(out_path, o), 0755)
shutil.copyfile(args[0], os.path.join(out_path, "source.xml"))
shutil.copyfile(os.path.join(pack_dir, "treeutils.py"),
os.path.join(out_path, "treeutils.py"))
shutil.copyfile(os.path.join(pack_dir, "version.py"),
os.path.join(out_path, "version.py"))
shutil.copyfile(os.path.join(pack_dir, "dump.py"),
os.path.join(out_path, "dump.py"))
shutil.copyfile(os.path.join(pack_dir, "hdimg.py"),
os.path.join(out_path, "hdimg.py"))
shutil.copyfile(os.path.join(pack_dir, "fstab.py"),
os.path.join(out_path, "fstab.py"))
def run_command(argv):
oparser = OptionParser(usage="usage: %prog show [options] <filename>")
oparser.add_option("--verbose",
action="store_true",
dest="verbose",
default=False,
help="show detailed project informations")
oparser.add_option("--skip-validation",
action="store_true",
dest="skip_validation",
default=False,
help="Skip xml schema validation")
(opt, args) = oparser.parse_args(argv)
if len(args) == 0:
print "No Filename specified"
oparser.print_help()
sys.exit(20)
if len(args) > 1:
print "too many filenames specified"
oparser.print_help()
sys.exit(20)
try:
if not opt.skip_validation:
validation = validate_xml(args[0])
if len(validation) != 0:
print "xml validation failed. Bailing out"
for i in validation:
print i
sys.exit(20)
xml = etree(args[0])
except:
print "Unable to open xml File. Bailing out"
sys.exit(20)
if not xml.has("./project"):
print "no project description available"
sys.exit(20)
print '== %s ==' % (args[0])
print 'Debian suite: %s' % (xml.text("./project/suite"))
for s in xml.text("./project/description").splitlines():
print '%s' % s.strip()
if opt.verbose:
print 'root password: %s' % xml.text("./target/passwd")
print 'primary_mirror: %s://%s%s' % (
xml.text("./project/mirror/primary_proto"),
xml.text("./project/mirror/primary_host"),
xml.text("./project/mirror/primary_path"))
if xml.has("./project/mirror/url-list"):
print 'additional mirrors:'
for url in xml.node("./project/mirror/url-list"):
if url.has("binary"):
print ' deb %s' % url.text("binary").strip()
if url.has("source"):
print ' deb-src %s' % url.text("source").strip()
print 'packages:'
for pkg in xml.node("./target/pkg-list"):
print ' %s' % pkg.et.text
print 'skip package validation: %s' % xml.has("./project/noauth")
print 'archive embedded? %s' % xml.has("./archive")
def run_command(argv):
oparser = OptionParser(
usage="usage: %prog check_updates [options] <source-xmlfile>")
oparser.add_option(
"-s",
"--script",
dest="script",
help="filename of script to run when an update is required")
oparser.add_option("--skip-validation",
action="store_true",
dest="skip_validation",
default=False,
help="Skip xml schema validation")
(opt, args) = oparser.parse_args(argv)
if len(args) != 1:
print "Wrong number of arguments"
oparser.print_help()
sys.exit(20)
if not opt.skip_validation:
validation = validate_xml(args[0])
if len(validation) != 0:
print "xml validation failed. Bailing out"
for i in validation:
print i
sys.exit(20)
print "checking %s" % args[0]
xml = etree(args[0])
if xml.has("project/buildtype"):
buildtype = xml.text("/project/buildtype")
else:
buildtype = "nodefaults"
defs = ElbeDefaults(buildtype)
arch = xml.text("project/buildimage/arch", default=defs, key="arch")
suite = xml.text("project/suite")
name = xml.text("project/name", default=defs, key="name")
apt_sources = xml.text("sources_list").replace("10.0.2.2", "localhost")
apt_prefs = xml.text("apt_prefs")
fullp = xml.node("fullpkgs")
v = virtapt.VirtApt(name, arch, suite, apt_sources, apt_prefs)
d = virtapt.apt_pkg.DepCache(v.cache)
d.read_pinfile(v.projectpath + "/etc/apt/preferences")
for p in fullp:
pname = p.et.text
pver = p.et.get('version')
pauto = p.et.get('auto')
if pauto != "true":
d.mark_install(v.cache[pname])
errors = 0
required_updates = 0
for p in fullp:
pname = p.et.text
pver = p.et.get('version')
pauto = p.et.get('auto')
if not pname in v.cache:
if pauto == 'false':
print pname, "does not exist in cache but is specified in pkg-list"
errors += 1
else:
print pname, "is no more required"
required_updates += 1
continue
centry = v.cache[pname]
if d.marked_install(centry):
cver = d.get_candidate_ver(v.cache[pname]).ver_str
if pver != cver:
print pname, "%s != %s" % (pver, cver)
required_updates += 1
sys.stdout.flush()
sys.stderr.flush()
if errors > 0:
print errors, "Errors occured, xml files needs fixing"
if opt.script:
os.system("%s ERRORS %s" % (opt.script, args[0]))
elif required_updates > 0:
print required_updates, "updates required"
if opt.script:
os.system("%s UPDATE %s" % (opt.script, args[0]))
else:
print "No Updates available"
def __init__(self):
self.outxml = etree(None)
self.pkglist = self.outxml.setroot('pkglicenses')
def run_command(argv):
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
oparser = OptionParser(usage="usage: %prog init [options] <filename>")
oparser.add_option("--skip-validation", action="store_true",
dest="skip_validation", default=False,
help="Skip xml schema validation")
oparser.add_option("--directory", dest="directory", default="./build",
help="Working directory (default is build)",
metavar="FILE")
oparser.add_option(
"--cdrom",
dest="cdrom",
help="Use FILE as cdrom iso, and use that to build the initvm",
metavar="FILE")
oparser.add_option("--buildtype", dest="buildtype",
help="Override the buildtype")
oparser.add_option(
"--debug",
dest="debug",
action="store_true",
default=False,
help="start qemu in graphical mode to enable console switch")
oparser.add_option(
"--devel",
dest="devel",
action="store_true",
default=False,
help="use devel mode, and install current builddir inside initvm")
oparser.add_option(
"--nesting",
dest="nesting",
action="store_true",
default=False,
help="allow initvm to support nested kvm. "
"This makes /proc/cpuinfo inside initvm differ per host.")
oparser.add_option(
"--skip-build-bin",
action="store_false",
dest="build_bin",
default=True,
help="Skip building Binary Repository CDROM, for exact Reproduction")
oparser.add_option(
"--skip-build-sources",
action="store_false",
dest="build_sources",
default=True,
help="Skip building Source CDROM")
(opt, args) = oparser.parse_args(argv)
if not args:
print("no filename specified")
oparser.print_help()
sys.exit(20)
elif len(args) > 1:
print("too many filenames specified")
oparser.print_help()
sys.exit(20)
with elbe_logging({"files": None}):
if opt.devel:
if not os.path.isdir(os.path.join(elbe_dir, "elbepack")):
logging.error("Devel Mode only valid, "
"when running from elbe checkout")
sys.exit(20)
if not opt.skip_validation:
validation = validate_xml(args[0])
if validation:
logging.error("xml validation failed. Bailing out")
for i in validation:
logging.error(i)
sys.exit(20)
xml = etree(args[0])
if not xml.has("initvm"):
logging.error("fatal error: "
"xml missing mandatory section 'initvm'")
sys.exit(20)
if opt.buildtype:
buildtype = opt.buildtype
elif xml.has("initvm/buildtype"):
buildtype = xml.text("/initvm/buildtype")
else:
buildtype = "nodefaults"
defs = ElbeDefaults(buildtype)
http_proxy = xml.text("/initvm/mirror/primary_proxy", default="")
http_proxy = http_proxy.strip().replace("LOCALMACHINE", "localhost")
if opt.cdrom:
mirror = xml.node("initvm/mirror")
mirror.clear()
cdrom = mirror.ensure_child("cdrom")
cdrom.set_text(os.path.abspath(opt.cdrom))
# this is a workaround for
# http://lists.linutronix.de/pipermail/elbe-devel/2017-July/000541.html
_, virt = command_out('test -x /usr/bin/systemd-detect-virt && '
'/usr/bin/systemd-detect-virt')
_, dist = command_out('lsb_release -cs')
if 'vmware' in virt and 'stretch' in dist:
machine_type = 'pc-i440fx-2.6'
else:
machine_type = 'pc'
try:
os.makedirs(opt.directory)
except OSError as e:
logging.error("unable to create project directory: %s (%s)",
opt.directory,
e.strerror)
sys.exit(30)
out_path = os.path.join(opt.directory, ".elbe-in")
try:
os.makedirs(out_path)
except OSError as e:
logging.error("unable to create subdirectory: %s (%s)",
out_path,
e.strerror)
sys.exit(30)
initvm_http_proxy = http_proxy.replace('http://localhost:',
'http://10.0.2.2:')
d = {"elbe_version": elbe_version,
"defs": defs,
"opt": opt,
"xml": xml,
"prj": xml.node("/initvm"),
"http_proxy": initvm_http_proxy,
"pkgs": xml.node("/initvm/pkg-list") or [],
"preseed": get_initvm_preseed(xml),
"machine_type": machine_type,
"cfg": cfg}
if http_proxy != "":
os.putenv("http_proxy", http_proxy)
os.putenv("https_proxy", http_proxy)
os.putenv("no_proxy", "localhost,127.0.0.1")
try:
copy_kinitrd(xml.node("/initvm"), out_path)
except NoKinitrdException as e:
msg = str(e)
logging.error("Failure to download kernel/initrd debian Package:")
logging.error("")
logging.error(msg)
logging.error("")
logging.error("Check Mirror configuration")
if 'SHA256SUMS' in msg:
logging.error("If you use debmirror please read "
"https://github.com/Linutronix/elbe/issues/188 "
"on how to work around the issue")
sys.exit(20)
templates = os.listdir(init_template_dir)
make_executable = ["init-elbe.sh.mako",
"preseed.cfg.mako"]
for t in templates:
o = t.replace(".mako", "")
if t in ("Makefile.mako", "libvirt.xml.mako"):
write_template(
os.path.join(
opt.directory, o), os.path.join(
init_template_dir, t), d, linebreak=True)
else:
write_template(
os.path.join(
out_path, o), os.path.join(
init_template_dir, t), d, linebreak=False)
if t in make_executable:
os.chmod(os.path.join(out_path, o), 0o755)
shutil.copyfile(args[0],
os.path.join(out_path, "source.xml"))
if opt.cdrom:
system('7z x -o%s "%s" elbe-keyring.gpg' % (out_path, opt.cdrom))
else:
keys = []
for key in xml.all(".//initvm/mirror/url-list/url/raw-key"):
keys.append(key.et.text)
import_keyring = os.path.join(out_path, "elbe-keyring")
do('gpg --no-options \
--no-default-keyring \
--keyring %s --import' % import_keyring,
stdin="".join(keys).encode('ascii'),
allow_fail=True,
env_add={'GNUPGHOME': out_path})
export_keyring = import_keyring + ".gpg"
# No need to set GNUPGHOME because both input and output
# keyring files are specified.
do('gpg --no-options \
--no-default-keyring \
--keyring %s \
--export \
--output %s' % (import_keyring, export_keyring))
if opt.devel:
out_real = os.path.realpath(out_path)
opts = []
if out_real.startswith(elbe_dir + os.sep):
opts.append('--exclude "%s"' %
os.path.relpath(out_path, start=elbe_dir))
opts.append("--exclude-vcs")
opts.append("--exclude-vcs-ignores")
opts.append("--exclude='elbe-build*'")
opts.append("--exclude='docs/*'")
tar_fname = os.path.join(out_path, "elbe-devel.tar.bz2")
system('tar cfj "%s" %s -C "%s" .' % (tar_fname,
" ".join(opts),
elbe_dir))
to_cpy = [("apt.conf", "etc/apt"),
("init-elbe.sh", ""),
("source.xml", ""),
("initrd-cdrom.gz", ""),
("vmlinuz", ""),
("preseed.cfg", "")]
elbe_in = Filesystem(out_path)
if opt.devel:
to_cpy.append(("elbe-devel.tar.bz2", ""))
# Convert relative rfs path to absolute in the system
to_cpy = [(elbe_in.fname(src), elbe_in.fname(os.path.join("initrd-tree", dst)))
for src, dst
in to_cpy]
# These are already absolute path!
keyrings = elbe_in.fname(os.path.join("initrd-tree", "usr/share/keyrings"))
for gpg in elbe_in.glob("*.gpg"):
to_cpy.append((gpg, keyrings))
for src, dst in to_cpy:
try:
os.makedirs(dst)
except FileExistsError:
pass
shutil.copy(src, dst)
def execute(self, initvmdir, opt, args):
# pylint: disable=too-many-branches
# pylint: disable=too-many-statements
if self.initvm is not None:
print("Initvm is already defined for the libvirt domain '%s'.\n" % cfg['initvm_domain'])
print("If you want to build in your old initvm, "
"use `elbe initvm submit <xml>`.")
print("If you want to remove your old initvm from libvirt "
"run `virsh --connect qemu:///system undefine %s`.\n" % cfg['initvm_domain'])
print("You can specify another libvirt domain by setting the "
"ELBE_INITVM_DOMAIN environment variable to an unused domain name.\n")
print("Note:")
print("\t1) You can reimport your old initvm via "
"`virsh --connect qemu:///system define <file>`")
print("\t where <file> is the corresponding libvirt.xml")
print("\t2) virsh --connect qemu:///system undefine does not delete the image "
"of your old initvm.")
sys.exit(20)
# Upgrade from older versions which used tmux
try:
system("tmux has-session -t ElbeInitVMSession 2>/dev/null")
print ("ElbeInitVMSession exists in tmux. "
"It may belong to an old elbe version. "
"Please stop it to prevent interfering with this version.", file=sys.stderr)
sys.exit(20)
except CommandError:
pass
# Init cdrom to None, if we detect it, we set it
cdrom = None
if len(args) == 1:
if args[0].endswith('.xml'):
# We have an xml file, use that for elbe init
xmlfile = args[0]
try:
xml = etree(xmlfile)
except ValidationError as e:
print("XML file is invalid: %s" % str(e))
# Use default XML if no initvm was specified
if not xml.has("initvm"):
xmlfile = os.path.join(
elbepack.__path__[0], "init/default-init.xml")
elif args[0].endswith('.iso'):
# We have an iso image, extract xml from there.
tmp = extract_cdrom(args[0])
xmlfile = tmp.fname('source.xml')
cdrom = args[0]
else:
print(
"Unknown file ending (use either xml or iso)",
file=sys.stderr)
sys.exit(20)
else:
# No xml File was specified, build the default elbe-init-with-ssh
xmlfile = os.path.join(
elbepack.__path__[0],
"init/default-init.xml")
try:
init_opts = ''
if opt.devel:
init_opts += ' --devel'
if opt.nesting:
init_opts += ' --nesting'
if not opt.build_bin:
init_opts += ' --skip-build-bin'
if not opt.build_sources:
init_opts += ' --skip-build-source'
if cdrom:
system('%s init %s --directory "%s" --cdrom "%s" "%s"' %
(elbe_exe, init_opts, initvmdir, cdrom, xmlfile))
else:
system(
'%s init %s --directory "%s" "%s"' %
(elbe_exe, init_opts, initvmdir, xmlfile))
except CommandError:
print("'elbe init' Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
# Read xml file for libvirt
with open(os.path.join(initvmdir, 'libvirt.xml')) as f:
xml = f.read()
# Register initvm in libvirt
try:
self.conn.defineXML(xml)
except CommandError:
print('Registering initvm in libvirt failed', file=sys.stderr)
print('Try `virsh --connect qemu:///system undefine %s` to delete existing initvm' % cfg['initvm_domain'],
file=sys.stderr)
sys.exit(20)
# Build initvm
try:
system('cd "%s"; make' % (initvmdir))
except CommandError:
print("Building the initvm Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
try:
system('%s initvm start' % elbe_exe)
except CommandError:
print("Starting the initvm Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
if len(args) == 1:
# if provided xml file has no initvm section xmlfile is set to a
# default initvm XML file. But we need the original file here
if args[0].endswith('.xml'):
# stop here if no project node was specified
try:
x = etree(args[0])
except ValidationError as e:
print("XML file is invalid: %s" % str(e))
sys.exit(20)
if not x.has('project'):
print("elbe initvm ready: use 'elbe initvm submit "
"myproject.xml' to build a project")
sys.exit(0)
xmlfile = args[0]
elif cdrom is not None:
xmlfile = tmp.fname('source.xml')
submit_and_dl_result(xmlfile, cdrom, opt)
def execute(self, initvmdir, opt, args):
# pylint: disable=too-many-branches
# pylint: disable=too-many-statements
if self.initvm is not None:
print("Initvm is already defined for the libvirt domain '%s'.\n" %
cfg['initvm_domain'])
print("If you want to build in your old initvm, "
"use `elbe initvm submit <xml>`.")
print("If you want to remove your old initvm from libvirt "
"run `virsh --connect qemu:///system undefine %s`.\n" %
cfg['initvm_domain'])
print(
"You can specify another libvirt domain by setting the "
"ELBE_INITVM_DOMAIN environment variable to an unused domain name.\n"
)
print("Note:")
print("\t1) You can reimport your old initvm via "
"`virsh --connect qemu:///system define <file>`")
print("\t where <file> is the corresponding libvirt.xml")
print(
"\t2) virsh --connect qemu:///system undefine does not delete the image "
"of your old initvm.")
sys.exit(20)
# Upgrade from older versions which used tmux
try:
system("tmux has-session -t ElbeInitVMSession 2>/dev/null")
print(
"ElbeInitVMSession exists in tmux. "
"It may belong to an old elbe version. "
"Please stop it to prevent interfering with this version.",
file=sys.stderr)
sys.exit(20)
except CommandError:
pass
# Init cdrom to None, if we detect it, we set it
cdrom = None
if len(args) == 1:
if args[0].endswith('.xml'):
# We have an xml file, use that for elbe init
xmlfile = args[0]
try:
xml = etree(xmlfile)
except ValidationError as e:
print("XML file is invalid: %s" % str(e))
# Use default XML if no initvm was specified
if not xml.has("initvm"):
xmlfile = os.path.join(elbepack.__path__[0],
"init/default-init.xml")
elif args[0].endswith('.iso'):
# We have an iso image, extract xml from there.
tmp = extract_cdrom(args[0])
xmlfile = tmp.fname('source.xml')
cdrom = args[0]
else:
print("Unknown file ending (use either xml or iso)",
file=sys.stderr)
sys.exit(20)
else:
# No xml File was specified, build the default elbe-init-with-ssh
xmlfile = os.path.join(elbepack.__path__[0],
"init/default-init.xml")
try:
init_opts = ''
if opt.devel:
init_opts += ' --devel'
if opt.nesting:
init_opts += ' --nesting'
if not opt.build_bin:
init_opts += ' --skip-build-bin'
if not opt.build_sources:
init_opts += ' --skip-build-source'
with PreprocessWrapper(xmlfile, opt) as ppw:
if cdrom:
system(
'%s init %s --directory "%s" --cdrom "%s" "%s"' %
(elbe_exe, init_opts, initvmdir, cdrom, ppw.preproc))
else:
system('%s init %s --directory "%s" "%s"' %
(elbe_exe, init_opts, initvmdir, ppw.preproc))
except CommandError:
print("'elbe init' Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
# Read xml file for libvirt
with open(os.path.join(initvmdir, 'libvirt.xml')) as f:
xml = f.read()
# Register initvm in libvirt
try:
self.conn.defineXML(xml)
except CommandError:
print('Registering initvm in libvirt failed', file=sys.stderr)
print(
'Try `virsh --connect qemu:///system undefine %s` to delete existing initvm'
% cfg['initvm_domain'],
file=sys.stderr)
sys.exit(20)
# Build initvm
try:
system('cd "%s"; make' % (initvmdir))
except CommandError:
print("Building the initvm Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
try:
system('%s initvm start' % elbe_exe)
except CommandError:
print("Starting the initvm Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
if len(args) == 1:
# if provided xml file has no initvm section xmlfile is set to a
# default initvm XML file. But we need the original file here
if args[0].endswith('.xml'):
# stop here if no project node was specified
try:
x = etree(args[0])
except ValidationError as e:
print("XML file is invalid: %s" % str(e))
sys.exit(20)
if not x.has('project'):
print("elbe initvm ready: use 'elbe initvm submit "
"myproject.xml' to build a project")
sys.exit(0)
xmlfile = args[0]
elif cdrom is not None:
xmlfile = tmp.fname('source.xml')
submit_and_dl_result(xmlfile, cdrom, opt)
def get_target_version(fname):
xml = etree(fname)
return xml.text("/project/version")
def _apply_update(fname, status):
try:
xml = etree(fname)
except:
return "read %s failed" % fname
fpl = xml.node("fullpkgs")
sources = apt_pkg.SourceList()
sources.read_main_list()
status.log("initialize apt")
apt_pkg.init()
cache = apt_pkg.Cache(progress=ElbeOpProgress(cb=status.log))
status.set_progress(1)
status.log("updating package cache")
cache.update(ElbeAcquireProgress(cb=status.log), sources)
# quote from python-apt api doc: "A call to this method does not affect the
# current Cache object, instead a new one should be created in order to use
# the changed index files."
cache = apt_pkg.Cache(progress=ElbeOpProgress(cb=status.log))
depcache = apt_pkg.DepCache(cache)
hl_cache = apt.cache.Cache(progress=ElbeOpProgress(cb=status.log))
hl_cache.update(fetch_progress=ElbeAcquireProgress(cb=status.log))
# go through package cache, if a package is in the fullpkg list of the XML
# mark the package for installation (with the specified version)
# if it is not mentioned in the fullpkg list purge the package out of the
# system.
status.set_progress(2)
status.log("calculating packages to install/remove")
count = len(hl_cache)
step = count / 10
i = 0
percent = 0
for p in hl_cache:
i = i + 1
if not (i % step):
percent = percent + 10
status.log(str(percent) + "% - " + str(i) + "/" + str(count))
status.set_progress(2, str(percent) + "%")
pkg = cache[p.name]
marked = False
for fpi in fpl:
if pkg.name == fpi.et.text:
mark_install(depcache, pkg, fpi.et.get('version'),
fpi.et.get('auto'), status)
marked = True
if not marked:
depcache.mark_delete(pkg, True)
status.set_progress(3)
status.log("applying snapshot")
depcache.commit(ElbeAcquireProgress(cb=status.log),
ElbeInstallProgress(cb=status.log))
del depcache
del hl_cache
del cache
del sources
version_file = open("/etc/updated_version", "w")
version_file.write(xml.text("/project/version"))
version_file.close()
def action_select(upd_file, status):
# pylint: disable=too-many-locals
# pylint: disable=too-many-return-statements
# pylint: disable=too-many-branches
# pylint: disable=too-many-statements
status.log("updating: " + upd_file)
try:
upd_file_z = ZipFile(upd_file)
except BadZipfile:
status.log("update aborted (bad zip file: %s)" % upd_file)
return
if "new.xml" not in upd_file_z.namelist():
status.log("update invalid (new.xml missing)")
return
with rw_access("/tmp", status):
upd_file_z.extract("new.xml", "/tmp/")
# prevent downgrades (if available)
if downgrade_prevention_feature_available:
try:
if reject_downgrade(status, "/tmp/new.xml"):
return
except Exception as e:
status.log('Error while reading XML files occurred: ' + str(e))
return
xml = etree("/tmp/new.xml")
prefix = status.repo_dir + "/" + fname_replace(xml.text("/project/name"))
prefix += "_" + fname_replace(xml.text("/project/version")) + "/"
status.log("preparing update: " + prefix)
with rw_access(prefix, status):
for i in upd_file_z.namelist():
try:
zi = upd_file_z.getinfo(i)
upd_file_z.extract(zi, prefix)
os.chmod(prefix + '/' + i, zi.external_attr >> 16)
except OSError:
status.log("extraction failed: %s" % sys.exc_info()[1])
return
with rw_access("/var/cache/elbe", status):
if os.path.isfile(prefix + '/' + 'pre.sh'):
try:
copy(prefix + '/' + 'pre.sh', '/var/cache/elbe/' + 'pre.sh')
except OSError as e:
status.log('presh-copy failed: ' + str(e))
except IOError as e:
status.log('presh-copy failed: ' + str(e))
if os.path.isfile(prefix + '/' + 'post.sh'):
try:
copy(prefix + '/' + 'post.sh', '/var/cache/elbe/' + 'post.sh')
except OSError as e:
status.log('postsh-copy failed: ' + str(e))
except IOError as e:
status.log('postsh-copy failed: ' + str(e))
if os.path.isdir(prefix + "conf"):
status.log("copying config files:")
for path, _, filenames in os.walk(prefix + "conf"):
dst = path[len(prefix + "conf"):]
with rw_access(dst, status):
for f in filenames:
src = os.path.join(path, f)
status.log("cp " + src + " " + dst)
try:
mkdir_p(dst)
copyfile(src, dst + '/' + f)
except OSError as e:
status.log('failed: ' + str(e))
except IOError as e:
status.log('failed: ' + str(e))
with rw_access(prefix + "conf", status):
rmtree(prefix + "conf")
if os.path.isdir(prefix + "cmd"):
status.log("executing scripts:")
for path, _, filenames in os.walk(prefix + "cmd"):
for f in filenames:
cmd = os.path.join(path, f)
if os.path.isfile(cmd):
status.log('exec: ' + cmd)
try:
execute(cmd, status)
except OSError as e:
status.log('exec: ' + cmd + ' - ' + str(e))
with rw_access(prefix + "cmd", status):
rmtree(prefix + "cmd")
if os.path.isdir(prefix + "repo"):
try:
update_sourceslist(xml, prefix + "repo", status)
except Exception as err:
status.log(str(err))
status.set_finished('error')
status.log("update apt sources list failed: " + prefix)
return
try:
apply_update("/tmp/new.xml", status)
except Exception as err:
status.log(str(err))
status.set_finished('error')
status.log("apply update failed: " + prefix)
return
status.set_finished('OK')
status.log("update done: " + prefix)
def run_command(argv):
oparser = OptionParser(usage="usage: %prog show [options] <filename>")
oparser.add_option(
"--verbose", action="store_true", dest="verbose", default=False, help="show detailed project informations"
)
oparser.add_option(
"--skip-validation",
action="store_true",
dest="skip_validation",
default=False,
help="Skip xml schema validation",
)
(opt, args) = oparser.parse_args(argv)
if len(args) == 0:
print "No Filename specified"
oparser.print_help()
sys.exit(20)
if len(args) > 1:
print "too many filenames specified"
oparser.print_help()
sys.exit(20)
try:
if not opt.skip_validation:
validation = validate_xml(args[0])
if len(validation) != 0:
print "xml validation failed. Bailing out"
for i in validation:
print i
sys.exit(20)
xml = etree(args[0])
except:
print "Unable to open xml File. Bailing out"
sys.exit(20)
if not xml.has("./project"):
print "no project description available"
sys.exit(20)
print "== %s ==" % (args[0])
print "Debian suite: %s" % (xml.text("./project/suite"))
for s in xml.text("./project/description").splitlines():
print "%s" % s.strip()
if opt.verbose:
print "root password: %s" % xml.text("./target/passwd")
print "primary_mirror: %s://%s%s" % (
xml.text("./project/mirror/primary_proto"),
xml.text("./project/mirror/primary_host"),
xml.text("./project/mirror/primary_path"),
)
if xml.has("./project/mirror/url-list"):
print "additional mirrors:"
for url in xml.node("./project/mirror/url-list"):
if url.has("binary"):
print " deb %s" % url.text("binary").strip()
if url.has("source"):
print " deb-src %s" % url.text("source").strip()
print "packages:"
for pkg in xml.node("./target/pkg-list"):
print " %s" % pkg.et.text
print "skip package validation: %s" % xml.has("./project/noauth")
print "archive embedded? %s" % xml.has("./archive")
def action_select(upd_file, status):
status.log("updating: " + upd_file)
try:
upd_file_z = ZipFile(upd_file)
except BadZipfile:
status.log("update aborted (bad zip file: %s)" % upd_file)
return
if not "new.xml" in upd_file_z.namelist():
status.log("update invalid (new.xml missing)")
return
with rw_access("/tmp", status):
upd_file_z.extract("new.xml", "/tmp/")
xml = etree("/tmp/new.xml")
prefix = status.repo_dir + "/" + fname_replace(xml.text("/project/name"))
prefix += "_" + fname_replace(xml.text("/project/version")) + "/"
status.log("preparing update: " + prefix)
with rw_access(prefix, status):
for i in upd_file_z.namelist():
(dirname, filename) = os.path.split(i)
try:
zi = upd_file_z.getinfo(i)
upd_file_z.extract(zi, prefix)
os.chmod(prefix + "/" + i, zi.external_attr >> 16)
except OSError:
status.log("extraction failed: %s" % sys.exc_info()[1])
return
with rw_access("/var/cache/elbe", status):
if os.path.isfile(prefix + "/" + "pre.sh"):
try:
copy(prefix + "/" + "pre.sh", "/var/cache/elbe/" + "pre.sh")
except OSError as e:
status.log("presh-copy failed: " + str(e))
except IOError as e:
status.log("presh-copy failed: " + str(e))
if os.path.isfile(prefix + "/" + "post.sh"):
try:
copy(prefix + "/" + "post.sh", "/var/cache/elbe/" + "post.sh")
except OSError as e:
status.log("postsh-copy failed: " + str(e))
except IOError as e:
status.log("postsh-copy failed: " + str(e))
if os.path.isdir(prefix + "conf"):
status.log("copying config files:")
for path, pathname, filenames in os.walk(prefix + "conf"):
dst = path[len(prefix + "conf") :]
with rw_access(dst, status):
for f in filenames:
src = os.path.join(path, f)
status.log("cp " + src + " " + dst)
try:
mkdir_p(dst)
copyfile(src, dst + "/" + f)
except OSError as e:
status.log("failed: " + str(e))
except IOError as e:
status.log("failed: " + str(e))
with rw_access(prefix + "conf", status):
rmtree(prefix + "conf")
if os.path.isdir(prefix + "cmd"):
status.log("executing scripts:")
for path, pathname, filenames in os.walk(prefix + "cmd"):
for f in filenames:
cmd = os.path.join(path, f)
if os.path.isfile(cmd):
status.log("exec: " + cmd)
try:
execute(cmd, status)
except OSError as e:
status.log("exec: " + cmd + " - " + str(e))
with rw_access(prefix + "cmd", status):
rmtree(prefix + "cmd")
if os.path.isdir(prefix + "repo"):
try:
update_sourceslist(xml, prefix + "repo", status)
except Exception, err:
status.log(str(err))
status.set_finished("error")
status.log("update apt sources list failed: " + prefix)
return
try:
apply_update("/tmp/new.xml", status)
except Exception, err:
status.log(str(err))
status.set_finished("error")
status.log("apply update failed: " + prefix)
return
def execute(self, initvmdir, opt, args):
try:
have_session = os.system(
"tmux has-session -t ElbeInitVMSession >/dev/null 2>&1")
except CommandError as e:
print(
"tmux execution failed, tmux version 1.9 or higher is required"
)
sys.exit(20)
if have_session == 0:
print("ElbeInitVMSession already exists in tmux.", file=sys.stderr)
print("", file=sys.stderr)
print(
"There can only exist a single ElbeInitVMSession, and this session",
file=sys.stderr)
print("can also be used to make your build.", file=sys.stderr)
print(
"See 'elbe initvm submit', 'elbe initvm attach' and 'elbe control'",
file=sys.stderr)
sys.exit(20)
# Init cdrom to None, if we detect it, we set it
cdrom = None
if len(args) == 1:
if args[0].endswith('.xml'):
# We have an xml file, use that for elbe init
exampl = args[0]
try:
xml = etree(exampl)
except ValidationError as e:
print('XML file is inavlid: ' + str(e))
# Use default XML if no initvm was specified
if not xml.has("initvm"):
exampl = os.path.join(elbepack.__path__[0],
"init/default-init.xml")
elif args[0].endswith('.iso'):
# We have an iso image, extract xml from there.
tmp = TmpdirFilesystem()
os.system('7z x -o%s "%s" source.xml' % (tmp.path, args[0]))
if not tmp.isfile('source.xml'):
print('Iso image does not contain a source.xml file',
file=sys.stderr)
print('This is not supported by "elbe initvm"',
file=sys.stderr)
print('', file=sys.stderr)
print('Exiting !!!', file=sys.stderr)
sys.exit(20)
try:
exml = ElbeXML(tmp.fname('source.xml'), skip_urlcheck=True)
except ValidationError as e:
print('Iso image does contain a source.xml file.',
file=sys.stderr)
print('But that xml does not validate correctly',
file=sys.stderr)
print('', file=sys.stderr)
print('Exiting !!!', file=sys.stderr)
sys.exit(20)
print('Iso Image with valid source.xml detected !')
print('Image was generated using Elbe Version %s' %
exml.get_elbe_version())
exampl = tmp.fname('source.xml')
cdrom = args[0]
else:
print('Unknown file ending (use either xml or iso)',
file=sys.stderr)
sys.exit(20)
else:
# No xml File was specified, build the default elbe-init-with-ssh
exampl = os.path.join(elbepack.__path__[0],
"init/default-init.xml")
try:
if opt.devel:
devel = ' --devel'
else:
devel = ''
if cdrom:
system('%s init %s --directory "%s" --cdrom "%s" "%s"' %
(elbe_exe, devel, initvmdir, cdrom, exampl))
else:
system('%s init %s --directory "%s" "%s"' %
(elbe_exe, devel, initvmdir, exampl))
except CommandError:
print("'elbe init' Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
try:
system('cd "%s"; make' % (initvmdir))
except CommandError:
print("Building the initvm Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
try:
system('%s initvm start --directory "%s"' % (elbe_exe, initvmdir))
except CommandError:
print("Starting the initvm Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
if len(args) == 1:
# if provided xml file has no initvm section exampl is set to a
# default initvm XML file. But we need the original file here
if args[0].endswith('.xml'):
# stop here if no project node was specified
try:
x = ElbeXML(args[0])
except ValidationError as e:
print('XML file is inavlid: ' + str(e))
sys.exit(20)
if not x.has('project'):
print(
'elbe initvm ready: use "elbe initvm submit myproject.xml" to build a project'
)
sys.exit(0)
ret, prjdir, err = command_out_stderr(
'%s control create_project "%s"' % (elbe_exe, args[0]))
else:
ret, prjdir, err = command_out_stderr(
'%s control create_project "%s"' % (elbe_exe, exampl))
if ret != 0:
print("elbe control create_project failed.", file=sys.stderr)
print(err, file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
prjdir = prjdir.strip()
if cdrom is not None:
print("Uploading CDROM. This might take a while")
try:
system('%s control set_cdrom "%s" "%s"' %
(elbe_exe, prjdir, cdrom))
except CommandError:
print("elbe control set_cdrom Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("Upload finished")
build_opts = ''
if opt.build_bin:
build_opts += '--build-bin '
if opt.build_sources:
build_opts += '--build-sources '
try:
system('%s control build "%s" %s' %
(elbe_exe, prjdir, build_opts))
except CommandError:
print("elbe control build Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
try:
system('%s control wait_busy "%s"' % (elbe_exe, prjdir))
except CommandError:
print("elbe control wait_busy Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print("Build finished !")
print("")
try:
system('%s control dump_file "%s" validation.txt' %
(elbe_exe, prjdir))
except CommandError:
print("Project failed to generate validation.txt",
file=sys.stderr)
print("Getting log.txt", file=sys.stderr)
try:
system('%s control dump_file "%s" log.txt' %
(elbe_exe, prjdir))
except CommandError:
print("Failed to dump log.txt", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
if opt.skip_download:
print("")
print("Listing available files:")
print("")
try:
system('%s control get_files "%s"' % (elbe_exe, prjdir))
except CommandError:
print("elbe control Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print('Get Files with: elbe control get_file "%s" <filename>' %
prjdir)
else:
ensure_outdir(wdfs, opt)
try:
system('%s control get_files --output "%s" "%s"' %
(elbe_exe, opt.outdir, prjdir))
except CommandError:
print("elbe control get_files Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
