def set_project_version( self, builddir, new_version = None):
if new_version == "":
raise ElbeDBError( "version number must not be empty" )
if not re.match("^[A-Za-z0-9_.-]{1,25}$", new_version):
raise ElbeDBError( "version number must contain valid characters [A-Za-z0-9_-.]" )
with session_scope(self.session) as s:
try:
p = s.query( Project ).filter( Project.builddir == builddir ).\
one()
except NoResultFound:
raise ElbeDBError( "project %s is not registered in the database" %
builddir )
if p.status == "empty_project" or p.status == "busy":
raise ElbeDBError( "project: " + builddir +
" invalid status: " + p.status )
xmlpath = os.path.join( builddir, "source.xml" )
xml = ElbeXML( xmlpath )
if not new_version is None:
xml.node( "/project/version" ).set_text( new_version )
xml.xml.write( xmlpath )
p.version = xml.text( "/project/version" )
def run_command(argv):
oparser = OptionParser(
usage="usage: %prog add [options] <xmlfile> <pkg1> [pkgN]")
(_, args) = oparser.parse_args(argv)
if len(args) < 2:
print("Wrong number of arguments")
oparser.print_help()
sys.exit(20)
try:
xml = ElbeXML(args[0])
except Exception as e:
print("Error reading xml file: %s" % str(e))
sys.exit(20)
for a in args[1:]:
try:
xml.add_target_package(a)
except Exception as e:
print("Error adding package %s: %s" % (a, str(e)))
sys.exit(20)
try:
xml.xml.write(args[0])
except BaseException:
print("Unable to write new xml file")
sys.exit(20)
def set_xml (self, xmlpath):
# Use supplied XML file, if given, otherwise change to source.xml
if not xmlpath:
xmlpath = os.path.join( self.builddir, "source.xml" )
newxml = ElbeXML( xmlpath, buildtype=self.override_buildtype,
skip_validate=self.skip_validate,
skip_urlcheck=self.skip_urlcheck )
# New XML file has to have the same architecture
oldarch = self.xml.text( "project/arch", key="arch" )
newarch = newxml.text( "project/arch", key="arch" )
if newarch != oldarch:
raise IncompatibleArchitectureException( oldarch, newarch )
# Throw away old APT cache, targetfs and buildenv
self._rpcaptcache = None
self.targetfs = None
self.buildenv = None
self.xml = newxml
# Create a new BuildEnv instance, if we have a build directory
if self.has_full_buildenv():
self.buildenv = BuildEnv( self.xml, self.log, self.chrootpath )
# Create TargetFs instance, if the target directory exists.
# We use the old content of the directory if no rebuild is done, so
# don't clean it (yet).
if os.path.exists( self.targetpath ):
self.targetfs = TargetFs( self.targetpath, self.log,
self.buildenv.xml, clean=False )
else:
self.targetfs = None
def __init__ (self, builddir, xmlpath = None, logpath = None, name = None,
override_buildtype = None, skip_validate = False,
skip_urlcheck = False, rpcaptcache_notifier = None,
private_data = None, postbuild_file = None, presh_file = None,
postsh_file = None, savesh_file = None):
self.builddir = os.path.abspath(str(builddir))
self.chrootpath = os.path.join(self.builddir, "chroot")
self.targetpath = os.path.join(self.builddir, "target")
self.name = name
self.override_buildtype = override_buildtype
self.skip_validate = skip_validate
self.skip_urlcheck = skip_urlcheck
self.postbuild_file = postbuild_file
self.presh_file = presh_file
self.postsh_file = postsh_file
self.savesh_file = savesh_file
self.private_data = private_data
# Apt-Cache will be created on demand with the specified notifier by
# the get_rpcaptcache method
self._rpcaptcache = None
self.rpcaptcache_notifier = rpcaptcache_notifier
# Initialise Repo Images to Empty list.
self.repo_images = []
# Use supplied XML file, if given, otherwise use the source.xml
# file of the project
if xmlpath:
self.xml = ElbeXML( xmlpath, buildtype=override_buildtype,
skip_validate=skip_validate, skip_urlcheck=skip_urlcheck )
else:
sourcexmlpath = os.path.join( self.builddir, "source.xml" )
self.xml = ElbeXML( sourcexmlpath, buildtype=override_buildtype,
skip_validate=skip_validate, skip_urlcheck=skip_urlcheck )
# If logpath is given, use an AsciiDocLog instance, otherwise log
# to stdout
if logpath:
self.log = ASCIIDocLog( logpath )
else:
self.log = StdoutLog()
# Create BuildEnv instance, if the chroot directory exists and
# has an etc/elbe_version
if self.has_full_buildenv():
self.buildenv = BuildEnv( self.xml, self.log, self.chrootpath )
else:
self.buildenv = None
self.targetfs = None
return
# Create TargetFs instance, if the target directory exists
if os.path.exists( self.targetpath ):
self.targetfs = TargetFs( self.targetpath, self.log,
self.buildenv.xml, clean=False )
else:
self.targetfs = None
def set_xml(self, builddir, xml_file):
# This method can throw: ElbeDBError, ValidationError, OSError
if not os.path.exists(builddir):
raise ElbeDBError("project directory does not exist")
srcxml_fname = os.path.join(builddir, "source.xml")
if xml_file is None:
xml_file = srcxml_fname
with session_scope(self.session) as s:
p = None
try:
p = s.query(Project). \
filter(Project.builddir == builddir).one()
except NoResultFound:
raise ElbeDBError(
"project %s is not registered in the database" %
builddir)
if p.status == "busy":
raise ElbeDBError(
"cannot set XML file while project %s is busy" %
builddir)
xml = ElbeXML(
xml_file,
url_validation=ValidationMode.NO_CHECK) # ValidationError
p.name = xml.text("project/name")
p.version = xml.text("project/version")
p.edit = datetime.utcnow()
if p.status == "empty_project" or p.status == "build_failed":
p.status = "needs_build"
elif p.status == "build_done":
p.status = "has_changes"
if xml_file != srcxml_fname:
copyfile(xml_file, srcxml_fname) # OSError
_update_project_file(
s,
builddir,
"source.xml",
"application/xml",
"ELBE recipe of the project")
def execute(self, client, _opt, args):
if len(args) != 2:
print(
"usage: elbe control set_xml <project_dir> <xml>",
file=sys.stderr)
sys.exit(20)
builddir = args[0]
filename = args[1]
try:
x = ElbeXML(
filename,
skip_validate=True,
url_validation=ValidationMode.NO_CHECK)
except IOError:
print("%s is not a valid elbe xml file" % filename)
sys.exit(20)
if not x.has('target'):
print("<target> is missing, this file can't be built in an initvm",
file=sys.stderr)
sys.exit(20)
size = 1024 * 1024
part = 0
with open(filename, "rb") as fp:
while True:
xml_base64 = binascii.b2a_base64(fp.read(size))
# finish upload
if len(xml_base64) == 1:
part = client.service.upload_file(builddir,
"source.xml",
xml_base64,
-1)
else:
part = client.service.upload_file(builddir,
"source.xml",
xml_base64,
part)
if part == -1:
print("project busy, upload not allowed")
return part
if part == -2:
print("upload of xml finished")
return 0
def extract_cdrom(cdrom):
""" Extract cdrom iso image
returns a TmpdirFilesystem() object containing
the source.xml, which is also validated.
"""
tmp = TmpdirFilesystem()
os.system('7z x -o%s "%s" source.xml' % (tmp.path, cdrom))
print("", file=sys.stderr)
if not tmp.isfile('source.xml'):
print(
"Iso image does not contain a source.xml file",
file=sys.stderr)
print(
"This is not supported by 'elbe initvm'",
file=sys.stderr)
print("", file=sys.stderr)
print("Exiting !!!", file=sys.stderr)
sys.exit(20)
try:
exml = ElbeXML(
tmp.fname('source.xml'),
url_validation=ValidationMode.NO_CHECK)
except ValidationError as e:
print(
"Iso image does contain a source.xml file.",
file=sys.stderr)
print(
"But that xml does not validate correctly",
file=sys.stderr)
print("", file=sys.stderr)
print("Exiting !!!", file=sys.stderr)
print(e)
sys.exit(20)
print("Iso Image with valid source.xml detected !")
print(
"Image was generated using Elbe Version %s" %
exml.get_elbe_version())
return tmp
def execute(self, client, opt, args):
if len (args) != 1:
print ("usage: elbe control create_project <xmlfile>", file=sys.stderr)
sys.exit(20)
filename = args[0]
if not os.path.isfile (filename):
print ("%s doesn't exist" % filename, file=sys.stderr)
sys.exit (20)
x = ElbeXML (filename, skip_validate=True, skip_urlcheck=True)
if not x.has ('target'):
print ("<target> is missing, this file can't be built in an initvm",
file=sys.stderr)
sys.exit (20)
with file (filename, "r") as fp:
xml_base64 = binascii.b2a_base64(fp.read ())
print (client.service.create_project ( xml_base64 ))
def set_xml(self, builddir, xml_file):
# This method can throw: ElbeDBError, ValidationError, OSError
if not os.path.exists(builddir):
raise ElbeDBError("project directory does not exist")
srcxml_fname = os.path.join(builddir, "source.xml")
if xml_file is None:
xml_file = srcxml_fname
with session_scope(self.session) as s:
p = None
try:
p = s.query(Project). \
filter(Project.builddir == builddir).one()
except NoResultFound:
raise ElbeDBError(
"project %s is not registered in the database" % builddir)
if p.status == "busy":
raise ElbeDBError(
"cannot set XML file while project %s is busy" % builddir)
xml = ElbeXML(
xml_file,
url_validation=ValidationMode.NO_CHECK) # ValidationError
p.name = xml.text("project/name")
p.version = xml.text("project/version")
p.edit = datetime.utcnow()
if p.status == "empty_project" or p.status == "build_failed":
p.status = "needs_build"
elif p.status == "build_done":
p.status = "has_changes"
if xml_file != srcxml_fname:
copyfile(xml_file, srcxml_fname) # OSError
_update_project_file(s, builddir, "source.xml", "application/xml",
"ELBE recipe of the project")
def execute(self, client, opt, args):
if len(args) != 1:
print("usage: elbe control create_project <xmlfile>",
file=sys.stderr)
sys.exit(20)
filename = args[0]
if not os.path.isfile(filename):
print("%s doesn't exist" % filename, file=sys.stderr)
sys.exit(20)
x = ElbeXML(filename, skip_validate=True, skip_urlcheck=True)
if not x.has('target'):
print("<target> is missing, this file can't be built in an initvm",
file=sys.stderr)
sys.exit(20)
with file(filename, "r") as fp:
xml_base64 = binascii.b2a_base64(fp.read())
print(client.service.create_project(xml_base64))
def execute(self, client, _opt, args):
if len(args) != 2:
print("usage: elbe control set_xml <project_dir> <xml>",
file=sys.stderr)
sys.exit(20)
builddir = args[0]
filename = args[1]
try:
x = ElbeXML(filename,
skip_validate=True,
url_validation=ValidationMode.NO_CHECK)
except IOError:
print("%s is not a valid elbe xml file" % filename)
sys.exit(20)
if not x.has('target'):
print("<target> is missing, this file can't be built in an initvm",
file=sys.stderr)
sys.exit(20)
size = 1024 * 1024
part = 0
with open(filename, "rb") as fp:
while True:
xml_base64 = binascii.b2a_base64(fp.read(size)).decode('ascii')
# finish upload
if len(xml_base64) == 1:
part = client.service.upload_file(builddir, "source.xml",
xml_base64, -1)
else:
part = client.service.upload_file(builddir, "source.xml",
xml_base64, part)
if part == -1:
print("project busy, upload not allowed")
return part
if part == -2:
print("upload of xml finished")
return 0
def set_xml(self, xmlpath):
# Use supplied XML file, if given, otherwise change to source.xml
if not xmlpath:
xmlpath = os.path.join(self.builddir, "source.xml")
newxml = ElbeXML(xmlpath,
buildtype=self.override_buildtype,
skip_validate=self.skip_validate,
url_validation=self.url_validation)
# New XML file has to have the same architecture
oldarch = self.xml.text("project/arch", key="arch")
newarch = newxml.text("project/arch", key="arch")
if newarch != oldarch:
raise IncompatibleArchitectureException(oldarch, newarch)
# Throw away old APT cache, targetfs and buildenv
self._rpcaptcache = None
self.targetfs = None
self.buildenv = None
self.xml = newxml
# Create a new BuildEnv instance, if we have a build directory
if self.has_full_buildenv():
self.buildenv = BuildEnv(self.xml,
self.log,
self.chrootpath,
clean=False)
# Create TargetFs instance, if the target directory exists.
# We use the old content of the directory if no rebuild is done, so
# don't clean it (yet).
if os.path.exists(self.targetpath):
self.targetfs = TargetFs(self.targetpath,
self.log,
self.xml,
clean=False)
else:
self.targetfs = None
def set_xml(self, xmlpath):
# Use supplied XML file, if given, otherwise change to source.xml
if not xmlpath:
xmlpath = os.path.join(self.builddir, "source.xml")
newxml = ElbeXML(xmlpath, buildtype=self.override_buildtype,
skip_validate=self.skip_validate,
url_validation=self.url_validation)
# New XML file has to have the same architecture
oldarch = self.xml.text("project/arch", key="arch")
newarch = newxml.text("project/arch", key="arch")
if newarch != oldarch:
raise IncompatibleArchitectureException(oldarch, newarch)
# Throw away old APT cache, targetfs and buildenv
self.targetfs = None
self.buildenv = None
# dont create sysroot instance, it should be build from scratch
# each time, because the pkglist including the -dev packages is
# tracked nowhere.
self.sysrootenv = None
self.log.do('rm -rf %s' % self.sysrootpath)
self.xml = newxml
# Create a new BuildEnv instance, if we have a build directory
if self.has_full_buildenv():
self.buildenv = BuildEnv(
self.xml, self.log, self.chrootpath, clean=False)
# Create TargetFs instance, if the target directory exists.
# We use the old content of the directory if no rebuild is done, so
# don't clean it (yet).
if os.path.exists(self.targetpath):
self.targetfs = TargetFs(self.targetpath, self.log,
self.xml, clean=False)
else:
self.targetfs = None
def save_version(self, builddir, description=None):
with session_scope(self.session) as s:
try:
p = s.query(Project).filter(Project.builddir == builddir).\
one()
except NoResultFound:
raise ElbeDBError(
"project %s is not registered in the database" %
builddir)
assert p.status == "busy"
sourcexmlpath = os.path.join(builddir, "source.xml")
sourcexml = ElbeXML(sourcexmlpath,
url_validation=ValidationMode.NO_CHECK)
version = sourcexml.text("project/version")
if s.query(ProjectVersion).\
filter(ProjectVersion.builddir == builddir).\
filter(ProjectVersion.version == version).count() > 0:
raise ElbeDBError(
"Version %s already exists for project in %s, "
"please change version number first" %
(version, builddir)
)
versionxmlname = get_versioned_filename(p.name, version,
".version.xml")
versionxmlpath = os.path.join(builddir, versionxmlname)
copyfile(sourcexmlpath, versionxmlpath)
v = ProjectVersion(builddir=builddir,
version=version,
description=description)
s.add(v)
_update_project_file(s, builddir, versionxmlname,
"application/xml",
"source.xml for version %s" % version)
def set_xml(self, xmlpath):
# Use supplied XML file, if given, otherwise change to source.xml
if not xmlpath:
xmlpath = os.path.join(self.builddir, "source.xml")
newxml = ElbeXML(xmlpath,
buildtype=self.override_buildtype,
skip_validate=self.skip_validate,
url_validation=self.url_validation)
# New XML file has to have the same architecture
oldarch = self.xml.text("project/arch", key="arch")
newarch = newxml.text("project/arch", key="arch")
if newarch != oldarch:
raise IncompatibleArchitectureException(oldarch, newarch)
# Throw away old APT cache, targetfs and buildenv
self.targetfs = None
self.buildenv = None
# don't create sysroot instance, it should be built from scratch
# each time, because the pkglist including the -dev packages is
# tracked nowhere.
self.sysrootenv = None
do('rm -rf %s' % self.sysrootpath)
self.xml = newxml
# Create a new BuildEnv instance, if we have a build directory
if self.has_full_buildenv():
self.buildenv = BuildEnv(self.xml, self.chrootpath, clean=False)
# Create TargetFs instance, if the target directory exists.
# We use the old content of the directory if no rebuild is done, so
# don't clean it (yet).
if os.path.exists(self.targetpath):
self.targetfs = TargetFs(self.targetpath, self.xml, clean=False)
else:
self.targetfs = None
def save_version(self, builddir, description=None):
with session_scope(self.session) as s:
try:
p = s.query( Project ).filter( Project.builddir == builddir).\
one()
except NoResultFound:
raise ElbeDBError(
"project %s is not registered in the database" % builddir)
assert p.status == "busy"
sourcexmlpath = os.path.join(builddir, "source.xml")
sourcexml = ElbeXML(sourcexmlpath,
url_validation=ValidationMode.NO_CHECK)
version = sourcexml.text("project/version")
if s.query( ProjectVersion ).\
filter( ProjectVersion.builddir == builddir ).\
filter( ProjectVersion.version == version ).count() > 0:
raise ElbeDBError(
"Version %s already exists for project in %s, "
"please change version number first" %\
(version, builddir)
)
versionxmlname = get_versioned_filename(p.name, version,
".version.xml")
versionxmlpath = os.path.join(builddir, versionxmlname)
copyfile(sourcexmlpath, versionxmlpath)
v = ProjectVersion(builddir=builddir,
version=version,
description=description)
s.add(v)
self._update_project_file(s, builddir, versionxmlname,
"application/xml",
"source.xml for version %s" % version)
def run_command(argv):
oparser = OptionParser(
usage="usage: %prog add [options] <xmlfile> <pkg1> [pkgN]")
(_, args) = oparser.parse_args(argv)
if len(args) < 2:
print("Wrong number of arguments")
oparser.print_help()
sys.exit(20)
xmlfile = args[0]
pkg_lst = args[1:]
try:
xml = ElbeXML(xmlfile)
except ValidationError as E:
print("Error while reading xml file %s: %s" % (xmlfile, E))
sys.exit(20)
for pkg in pkg_lst:
try:
xml.add_target_package(pkg)
except ValueError as E:
print("Error while adding package %s to %s: %s" %
(pkg, xmlfile, E))
sys.exit(20)
try:
xml.xml.write(xmlfile)
sys.exit(0)
# TODO:py3 - Change exception to PermissionError
except IOError as E:
print("Unable to truncate file %s: %s" % (xmlfile, E))
sys.exit(20)
def copy_initvmnode(self):
source = self.xml
source_path = "/var/cache/elbe/source.xml"
try:
initxml = ElbeXML(source_path,
skip_validate=self.skip_validate,
url_validation=ValidationMode.NO_CHECK)
self.xml.get_initvmnode_from(initxml)
except ValidationError:
logging.exception(
"%s validation failed. "
"Will not copy initvm node", source_path)
except IOError:
logging.exception("%s not available. "
"Can not copy initvm node", source_path)
except NoInitvmNode:
logging.exception(
"%s is available. But it does not "
"contain an initvm node", source_path)
def run_command(argv):
# TODO - Set threshold and remove pylint directives
#
# We might want to make the threshold higher for certain
# files/directories or just globaly.
# pylint: disable=too-many-locals
# pylint: disable=too-many-branches
# pylint: disable=too-many-statements
oparser = OptionParser(
usage="usage: %prog fetch_initvm_pkgs [options] <xmlfile>")
oparser.add_option("-b",
"--binrepo",
dest="binrepo",
default="/var/cache/elbe/initvm-bin-repo",
help="directory where the bin repo should reside")
oparser.add_option("-s",
"--srcrepo",
dest="srcrepo",
default="/var/cache/elbe/initvm-src-repo",
help="directory where the src repo should reside")
oparser.add_option("--skip-validation",
action="store_true",
dest="skip_validation",
default=False,
help="Skip xml schema validation")
oparser.add_option("--cdrom-mount-path",
dest="cdrom_path",
help="path where cdrom is mounted")
oparser.add_option("--cdrom-device",
dest="cdrom_device",
help="cdrom device, in case it has to be mounted")
oparser.add_option("--apt-archive",
dest="archive",
default="/var/cache/elbe/binaries/main",
help="path where binary packages are downloaded to.")
oparser.add_option("--src-archive",
dest="srcarchive",
default="/var/cache/elbe/sources",
help="path where src packages are downloaded to.")
oparser.add_option("--skip-build-sources",
action="store_false",
dest="build_sources",
default=True,
help="Skip downloading Source Packages")
oparser.add_option("--skip-build-bin",
action="store_false",
dest="build_bin",
default=True,
help="Skip downloading binary packages")
(opt, args) = oparser.parse_args(argv)
if len(args) != 1:
print("wrong number of arguments")
oparser.print_help()
sys.exit(20)
try:
xml = ElbeXML(args[0], skip_validate=opt.skip_validation)
except ValidationError as e:
print(str(e))
print("xml validation failed. Bailing out")
sys.exit(20)
with elbe_logging({"streams": sys.stdout}):
if opt.cdrom_path:
if opt.cdrom_device:
do('mount "%s" "%s"' % (opt.cdrom_device, opt.cdrom_path))
# a cdrom build is identified by the cdrom option
# the xml file that is copied into the initvm
# by the initrd does not have the cdrom tags setup.
mirror = "file://%s" % opt.cdrom_path
else:
mirror = xml.get_initvm_primary_mirror(opt.cdrom_path)
init_codename = xml.get_initvm_codename()
# Binary Repo
#
repo = CdromInitRepo(init_codename, opt.binrepo, mirror)
hostfs.mkdir_p(opt.archive)
if opt.build_bin:
pkglist = get_initvm_pkglist()
cache = Cache()
cache.open()
for pkg in pkglist:
pkg_id = "%s-%s" % (pkg.name, pkg.installed_version)
try:
p = cache[pkg.name]
pkgver = p.installed
deb = fetch_binary(pkgver, opt.archive,
ElbeAcquireProgress(cb=None))
repo.includedeb(deb, 'main')
except ValueError:
logging.exception('No package "%s"', pkg_id)
except FetchError:
logging.exception(
'Package "%s-%s" could not be downloaded', pkg.name,
pkgver.version)
except TypeError:
logging.exception('Package "%s" missing name or version',
pkg_id)
repo.finalize()
# Source Repo
#
repo = CdromSrcRepo(init_codename, init_codename, opt.srcrepo, 0,
mirror)
hostfs.mkdir_p(opt.srcarchive)
# a cdrom build does not have sources
# skip adding packages to the source repo
#
# FIXME: we need a way to add source cdroms later on
if opt.cdrom_path:
opt.build_sources = False
if opt.build_sources:
for pkg in pkglist:
pkg_id = "%s-%s" % (pkg.name, pkg.installed_version)
try:
p = cache[pkg.name]
pkgver = p.installed
dsc = pkgver.fetch_source(opt.srcarchive,
ElbeAcquireProgress(cb=None),
unpack=False)
repo.include_init_dsc(dsc, 'initvm')
except ValueError:
logging.exception('No package "%s"', pkg_id)
except FetchError:
logging.exception(
'Package "%s-%s" could not be downloaded', pkg.name,
pkgver.version)
except TypeError:
logging.exception('Package "%s" missing name or version',
pkg_id)
repo.finalize()
if opt.cdrom_device:
do('umount "%s"' % opt.cdrom_device)
def run_command(argv):
oparser = OptionParser(
usage="usage: %prog pkgdiff [options] <rfs1> <rfs2>")
oparser.add_option("--noauto",
action="store_true",
dest="noauto",
default=False,
help="Dont compare automatically installed Packages")
(opt, args) = oparser.parse_args(argv)
if len(args) != 2:
print "Wrong number of arguments"
oparser.print_help()
sys.exit(20)
gen_rfs = args[0]
fix_rfs = args[1]
x = os.path.join(gen_rfs, 'etc/elbe_base.xml')
xml = ElbeXML(x, skip_validate=True, skip_urlcheck=True)
arch = xml.text('project/arch', key='arch')
apt_pkg.init_config()
apt_pkg.config.set('RootDir', gen_rfs)
apt_pkg.config.set('APT::Architecture', arch)
apt_pkg.init_system()
gen_cache = apt_pkg.Cache(apt.progress.base.OpProgress())
gc = apt.Cache()
gen_pkgs = {}
for p in gen_cache.packages:
if opt.noauto:
if p.current_ver and not gc[
p.name].is_auto_installed and not p.essential:
gen_pkgs[p.name] = p.current_ver
else:
if p.current_ver and not p.essential:
gen_pkgs[p.name] = p.current_ver
apt_pkg.init_config()
apt_pkg.config.set('RootDir', fix_rfs)
apt_pkg.config.set('APT::Architecture', arch)
apt_pkg.init_system()
fix_cache = apt_pkg.Cache(apt.progress.base.OpProgress())
fc = apt.Cache()
fix_pkgs = {}
for p in fix_cache.packages:
if opt.noauto:
if p.current_ver and not fc[
p.name].is_auto_installed and not p.essential:
fix_pkgs[p.name] = p.current_ver
else:
if p.current_ver and not p.essential:
fix_pkgs[p.name] = p.current_ver
for p in fix_pkgs.keys():
if not p in gen_pkgs.keys():
print "+<pkg>%s</pkg>" % p
for p in gen_pkgs.keys():
if not p in fix_pkgs.keys():
print "-<pkg>%s</pkg>" % p
for p in fix_pkgs.keys():
if p in gen_pkgs.keys() and fix_pkgs[p] != gen_pkgs[p]:
print "%s: Version mismatch %s != %s" % (p, fix_pkgs[p],
gen_pkgs[p])
def execute(self, initvmdir, opt, args):
try:
have_session = os.system(
"tmux has-session -t ElbeInitVMSession >/dev/null 2>&1")
except CommandError as e:
print(
"tmux execution failed, tmux version 1.9 or higher is required"
)
sys.exit(20)
if have_session == 0:
print("ElbeInitVMSession already exists in tmux.", file=sys.stderr)
print("", file=sys.stderr)
print(
"There can only exist a single ElbeInitVMSession, and this session",
file=sys.stderr)
print("can also be used to make your build.", file=sys.stderr)
print(
"See 'elbe initvm submit', 'elbe initvm attach' and 'elbe control'",
file=sys.stderr)
sys.exit(20)
# Init cdrom to None, if we detect it, we set it
cdrom = None
if len(args) == 1:
if args[0].endswith('.xml'):
# We have an xml file, use that for elbe init
exampl = args[0]
try:
xml = etree(exampl)
except ValidationError as e:
print('XML file is inavlid: ' + str(e))
# Use default XML if no initvm was specified
if not xml.has("initvm"):
exampl = os.path.join(elbepack.__path__[0],
"init/default-init.xml")
elif args[0].endswith('.iso'):
# We have an iso image, extract xml from there.
tmp = TmpdirFilesystem()
os.system('7z x -o%s "%s" source.xml' % (tmp.path, args[0]))
if not tmp.isfile('source.xml'):
print('Iso image does not contain a source.xml file',
file=sys.stderr)
print('This is not supported by "elbe initvm"',
file=sys.stderr)
print('', file=sys.stderr)
print('Exiting !!!', file=sys.stderr)
sys.exit(20)
try:
exml = ElbeXML(tmp.fname('source.xml'), skip_urlcheck=True)
except ValidationError as e:
print('Iso image does contain a source.xml file.',
file=sys.stderr)
print('But that xml does not validate correctly',
file=sys.stderr)
print('', file=sys.stderr)
print('Exiting !!!', file=sys.stderr)
sys.exit(20)
print('Iso Image with valid source.xml detected !')
print('Image was generated using Elbe Version %s' %
exml.get_elbe_version())
exampl = tmp.fname('source.xml')
cdrom = args[0]
else:
print('Unknown file ending (use either xml or iso)',
file=sys.stderr)
sys.exit(20)
else:
# No xml File was specified, build the default elbe-init-with-ssh
exampl = os.path.join(elbepack.__path__[0],
"init/default-init.xml")
try:
if opt.devel:
devel = ' --devel'
else:
devel = ''
if cdrom:
system('%s init %s --directory "%s" --cdrom "%s" "%s"' %
(elbe_exe, devel, initvmdir, cdrom, exampl))
else:
system('%s init %s --directory "%s" "%s"' %
(elbe_exe, devel, initvmdir, exampl))
except CommandError:
print("'elbe init' Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
try:
system('cd "%s"; make' % (initvmdir))
except CommandError:
print("Building the initvm Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
try:
system('%s initvm start --directory "%s"' % (elbe_exe, initvmdir))
except CommandError:
print("Starting the initvm Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
if len(args) == 1:
# if provided xml file has no initvm section exampl is set to a
# default initvm XML file. But we need the original file here
if args[0].endswith('.xml'):
# stop here if no project node was specified
try:
x = ElbeXML(args[0])
except ValidationError as e:
print('XML file is inavlid: ' + str(e))
sys.exit(20)
if not x.has('project'):
print(
'elbe initvm ready: use "elbe initvm submit myproject.xml" to build a project'
)
sys.exit(0)
ret, prjdir, err = command_out_stderr(
'%s control create_project "%s"' % (elbe_exe, args[0]))
else:
ret, prjdir, err = command_out_stderr(
'%s control create_project "%s"' % (elbe_exe, exampl))
if ret != 0:
print("elbe control create_project failed.", file=sys.stderr)
print(err, file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
prjdir = prjdir.strip()
if cdrom is not None:
print("Uploading CDROM. This might take a while")
try:
system('%s control set_cdrom "%s" "%s"' %
(elbe_exe, prjdir, cdrom))
except CommandError:
print("elbe control set_cdrom Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("Upload finished")
build_opts = ''
if opt.build_bin:
build_opts += '--build-bin '
if opt.build_sources:
build_opts += '--build-sources '
try:
system('%s control build "%s" %s' %
(elbe_exe, prjdir, build_opts))
except CommandError:
print("elbe control build Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
try:
system('%s control wait_busy "%s"' % (elbe_exe, prjdir))
except CommandError:
print("elbe control wait_busy Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print("Build finished !")
print("")
try:
system('%s control dump_file "%s" validation.txt' %
(elbe_exe, prjdir))
except CommandError:
print("Project failed to generate validation.txt",
file=sys.stderr)
print("Getting log.txt", file=sys.stderr)
try:
system('%s control dump_file "%s" log.txt' %
(elbe_exe, prjdir))
except CommandError:
print("Failed to dump log.txt", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
if opt.skip_download:
print("")
print("Listing available files:")
print("")
try:
system('%s control get_files "%s"' % (elbe_exe, prjdir))
except CommandError:
print("elbe control Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print('Get Files with: elbe control get_file "%s" <filename>' %
prjdir)
else:
ensure_outdir(wdfs, opt)
try:
system('%s control get_files --output "%s" "%s"' %
(elbe_exe, opt.outdir, prjdir))
except CommandError:
print("elbe control get_files Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
def execute(self, initvmdir, opt, args):
try:
have_session = os.system(
"tmux has-session -t ElbeInitVMSession >/dev/null 2>&1")
except CommandError as e:
print(
"tmux execution failed, tmux version 1.9 or higher is required"
)
sys.exit(20)
if have_session == 256:
print("ElbeInitVMSession does not exist in tmux.", file=sys.stderr)
print("Try 'elbe initvm start' to start the session.",
file=sys.stderr)
sys.exit(20)
try:
system('%s initvm ensure --directory "%s"' % (elbe_exe, initvmdir))
except CommandError:
print("Starting the initvm Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
# Init cdrom to None, if we detect it, we set it
cdrom = None
if len(args) == 1:
if args[0].endswith('.xml'):
# We have an xml file, use that for elbe init
xmlfile = args[0]
elif args[0].endswith('.iso'):
# We have an iso image, extract xml from there.
tmp = TmpdirFilesystem()
os.system('7z x -o%s "%s" source.xml' % (tmp.path, args[0]))
print('', file=sys.stderr)
if not tmp.isfile('source.xml'):
print('Iso image does not contain a source.xml file',
file=sys.stderr)
print('This is not supported by "elbe initvm"',
file=sys.stderr)
print('', file=sys.stderr)
print('Exiting !!!', file=sys.stderr)
sys.exit(20)
try:
exml = ElbeXML(tmp.fname('source.xml'), skip_urlcheck=True)
except ValidationError as e:
print('Iso image does contain a source.xml file.',
file=sys.stderr)
print('But that xml does not validate correctly',
file=sys.stderr)
print('', file=sys.stderr)
print('Exiting !!!', file=sys.stderr)
sys.exit(20)
print('Iso Image with valid source.xml detected !')
print('Image was generated using Elbe Version %s' %
exml.get_elbe_version())
xmlfile = tmp.fname('source.xml')
cdrom = args[0]
else:
print('Unknown file ending (use either xml or iso)',
file=sys.stderr)
sys.exit(20)
ret, prjdir, err = command_out_stderr(
'%s control create_project --retries 60 "%s"' %
(elbe_exe, xmlfile))
if ret != 0:
print("elbe control create_project failed.", file=sys.stderr)
print(err, file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
prjdir = prjdir.strip()
if cdrom is not None:
print("Uploading CDROM. This might take a while")
try:
system('%s control set_cdrom "%s" "%s"' %
(elbe_exe, prjdir, cdrom))
except CommandError:
print("elbe control set_cdrom Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("Upload finished")
build_opts = ''
if opt.build_bin:
build_opts += '--build-bin '
if opt.build_sources:
build_opts += '--build-sources '
try:
system('%s control build "%s" %s' %
(elbe_exe, prjdir, build_opts))
except CommandError:
print("elbe control build Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("Build started, waiting till it finishes")
try:
system('%s control wait_busy "%s"' % (elbe_exe, prjdir))
except CommandError:
print("elbe control wait_busy Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print("Build finished !")
print("")
try:
system('%s control dump_file "%s" validation.txt' %
(elbe_exe, prjdir))
except CommandError:
print("Project failed to generate validation.txt",
file=sys.stderr)
print("Getting log.txt", file=sys.stderr)
try:
system('%s control dump_file "%s" log.txt' %
(elbe_exe, prjdir))
except CommandError:
print("Failed to dump log.txt", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
if opt.skip_download:
print("")
print("Listing available files:")
print("")
try:
system('%s control get_files "%s"' % (elbe_exe, prjdir))
except CommandError:
print("elbe control get_files Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print('Get Files with: elbe control get_file "%s" <filename>' %
prjdir)
else:
print("")
print("Getting generated Files")
print("")
ensure_outdir(wdfs, opt)
try:
system('%s control get_files --output "%s" "%s"' %
(elbe_exe, opt.outdir, prjdir))
except CommandError:
print("elbe control get_files Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
if not opt.keep_files:
try:
system('%s control del_project "%s"' %
(elbe_exe, prjdir))
except CommandError:
print("remove project from initvm failed",
file=sys.stderr)
sys.exit(20)
def execute(self, initvmdir, opt, args):
try:
have_session = os.system( "tmux has-session -t ElbeInitVMSession >/dev/null 2>&1" )
except CommandError as e:
print ("tmux execution failed, tmux version 1.9 or higher is required")
sys.exit(20)
if have_session == 256:
print ("ElbeInitVMSession does not exist in tmux.", file=sys.stderr)
print ("Try 'elbe initvm start' to start the session.", file=sys.stderr)
sys.exit(20)
try:
system ('%s initvm ensure --directory "%s"' % (elbe_exe, initvmdir))
except CommandError:
print ("Starting the initvm Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
# Init cdrom to None, if we detect it, we set it
cdrom = None
if len(args) == 1:
if args[0].endswith ('.xml'):
# We have an xml file, use that for elbe init
xmlfile = args[0]
elif args[0].endswith ('.iso'):
# We have an iso image, extract xml from there.
tmp = TmpdirFilesystem ()
os.system ('7z x -o%s "%s" source.xml' % (tmp.path, args[0]))
print ('', file=sys.stderr)
if not tmp.isfile ('source.xml'):
print ('Iso image does not contain a source.xml file', file=sys.stderr)
print ('This is not supported by "elbe initvm"', file=sys.stderr)
print ('', file=sys.stderr)
print ('Exiting !!!', file=sys.stderr)
sys.exit (20)
try:
exml = ElbeXML (tmp.fname ('source.xml'), skip_urlcheck=True)
except ValidationError as e:
print ('Iso image does contain a source.xml file.', file=sys.stderr)
print ('But that xml does not validate correctly', file=sys.stderr)
print ('', file=sys.stderr)
print ('Exiting !!!', file=sys.stderr)
sys.exit (20)
print ('Iso Image with valid source.xml detected !')
print ('Image was generated using Elbe Version %s' % exml.get_elbe_version ())
xmlfile = tmp.fname ('source.xml')
cdrom = args[0]
else:
print ('Unknown file ending (use either xml or iso)', file=sys.stderr)
sys.exit (20)
ret, prjdir, err = command_out_stderr ('%s control create_project --retries 60 "%s"' % (elbe_exe, xmlfile))
if ret != 0:
print ("elbe control create_project failed.", file=sys.stderr)
print (err, file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
prjdir = prjdir.strip()
if cdrom is not None:
print ("Uploading CDROM. This might take a while")
try:
system ('%s control set_cdrom "%s" "%s"' % (elbe_exe, prjdir, cdrom) )
except CommandError:
print ("elbe control set_cdrom Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
print ("Upload finished")
build_opts = ''
if opt.build_bin:
build_opts += '--build-bin '
if opt.build_sources:
build_opts += '--build-sources '
try:
system ('%s control build "%s" %s' % (elbe_exe, prjdir, build_opts) )
except CommandError:
print ("elbe control build Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
print ("Build started, waiting till it finishes")
try:
system ('%s control wait_busy "%s"' % (elbe_exe, prjdir) )
except CommandError:
print ("elbe control wait_busy Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
print ("")
print ("Build finished !")
print ("")
try:
system ('%s control dump_file "%s" validation.txt' % (elbe_exe, prjdir) )
except CommandError:
print ("Project failed to generate validation.txt", file=sys.stderr)
print ("Getting log.txt", file=sys.stderr)
try:
system ('%s control dump_file "%s" log.txt' % (elbe_exe, prjdir) )
except CommandError:
print ("Failed to dump log.txt", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
if opt.skip_download:
print ("")
print ("Listing available files:")
print ("")
try:
system ('%s control get_files "%s"' % (elbe_exe, prjdir) )
except CommandError:
print ("elbe control get_files Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
print ("")
print ('Get Files with: elbe control get_file "%s" <filename>' % prjdir)
else:
print ("")
print ("Getting generated Files")
print ("")
ensure_outdir (wdfs, opt)
try:
system ('%s control get_files --output "%s" "%s"' % (
elbe_exe, opt.outdir, prjdir ))
except CommandError:
print ("elbe control get_files Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
if not opt.keep_files:
try:
system ('%s control del_project "%s"' % (
elbe_exe, prjdir))
except CommandError:
print ("remove project from initvm failed",
file=sys.stderr)
sys.exit(20)
class ElbeProject (object):
def __init__ (self, builddir, xmlpath = None, logpath = None, name = None,
override_buildtype = None, skip_validate = False,
skip_urlcheck = False, rpcaptcache_notifier = None,
private_data = None, postbuild_file = None, presh_file = None,
postsh_file = None, savesh_file = None):
self.builddir = os.path.abspath(str(builddir))
self.chrootpath = os.path.join(self.builddir, "chroot")
self.targetpath = os.path.join(self.builddir, "target")
self.name = name
self.override_buildtype = override_buildtype
self.skip_validate = skip_validate
self.skip_urlcheck = skip_urlcheck
self.postbuild_file = postbuild_file
self.presh_file = presh_file
self.postsh_file = postsh_file
self.savesh_file = savesh_file
self.private_data = private_data
# Apt-Cache will be created on demand with the specified notifier by
# the get_rpcaptcache method
self._rpcaptcache = None
self.rpcaptcache_notifier = rpcaptcache_notifier
# Initialise Repo Images to Empty list.
self.repo_images = []
# Use supplied XML file, if given, otherwise use the source.xml
# file of the project
if xmlpath:
self.xml = ElbeXML( xmlpath, buildtype=override_buildtype,
skip_validate=skip_validate, skip_urlcheck=skip_urlcheck )
else:
sourcexmlpath = os.path.join( self.builddir, "source.xml" )
self.xml = ElbeXML( sourcexmlpath, buildtype=override_buildtype,
skip_validate=skip_validate, skip_urlcheck=skip_urlcheck )
# If logpath is given, use an AsciiDocLog instance, otherwise log
# to stdout
if logpath:
self.log = ASCIIDocLog( logpath )
else:
self.log = StdoutLog()
# Create BuildEnv instance, if the chroot directory exists and
# has an etc/elbe_version
if self.has_full_buildenv():
self.buildenv = BuildEnv( self.xml, self.log, self.chrootpath )
else:
self.buildenv = None
self.targetfs = None
return
# Create TargetFs instance, if the target directory exists
if os.path.exists( self.targetpath ):
self.targetfs = TargetFs( self.targetpath, self.log,
self.buildenv.xml, clean=False )
else:
self.targetfs = None
def build_sysroot (self):
debootstrap_pkgs = [p.et.text for p in self.xml.node ("debootstrappkgs")]
with self.buildenv:
try:
self.get_rpcaptcache().mark_install_devpkgs(debootstrap_pkgs)
except KeyError:
self.log.printo( "No Package " + p )
except SystemError:
self.log.printo( "Unable to correct problems " + p )
try:
self.get_rpcaptcache().commit()
except SystemError:
self.log.printo( "commiting changes failed" )
raise AptCacheCommitError ()
sysrootfilelist = os.path.join(self.builddir, "sysroot-filelist")
with self.buildenv.rfs:
self.log.do( "chroot %s /usr/bin/symlinks -cr /usr/lib" %
self.chrootpath )
triplet = self.xml.defs["triplet"]
paths = [ './usr/include', './lib/*.so', './lib/*.so.*',
'./lib/' + triplet, './usr/lib/*.so', './usr/lib/*.so',
'./usr/lib/*.so.*', './usr/lib/' + triplet ]
self.log.do( "rm %s" % sysrootfilelist, allow_fail=True)
os.chdir( self.chrootpath )
for p in paths:
self.log.do( 'find -path "%s" >> %s' % (p, sysrootfilelist) )
self.log.do( "tar cvfJ %s/sysroot.tar.xz -C %s -T %s" %
(self.builddir, self.chrootpath, sysrootfilelist) )
def build (self, skip_debootstrap = False, build_bin = False,
build_sources = False, cdrom_size = None, debug = False, skip_pkglist = False):
# Write the log header
self.write_log_header()
# Create the build environment, if it does not exist yet
if not self.buildenv:
self.log.do( 'mkdir -p "%s"' % self.chrootpath )
self.buildenv = BuildEnv( self.xml, self.log, self.chrootpath, build_sources = build_sources )
skip_pkglist = False
# Install packages
if not skip_pkglist:
self.install_packages()
try:
self.buildenv.rfs.dump_elbeversion (self.xml)
except IOError:
self.log.printo ("dump elbeversion failed")
# Extract target FS. We always create a new instance here with
# clean=true, because we want a pristine directory.
self.targetfs = TargetFs( self.targetpath, self.log,
self.buildenv.xml, clean=True )
os.chdir( self.buildenv.rfs.fname( '' ) )
extract_target( self.buildenv.rfs, self.xml, self.targetfs,
self.log, self.get_rpcaptcache() )
# Package validation and package list
if not skip_pkglist:
validationpath = os.path.join( self.builddir, "validation.txt" )
pkgs = self.xml.xml.node( "/target/pkg-list" )
if self.xml.has( "fullpkgs" ):
check_full_pkgs( pkgs, self.xml.xml.node( "/fullpkgs" ),
validationpath, self.get_rpcaptcache() )
else:
check_full_pkgs( pkgs, None, validationpath,
self.get_rpcaptcache() )
dump_fullpkgs( self.xml, self.buildenv.rfs, self.get_rpcaptcache() )
self.xml.dump_elbe_version ()
self.targetfs.write_fstab (self.xml )
# Dump ELBE version
try:
self.targetfs.dump_elbeversion( self.xml )
except MemoryError:
self.log.printo( "dump elbeversion failed" )
# install packages for buildenv
if not skip_pkglist:
self.install_packages(buildenv=True)
# Write source.xml
try:
sourcexmlpath = os.path.join( self.builddir, "source.xml" )
self.xml.xml.write( sourcexmlpath )
except MemoryError:
self.log.printo( "write source.xml failed (archive to huge?)" )
# Elbe report
reportpath = os.path.join( self.builddir, "elbe-report.txt" )
elbe_report( self.xml, self.buildenv, self.get_rpcaptcache(),
reportpath, self.targetfs )
# Licenses
f = open( os.path.join( self.builddir, "licence.txt" ), "w+" )
self.buildenv.rfs.write_licenses(f, self.log)
f.close()
# Read arch and codename from xml
arch = self.xml.text( "project/arch", key="arch" )
codename = self.xml.text( "project/suite" )
# Use some handwaving to determine grub version
# jessie and wheezy grubs are 2.0 but differ in behaviour
#
# We might also want support for legacy grub
if self.get_rpcaptcache().is_installed( 'grub-pc' ):
if codename == "jessie":
grub_version = 202
else:
grub_version = 199
elif self.get_rpcaptcache().is_installed( 'grub-legacy' ):
self.log.printo( "package grub-legacy is installed, this is obsolete, skipping grub" )
grub_version = 0
else:
self.log.printo( "package grub-pc is not installed, skipping grub" )
# version 0 == skip_grub
grub_version = 0
self.targetfs.part_target( self.builddir, grub_version )
# Build cdrom images
self.repo_images = []
with self.buildenv:
init_codename = self.xml.get_initvm_codename()
if build_bin:
self.repo_images += mk_binary_cdrom( self.buildenv.rfs,
arch,
codename,
init_codename,
self.xml,
self.builddir,
self.log,
cdrom_size=cdrom_size )
if build_sources:
try:
self.repo_images += mk_source_cdrom( self.buildenv.rfs,
arch,
codename,
init_codename,
self.builddir,
self.log,
cdrom_size=cdrom_size )
except SystemError as e:
# e.g. no deb-src urls specified
self.log.printo( str (e) )
if self.postbuild_file:
self.log.h2 ("postbuild script:")
self.log.do (self.postbuild_file + ' "%s %s %s"' % (
self.builddir,
self.xml.text ("project/version"),
self.xml.text ("project/name")),
allow_fail=True)
os.system( 'cat "%s"' % os.path.join( self.builddir, "validation.txt" ) )
def pdebuild (self):
# Remove pdebuilder directory, containing last build results
self.log.do ('rm -rf "%s"' % os.path.join (self.builddir, "pdebuilder"))
# Remove pbuilder/result directory
self.log.do ('rm -rf "%s"' % os.path.join (self.builddir, "pbuilder", "result"))
# Recreate the directories removed
self.log.do ('mkdir -p "%s"' % os.path.join (self.builddir, "pbuilder", "result"))
self.log.do ('mkdir -p "%s"' % os.path.join (self.builddir, "pdebuilder", "current"))
# Untar current_pdebuild.tar.gz into pdebuilder/current
self.log.do ('tar xvfz "%s" -C "%s"' % (os.path.join (self.builddir, "current_pdebuild.tar.gz"),
os.path.join (self.builddir, "pdebuilder", "current")))
# Run pdebuild
try:
self.log.do ('cd "%s"; pdebuild --configfile "%s" --use-pdebuild-internal --buildresult "%s"' % (
os.path.join (self.builddir, "pdebuilder", "current"),
os.path.join (self.builddir, "pbuilderrc"),
os.path.join (self.builddir, "pbuilder", "result")))
except CommandError as e:
self.log.printo ('')
self.log.printo ('Package fails to build.')
self.log.printo ('Please make sure, that the submitted package builds in pbuilder')
def create_pbuilder (self):
# Remove old pbuilder directory, if it exists
self.log.do ('rm -rf "%s"' % os.path.join (self.builddir, "pbuilder"))
# make hooks.d and pbuilder directory
self.log.do ('mkdir -p "%s"' % os.path.join (self.builddir, "pbuilder", "hooks.d"))
# write config files
pbuilder_write_config (self.builddir, self.xml, self.log)
pbuilder_write_repo_hook (self.builddir, self.xml)
self.log.do ('chmod 755 "%s"' % os.path.join (self.builddir, "pbuilder", "hooks.d", "D10elbe_apt_sources"))
# Run pbuilder --create
self.log.do ('pbuilder --create --configfile "%s" --extrapackages git' % os.path.join (self.builddir, "pbuilderrc"))
def sync_xml_to_disk (self):
try:
sourcexmlpath = os.path.join( self.builddir, "source.xml" )
self.xml.xml.write( sourcexmlpath )
except MemoryError:
self.log.printo( "write source.xml failed (archive to huge?)" )
def get_rpcaptcache (self):
if self._rpcaptcache is None:
self._rpcaptcache = get_rpcaptcache( self.buildenv.rfs,
"aptcache.log",
self.xml.text( "project/arch", key="arch" ),
self.rpcaptcache_notifier )
return self._rpcaptcache
def drop_rpcaptcache (self):
self._rpcaptcache = None;
def has_full_buildenv (self):
if os.path.exists( self.chrootpath ):
elbeversionpath = os.path.join( self.chrootpath,
"etc", "elbe_version" )
if os.path.isfile( elbeversionpath ):
return True
else:
self.log.printo( "%s exists, but it does not have an etc/elbe_version file." %
self.chrootpath )
# Apparently we do not have a functional build environment
return False
else:
return False
def set_xml (self, xmlpath):
# Use supplied XML file, if given, otherwise change to source.xml
if not xmlpath:
xmlpath = os.path.join( self.builddir, "source.xml" )
newxml = ElbeXML( xmlpath, buildtype=self.override_buildtype,
skip_validate=self.skip_validate,
skip_urlcheck=self.skip_urlcheck )
# New XML file has to have the same architecture
oldarch = self.xml.text( "project/arch", key="arch" )
newarch = newxml.text( "project/arch", key="arch" )
if newarch != oldarch:
raise IncompatibleArchitectureException( oldarch, newarch )
# Throw away old APT cache, targetfs and buildenv
self._rpcaptcache = None
self.targetfs = None
self.buildenv = None
self.xml = newxml
# Create a new BuildEnv instance, if we have a build directory
if self.has_full_buildenv():
self.buildenv = BuildEnv( self.xml, self.log, self.chrootpath )
# Create TargetFs instance, if the target directory exists.
# We use the old content of the directory if no rebuild is done, so
# don't clean it (yet).
if os.path.exists( self.targetpath ):
self.targetfs = TargetFs( self.targetpath, self.log,
self.buildenv.xml, clean=False )
else:
self.targetfs = None
def write_log_header (self):
if self.name:
self.log.h1( "ELBE Report for Project " + self.name)
else:
self.log.h1( "ELBE Report" )
self.log.printo( "report timestamp: " +
datetime.datetime.now().strftime("%Y%m%d-%H%M%S") )
def install_packages (self, buildenv=False):
with self.buildenv:
# First update the apt cache
try:
self.get_rpcaptcache().update()
except:
self.log.printo( "update cache failed" )
raise AptCacheUpdateError ()
# Then dump the debootstrap packages
if self.buildenv.fresh_debootstrap:
if self.buildenv.need_dumpdebootstrap:
dump_debootstrappkgs (self.xml, self.get_rpcaptcache ())
dump_initvmpkgs (self.xml)
self.buildenv.need_dumpdebootstrap = False
source = self.xml
try:
initxml = ElbeXML( "/var/cache/elbe/source.xml",
skip_validate=self.skip_validate, skip_urlcheck=True )
self.xml.get_initvmnode_from( initxml )
except ValidationError as e:
self.log.printo( "/var/cache/elbe/source.xml validation failed" )
self.log.printo( str(e) )
self.log.printo( "will not copy initvm node" )
except IOError:
self.log.printo( "/var/cache/elbe/source.xml not available" )
self.log.printo( "can not copy initvm node" )
except NoInitvmNode:
self.log.printo( "/var/cache/elbe/source.xml is available" )
self.log.printo( "But it does not contain an initvm node" )
else:
sourcepath = os.path.join( self.builddir, "source.xml" )
source = ElbeXML( sourcepath,
buildtype=self.override_buildtype,
skip_validate=self.skip_validate,
skip_urlcheck=self.skip_urlcheck )
self.xml.get_debootstrappkgs_from( source )
try:
self.xml.get_initvmnode_from( source )
except NoInitvmNode:
self.log.printo( "source.xml is available" )
self.log.printo( "But it does not contain an initvm node" )
# Seed /etc, we need /etc/hosts for hostname -f to work correctly
if not buildenv:
self.buildenv.seed_etc()
# remove all non-essential packages to ensure that on a incremental
# build packages can be removed
debootstrap_pkgs = []
for p in self.xml.node("debootstrappkgs"):
debootstrap_pkgs.append (p.et.text)
pkgs = self.buildenv.xml.get_target_packages()
if buildenv:
pkgs = pkgs + self.buildenv.xml.get_buildenv_packages()
# Now install requested packages
for p in pkgs:
try:
self.get_rpcaptcache().mark_install( p, None )
except KeyError:
self.log.printo( "No Package " + p )
except SystemError:
self.log.printo( "Unable to correct problems " + p )
# temporary disabled because of
# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=776057
# the functions cleans up to much
# self.get_rpcaptcache().cleanup(debootstrap_pkgs + pkgs)
try:
self.get_rpcaptcache().commit()
except SystemError:
self.log.printo( "commiting changes failed" )
raise AptCacheCommitError ()
def __init__(self,
builddir,
xmlpath=None,
logpath=None,
name=None,
override_buildtype=None,
skip_validate=False,
skip_urlcheck=False,
rpcaptcache_notifier=None,
private_data=None,
postbuild_file=None,
presh_file=None,
postsh_file=None,
savesh_file=None):
self.builddir = os.path.abspath(str(builddir))
self.chrootpath = os.path.join(self.builddir, "chroot")
self.targetpath = os.path.join(self.builddir, "target")
self.name = name
self.override_buildtype = override_buildtype
self.skip_validate = skip_validate
self.skip_urlcheck = skip_urlcheck
self.postbuild_file = postbuild_file
self.presh_file = presh_file
self.postsh_file = postsh_file
self.savesh_file = savesh_file
self.private_data = private_data
# Apt-Cache will be created on demand with the specified notifier by
# the get_rpcaptcache method
self._rpcaptcache = None
self.rpcaptcache_notifier = rpcaptcache_notifier
# Initialise Repo Images to Empty list.
self.repo_images = []
# Use supplied XML file, if given, otherwise use the source.xml
# file of the project
if xmlpath:
self.xml = ElbeXML(xmlpath,
buildtype=override_buildtype,
skip_validate=skip_validate,
skip_urlcheck=skip_urlcheck)
else:
sourcexmlpath = os.path.join(self.builddir, "source.xml")
self.xml = ElbeXML(sourcexmlpath,
buildtype=override_buildtype,
skip_validate=skip_validate,
skip_urlcheck=skip_urlcheck)
# If logpath is given, use an AsciiDocLog instance, otherwise log
# to stdout
if logpath:
self.log = ASCIIDocLog(logpath)
else:
self.log = StdoutLog()
# Create BuildEnv instance, if the chroot directory exists and
# has an etc/elbe_version
if self.has_full_buildenv():
self.buildenv = BuildEnv(self.xml, self.log, self.chrootpath)
else:
self.buildenv = None
self.targetfs = None
return
# Create TargetFs instance, if the target directory exists
if os.path.exists(self.targetpath):
self.targetfs = TargetFs(self.targetpath,
self.log,
self.buildenv.xml,
clean=False)
else:
self.targetfs = None
class ElbeProject(object):
def __init__(self,
builddir,
xmlpath=None,
logpath=None,
name=None,
override_buildtype=None,
skip_validate=False,
url_validation=ValidationMode.CHECK_ALL,
rpcaptcache_notifier=None,
private_data=None,
postbuild_file=None,
presh_file=None,
postsh_file=None,
savesh_file=None):
self.builddir = os.path.abspath(str(builddir))
self.chrootpath = os.path.join(self.builddir, "chroot")
self.targetpath = os.path.join(self.builddir, "target")
self.name = name
self.override_buildtype = override_buildtype
self.skip_validate = skip_validate
self.url_validation = url_validation
self.postbuild_file = postbuild_file
self.presh_file = presh_file
self.postsh_file = postsh_file
self.savesh_file = savesh_file
self.private_data = private_data
# Apt-Cache will be created on demand with the specified notifier by
# the get_rpcaptcache method
self._rpcaptcache = None
self.rpcaptcache_notifier = rpcaptcache_notifier
# Initialise Repo Images to Empty list.
self.repo_images = []
self.orig_fname = None
self.orig_files = []
# Use supplied XML file, if given, otherwise use the source.xml
# file of the project
if xmlpath:
self.xml = ElbeXML(xmlpath,
buildtype=override_buildtype,
skip_validate=skip_validate,
url_validation=url_validation)
else:
sourcexmlpath = os.path.join(self.builddir, "source.xml")
self.xml = ElbeXML(sourcexmlpath,
buildtype=override_buildtype,
skip_validate=skip_validate,
url_validation=url_validation)
self.arch = self.xml.text("project/arch", key="arch")
self.codename = self.xml.text("project/suite")
# If logpath is given, use an AsciiDocLog instance, otherwise log
# to stdout
if logpath:
self.log = ASCIIDocLog(logpath)
else:
self.log = StdoutLog()
self.repo = ProjectRepo(self.arch, self.codename,
os.path.join(self.builddir, "repo"), self.log)
# Create BuildEnv instance, if the chroot directory exists and
# has an etc/elbe_version
if os.path.exists(self.chrootpath):
self.buildenv = BuildEnv(self.xml,
self.log,
self.chrootpath,
clean=False)
else:
self.buildenv = None
# Create TargetFs instance, if the target directory exists
if os.path.exists(self.targetpath):
self.targetfs = TargetFs(self.targetpath,
self.log,
self.buildenv.xml,
clean=False)
else:
self.targetfs = None
def build_chroottarball(self):
self.log.do("tar cJf %s/chroot.tar.xz \
--exclude=./tmp/* --exclude=./dev/* \
--exclude=./run/* --exclude=./sys/* \
--exclude=./proc/* --exclude=./var/cache/* \
-C %s ." % (self.builddir, self.chrootpath))
def build_sysroot(self):
# ignore packages from debootstrap
ignore_pkgs = [p.et.text for p in self.xml.node("debootstrappkgs")]
ignore_dev_pkgs = []
if self.xml.has('target/pkg-blacklist/sysroot'):
ignore_dev_pkgs = [
p.et.text
for p in self.xml.node("target/pkg-blacklist/sysroot")
]
with self.buildenv:
try:
self.get_rpcaptcache().mark_install_devpkgs(
ignore_pkgs, ignore_dev_pkgs)
except SystemError as e:
self.log.printo("mark install devpkgs failed: %s" % str(e))
try:
self.get_rpcaptcache().commit()
except SystemError as e:
self.log.printo("commiting changes failed: %s" % str(e))
raise AptCacheCommitError(str(e))
sysrootfilelist = os.path.join(self.builddir, "sysroot-filelist")
with self.buildenv.rfs:
self.log.do("chroot %s /usr/bin/symlinks -cr /usr/lib" %
self.chrootpath)
triplet = self.xml.defs["triplet"]
paths = [
'./usr/include', './usr/include/' + triplet, './etc/ld.so.conf*',
'./opt/*/lib/*.so', '/opt/*lib/*.so.*', './opt/*/include/',
'./opt/*/lib/' + triplet, './opt/*/include/' + triplet,
'./lib/*.so', './lib/*.so.*', './lib/' + triplet, './usr/lib/*.so',
'./usr/lib/*.so', './usr/lib/*.so.*', './usr/lib/' + triplet
]
self.log.do("rm %s" % sysrootfilelist, allow_fail=True)
os.chdir(self.chrootpath)
for p in paths:
self.log.do('find -path "%s" >> %s' % (p, sysrootfilelist))
self.log.do("tar cfJ %s/sysroot.tar.xz -C %s -T %s" %
(self.builddir, self.chrootpath, sysrootfilelist))
# chroot is invalid after adding all the -dev packages
# it shouldn't be used to create an incremental image
#
# We only remove /etc/elbe_version here.
# So we can still elbe chroot into the fs
self.buildenv.rfs.remove("/etc/elbe_version", noerr=True)
def pbuild(self, p):
self.pdebuild_init()
src_path = os.path.join(self.builddir, "pdebuilder", "current")
self.log.printo("retrieve pbuild sources: %s" % p.text('.').strip())
if p.tag == 'git':
self.log.do("git clone %s %s" % (p.text('.').strip(), src_path))
try:
self.log.do("cd %s; git reset --hard %s" %
(src_path, p.et.attrib['revision']))
except IndexError:
pass
elif p.tag == 'svn':
self.log.do("svn co --non-interactive %s %s" %
(p.text('.').strip(), src_path))
else:
self.log.printo("unknown pbuild source vcs: %s" % p.tag)
self.pdebuild_build()
def build(self,
skip_debootstrap=False,
build_bin=False,
build_sources=False,
cdrom_size=None,
debug=False,
skip_pkglist=False,
skip_pbuild=False):
# Write the log header
self.write_log_header()
# Validate Apt Sources
m = ValidationMode.NO_CHECK
if build_bin:
m = ValidationMode.CHECK_BINARIES
if build_sources:
m = ValidationMode.CHECK_ALL
self.xml.validate_apt_sources(m, self.arch)
if (self.xml.has('target/pbuilder') and not skip_pbuild):
if not os.path.exists(os.path.join(self.builddir, "pbuilder")):
self.create_pbuilder()
for p in self.xml.node('target/pbuilder'):
self.pbuild(p)
# the package might be needed by a following pbuild, so update
# the project repo that it can be installed in as
# build-dependency
self.repo.finalize()
# To avoid update cache errors, the project repo needs to have
# Release and Packages files, even if it's empty. So don't do this
# in the if case above!
self.repo.finalize()
# Create the build environment, if it does not a valid one
# self.buildenv might be set when we come here.
# However, if its not a full_buildenv, we specify clean here,
# so it gets rebuilt properly.
if not self.has_full_buildenv():
self.log.do('mkdir -p "%s"' % self.chrootpath)
self.buildenv = BuildEnv(self.xml,
self.log,
self.chrootpath,
build_sources=build_sources,
clean=True)
skip_pkglist = False
# Import keyring
self.buildenv.import_keys()
self.log.printo("Keys imported")
# Install packages
if not skip_pkglist:
self.install_packages()
try:
self.buildenv.rfs.dump_elbeversion(self.xml)
except IOError:
self.log.printo("dump elbeversion failed")
# Extract target FS. We always create a new instance here with
# clean=true, because we want a pristine directory.
self.targetfs = TargetFs(self.targetpath,
self.log,
self.buildenv.xml,
clean=True)
os.chdir(self.buildenv.rfs.fname(''))
extract_target(self.buildenv.rfs, self.xml, self.targetfs, self.log,
self.get_rpcaptcache())
# Package validation and package list
if not skip_pkglist:
validationpath = os.path.join(self.builddir, "validation.txt")
pkgs = self.xml.xml.node("/target/pkg-list")
if self.xml.has("fullpkgs"):
check_full_pkgs(pkgs, self.xml.xml.node("/fullpkgs"),
validationpath, self.get_rpcaptcache())
else:
check_full_pkgs(pkgs, None, validationpath,
self.get_rpcaptcache())
dump_fullpkgs(self.xml, self.buildenv.rfs, self.get_rpcaptcache())
self.xml.dump_elbe_version()
self.targetfs.write_fstab(self.xml)
# Dump ELBE version
try:
self.targetfs.dump_elbeversion(self.xml)
except MemoryError:
self.log.printo("dump elbeversion failed")
# install packages for buildenv
if not skip_pkglist:
self.install_packages(buildenv=True)
# Write source.xml
try:
sourcexmlpath = os.path.join(self.builddir, "source.xml")
self.xml.xml.write(sourcexmlpath)
except MemoryError:
self.log.printo("write source.xml failed (archive to huge?)")
# Elbe report
reportpath = os.path.join(self.builddir, "elbe-report.txt")
elbe_report(self.xml, self.buildenv, self.get_rpcaptcache(),
reportpath, self.targetfs)
# Licenses
f = io.open(os.path.join(self.builddir, "licence.txt"),
"w+",
encoding='utf-8')
self.buildenv.rfs.write_licenses(
f, self.log, os.path.join(self.builddir, "licence.xml"))
f.close()
# Use some handwaving to determine grub version
# jessie and wheezy grubs are 2.0 but differ in behaviour
#
# We might also want support for legacy grub
if self.get_rpcaptcache().is_installed('grub-pc'):
if self.codename == "wheezy":
grub_version = 199
else:
grub_version = 202
elif self.get_rpcaptcache().is_installed('grub-legacy'):
self.log.printo(
"package grub-legacy is installed, this is obsolete, skipping grub"
)
grub_version = 0
else:
self.log.printo("package grub-pc is not installed, skipping grub")
# version 0 == skip_grub
grub_version = 0
self.targetfs.part_target(self.builddir, grub_version)
# Build cdrom images
self.repo_images = []
with self.buildenv:
init_codename = self.xml.get_initvm_codename()
if build_bin:
self.repo_images += mk_binary_cdrom(self.buildenv.rfs,
self.arch,
self.codename,
init_codename,
self.xml,
self.builddir,
self.log,
cdrom_size=cdrom_size)
if build_sources:
try:
self.repo_images += mk_source_cdrom(self.buildenv.rfs,
self.arch,
self.codename,
init_codename,
self.builddir,
self.log,
cdrom_size=cdrom_size,
xml=self.xml)
except SystemError as e:
# e.g. no deb-src urls specified
self.log.printo(str(e))
if self.postbuild_file:
self.log.h2("postbuild script:")
self.log.do(self.postbuild_file + ' "%s %s %s"' %
(self.builddir, self.xml.text("project/version"),
self.xml.text("project/name")),
allow_fail=True)
os.system('cat "%s"' % os.path.join(self.builddir, "validation.txt"))
def pdebuild_init(self):
# Remove pdebuilder directory, containing last build results
self.log.do('rm -rf "%s"' % os.path.join(self.builddir, "pdebuilder"))
# Remove pbuilder/result directory
self.log.do('rm -rf "%s"' %
os.path.join(self.builddir, "pbuilder", "result"))
# Recreate the directories removed
self.log.do('mkdir -p "%s"' %
os.path.join(self.builddir, "pbuilder", "result"))
def pdebuild(self):
self.pdebuild_init()
pbdir = os.path.join(self.builddir, "pdebuilder", "current")
self.log.do('mkdir -p "%s"' % os.path.join(pbdir))
try:
for orig_fname in self.orig_files:
ofname = os.path.join(self.builddir, orig_fname)
self.log.do(
'mv "%s" "%s"' %
(ofname, os.path.join(self.builddir, "pdebuilder")))
finally:
self.orig_fname = None
self.orig_files = []
# Untar current_pdebuild.tar.gz into pdebuilder/current
self.log.do(
'tar xfz "%s" -C "%s"' %
(os.path.join(self.builddir, "current_pdebuild.tar.gz"), pbdir))
self.pdebuild_build()
self.repo.finalize()
def pdebuild_build(self):
try:
self.log.do(
'cd "%s"; pdebuild --debbuildopts "-j%s -sa" --configfile "%s" --use-pdebuild-internal --buildresult "%s"'
% (os.path.join(self.builddir, "pdebuilder",
"current"), cfg['pbuilder_jobs'],
os.path.join(self.builddir, "pbuilderrc"),
os.path.join(self.builddir, "pbuilder", "result")))
self.repo.remove(
os.path.join(self.builddir, "pdebuilder", "current", "debian",
"control"))
self.repo.include(
os.path.join(self.builddir, "pbuilder", "result", "*.changes"))
except CommandError as e:
self.log.printo('')
self.log.printo('Package fails to build.')
self.log.printo(
'Please make sure, that the submitted package builds in pbuilder'
)
def create_pbuilder(self):
# Remove old pbuilder directory, if it exists
self.log.do('rm -rf "%s"' % os.path.join(self.builddir, "pbuilder"))
# make hooks.d and pbuilder directory
self.log.do('mkdir -p "%s"' %
os.path.join(self.builddir, "pbuilder", "hooks.d"))
self.log.do('mkdir -p "%s"' %
os.path.join(self.builddir, "aptconfdir", "apt.conf.d"))
# write config files
pbuilder_write_config(self.builddir, self.xml, self.log)
pbuilder_write_apt_conf(self.builddir, self.xml)
pbuilder_write_repo_hook(self.builddir, self.xml)
self.log.do('chmod 755 "%s"' % os.path.join(
self.builddir, "pbuilder", "hooks.d", "D10elbe_apt_sources"))
# Run pbuilder --create
self.log.do(
'pbuilder --create --configfile "%s" --aptconfdir "%s" --extrapackages git'
% (os.path.join(self.builddir, "pbuilderrc"),
os.path.join(self.builddir, "aptconfdir")))
def sync_xml_to_disk(self):
try:
sourcexmlpath = os.path.join(self.builddir, "source.xml")
self.xml.xml.write(sourcexmlpath)
except MemoryError:
self.log.printo("write source.xml failed (archive to huge?)")
def get_rpcaptcache(self):
if self._rpcaptcache is None:
self._rpcaptcache = get_rpcaptcache(
self.buildenv.rfs, self.log.fp.name, self.arch,
self.rpcaptcache_notifier, self.xml.prj.has('norecommend'),
self.xml.prj.has('noauth'))
return self._rpcaptcache
def drop_rpcaptcache(self):
self._rpcaptcache = None
def has_full_buildenv(self):
if os.path.exists(self.chrootpath):
elbeversionpath = os.path.join(self.chrootpath, "etc",
"elbe_version")
if os.path.isfile(elbeversionpath):
return True
else:
self.log.printo(
"%s exists, but it does not have an etc/elbe_version file."
% self.chrootpath)
# Apparently we do not have a functional build environment
return False
else:
return False
def set_xml(self, xmlpath):
# Use supplied XML file, if given, otherwise change to source.xml
if not xmlpath:
xmlpath = os.path.join(self.builddir, "source.xml")
newxml = ElbeXML(xmlpath,
buildtype=self.override_buildtype,
skip_validate=self.skip_validate,
url_validation=self.url_validation)
# New XML file has to have the same architecture
oldarch = self.xml.text("project/arch", key="arch")
newarch = newxml.text("project/arch", key="arch")
if newarch != oldarch:
raise IncompatibleArchitectureException(oldarch, newarch)
# Throw away old APT cache, targetfs and buildenv
self._rpcaptcache = None
self.targetfs = None
self.buildenv = None
self.xml = newxml
# Create a new BuildEnv instance, if we have a build directory
if self.has_full_buildenv():
self.buildenv = BuildEnv(self.xml,
self.log,
self.chrootpath,
clean=False)
# Create TargetFs instance, if the target directory exists.
# We use the old content of the directory if no rebuild is done, so
# don't clean it (yet).
if os.path.exists(self.targetpath):
self.targetfs = TargetFs(self.targetpath,
self.log,
self.xml,
clean=False)
else:
self.targetfs = None
def write_log_header(self):
if self.name:
self.log.h1("ELBE Report for Project " + self.name)
else:
self.log.h1("ELBE Report")
self.log.printo("report timestamp: " +
datetime.datetime.now().strftime("%Y%m%d-%H%M%S"))
def install_packages(self, buildenv=False):
with self.buildenv:
# First update the apt cache
try:
self.get_rpcaptcache().update()
except Exception as e:
raise AptCacheUpdateError(e)
# Then dump the debootstrap packages
if self.buildenv.fresh_debootstrap:
if self.buildenv.need_dumpdebootstrap:
dump_debootstrappkgs(self.xml, self.get_rpcaptcache())
dump_initvmpkgs(self.xml)
self.buildenv.need_dumpdebootstrap = False
source = self.xml
try:
initxml = ElbeXML("/var/cache/elbe/source.xml",
skip_validate=self.skip_validate,
url_validation=ValidationMode.NO_CHECK)
self.xml.get_initvmnode_from(initxml)
except ValidationError as e:
self.log.printo(
"/var/cache/elbe/source.xml validation failed")
self.log.printo(str(e))
self.log.printo("will not copy initvm node")
except IOError:
self.log.printo("/var/cache/elbe/source.xml not available")
self.log.printo("can not copy initvm node")
except NoInitvmNode:
self.log.printo("/var/cache/elbe/source.xml is available")
self.log.printo("But it does not contain an initvm node")
else:
sourcepath = os.path.join(self.builddir, "source.xml")
source = ElbeXML(sourcepath,
buildtype=self.override_buildtype,
skip_validate=self.skip_validate,
url_validation=self.url_validation)
self.xml.get_debootstrappkgs_from(source)
try:
self.xml.get_initvmnode_from(source)
except NoInitvmNode:
self.log.printo("source.xml is available")
self.log.printo("But it does not contain an initvm node")
# Seed /etc, we need /etc/hosts for hostname -f to work correctly
if not buildenv:
self.buildenv.seed_etc()
# remove all non-essential packages to ensure that on a incremental
# build packages can be removed
debootstrap_pkgs = []
for p in self.xml.node("debootstrappkgs"):
debootstrap_pkgs.append(p.et.text)
pkgs = self.buildenv.xml.get_target_packages()
if buildenv:
pkgs = pkgs + self.buildenv.xml.get_buildenv_packages()
# Now install requested packages
for p in pkgs:
try:
self.get_rpcaptcache().mark_install(p, None)
except KeyError:
self.log.printo("No Package " + p)
except SystemError:
self.log.printo("Unable to correct problems " + p)
# temporary disabled because of
# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=776057
# the functions cleans up to much
# self.get_rpcaptcache().cleanup(debootstrap_pkgs + pkgs)
try:
self.get_rpcaptcache().commit()
except SystemError as e:
self.log.printo("commiting changes failed: %s" % str(e))
raise AptCacheCommitError(str(e))
def install_packages(self, buildenv=False):
with self.buildenv:
# First update the apt cache
try:
self.get_rpcaptcache().update()
except:
self.log.printo("update cache failed")
raise AptCacheUpdateError()
# Then dump the debootstrap packages
if self.buildenv.fresh_debootstrap:
if self.buildenv.need_dumpdebootstrap:
dump_debootstrappkgs(self.xml, self.get_rpcaptcache())
dump_initvmpkgs(self.xml)
self.buildenv.need_dumpdebootstrap = False
source = self.xml
try:
initxml = ElbeXML("/var/cache/elbe/source.xml",
skip_validate=self.skip_validate,
skip_urlcheck=True)
self.xml.get_initvmnode_from(initxml)
except ValidationError as e:
self.log.printo(
"/var/cache/elbe/source.xml validation failed")
self.log.printo(str(e))
self.log.printo("will not copy initvm node")
except IOError:
self.log.printo("/var/cache/elbe/source.xml not available")
self.log.printo("can not copy initvm node")
except NoInitvmNode:
self.log.printo("/var/cache/elbe/source.xml is available")
self.log.printo("But it does not contain an initvm node")
else:
sourcepath = os.path.join(self.builddir, "source.xml")
source = ElbeXML(sourcepath,
buildtype=self.override_buildtype,
skip_validate=self.skip_validate,
skip_urlcheck=self.skip_urlcheck)
self.xml.get_debootstrappkgs_from(source)
try:
self.xml.get_initvmnode_from(source)
except NoInitvmNode:
self.log.printo("source.xml is available")
self.log.printo("But it does not contain an initvm node")
# Seed /etc, we need /etc/hosts for hostname -f to work correctly
if not buildenv:
self.buildenv.seed_etc()
# remove all non-essential packages to ensure that on a incremental
# build packages can be removed
debootstrap_pkgs = []
for p in self.xml.node("debootstrappkgs"):
debootstrap_pkgs.append(p.et.text)
pkgs = self.buildenv.xml.get_target_packages()
if buildenv:
pkgs = pkgs + self.buildenv.xml.get_buildenv_packages()
# Now install requested packages
for p in pkgs:
try:
self.get_rpcaptcache().mark_install(p, None)
except KeyError:
self.log.printo("No Package " + p)
except SystemError:
self.log.printo("Unable to correct problems " + p)
# temporary disabled because of
# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=776057
# the functions cleans up to much
# self.get_rpcaptcache().cleanup(debootstrap_pkgs + pkgs)
try:
self.get_rpcaptcache().commit()
except SystemError:
self.log.printo("commiting changes failed")
raise AptCacheCommitError()
class ElbeProject(object):
def __init__(self,
builddir,
xmlpath=None,
logpath=None,
name=None,
override_buildtype=None,
skip_validate=False,
skip_urlcheck=False,
rpcaptcache_notifier=None,
private_data=None,
postbuild_file=None,
presh_file=None,
postsh_file=None,
savesh_file=None):
self.builddir = os.path.abspath(str(builddir))
self.chrootpath = os.path.join(self.builddir, "chroot")
self.targetpath = os.path.join(self.builddir, "target")
self.name = name
self.override_buildtype = override_buildtype
self.skip_validate = skip_validate
self.skip_urlcheck = skip_urlcheck
self.postbuild_file = postbuild_file
self.presh_file = presh_file
self.postsh_file = postsh_file
self.savesh_file = savesh_file
self.private_data = private_data
# Apt-Cache will be created on demand with the specified notifier by
# the get_rpcaptcache method
self._rpcaptcache = None
self.rpcaptcache_notifier = rpcaptcache_notifier
# Initialise Repo Images to Empty list.
self.repo_images = []
# Use supplied XML file, if given, otherwise use the source.xml
# file of the project
if xmlpath:
self.xml = ElbeXML(xmlpath,
buildtype=override_buildtype,
skip_validate=skip_validate,
skip_urlcheck=skip_urlcheck)
else:
sourcexmlpath = os.path.join(self.builddir, "source.xml")
self.xml = ElbeXML(sourcexmlpath,
buildtype=override_buildtype,
skip_validate=skip_validate,
skip_urlcheck=skip_urlcheck)
# If logpath is given, use an AsciiDocLog instance, otherwise log
# to stdout
if logpath:
self.log = ASCIIDocLog(logpath)
else:
self.log = StdoutLog()
# Create BuildEnv instance, if the chroot directory exists and
# has an etc/elbe_version
if self.has_full_buildenv():
self.buildenv = BuildEnv(self.xml, self.log, self.chrootpath)
else:
self.buildenv = None
self.targetfs = None
return
# Create TargetFs instance, if the target directory exists
if os.path.exists(self.targetpath):
self.targetfs = TargetFs(self.targetpath,
self.log,
self.buildenv.xml,
clean=False)
else:
self.targetfs = None
def build_sysroot(self):
debootstrap_pkgs = [
p.et.text for p in self.xml.node("debootstrappkgs")
]
with self.buildenv:
try:
self.get_rpcaptcache().mark_install_devpkgs(debootstrap_pkgs)
except KeyError:
self.log.printo("No Package " + p)
except SystemError:
self.log.printo("Unable to correct problems " + p)
try:
self.get_rpcaptcache().commit()
except SystemError:
self.log.printo("commiting changes failed")
raise AptCacheCommitError()
sysrootfilelist = os.path.join(self.builddir, "sysroot-filelist")
with self.buildenv.rfs:
self.log.do("chroot %s /usr/bin/symlinks -cr /usr/lib" %
self.chrootpath)
triplet = self.xml.defs["triplet"]
paths = [
'./usr/include', './lib/*.so', './lib/*.so.*', './lib/' + triplet,
'./usr/lib/*.so', './usr/lib/*.so', './usr/lib/*.so.*',
'./usr/lib/' + triplet
]
self.log.do("rm %s" % sysrootfilelist, allow_fail=True)
os.chdir(self.chrootpath)
for p in paths:
self.log.do('find -path "%s" >> %s' % (p, sysrootfilelist))
self.log.do("tar cvfJ %s/sysroot.tar.xz -C %s -T %s" %
(self.builddir, self.chrootpath, sysrootfilelist))
def build(self,
skip_debootstrap=False,
build_bin=False,
build_sources=False,
cdrom_size=None,
debug=False,
skip_pkglist=False):
# Write the log header
self.write_log_header()
# Create the build environment, if it does not exist yet
if not self.buildenv:
self.log.do('mkdir -p "%s"' % self.chrootpath)
self.buildenv = BuildEnv(self.xml,
self.log,
self.chrootpath,
build_sources=build_sources)
skip_pkglist = False
# Install packages
if not skip_pkglist:
self.install_packages()
try:
self.buildenv.rfs.dump_elbeversion(self.xml)
except IOError:
self.log.printo("dump elbeversion failed")
# Extract target FS. We always create a new instance here with
# clean=true, because we want a pristine directory.
self.targetfs = TargetFs(self.targetpath,
self.log,
self.buildenv.xml,
clean=True)
os.chdir(self.buildenv.rfs.fname(''))
extract_target(self.buildenv.rfs, self.xml, self.targetfs, self.log,
self.get_rpcaptcache())
# Package validation and package list
if not skip_pkglist:
validationpath = os.path.join(self.builddir, "validation.txt")
pkgs = self.xml.xml.node("/target/pkg-list")
if self.xml.has("fullpkgs"):
check_full_pkgs(pkgs, self.xml.xml.node("/fullpkgs"),
validationpath, self.get_rpcaptcache())
else:
check_full_pkgs(pkgs, None, validationpath,
self.get_rpcaptcache())
dump_fullpkgs(self.xml, self.buildenv.rfs, self.get_rpcaptcache())
self.xml.dump_elbe_version()
self.targetfs.write_fstab(self.xml)
# Dump ELBE version
try:
self.targetfs.dump_elbeversion(self.xml)
except MemoryError:
self.log.printo("dump elbeversion failed")
# install packages for buildenv
if not skip_pkglist:
self.install_packages(buildenv=True)
# Write source.xml
try:
sourcexmlpath = os.path.join(self.builddir, "source.xml")
self.xml.xml.write(sourcexmlpath)
except MemoryError:
self.log.printo("write source.xml failed (archive to huge?)")
# Elbe report
reportpath = os.path.join(self.builddir, "elbe-report.txt")
elbe_report(self.xml, self.buildenv, self.get_rpcaptcache(),
reportpath, self.targetfs)
# Licenses
f = open(os.path.join(self.builddir, "licence.txt"), "w+")
self.buildenv.rfs.write_licenses(f, self.log)
f.close()
# Read arch and codename from xml
arch = self.xml.text("project/arch", key="arch")
codename = self.xml.text("project/suite")
# Use some handwaving to determine grub version
# jessie and wheezy grubs are 2.0 but differ in behaviour
#
# We might also want support for legacy grub
if self.get_rpcaptcache().is_installed('grub-pc'):
if codename == "jessie":
grub_version = 202
else:
grub_version = 199
elif self.get_rpcaptcache().is_installed('grub-legacy'):
self.log.printo(
"package grub-legacy is installed, this is obsolete, skipping grub"
)
grub_version = 0
else:
self.log.printo("package grub-pc is not installed, skipping grub")
# version 0 == skip_grub
grub_version = 0
self.targetfs.part_target(self.builddir, grub_version)
# Build cdrom images
self.repo_images = []
with self.buildenv:
init_codename = self.xml.get_initvm_codename()
if build_bin:
self.repo_images += mk_binary_cdrom(self.buildenv.rfs,
arch,
codename,
init_codename,
self.xml,
self.builddir,
self.log,
cdrom_size=cdrom_size)
if build_sources:
try:
self.repo_images += mk_source_cdrom(self.buildenv.rfs,
arch,
codename,
init_codename,
self.builddir,
self.log,
cdrom_size=cdrom_size)
except SystemError as e:
# e.g. no deb-src urls specified
self.log.printo(str(e))
if self.postbuild_file:
self.log.h2("postbuild script:")
self.log.do(self.postbuild_file + ' "%s %s %s"' %
(self.builddir, self.xml.text("project/version"),
self.xml.text("project/name")),
allow_fail=True)
os.system('cat "%s"' % os.path.join(self.builddir, "validation.txt"))
def pdebuild(self):
# Remove pdebuilder directory, containing last build results
self.log.do('rm -rf "%s"' % os.path.join(self.builddir, "pdebuilder"))
# Remove pbuilder/result directory
self.log.do('rm -rf "%s"' %
os.path.join(self.builddir, "pbuilder", "result"))
# Recreate the directories removed
self.log.do('mkdir -p "%s"' %
os.path.join(self.builddir, "pbuilder", "result"))
self.log.do('mkdir -p "%s"' %
os.path.join(self.builddir, "pdebuilder", "current"))
# Untar current_pdebuild.tar.gz into pdebuilder/current
self.log.do('tar xvfz "%s" -C "%s"' %
(os.path.join(self.builddir, "current_pdebuild.tar.gz"),
os.path.join(self.builddir, "pdebuilder", "current")))
# Run pdebuild
try:
self.log.do(
'cd "%s"; pdebuild --configfile "%s" --use-pdebuild-internal --buildresult "%s"'
% (os.path.join(self.builddir, "pdebuilder", "current"),
os.path.join(self.builddir, "pbuilderrc"),
os.path.join(self.builddir, "pbuilder", "result")))
except CommandError as e:
self.log.printo('')
self.log.printo('Package fails to build.')
self.log.printo(
'Please make sure, that the submitted package builds in pbuilder'
)
def create_pbuilder(self):
# Remove old pbuilder directory, if it exists
self.log.do('rm -rf "%s"' % os.path.join(self.builddir, "pbuilder"))
# make hooks.d and pbuilder directory
self.log.do('mkdir -p "%s"' %
os.path.join(self.builddir, "pbuilder", "hooks.d"))
# write config files
pbuilder_write_config(self.builddir, self.xml, self.log)
pbuilder_write_repo_hook(self.builddir, self.xml)
self.log.do('chmod 755 "%s"' % os.path.join(
self.builddir, "pbuilder", "hooks.d", "D10elbe_apt_sources"))
# Run pbuilder --create
self.log.do('pbuilder --create --configfile "%s" --extrapackages git' %
os.path.join(self.builddir, "pbuilderrc"))
def sync_xml_to_disk(self):
try:
sourcexmlpath = os.path.join(self.builddir, "source.xml")
self.xml.xml.write(sourcexmlpath)
except MemoryError:
self.log.printo("write source.xml failed (archive to huge?)")
def get_rpcaptcache(self):
if self._rpcaptcache is None:
self._rpcaptcache = get_rpcaptcache(
self.buildenv.rfs, "aptcache.log",
self.xml.text("project/arch", key="arch"),
self.rpcaptcache_notifier)
return self._rpcaptcache
def drop_rpcaptcache(self):
self._rpcaptcache = None
def has_full_buildenv(self):
if os.path.exists(self.chrootpath):
elbeversionpath = os.path.join(self.chrootpath, "etc",
"elbe_version")
if os.path.isfile(elbeversionpath):
return True
else:
self.log.printo(
"%s exists, but it does not have an etc/elbe_version file."
% self.chrootpath)
# Apparently we do not have a functional build environment
return False
else:
return False
def set_xml(self, xmlpath):
# Use supplied XML file, if given, otherwise change to source.xml
if not xmlpath:
xmlpath = os.path.join(self.builddir, "source.xml")
newxml = ElbeXML(xmlpath,
buildtype=self.override_buildtype,
skip_validate=self.skip_validate,
skip_urlcheck=self.skip_urlcheck)
# New XML file has to have the same architecture
oldarch = self.xml.text("project/arch", key="arch")
newarch = newxml.text("project/arch", key="arch")
if newarch != oldarch:
raise IncompatibleArchitectureException(oldarch, newarch)
# Throw away old APT cache, targetfs and buildenv
self._rpcaptcache = None
self.targetfs = None
self.buildenv = None
self.xml = newxml
# Create a new BuildEnv instance, if we have a build directory
if self.has_full_buildenv():
self.buildenv = BuildEnv(self.xml, self.log, self.chrootpath)
# Create TargetFs instance, if the target directory exists.
# We use the old content of the directory if no rebuild is done, so
# don't clean it (yet).
if os.path.exists(self.targetpath):
self.targetfs = TargetFs(self.targetpath,
self.log,
self.buildenv.xml,
clean=False)
else:
self.targetfs = None
def write_log_header(self):
if self.name:
self.log.h1("ELBE Report for Project " + self.name)
else:
self.log.h1("ELBE Report")
self.log.printo("report timestamp: " +
datetime.datetime.now().strftime("%Y%m%d-%H%M%S"))
def install_packages(self, buildenv=False):
with self.buildenv:
# First update the apt cache
try:
self.get_rpcaptcache().update()
except:
self.log.printo("update cache failed")
raise AptCacheUpdateError()
# Then dump the debootstrap packages
if self.buildenv.fresh_debootstrap:
if self.buildenv.need_dumpdebootstrap:
dump_debootstrappkgs(self.xml, self.get_rpcaptcache())
dump_initvmpkgs(self.xml)
self.buildenv.need_dumpdebootstrap = False
source = self.xml
try:
initxml = ElbeXML("/var/cache/elbe/source.xml",
skip_validate=self.skip_validate,
skip_urlcheck=True)
self.xml.get_initvmnode_from(initxml)
except ValidationError as e:
self.log.printo(
"/var/cache/elbe/source.xml validation failed")
self.log.printo(str(e))
self.log.printo("will not copy initvm node")
except IOError:
self.log.printo("/var/cache/elbe/source.xml not available")
self.log.printo("can not copy initvm node")
except NoInitvmNode:
self.log.printo("/var/cache/elbe/source.xml is available")
self.log.printo("But it does not contain an initvm node")
else:
sourcepath = os.path.join(self.builddir, "source.xml")
source = ElbeXML(sourcepath,
buildtype=self.override_buildtype,
skip_validate=self.skip_validate,
skip_urlcheck=self.skip_urlcheck)
self.xml.get_debootstrappkgs_from(source)
try:
self.xml.get_initvmnode_from(source)
except NoInitvmNode:
self.log.printo("source.xml is available")
self.log.printo("But it does not contain an initvm node")
# Seed /etc, we need /etc/hosts for hostname -f to work correctly
if not buildenv:
self.buildenv.seed_etc()
# remove all non-essential packages to ensure that on a incremental
# build packages can be removed
debootstrap_pkgs = []
for p in self.xml.node("debootstrappkgs"):
debootstrap_pkgs.append(p.et.text)
pkgs = self.buildenv.xml.get_target_packages()
if buildenv:
pkgs = pkgs + self.buildenv.xml.get_buildenv_packages()
# Now install requested packages
for p in pkgs:
try:
self.get_rpcaptcache().mark_install(p, None)
except KeyError:
self.log.printo("No Package " + p)
except SystemError:
self.log.printo("Unable to correct problems " + p)
# temporary disabled because of
# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=776057
# the functions cleans up to much
# self.get_rpcaptcache().cleanup(debootstrap_pkgs + pkgs)
try:
self.get_rpcaptcache().commit()
except SystemError:
self.log.printo("commiting changes failed")
raise AptCacheCommitError()
def run_command(argv):
# pylint: disable=too-many-locals
# pylint: disable=too-many-branches
oparser = OptionParser(
usage="usage: %prog pkgdiff [options] <rfs1> <rfs2>")
oparser.add_option(
"--noauto",
action="store_true",
dest="noauto",
default=False,
help="Dont compare automatically installed Packages")
(opt, args) = oparser.parse_args(argv)
if len(args) != 2:
print("Wrong number of arguments")
oparser.print_help()
sys.exit(20)
gen_rfs = args[0]
fix_rfs = args[1]
x = os.path.join(gen_rfs, 'etc/elbe_base.xml')
xml = ElbeXML(
x,
skip_validate=True,
url_validation=ValidationMode.NO_CHECK)
arch = xml.text('project/arch', key='arch')
apt_pkg.init_config()
apt_pkg.config.set('RootDir', gen_rfs)
apt_pkg.config.set('APT::Architecture', arch)
apt_pkg.init_system()
gen_cache = apt_pkg.Cache(apt.progress.base.OpProgress())
gc = apt.Cache()
gen_pkgs = {}
# pylint: disable=E1133
for p in gen_cache.packages:
if opt.noauto:
if p.current_ver and not \
gc[p.name].is_auto_installed and not \
p.essential:
gen_pkgs[p.name] = p.current_ver
else:
if p.current_ver and not p.essential:
gen_pkgs[p.name] = p.current_ver
apt_pkg.init_config()
apt_pkg.config.set('RootDir', fix_rfs)
apt_pkg.config.set('APT::Architecture', arch)
apt_pkg.init_system()
fix_cache = apt_pkg.Cache(apt.progress.base.OpProgress())
fc = apt.Cache()
fix_pkgs = {}
# pylint: disable=E1133
for p in fix_cache.packages:
if opt.noauto:
if p.current_ver and not \
fc[p.name].is_auto_installed and not \
p.essential:
fix_pkgs[p.name] = p.current_ver
else:
if p.current_ver and not p.essential:
fix_pkgs[p.name] = p.current_ver
for p in fix_pkgs:
if p not in gen_pkgs:
print("+<pkg>%s</pkg>" % p)
for p in gen_pkgs:
if p not in fix_pkgs.keys():
print("-<pkg>%s</pkg>" % p)
for p in fix_pkgs:
if p in gen_pkgs.keys() and fix_pkgs[p] != gen_pkgs[p]:
print(
"%s: Version mismatch %s != %s" %
(p, fix_pkgs[p], gen_pkgs[p]))
def __init__(
self,
builddir,
xmlpath=None,
logpath=None,
name=None,
override_buildtype=None,
skip_validate=False,
url_validation=ValidationMode.CHECK_ALL,
rpcaptcache_notifier=None,
private_data=None,
postbuild_file=None,
presh_file=None,
postsh_file=None,
savesh_file=None):
# pylint: disable=too-many-arguments
self.builddir = os.path.abspath(str(builddir))
self.chrootpath = os.path.join(self.builddir, "chroot")
self.targetpath = os.path.join(self.builddir, "target")
self.sysrootpath = os.path.join(self.builddir, "sysroot")
self.sdkpath = os.path.join(self.builddir, "sdk")
self.validationpath = os.path.join(self.builddir, "validation.txt")
self.name = name
self.override_buildtype = override_buildtype
self.skip_validate = skip_validate
self.url_validation = url_validation
self.postbuild_file = postbuild_file
self.presh_file = presh_file
self.postsh_file = postsh_file
self.savesh_file = savesh_file
self.private_data = private_data
# Apt-Cache will be created on demand with the specified notifier by
# the get_rpcaptcache method
self._rpcaptcache = None
self.rpcaptcache_notifier = rpcaptcache_notifier
# Initialise Repo Images to Empty list.
self.repo_images = []
self.orig_fname = None
self.orig_files = []
# Use supplied XML file, if given, otherwise use the source.xml
# file of the project
if xmlpath:
self.xml = ElbeXML(
xmlpath,
buildtype=override_buildtype,
skip_validate=skip_validate,
url_validation=url_validation)
else:
sourcexmlpath = os.path.join(self.builddir, "source.xml")
self.xml = ElbeXML(
sourcexmlpath,
buildtype=override_buildtype,
skip_validate=skip_validate,
url_validation=url_validation)
self.arch = self.xml.text("project/arch", key="arch")
self.codename = self.xml.text("project/suite")
if not self.name:
self.name = self.xml.text("project/name")
# If logpath is given, use an AsciiDocLog instance, otherwise log
# to stdout
if logpath:
self.log = ASCIIDocLog(logpath)
else:
self.log = StdoutLog()
self.repo = ProjectRepo(self.arch, self.codename,
os.path.join(self.builddir, "repo"), self.log)
# Create BuildEnv instance, if the chroot directory exists and
# has an etc/elbe_version
if os.path.exists(self.chrootpath):
self.buildenv = BuildEnv(
self.xml, self.log, self.chrootpath, clean=False)
else:
self.buildenv = None
# Create TargetFs instance, if the target directory exists
if os.path.exists(self.targetpath) and self.buildenv:
self.targetfs = TargetFs(self.targetpath, self.log,
self.buildenv.xml, clean=False)
else:
self.targetfs = None
# dont create sysroot instance, it should be build from scratch
# each time, because the pkglist including the -dev packages is
# tracked nowhere.
self.sysrootenv = None
self.log.do('rm -rf %s' % self.sysrootpath)
# same for host_sysroot instance recreate it in any case
self.host_sysrootenv = None
class ElbeProject (object):
# pylint: disable=too-many-instance-attributes
def __init__(
self,
builddir,
xmlpath=None,
logpath=None,
name=None,
override_buildtype=None,
skip_validate=False,
url_validation=ValidationMode.CHECK_ALL,
rpcaptcache_notifier=None,
private_data=None,
postbuild_file=None,
presh_file=None,
postsh_file=None,
savesh_file=None):
# pylint: disable=too-many-arguments
self.builddir = os.path.abspath(str(builddir))
self.chrootpath = os.path.join(self.builddir, "chroot")
self.targetpath = os.path.join(self.builddir, "target")
self.sysrootpath = os.path.join(self.builddir, "sysroot")
self.sdkpath = os.path.join(self.builddir, "sdk")
self.validationpath = os.path.join(self.builddir, "validation.txt")
self.name = name
self.override_buildtype = override_buildtype
self.skip_validate = skip_validate
self.url_validation = url_validation
self.postbuild_file = postbuild_file
self.presh_file = presh_file
self.postsh_file = postsh_file
self.savesh_file = savesh_file
self.private_data = private_data
# Apt-Cache will be created on demand with the specified notifier by
# the get_rpcaptcache method
self._rpcaptcache = None
self.rpcaptcache_notifier = rpcaptcache_notifier
# Initialise Repo Images to Empty list.
self.repo_images = []
self.orig_fname = None
self.orig_files = []
# Use supplied XML file, if given, otherwise use the source.xml
# file of the project
if xmlpath:
self.xml = ElbeXML(
xmlpath,
buildtype=override_buildtype,
skip_validate=skip_validate,
url_validation=url_validation)
else:
sourcexmlpath = os.path.join(self.builddir, "source.xml")
self.xml = ElbeXML(
sourcexmlpath,
buildtype=override_buildtype,
skip_validate=skip_validate,
url_validation=url_validation)
self.arch = self.xml.text("project/arch", key="arch")
self.codename = self.xml.text("project/suite")
if not self.name:
self.name = self.xml.text("project/name")
# If logpath is given, use an AsciiDocLog instance, otherwise log
# to stdout
if logpath:
self.log = ASCIIDocLog(logpath)
else:
self.log = StdoutLog()
self.repo = ProjectRepo(self.arch, self.codename,
os.path.join(self.builddir, "repo"), self.log)
# Create BuildEnv instance, if the chroot directory exists and
# has an etc/elbe_version
if os.path.exists(self.chrootpath):
self.buildenv = BuildEnv(
self.xml, self.log, self.chrootpath, clean=False)
else:
self.buildenv = None
# Create TargetFs instance, if the target directory exists
if os.path.exists(self.targetpath) and self.buildenv:
self.targetfs = TargetFs(self.targetpath, self.log,
self.buildenv.xml, clean=False)
else:
self.targetfs = None
# dont create sysroot instance, it should be build from scratch
# each time, because the pkglist including the -dev packages is
# tracked nowhere.
self.sysrootenv = None
self.log.do('rm -rf %s' % self.sysrootpath)
# same for host_sysroot instance recreate it in any case
self.host_sysrootenv = None
def build_chroottarball(self):
self.log.do("tar cJf %s/chroot.tar.xz \
--exclude=./tmp/* --exclude=./dev/* \
--exclude=./run/* --exclude=./sys/* \
--exclude=./proc/* --exclude=./var/cache/* \
-C %s ." % (self.builddir, self.chrootpath))
def get_sysroot_paths(self):
triplet = self.xml.defs["triplet"]
paths = [
'./usr/include',
'./usr/include/' + triplet,
'./etc/ld.so.conf*',
'./opt/*/lib/*.so',
'./opt/*lib/*.so.*',
'./opt/*/include/',
'./opt/*/lib/' + triplet,
'./opt/*/include/' + triplet,
'./lib/*.so',
'./lib/*.so.*',
'./lib/' + triplet,
'./usr/lib/*.so',
'./usr/lib/*.so',
'./usr/lib/*.so.*',
'./usr/lib/' + triplet]
return paths
def build_sysroot(self):
self.log.do('rm -rf %s; mkdir "%s"' % (self.sysrootpath,
self.sysrootpath))
self.sysrootenv = BuildEnv(self.xml,
self.log,
self.sysrootpath,
clean=True)
# Import keyring
self.sysrootenv.import_keys()
self.log.printo("Keys imported")
self.install_packages(self.sysrootenv, buildenv=False)
# ignore packages from debootstrap
tpkgs = self.xml.get_target_packages()
bspkgs = self.xml.node("debootstrappkgs")
ignore_pkgs = [p.et.text for p in bspkgs if p.et.text not in tpkgs]
ignore_dev_pkgs = []
if self.xml.has('target/pkg-blacklist/sysroot'):
ignore_dev_pkgs = [p.et.text for p in self.xml.node(
"target/pkg-blacklist/sysroot")]
with self.sysrootenv:
try:
self.get_rpcaptcache(env=self.sysrootenv).update()
except Exception as e:
raise AptCacheUpdateError(e)
try:
self.get_rpcaptcache(
env=self.sysrootenv).mark_install_devpkgs(
set(ignore_pkgs), set(ignore_dev_pkgs))
except SystemError as e:
self.log.printo("mark install devpkgs failed: %s" % str(e))
try:
self.get_rpcaptcache(env=self.sysrootenv).commit()
except SystemError as e:
self.log.printo("commiting changes failed: %s" % str(e))
raise AptCacheCommitError(str(e))
try:
self.sysrootenv.rfs.dump_elbeversion(self.xml)
except IOError:
self.log.printo("dump elbeversion into sysroot failed")
sysrootfilelist = os.path.join(self.builddir, "sysroot-filelist")
with self.sysrootenv.rfs:
self.log.do("chroot %s /usr/bin/symlinks -cr /usr/lib" %
self.sysrootpath)
paths = self.get_sysroot_paths()
self.log.do("rm %s" % sysrootfilelist, allow_fail=True)
os.chdir(self.sysrootpath)
for p in paths:
self.log.do('find -path "%s" >> %s' % (p, sysrootfilelist))
self.log.do("tar cfJ %s/sysroot.tar.xz -C %s -T %s" %
(self.builddir, self.sysrootpath, sysrootfilelist))
def build_host_sysroot(self, pkgs, hostsysrootpath):
self.log.do('rm -rf %s; mkdir "%s"' % (hostsysrootpath,
hostsysrootpath))
self.host_sysrootenv = BuildEnv(self.xml,
self.log,
hostsysrootpath,
clean=True,
arch="amd64")
# Import keyring
self.host_sysrootenv.import_keys()
self.log.printo("Keys imported")
with self.host_sysrootenv:
try:
cache = self.get_rpcaptcache(env=self.host_sysrootenv,
norecommend=True)
cache.update()
except Exception as e:
raise AptCacheUpdateError(e)
for p in pkgs:
try:
cache.mark_install(p, None)
except KeyError:
self.log.printo("No Package " + p)
except SystemError as e:
self.log.printo("Error: Unable to correct problems "
"in package %s (%s)" % (p, str(e)))
try:
cache.commit()
except SystemError as e:
self.log.printo("commiting changes failed: %s" % str(e))
raise AptCacheCommitError(str(e))
# This is just a sysroot, some directories
# need to be removed.
#
# This can move into finetuning in the
# second implementation step.
self.host_sysrootenv.rfs.rmtree('/boot')
self.host_sysrootenv.rfs.rmtree('/dev')
self.host_sysrootenv.rfs.rmtree('/etc')
self.host_sysrootenv.rfs.rmtree('/home')
self.host_sysrootenv.rfs.rmtree('/media')
self.host_sysrootenv.rfs.rmtree('/mnt')
self.host_sysrootenv.rfs.rmtree('/proc')
self.host_sysrootenv.rfs.rmtree('/root')
self.host_sysrootenv.rfs.rmtree('/run')
self.host_sysrootenv.rfs.rmtree('/sys')
self.host_sysrootenv.rfs.rmtree('/tmp')
self.host_sysrootenv.rfs.rmtree('/var')
def build_sdk(self):
triplet = self.xml.defs["triplet"]
elfcode = self.xml.defs["elfcode"]
host_pkglist = []
if self.xml.tgt.has('hostsdk-pkg-list'):
for p in self.xml.tgt.node('hostsdk-pkg-list'):
if p.tag == 'pkg':
host_pkglist.append(p.et.text.strip())
else:
try:
host_pkglist.append("g++-%s" % self.xml.defs["sdkarch"])
except KeyError:
raise UnsupportedSDKException(triplet)
host_pkglist.append('gdb-multiarch')
# build target sysroot including libs and headers for the target
self.build_sysroot()
sdktargetpath = os.path.join(self.sdkpath, "sysroots", "target")
self.log.do("mkdir -p %s" % sdktargetpath)
self.log.do("tar xJf %s/sysroot.tar.xz -C %s" % (self.builddir,
sdktargetpath))
# build host sysroot including cross compiler
hostsysrootpath = os.path.join(self.sdkpath, 'sysroots', 'host')
self.build_host_sysroot(host_pkglist, hostsysrootpath)
n = gen_sdk_scripts(triplet,
elfcode,
self.name,
self.xml.text("project/version"),
self.builddir,
self.sdkpath)
# create sdk tar and append it to setup script
self.log.do("cd %s; tar cJf ../sdk.txz ." % self.sdkpath)
self.log.do("cd %s; rm -rf sdk" % self.builddir)
self.log.do("cd %s; cat sdk.txz >> %s" % (self.builddir, n))
self.log.do("cd %s; chmod +x %s" % (self.builddir, n))
self.log.do("cd %s; rm sdk.txz" % self.builddir)
def pbuild(self, p):
self.pdebuild_init()
src_path = os.path.join(self.builddir, "pdebuilder", "current")
src_uri = p.text('.').replace("LOCALMACHINE", "10.0.2.2").strip()
self.log.printo("retrieve pbuild sources: %s" % src_uri)
if p.tag == 'git':
self.log.do("git clone %s %s" % (src_uri, src_path))
try:
self.log.do(
"cd %s; git reset --hard %s" %
(src_path, p.et.attrib['revision']))
except IndexError:
pass
elif p.tag == 'svn':
self.log.do("svn co --non-interactive %s %s" % (src_uri, src_path))
else:
self.log.printo("unknown pbuild source vcs: %s" % p.tag)
# pdebuild_build(-1) means use all cpus
self.pdebuild_build(cpuset=-1, profile="")
def build_cdroms(self, build_bin=True,
build_sources=False, cdrom_size=None):
self.repo_images = []
elog = ASCIIDocLog(self.validationpath, True)
env = None
sysrootstr = ""
if os.path.exists(self.sysrootpath):
sysrootstr = "(including sysroot packages)"
env = BuildEnv(self.xml, self.log, self.sysrootpath,
build_sources=build_sources, clean=False)
else:
env = BuildEnv(self.xml, self.log, self.chrootpath,
build_sources=build_sources, clean=False)
# ensure the /etc/apt/sources.list is created according to
# buil_sources, # build_bin flag, ensure to reopen it with
# the new 'sources.list'
with env:
env.seed_etc()
self.drop_rpcaptcache(env=env)
with env:
init_codename = self.xml.get_initvm_codename()
if build_bin:
elog.h1("Binary CD %s" % sysrootstr)
self.repo_images += mk_binary_cdrom(env.rfs,
self.arch,
self.codename,
init_codename,
self.xml,
self.builddir,
self.log,
cdrom_size=cdrom_size)
if build_sources:
elog.h1("Source CD %s" % sysrootstr)
try:
self.repo_images += mk_source_cdrom(env.rfs,
self.arch,
self.codename,
init_codename,
self.builddir,
self.log,
cdrom_size=cdrom_size,
xml=self.xml)
except SystemError as e:
# e.g. no deb-src urls specified
elog.printo(str(e))
def build(self, build_bin=False, build_sources=False, cdrom_size=None,
skip_pkglist=False, skip_pbuild=False):
# pylint: disable=too-many-arguments
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
# Write the log header
self.write_log_header()
# Validate Apt Sources
if build_sources:
m = ValidationMode.CHECK_ALL
else:
m = ValidationMode.CHECK_BINARIES
self.xml.validate_apt_sources(m, self.arch)
if self.xml.has('target/pbuilder') and not skip_pbuild:
if not os.path.exists(os.path.join(self.builddir, "pbuilder")):
self.create_pbuilder()
for p in self.xml.node('target/pbuilder'):
self.pbuild(p)
# the package might be needed by a following pbuild, so update
# the project repo that it can be installed in as
# build-dependency
self.repo.finalize()
# To avoid update cache errors, the project repo needs to have
# Release and Packages files, even if it's empty. So don't do this
# in the if case above!
self.repo.finalize()
# Create the build environment, if it does not a valid one
# self.buildenv might be set when we come here.
# However, if its not a full_buildenv, we specify clean here,
# so it gets rebuilt properly.
if not self.has_full_buildenv():
self.log.do('mkdir -p "%s"' % self.chrootpath)
self.buildenv = BuildEnv(self.xml, self.log, self.chrootpath,
build_sources=build_sources, clean=True)
skip_pkglist = False
# Import keyring
self.buildenv.import_keys()
self.log.printo("Keys imported")
# Install packages
if not skip_pkglist:
self.install_packages(self.buildenv)
try:
self.buildenv.rfs.dump_elbeversion(self.xml)
except IOError:
self.log.printo("dump elbeversion failed")
# Extract target FS. We always create a new instance here with
# clean=true, because we want a pristine directory.
self.targetfs = TargetFs(self.targetpath, self.log,
self.buildenv.xml, clean=True)
os.chdir(self.buildenv.rfs.fname(''))
extract_target(self.buildenv.rfs, self.xml, self.targetfs,
self.log, self.get_rpcaptcache())
# The validation file is created using check_full_pkgs() and
# elbe_report(), both opening the file in append mode. So if an
# old validation file already exists, it must be deleted first.
if os.path.isfile(self.validationpath):
os.unlink(self.validationpath)
# Package validation and package list
if not skip_pkglist:
pkgs = self.xml.xml.node("/target/pkg-list")
if self.xml.has("fullpkgs"):
check_full_pkgs(pkgs, self.xml.xml.node("/fullpkgs"),
self.validationpath, self.get_rpcaptcache())
else:
check_full_pkgs(pkgs, None, self.validationpath,
self.get_rpcaptcache())
dump_fullpkgs(self.xml, self.buildenv.rfs, self.get_rpcaptcache())
self.xml.dump_elbe_version()
self.targetfs.write_fstab(self.xml)
# Dump ELBE version
try:
self.targetfs.dump_elbeversion(self.xml)
except MemoryError:
self.log.printo("dump elbeversion failed")
# install packages for buildenv
if not skip_pkglist:
self.install_packages(self.buildenv, buildenv=True)
# Write source.xml
try:
sourcexmlpath = os.path.join(self.builddir, "source.xml")
self.xml.xml.write(sourcexmlpath)
except MemoryError:
self.log.printo("write source.xml failed (archive to huge?)")
# Elbe report
reportpath = os.path.join(self.builddir, "elbe-report.txt")
elbe_report(self.xml, self.buildenv, self.get_rpcaptcache(),
reportpath, self.validationpath, self.targetfs)
# the current license code raises an exception that interrupts the hole
# build if a licence can't be converted to utf-8. Exception handling
# can be removed as soon as the licence code is more stable
lic_err = False
try:
f = io.open(
os.path.join(
self.builddir,
"licence.txt"),
"w+",
encoding='utf-8')
self.buildenv.rfs.write_licenses(
f, self.log, os.path.join(
self.builddir, "licence.xml"))
except Exception:
self.log.printo("error during generating licence.txt/xml")
self.log.printo(sys.exc_info()[0])
lic_err = True
finally:
f.close()
if lic_err:
os.remove(os.path.join(self.builddir, "licence.txt"))
os.remove(os.path.join(self.builddir, "licence.xml"))
# Use some handwaving to determine grub version
# jessie and wheezy grubs are 2.0 but differ in behaviour
#
# We might also want support for legacy grub
if (self.get_rpcaptcache().is_installed('grub-pc') and
self.get_rpcaptcache().is_installed('grub-efi-amd64-bin')):
grub_version = 202
grub_fw_type = "hybrid"
elif self.get_rpcaptcache().is_installed('grub-pc'):
if self.codename == "wheezy":
grub_version = 199
else:
grub_version = 202
grub_fw_type = "bios"
elif self.get_rpcaptcache().is_installed('grub-efi-amd64'):
grub_version = 202
grub_fw_type = "efi"
elif self.get_rpcaptcache().is_installed('grub-legacy'):
self.log.printo("package grub-legacy is installed, "
"this is obsolete, skipping grub")
grub_version = 0
grub_fw_type = ""
else:
self.log.printo("package grub-pc is not installed, skipping grub")
# version 0 == skip_grub
grub_version = 0
grub_fw_type = ""
self.targetfs.part_target(self.builddir, grub_version, grub_fw_type)
self.build_cdroms(build_bin, build_sources, cdrom_size)
if self.postbuild_file:
self.log.h2("postbuild script:")
self.log.do(self.postbuild_file + ' "%s %s %s"' % (
self.builddir,
self.xml.text("project/version"),
self.xml.text("project/name")),
allow_fail=True)
do_prj_finetuning(self.xml,
self.log,
self.buildenv,
self.targetfs,
self.builddir)
self.targetfs.pack_images(self.builddir)
os.system('cat "%s"' % self.validationpath)
def pdebuild_init(self):
# Remove pdebuilder directory, containing last build results
self.log.do('rm -rf "%s"' % os.path.join(self.builddir,
"pdebuilder"))
# Remove pbuilder/result directory
self.log.do('rm -rf "%s"' % os.path.join(self.builddir,
"pbuilder", "result"))
# Recreate the directories removed
self.log.do('mkdir -p "%s"' % os.path.join(self.builddir,
"pbuilder", "result"))
def pdebuild(self, cpuset, profile):
self.pdebuild_init()
pbdir = os.path.join(self.builddir, "pdebuilder", "current")
self.log.do('mkdir -p "%s"' % os.path.join(pbdir))
try:
for orig_fname in self.orig_files:
ofname = os.path.join(self.builddir, orig_fname)
self.log.do('mv "%s" "%s"' % (ofname,
os.path.join(self.builddir,
"pdebuilder")))
finally:
self.orig_fname = None
self.orig_files = []
# Untar current_pdebuild.tar.gz into pdebuilder/current
self.log.do(
'tar xfz "%s" -C "%s"' %
(os.path.join(
self.builddir,
"current_pdebuild.tar.gz"),
pbdir))
self.pdebuild_build(cpuset, profile)
self.repo.finalize()
def pdebuild_build(self, cpuset, profile):
# check whether we have to use taskset to run pdebuild
# this might be useful, when things like java dont
# work with multithreading
#
if cpuset != -1:
cpuset_cmd = 'taskset %d ' % cpuset
else:
# cpuset == -1 means empty cpuset_cmd
cpuset_cmd = ''
try:
self.log.do('cd "%s"; %s pdebuild --debbuildopts "-j%s -sa" '
'--configfile "%s" '
'--use-pdebuild-internal --buildresult "%s"' % (
os.path.join(self.builddir,
"pdebuilder",
"current"),
cpuset_cmd,
cfg['pbuilder_jobs'],
os.path.join(self.builddir, "pbuilderrc"),
os.path.join(self.builddir, "pbuilder", "result")),
env_add={'DEB_BUILD_PROFILES': profile})
self.repo.remove(os.path.join(self.builddir,
"pdebuilder",
"current",
"debian",
"control"))
self.repo.include(os.path.join(self.builddir,
"pbuilder", "result", "*.changes"))
except CommandError:
self.log.printo('')
self.log.printo('Package fails to build.')
self.log.printo('Please make sure, that the submitted package '
'builds in pbuilder')
def update_pbuilder(self):
self.log.do(
'pbuilder --update --configfile "%s" --aptconfdir "%s"' %
(os.path.join(
self.builddir, "pbuilderrc"), os.path.join(
self.builddir, "aptconfdir")))
def create_pbuilder(self):
# Remove old pbuilder directory, if it exists
self.log.do('rm -rf "%s"' % os.path.join(self.builddir, "pbuilder"))
# make hooks.d and pbuilder directory
self.log.do(
'mkdir -p "%s"' %
os.path.join(
self.builddir,
"pbuilder",
"hooks.d"))
self.log.do(
'mkdir -p "%s"' %
os.path.join(
self.builddir,
"pbuilder",
"aptcache"))
self.log.do(
'mkdir -p "%s"' %
os.path.join(
self.builddir,
"aptconfdir",
"apt.conf.d"))
# write config files
pbuilder_write_config(self.builddir, self.xml, self.log)
pbuilder_write_apt_conf(self.builddir, self.xml)
pbuilder_write_repo_hook(self.builddir, self.xml)
self.log.do(
'chmod -R 755 "%s"' %
os.path.join(
self.builddir,
"pbuilder",
"hooks.d"))
# Run pbuilder --create
self.log.do('pbuilder --create --configfile "%s" --aptconfdir "%s" '
'--debootstrapopts --include="git gnupg2"' % (
os.path.join(self.builddir, "pbuilderrc"),
os.path.join(self.builddir, "aptconfdir")))
def sync_xml_to_disk(self):
try:
sourcexmlpath = os.path.join(self.builddir, "source.xml")
self.xml.xml.write(sourcexmlpath)
except MemoryError:
self.log.printo("write source.xml failed (archive to huge?)")
def get_rpcaptcache(self, env=None, norecommend=None):
if not env:
env = self.buildenv
if norecommend is None:
norecommend = not self.xml.prj.has('install-recommends'),
if env.arch == "default":
arch = self.arch
else:
arch = env.arch
if env.rpcaptcache is None:
env.rpcaptcache = get_rpcaptcache(
env.rfs,
self.log.fp.name,
arch,
self.rpcaptcache_notifier,
norecommend,
self.xml.prj.has('noauth'))
return env.rpcaptcache
def drop_rpcaptcache(self, env=None):
if not env:
env = self.buildenv
env.rpcaptcache = None
def has_full_buildenv(self):
if os.path.exists(self.chrootpath):
elbeversionpath = os.path.join(self.chrootpath,
"etc", "elbe_version")
if os.path.isfile(elbeversionpath):
return True
self.log.printo("%s exists, but it does not have "
"an etc/elbe_version file." % self.chrootpath)
# Apparently we do not have a functional build environment
return False
return False
def set_xml(self, xmlpath):
# Use supplied XML file, if given, otherwise change to source.xml
if not xmlpath:
xmlpath = os.path.join(self.builddir, "source.xml")
newxml = ElbeXML(xmlpath, buildtype=self.override_buildtype,
skip_validate=self.skip_validate,
url_validation=self.url_validation)
# New XML file has to have the same architecture
oldarch = self.xml.text("project/arch", key="arch")
newarch = newxml.text("project/arch", key="arch")
if newarch != oldarch:
raise IncompatibleArchitectureException(oldarch, newarch)
# Throw away old APT cache, targetfs and buildenv
self.targetfs = None
self.buildenv = None
# dont create sysroot instance, it should be build from scratch
# each time, because the pkglist including the -dev packages is
# tracked nowhere.
self.sysrootenv = None
self.log.do('rm -rf %s' % self.sysrootpath)
self.xml = newxml
# Create a new BuildEnv instance, if we have a build directory
if self.has_full_buildenv():
self.buildenv = BuildEnv(
self.xml, self.log, self.chrootpath, clean=False)
# Create TargetFs instance, if the target directory exists.
# We use the old content of the directory if no rebuild is done, so
# don't clean it (yet).
if os.path.exists(self.targetpath):
self.targetfs = TargetFs(self.targetpath, self.log,
self.xml, clean=False)
else:
self.targetfs = None
def write_log_header(self):
if self.name:
self.log.h1("ELBE Report for Project " + self.name)
else:
self.log.h1("ELBE Report")
self.log.printo("report timestamp: " +
datetime.datetime.now().strftime("%Y%m%d-%H%M%S"))
def install_packages(self, target, buildenv=False):
# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
with target:
# First update the apt cache
try:
self.get_rpcaptcache(env=target).update()
except Exception as e:
raise AptCacheUpdateError(e)
# Then dump the debootstrap packages
if target.fresh_debootstrap:
if target.need_dumpdebootstrap:
dump_debootstrappkgs(self.xml,
self.get_rpcaptcache(env=target))
dump_initvmpkgs(self.xml)
target.need_dumpdebootstrap = False
source = self.xml
try:
initxml = ElbeXML(
"/var/cache/elbe/source.xml",
skip_validate=self.skip_validate,
url_validation=ValidationMode.NO_CHECK)
self.xml.get_initvmnode_from(initxml)
except ValidationError as e:
self.log.printo(
"/var/cache/elbe/source.xml validation failed")
self.log.printo(str(e))
self.log.printo("will not copy initvm node")
except IOError:
self.log.printo("/var/cache/elbe/source.xml not available")
self.log.printo("can not copy initvm node")
except NoInitvmNode:
self.log.printo("/var/cache/elbe/source.xml is available")
self.log.printo("But it does not contain an initvm node")
else:
sourcepath = os.path.join(self.builddir, "source.xml")
source = ElbeXML(sourcepath,
buildtype=self.override_buildtype,
skip_validate=self.skip_validate,
url_validation=self.url_validation)
self.xml.get_debootstrappkgs_from(source)
try:
self.xml.get_initvmnode_from(source)
except NoInitvmNode:
self.log.printo("source.xml is available")
self.log.printo("But it does not contain an initvm node")
# Seed /etc, we need /etc/hosts for hostname -f to work correctly
if not buildenv:
target.seed_etc()
# remove all non-essential packages to ensure that on a incremental
# build packages can be removed
debootstrap_pkgs = []
for p in self.xml.node("debootstrappkgs"):
debootstrap_pkgs.append(p.et.text)
pkgs = target.xml.get_target_packages() + debootstrap_pkgs
if buildenv:
pkgs = pkgs + target.xml.get_buildenv_packages()
# Now install requested packages
for p in pkgs:
try:
self.get_rpcaptcache(env=target).mark_install(p, None)
except KeyError:
self.log.printo("No Package " + p)
except SystemError as e:
self.log.printo("Error: Unable to correct problems in "
"package %s (%s)" % (p, str(e)))
# temporary disabled because of
# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=776057
# the functions cleans up to much
# self.get_rpcaptcache().cleanup(debootstrap_pkgs + pkgs)
try:
self.get_rpcaptcache(env=target).commit()
except SystemError as e:
self.log.printo("commiting changes failed: %s" % str(e))
raise AptCacheCommitError(str(e))
class ElbeProject(object):
# pylint: disable=too-many-instance-attributes
# pylint: disable=too-many-public-methods
def __init__(self,
builddir,
xmlpath=None,
name=None,
override_buildtype=None,
skip_validate=False,
url_validation=ValidationMode.CHECK_ALL,
rpcaptcache_notifier=None,
private_data=None,
postbuild_file=None,
presh_file=None,
postsh_file=None,
savesh_file=None):
# pylint: disable=too-many-arguments
self.builddir = os.path.abspath(str(builddir))
self.chrootpath = os.path.join(self.builddir, "chroot")
self.targetpath = os.path.join(self.builddir, "target")
self.sysrootpath = os.path.join(self.builddir, "sysroot")
self.sdkpath = os.path.join(self.builddir, "sdk")
self.validationpath = os.path.join(self.builddir, "validation.txt")
self.name = name
self.override_buildtype = override_buildtype
self.skip_validate = skip_validate
self.url_validation = url_validation
self.postbuild_file = postbuild_file
self.presh_file = presh_file
self.postsh_file = postsh_file
self.savesh_file = savesh_file
self.private_data = private_data
# Apt-Cache will be created on demand with the specified notifier by
# the get_rpcaptcache method
self._rpcaptcache = None
self.rpcaptcache_notifier = rpcaptcache_notifier
# Initialise Repo Images to Empty list.
self.repo_images = []
self.orig_fname = None
self.orig_files = []
# Use supplied XML file, if given, otherwise use the source.xml
# file of the project
if xmlpath:
self.xml = ElbeXML(xmlpath,
buildtype=override_buildtype,
skip_validate=skip_validate,
url_validation=url_validation)
else:
sourcexmlpath = os.path.join(self.builddir, "source.xml")
self.xml = ElbeXML(sourcexmlpath,
buildtype=override_buildtype,
skip_validate=skip_validate,
url_validation=url_validation)
self.arch = self.xml.text("project/arch", key="arch")
self.codename = self.xml.text("project/suite")
if not self.name:
self.name = self.xml.text("project/name")
self.repo = ProjectRepo(self.arch, self.codename,
os.path.join(self.builddir, "repo"))
# Create BuildEnv instance, if the chroot directory exists and
# has an etc/elbe_version
if os.path.exists(self.chrootpath):
self.buildenv = BuildEnv(self.xml, self.chrootpath, clean=False)
else:
self.buildenv = None
# Create TargetFs instance, if the target directory exists
if os.path.exists(self.targetpath) and self.buildenv:
self.targetfs = TargetFs(self.targetpath,
self.buildenv.xml,
clean=False)
else:
self.targetfs = None
# don't create sysroot instance, it should be built from scratch
# each time, because the pkglist including the -dev packages is
# tracked nowhere.
self.sysrootenv = None
do('rm -rf %s' % self.sysrootpath)
# same for host_sysroot instance recreate it in any case
self.host_sysrootenv = None
def build_chroottarball(self):
do("tar cJf %s/chroot.tar.xz "
"--exclude=./tmp/* --exclude=./dev/* "
"--exclude=./run/* --exclude=./sys/* "
"--exclude=./proc/* --exclude=./var/cache/* "
"-C %s ." % (self.builddir, self.chrootpath))
def get_sysroot_paths(self):
triplet = self.xml.defs["triplet"]
paths = [
'./usr/include', './usr/include/' + triplet, './etc/ld.so.conf*',
'./opt/*/lib/*.so', './opt/*lib/*.so.*', './opt/*/include',
'./opt/*/lib/' + triplet, './opt/*/include/' + triplet,
'./lib/*.so', './lib/*.so.*', './lib/' + triplet,
'./usr/lib/debug/.build-id/*/*.debug', './usr/lib/*.so',
'./usr/lib/*.so.*', './usr/lib/' + triplet
]
return paths
def build_sysroot(self):
do('rm -rf %s; mkdir "%s"' % (self.sysrootpath, self.sysrootpath))
self.sysrootenv = BuildEnv(self.xml, self.sysrootpath, clean=True)
# Import keyring
self.sysrootenv.import_keys()
logging.info("Keys imported")
self.install_packages(self.sysrootenv, buildenv=False)
# ignore packages from debootstrap
tpkgs = self.xml.get_target_packages()
bspkgs = self.xml.node("debootstrappkgs")
ignore_pkgs = [p.et.text for p in bspkgs if p.et.text not in tpkgs]
ignore_dev_pkgs = []
if self.xml.has('target/pkg-blacklist/sysroot'):
ignore_dev_pkgs = [
p.et.text
for p in self.xml.node("target/pkg-blacklist/sysroot")
]
with self.sysrootenv:
try:
cache = self.get_rpcaptcache(env=self.sysrootenv)
cache.update()
except Exception as e:
raise AptCacheUpdateError(e)
try:
cache.mark_install_devpkgs(set(ignore_pkgs),
set(ignore_dev_pkgs))
except SystemError as e:
logging.exception("Mark install devpkgs failed")
try:
cache.commit()
except SystemError as e:
logging.exception("Commiting changes failed")
raise AptCacheCommitError(str(e))
self.gen_licenses("sysroot-target", self.sysrootenv,
[p.name for p in cache.get_installed_pkgs()])
try:
self.sysrootenv.rfs.dump_elbeversion(self.xml)
except IOError:
logging.exception("Dump elbeversion into sysroot failed")
sysrootfilelist = os.path.join(self.builddir, "sysroot-filelist")
with self.sysrootenv.rfs:
chroot(self.sysrootpath, "/usr/bin/symlinks -cr /usr/lib")
paths = self.get_sysroot_paths()
do("rm %s" % sysrootfilelist, allow_fail=True)
os.chdir(self.sysrootpath)
for p in paths:
do('find -path "%s" >> %s' % (p, sysrootfilelist))
# include /lib if it is a symlink (buster and later)
if os.path.islink(self.sysrootpath + '/lib'):
with open(sysrootfilelist, 'a') as filelist_fd:
filelist_fd.write('./lib')
do("tar cfJ %s/sysroot.tar.xz -C %s -T %s" %
(self.builddir, self.sysrootpath, sysrootfilelist))
def build_host_sysroot(self, pkgs, hostsysrootpath):
do('rm -rf %s; mkdir "%s"' % (hostsysrootpath, hostsysrootpath))
self.host_sysrootenv = BuildEnv(self.xml,
hostsysrootpath,
clean=True,
arch="amd64")
# Import keyring
self.host_sysrootenv.import_keys()
logging.info("Keys imported")
with self.host_sysrootenv:
try:
cache = self.get_rpcaptcache(env=self.host_sysrootenv,
norecommend=True)
cache.update()
except Exception as e:
raise AptCacheUpdateError(e)
for p in pkgs:
try:
cache.mark_install(p, None)
except KeyError:
logging.exception("No Package %s", p)
except SystemError:
logging.exception(
"Unable to correct problems in "
"package %s", p)
try:
cache.commit()
except SystemError as e:
logging.exception("Commiting changes failed")
raise AptCacheCommitError(str(e))
self.gen_licenses("sysroot-host", self.host_sysrootenv,
[p.name for p in cache.get_installed_pkgs()])
# This is just a sysroot, some directories
# need to be removed.
#
# This can move into finetuning in the
# second implementation step.
self.host_sysrootenv.rfs.rmtree('/boot')
self.host_sysrootenv.rfs.rmtree('/dev')
self.host_sysrootenv.rfs.rmtree('/etc')
self.host_sysrootenv.rfs.rmtree('/home')
self.host_sysrootenv.rfs.rmtree('/media')
self.host_sysrootenv.rfs.rmtree('/mnt')
self.host_sysrootenv.rfs.rmtree('/proc')
self.host_sysrootenv.rfs.rmtree('/root')
self.host_sysrootenv.rfs.rmtree('/run')
self.host_sysrootenv.rfs.rmtree('/sys')
self.host_sysrootenv.rfs.rmtree('/tmp')
self.host_sysrootenv.rfs.rmtree('/var')
def build_sdk(self):
triplet = self.xml.defs["triplet"]
elfcode = self.xml.defs["elfcode"]
host_pkglist = []
if self.xml.tgt.has('hostsdk-pkg-list'):
for p in self.xml.tgt.node('hostsdk-pkg-list'):
if p.tag == 'pkg':
host_pkglist.append(p.et.text.strip())
else:
try:
host_pkglist.append(self.xml.defs["sdkgccpkg"])
except KeyError:
raise UnsupportedSDKException(triplet)
host_pkglist.append('gdb-multiarch')
# build target sysroot including libs and headers for the target
self.build_sysroot()
sdktargetpath = os.path.join(self.sdkpath, "sysroots", "target")
do("mkdir -p %s" % sdktargetpath)
do("tar xJf %s/sysroot.tar.xz -C %s" % (self.builddir, sdktargetpath))
# build host sysroot including cross compiler
hostsysrootpath = os.path.join(self.sdkpath, 'sysroots', 'host')
self.build_host_sysroot(host_pkglist, hostsysrootpath)
n = gen_sdk_scripts(triplet, elfcode, self.name,
self.xml.text("project/version"), self.builddir,
self.sdkpath)
# create sdk tar and append it to setup script
do("cd %s; tar cJf ../sdk.txz ." % self.sdkpath)
do("cd %s; rm -rf sdk" % self.builddir)
do("cd %s; cat sdk.txz >> %s" % (self.builddir, n))
do("cd %s; chmod +x %s" % (self.builddir, n))
do("cd %s; rm sdk.txz" % self.builddir)
def pbuild(self, p):
self.pdebuild_init()
src_path = os.path.join(self.builddir, "pdebuilder", "current")
src_uri = p.text('.').replace("LOCALMACHINE", "10.0.2.2").strip()
logging.info("Retrieve pbuild sources: %s", src_uri)
if p.tag == 'git':
do("git clone %s %s" % (src_uri, src_path))
try:
do("cd %s; git reset --hard %s" %
(src_path, p.et.attrib['revision']))
except IndexError:
pass
elif p.tag == 'svn':
do("svn co --non-interactive %s %s" % (src_uri, src_path))
elif p.tag == 'src-pkg':
pdb_path = os.path.join(self.builddir, "pdebuilder")
os.mkdir(pdb_path)
apt_args = '--yes -q --download-only'
if self.xml.prj.has('noauth'):
apt_args += ' --allow-unauthenticated'
do('cd "%s";apt-get source %s "%s"' %
(pdb_path, apt_args, src_uri))
do('dpkg-source -x %s/*.dsc "%s"' % (pdb_path, src_path))
else:
logging.info("Unknown pbuild source: %s", p.tag)
# pdebuild_build(-1) means use all cpus
self.pdebuild_build(cpuset=-1, profile="", cross=False)
def build_cdroms(self,
build_bin=True,
build_sources=False,
cdrom_size=None,
tgt_pkg_lst=None):
# pylint: disable=too-many-branches
# pylint: disable=too-many-locals
self.repo_images = []
env = None
sysrootstr = ""
if os.path.exists(self.sysrootpath):
sysrootstr = "(including sysroot packages)"
env = BuildEnv(self.xml,
self.sysrootpath,
build_sources=build_sources,
clean=False)
else:
env = BuildEnv(self.xml,
self.chrootpath,
build_sources=build_sources,
clean=False)
# ensure the /etc/apt/sources.list is created according to
# buil_sources, # build_bin flag, ensure to reopen it with
# the new 'sources.list'
with env:
env.seed_etc()
self.drop_rpcaptcache(env=env)
with env:
init_codename = self.xml.get_initvm_codename()
if build_bin:
validation.info("Binary CD %s", sysrootstr)
self.repo_images += mk_binary_cdrom(env.rfs, self.arch,
self.codename,
init_codename, self.xml,
self.builddir)
if build_sources:
if not cdrom_size and self.xml.has("src-cdrom/size"):
cdrom_size = size_to_int(self.xml.text("src-cdrom/size"))
validation.info("Source CD %s", sysrootstr)
# Target component
cache = self.get_rpcaptcache(env=self.buildenv)
tgt_lst = cache.get_corresponding_source_packages(
pkg_lst=tgt_pkg_lst)
components = {"target": (self.targetfs, cache, tgt_lst)}
# Main component
main_lst = []
if self.xml is not None:
tmp_lst = []
for pkg_node in self.xml.node("debootstrappkgs"):
pkg = XMLPackage(pkg_node, self.arch)
tmp_lst.append(pkg.name)
main_lst = cache.get_corresponding_source_packages(
pkg_lst=tmp_lst)
components["main"] = (env.rfs, cache, main_lst)
# Added component
other_components = [(env, "added")]
# Let's build a list of (build_env, name) for the
# other RFS if they exist
host_sysroot_path = os.path.join(self.sdkpath, "sysroots",
"host")
for path, name in [(self.chrootpath, "chroot"),
(host_sysroot_path, "sysroot-host")]:
if os.path.exists(path) and env.path != path:
tmp_env = BuildEnv(self.xml, path)
with tmp_env:
tmp_env.seed_etc()
other_components.append((tmp_env, name))
# Now let's generate the correct (rfs, cache, pkg_lst)
# components using the full installed packages
for build_env, name in other_components:
cache = self.get_rpcaptcache(env=build_env)
src_lst = cache.get_corresponding_source_packages()
components[name] = (build_env.rfs, cache, src_lst)
try:
# Using kwargs here allows us to avoid making
# special case for when self.xml is None
kwargs = {"cdrom_size": cdrom_size, "xml": self.xml}
if self.xml is not None:
kwargs["mirror"] = self.xml.get_primary_mirror(
env.rfs.fname("cdrom"))
self.repo_images += mk_source_cdrom(
components, self.codename, init_codename,
self.builddir, **kwargs)
except SystemError as e:
# e.g. no deb-src urls specified
validation.error(str(e))
def build(self,
build_bin=False,
build_sources=False,
cdrom_size=None,
skip_pkglist=False,
skip_pbuild=False):
# pylint: disable=too-many-arguments
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
# Write the log header
self.write_log_header()
# Validate Apt Sources
if build_sources:
m = ValidationMode.CHECK_ALL
else:
m = ValidationMode.CHECK_BINARIES
self.xml.validate_apt_sources(m, self.arch)
if self.xml.has('target/pbuilder') and not skip_pbuild:
if not os.path.exists(os.path.join(self.builddir, "pbuilder")):
self.create_pbuilder(cross=False)
for p in self.xml.node('target/pbuilder'):
self.pbuild(p)
# the package might be needed by a following pbuild, so update
# the project repo that it can be installed in as
# build-dependency
self.repo.finalize()
# To avoid update cache errors, the project repo needs to have
# Release and Packages files, even if it's empty. So don't do this
# in the if case above!
self.repo.finalize()
# Create the build environment, if it does not a valid one
# self.buildenv might be set when we come here.
# However, if its not a full_buildenv, we specify clean here,
# so it gets rebuilt properly.
if not self.has_full_buildenv():
do('mkdir -p "%s"' % self.chrootpath)
self.buildenv = BuildEnv(self.xml,
self.chrootpath,
build_sources=build_sources,
clean=True)
skip_pkglist = False
# Import keyring
self.buildenv.import_keys()
logging.info("Keys imported")
# Install packages
if not skip_pkglist:
self.install_packages(self.buildenv)
try:
self.buildenv.rfs.dump_elbeversion(self.xml)
except IOError:
logging.exception("Dump elbeversion failed")
# Extract target FS. We always create a new instance here with
# clean=true, because we want a pristine directory.
self.targetfs = TargetFs(self.targetpath,
self.buildenv.xml,
clean=True)
os.chdir(self.buildenv.rfs.fname(''))
extract_target(self.buildenv.rfs, self.xml, self.targetfs,
self.get_rpcaptcache())
# Package validation and package list
if not skip_pkglist:
pkgs = self.xml.xml.node("/target/pkg-list")
if self.xml.has("fullpkgs"):
check_full_pkgs(pkgs, self.xml.xml.node("/fullpkgs"),
self.get_rpcaptcache())
else:
check_full_pkgs(pkgs, None, self.get_rpcaptcache())
dump_fullpkgs(self.xml, self.buildenv.rfs, self.get_rpcaptcache())
self.xml.dump_elbe_version()
self.targetfs.write_fstab(self.xml)
# Dump ELBE version
try:
self.targetfs.dump_elbeversion(self.xml)
except MemoryError:
logging.exception("Dump elbeversion failed")
# install packages for buildenv
if not skip_pkglist:
self.install_packages(self.buildenv, buildenv=True)
# Write source.xml
try:
sourcexmlpath = os.path.join(self.builddir, "source.xml")
self.xml.xml.write(sourcexmlpath)
except MemoryError:
logging.exception("Write source.xml failed (archive to huge?)")
# Elbe report
cache = self.get_rpcaptcache()
tgt_pkgs = elbe_report(self.xml, self.buildenv, cache, self.targetfs)
# chroot' licenses
self.gen_licenses("chroot", self.buildenv,
[p.name for p in cache.get_installed_pkgs()])
self.gen_licenses("target", self.buildenv, tgt_pkgs)
# Use some handwaving to determine grub version
grub_arch = "ia32" if self.arch == "i386" else self.arch
grub_fw_type = []
grub_version = 0
if self.get_rpcaptcache().is_installed('grub-pc'):
grub_version = 202
grub_fw_type.append("bios")
if self.get_rpcaptcache().is_installed('grub-efi-%s-bin' % grub_arch):
grub_version = 202
grub_tgt = "x86_64" if self.arch == "amd64" else self.arch
grub_fw_type.extend(["efi", grub_tgt + "-efi"])
if (self.get_rpcaptcache().is_installed('shim-signed')
and self.get_rpcaptcache().is_installed(
'grub-efi-%s-signed' % grub_arch)):
grub_version = 202
grub_fw_type.append("shimfix")
if self.get_rpcaptcache().is_installed('grub-legacy'):
logging.warning("package grub-legacy is installed, "
"this is obsolete.")
grub_version = 97
grub_fw_type.append("bios")
elif not grub_fw_type:
logging.warning(
"neither package grub-pc nor grub-efi-%s-bin "
"are installed, skipping grub", grub_arch)
self.targetfs.part_target(self.builddir, grub_version, grub_fw_type)
self.build_cdroms(build_bin,
build_sources,
cdrom_size,
tgt_pkg_lst=tgt_pkgs)
if self.postbuild_file:
logging.info("Postbuild script")
cmd = ' "%s %s %s"' % (self.builddir,
self.xml.text("project/version"),
self.xml.text("project/name"))
do(self.postbuild_file + cmd, allow_fail=True)
do_prj_finetuning(self.xml, self.buildenv, self.targetfs,
self.builddir)
self.targetfs.pack_images(self.builddir)
system('cat "%s"' % self.validationpath)
def pdebuild_init(self):
# Remove pdebuilder directory, containing last build results
do('rm -rf "%s"' % os.path.join(self.builddir, "pdebuilder"))
# Remove pbuilder/result directory
do('rm -rf "%s" "%s"' %
(os.path.join(self.builddir, "pbuilder", "result"),
os.path.join(self.builddir, "pbuilder_cross", "result")))
# Recreate the directories removed
if os.path.exists(os.path.join(self.builddir, "pbuilder_cross")):
do('mkdir -p "%s"' %
os.path.join(self.builddir, "pbuilder_cross", "result"))
else:
do('mkdir -p "%s"' %
os.path.join(self.builddir, "pbuilder", "result"))
def pdebuild(self, cpuset, profile, cross):
cross_pbuilderrc = os.path.join(self.builddir, "cross_pbuilderrc")
if cross and not os.path.exists(cross_pbuilderrc):
logging.error("Please make sure that you create the pbuilder "
"environment with the --cross option if you want to "
"use the build command with --cross.")
sys.exit(20)
if os.path.exists(cross_pbuilderrc) and not cross:
logging.error(
"Please make sure that if you created the pbuilder "
"environment without the --cross option, you use the "
"build command without --cross too.")
sys.exit(20)
self.pdebuild_init()
pbdir = os.path.join(self.builddir, "pdebuilder", "current")
do('mkdir -p "%s"' % os.path.join(pbdir))
try:
for orig_fname in self.orig_files:
ofname = os.path.join(self.builddir, orig_fname)
do('mv "%s" "%s"' %
(ofname, os.path.join(self.builddir, "pdebuilder")))
finally:
self.orig_fname = None
self.orig_files = []
# Untar current_pdebuild.tar.gz into pdebuilder/current
do('tar xfz "%s" -C "%s"' %
(os.path.join(self.builddir, "current_pdebuild.tar.gz"), pbdir))
self.pdebuild_build(cpuset, profile, cross)
self.repo.finalize()
def pdebuild_build(self, cpuset, profile, cross):
# check whether we have to use taskset to run pdebuild
# this might be useful, when things like java dont
# work with multithreading
#
if cpuset != -1:
cpuset_cmd = 'taskset %d ' % cpuset
else:
# cpuset == -1 means empty cpuset_cmd
cpuset_cmd = ''
try:
if cross:
do('cd "%s"; dpkg-source -b .; %s '
'pbuilder build --host-arch %s --configfile "%s" '
'--basetgz "%s" --buildresult "%s" '
'../*.dsc' %
(os.path.join(self.builddir, "pdebuilder",
"current"), cpuset_cmd, self.arch,
os.path.join(self.builddir, "cross_pbuilderrc"),
os.path.join(self.builddir, "pbuilder_cross", "base.tgz"),
os.path.join(self.builddir, "pbuilder_cross", "result")),
env_add={'DEB_BUILD_PROFILES': profile.replace(",", " ")})
self.repo.include(
os.path.join(self.builddir, "pbuilder_cross", "result",
"*.changes"))
else:
do('cd "%s"; %s pdebuild --debbuildopts "-j%s -sa" '
'--configfile "%s" '
'--use-pdebuild-internal --buildresult "%s"' %
(os.path.join(self.builddir, "pdebuilder",
"current"), cpuset_cmd, cfg['pbuilder_jobs'],
os.path.join(self.builddir, "pbuilderrc"),
os.path.join(self.builddir, "pbuilder", "result")),
env_add={'DEB_BUILD_PROFILES': profile.replace(",", " ")})
self.repo.include(
os.path.join(self.builddir, "pbuilder", "result",
"*.changes"))
self.repo.remove(
os.path.join(self.builddir, "pdebuilder", "current", "debian",
"control"))
except CommandError:
logging.exception("Package fails to build.\n"
"Please make sure, that the submitted package "
"builds in pbuilder")
def update_pbuilder(self):
do('pbuilder --update --configfile "%s" --aptconfdir "%s"' %
(os.path.join(self.builddir, "pbuilderrc"),
os.path.join(self.builddir, "aptconfdir")))
def create_pbuilder(self, cross):
# Remove old pbuilder directory, if it exists
do('rm -rf "%s" "%s"' %
(os.path.join(self.builddir, "pbuilder"),
os.path.join(self.builddir, "pbuilder_cross")))
# make hooks.d and pbuilder directory
if cross:
do('mkdir -p "%s"' %
os.path.join(self.builddir, "pbuilder_cross", "hooks.d"))
do('mkdir -p "%s"' %
os.path.join(self.builddir, "pbuilder_cross", "aptcache"))
else:
do('mkdir -p "%s"' %
os.path.join(self.builddir, "pbuilder", "hooks.d"))
do('mkdir -p "%s"' %
os.path.join(self.builddir, "pbuilder", "aptcache"))
do('mkdir -p "%s"' %
os.path.join(self.builddir, "aptconfdir", "apt.conf.d"))
# write config files
if cross:
pbuilder_write_cross_config(self.builddir, self.xml)
pbuilder_write_repo_hook(self.builddir, self.xml, cross)
do('chmod -R 755 "%s"' %
os.path.join(self.builddir, "pbuilder_cross", "hooks.d"))
else:
pbuilder_write_config(self.builddir, self.xml)
pbuilder_write_repo_hook(self.builddir, self.xml, cross)
do('chmod -R 755 "%s"' %
os.path.join(self.builddir, "pbuilder", "hooks.d"))
pbuilder_write_apt_conf(self.builddir, self.xml)
# Run pbuilder --create
if cross:
do('pbuilder --create --buildplace "%s" '
'--configfile "%s" --aptconfdir "%s" '
'--debootstrapopts --include="git gnupg";' %
(os.path.join(self.builddir, "pbuilder_cross"),
os.path.join(self.builddir, "cross_pbuilderrc"),
os.path.join(self.builddir, "aptconfdir")))
else:
do('pbuilder --create --configfile "%s" --aptconfdir "%s" '
'--debootstrapopts --include="git gnupg"' %
(os.path.join(self.builddir, "pbuilderrc"),
os.path.join(self.builddir, "aptconfdir")))
def sync_xml_to_disk(self):
try:
sourcexmlpath = os.path.join(self.builddir, "source.xml")
self.xml.xml.write(sourcexmlpath)
except MemoryError:
logging.exception("write source.xml failed (archive to huge?)")
def get_rpcaptcache(self, env=None, norecommend=None):
if not env:
env = self.buildenv
if norecommend is None:
norecommend = not self.xml.prj.has('install-recommends')
if env.arch == "default":
arch = self.arch
else:
arch = env.arch
if env.rpcaptcache is None:
env.rpcaptcache = get_rpcaptcache(env.rfs, arch,
self.rpcaptcache_notifier,
norecommend,
self.xml.prj.has('noauth'))
return env.rpcaptcache
def drop_rpcaptcache(self, env=None):
if not env:
env = self.buildenv
env.rpcaptcache = None
def has_full_buildenv(self):
if os.path.exists(self.chrootpath):
elbeversionpath = os.path.join(self.chrootpath, "etc",
"elbe_version")
if os.path.isfile(elbeversionpath):
return True
logging.warning(
"%s exists, but it does not have "
"an etc/elbe_version file.", self.chrootpath)
# Apparently we do not have a functional build environment
return False
return False
def set_xml(self, xmlpath):
# Use supplied XML file, if given, otherwise change to source.xml
if not xmlpath:
xmlpath = os.path.join(self.builddir, "source.xml")
newxml = ElbeXML(xmlpath,
buildtype=self.override_buildtype,
skip_validate=self.skip_validate,
url_validation=self.url_validation)
# New XML file has to have the same architecture
oldarch = self.xml.text("project/arch", key="arch")
newarch = newxml.text("project/arch", key="arch")
if newarch != oldarch:
raise IncompatibleArchitectureException(oldarch, newarch)
# Throw away old APT cache, targetfs and buildenv
self.targetfs = None
self.buildenv = None
# don't create sysroot instance, it should be built from scratch
# each time, because the pkglist including the -dev packages is
# tracked nowhere.
self.sysrootenv = None
do('rm -rf %s' % self.sysrootpath)
self.xml = newxml
# Create a new BuildEnv instance, if we have a build directory
if self.has_full_buildenv():
self.buildenv = BuildEnv(self.xml, self.chrootpath, clean=False)
# Create TargetFs instance, if the target directory exists.
# We use the old content of the directory if no rebuild is done, so
# don't clean it (yet).
if os.path.exists(self.targetpath):
self.targetfs = TargetFs(self.targetpath, self.xml, clean=False)
else:
self.targetfs = None
def write_log_header(self):
logging.info("ELBE Report for Project %s\n"
"Report timestamp: %s", self.name,
datetime.datetime.now().strftime("%Y%m%d-%H%M%S"))
def install_packages(self, target, buildenv=False):
# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
with target:
# First update the apt cache
try:
self.get_rpcaptcache(env=target).update()
except Exception as e:
raise AptCacheUpdateError(e)
# Then dump the debootstrap packages
if target.fresh_debootstrap:
if target.need_dumpdebootstrap:
dump_debootstrappkgs(self.xml,
self.get_rpcaptcache(env=target))
dump_initvmpkgs(self.xml)
target.need_dumpdebootstrap = False
source = self.xml
source_path = "/var/cache/elbe/source.xml"
try:
initxml = ElbeXML(source_path,
skip_validate=self.skip_validate,
url_validation=ValidationMode.NO_CHECK)
self.xml.get_initvmnode_from(initxml)
except ValidationError:
logging.exception(
"%s validation failed. "
"Will not copy initvm node", source_path)
except IOError:
logging.exception(
"%s not available. "
"Can not copy initvm node", source_path)
except NoInitvmNode:
logging.exception(
"%s is available. But it does not "
"contain an initvm node", source_path)
else:
sourcepath = os.path.join(self.builddir, "source.xml")
source = ElbeXML(sourcepath,
buildtype=self.override_buildtype,
skip_validate=self.skip_validate,
url_validation=self.url_validation)
self.xml.get_debootstrappkgs_from(source)
try:
self.xml.get_initvmnode_from(source)
except NoInitvmNode:
logging.exception("source.xml is available. "
"But it does not contain an initvm node")
# Seed /etc, we need /etc/hosts for hostname -f to work correctly
if not buildenv:
target.seed_etc()
# remove all non-essential packages to ensure that on a incremental
# build packages can be removed
debootstrap_pkgs = []
for p in self.xml.node("debootstrappkgs"):
debootstrap_pkgs.append(p.et.text)
pkgs = target.xml.get_target_packages() + debootstrap_pkgs
if buildenv:
pkgs = pkgs + target.xml.get_buildenv_packages()
# Now install requested packages
for p in pkgs:
try:
self.get_rpcaptcache(env=target).mark_install(p, None)
except KeyError:
logging.exception("No Package %s", p)
except SystemError:
logging.exception(
"Unable to correct problems "
"in package %s", p)
# temporary disabled because of
# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=776057
# the functions cleans up to much
# self.get_rpcaptcache().cleanup(debootstrap_pkgs + pkgs)
try:
self.get_rpcaptcache(env=target).commit()
except SystemError as e:
logging.exception("Commiting changes failed")
raise AptCacheCommitError(str(e))
def gen_licenses(self, rfs, env, pkg_list):
lic_txt_fname = os.path.join(self.builddir, "licence-%s.txt" % rfs)
lic_xml_fname = os.path.join(self.builddir, "licence-%s.xml" % rfs)
pkg_list.sort()
with io.open(lic_txt_fname, 'w+', encoding='utf-8',
errors='replace') as f:
env.rfs.write_licenses(f, pkg_list, lic_xml_fname)
def run_command(argv):
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
oparser = OptionParser(
usage="usage: %prog check_updates [options] <source-xmlfile>")
oparser.add_option(
"-s",
"--script",
dest="script",
help="filename of script to run when an update is required")
oparser.add_option("--skip-validation",
action="store_true",
dest="skip_validation",
default=False,
help="Skip xml schema validation")
oparser.add_option("-c",
"--changelogs",
dest="changelogs",
help="filename of changelog xml file")
(opt, args) = oparser.parse_args(argv)
if len(args) != 1:
print("Wrong number of arguments")
oparser.print_help()
sys.exit(20)
if not opt.skip_validation:
validation = validate_xml(args[0])
if validation:
print("xml validation failed. Bailing out")
for i in validation:
print(i)
sys.exit(20)
print("checking %s" % args[0])
xml = ElbeXML(args[0])
fullp = xml.node("fullpkgs")
arch = xml.text("project/buildimage/arch", key="arch")
v = virtapt.VirtApt(xml)
for p in fullp:
pname = p.et.text
pauto = p.et.get('auto')
if pauto != "true":
v.mark_install(pname)
errors = 0
required_updates = 0
update_packages = []
for p in fullp:
xp = XMLPackage(p, arch)
pname = p.et.text
pauto = p.et.get('auto')
if not v.has_pkg(xp.name):
if not xp.is_auto_installed:
print(
"%s does not exist in cache but is specified in pkg-list" %
xp.name)
errors += 1
else:
print("%s is no more required" % xp.name)
required_updates += 1
continue
if v.marked_install(xp.name):
cver = v.get_candidate_ver(xp.name)
if xp.installed_version != cver:
print("%s: %s != %s" % (xp.name, xp.installed_version, cver))
required_updates += 1
if opt.changelogs:
v.mark_pkg_download(xp.name)
xp.candidate_version = cver
update_packages.append(xp)
sys.stdout.flush()
sys.stderr.flush()
if errors > 0:
print("%d Errors occured, xml files needs fixing" % errors)
if opt.script:
system("%s ERRORS %s" % (opt.script, args[0]), allow_fail=True)
elif required_updates > 0:
print("%d updates required" % required_updates)
if opt.changelogs:
build_changelog_xml(v, opt, update_packages)
if opt.script:
system("%s UPDATE %s" % (opt.script, args[0]), allow_fail=True)
else:
print("No Updates available")
def __init__(self,
builddir,
xmlpath=None,
name=None,
override_buildtype=None,
skip_validate=False,
url_validation=ValidationMode.CHECK_ALL,
rpcaptcache_notifier=None,
private_data=None,
postbuild_file=None,
presh_file=None,
postsh_file=None,
savesh_file=None):
# pylint: disable=too-many-arguments
self.builddir = os.path.abspath(str(builddir))
self.chrootpath = os.path.join(self.builddir, "chroot")
self.targetpath = os.path.join(self.builddir, "target")
self.sysrootpath = os.path.join(self.builddir, "sysroot")
self.sdkpath = os.path.join(self.builddir, "sdk")
self.validationpath = os.path.join(self.builddir, "validation.txt")
self.name = name
self.override_buildtype = override_buildtype
self.skip_validate = skip_validate
self.url_validation = url_validation
self.postbuild_file = postbuild_file
self.presh_file = presh_file
self.postsh_file = postsh_file
self.savesh_file = savesh_file
self.private_data = private_data
# Apt-Cache will be created on demand with the specified notifier by
# the get_rpcaptcache method
self._rpcaptcache = None
self.rpcaptcache_notifier = rpcaptcache_notifier
# Initialise Repo Images to Empty list.
self.repo_images = []
self.orig_fname = None
self.orig_files = []
# Use supplied XML file, if given, otherwise use the source.xml
# file of the project
if xmlpath:
self.xml = ElbeXML(xmlpath,
buildtype=override_buildtype,
skip_validate=skip_validate,
url_validation=url_validation)
else:
sourcexmlpath = os.path.join(self.builddir, "source.xml")
self.xml = ElbeXML(sourcexmlpath,
buildtype=override_buildtype,
skip_validate=skip_validate,
url_validation=url_validation)
self.arch = self.xml.text("project/arch", key="arch")
self.codename = self.xml.text("project/suite")
if not self.name:
self.name = self.xml.text("project/name")
self.repo = ProjectRepo(self.arch, self.codename,
os.path.join(self.builddir, "repo"))
# Create BuildEnv instance, if the chroot directory exists and
# has an etc/elbe_version
if os.path.exists(self.chrootpath):
self.buildenv = BuildEnv(self.xml, self.chrootpath, clean=False)
else:
self.buildenv = None
# Create TargetFs instance, if the target directory exists
if os.path.exists(self.targetpath) and self.buildenv:
self.targetfs = TargetFs(self.targetpath,
self.buildenv.xml,
clean=False)
else:
self.targetfs = None
# don't create sysroot instance, it should be built from scratch
# each time, because the pkglist including the -dev packages is
# tracked nowhere.
self.sysrootenv = None
do('rm -rf %s' % self.sysrootpath)
# same for host_sysroot instance recreate it in any case
self.host_sysrootenv = None
class ElbeProject(object):
# pylint: disable=too-many-instance-attributes
def __init__(self,
builddir,
xmlpath=None,
logpath=None,
name=None,
override_buildtype=None,
skip_validate=False,
url_validation=ValidationMode.CHECK_ALL,
rpcaptcache_notifier=None,
private_data=None,
postbuild_file=None,
presh_file=None,
postsh_file=None,
savesh_file=None):
# pylint: disable=too-many-arguments
self.builddir = os.path.abspath(str(builddir))
self.chrootpath = os.path.join(self.builddir, "chroot")
self.targetpath = os.path.join(self.builddir, "target")
self.sysrootpath = os.path.join(self.builddir, "sysroot")
self.sdkpath = os.path.join(self.builddir, "sdk")
self.validationpath = os.path.join(self.builddir, "validation.txt")
self.name = name
self.override_buildtype = override_buildtype
self.skip_validate = skip_validate
self.url_validation = url_validation
self.postbuild_file = postbuild_file
self.presh_file = presh_file
self.postsh_file = postsh_file
self.savesh_file = savesh_file
self.private_data = private_data
# Apt-Cache will be created on demand with the specified notifier by
# the get_rpcaptcache method
self._rpcaptcache = None
self.rpcaptcache_notifier = rpcaptcache_notifier
# Initialise Repo Images to Empty list.
self.repo_images = []
self.orig_fname = None
self.orig_files = []
# Use supplied XML file, if given, otherwise use the source.xml
# file of the project
if xmlpath:
self.xml = ElbeXML(xmlpath,
buildtype=override_buildtype,
skip_validate=skip_validate,
url_validation=url_validation)
else:
sourcexmlpath = os.path.join(self.builddir, "source.xml")
self.xml = ElbeXML(sourcexmlpath,
buildtype=override_buildtype,
skip_validate=skip_validate,
url_validation=url_validation)
self.arch = self.xml.text("project/arch", key="arch")
self.codename = self.xml.text("project/suite")
if not self.name:
self.name = self.xml.text("project/name")
# If logpath is given, use an AsciiDocLog instance, otherwise log
# to stdout
if logpath:
self.log = ASCIIDocLog(logpath)
else:
self.log = StdoutLog()
self.repo = ProjectRepo(self.arch, self.codename,
os.path.join(self.builddir, "repo"), self.log)
# Create BuildEnv instance, if the chroot directory exists and
# has an etc/elbe_version
if os.path.exists(self.chrootpath):
self.buildenv = BuildEnv(self.xml,
self.log,
self.chrootpath,
clean=False)
else:
self.buildenv = None
# Create TargetFs instance, if the target directory exists
if os.path.exists(self.targetpath) and self.buildenv:
self.targetfs = TargetFs(self.targetpath,
self.log,
self.buildenv.xml,
clean=False)
else:
self.targetfs = None
# dont create sysroot instance, it should be build from scratch
# each time, because the pkglist including the -dev packages is
# tracked nowhere.
self.sysrootenv = None
self.log.do('rm -rf %s' % self.sysrootpath)
# same for host_sysroot instance recreate it in any case
self.host_sysrootenv = None
def build_chroottarball(self):
self.log.do("tar cJf %s/chroot.tar.xz \
--exclude=./tmp/* --exclude=./dev/* \
--exclude=./run/* --exclude=./sys/* \
--exclude=./proc/* --exclude=./var/cache/* \
-C %s ." % (self.builddir, self.chrootpath))
def get_sysroot_paths(self):
triplet = self.xml.defs["triplet"]
paths = [
'./usr/include', './usr/include/' + triplet, './etc/ld.so.conf*',
'./opt/*/lib/*.so', './opt/*lib/*.so.*', './opt/*/include/',
'./opt/*/lib/' + triplet, './opt/*/include/' + triplet,
'./lib/*.so', './lib/*.so.*', './lib/' + triplet, './usr/lib/*.so',
'./usr/lib/*.so', './usr/lib/*.so.*', './usr/lib/' + triplet
]
return paths
def build_sysroot(self):
self.log.do('rm -rf %s; mkdir "%s"' %
(self.sysrootpath, self.sysrootpath))
self.sysrootenv = BuildEnv(self.xml,
self.log,
self.sysrootpath,
clean=True)
# Import keyring
self.sysrootenv.import_keys()
self.log.printo("Keys imported")
self.install_packages(self.sysrootenv, buildenv=False)
# ignore packages from debootstrap
tpkgs = self.xml.get_target_packages()
bspkgs = self.xml.node("debootstrappkgs")
ignore_pkgs = [p.et.text for p in bspkgs if p.et.text not in tpkgs]
ignore_dev_pkgs = []
if self.xml.has('target/pkg-blacklist/sysroot'):
ignore_dev_pkgs = [
p.et.text
for p in self.xml.node("target/pkg-blacklist/sysroot")
]
with self.sysrootenv:
try:
self.get_rpcaptcache(env=self.sysrootenv).update()
except Exception as e:
raise AptCacheUpdateError(e)
try:
self.get_rpcaptcache(env=self.sysrootenv).mark_install_devpkgs(
set(ignore_pkgs), set(ignore_dev_pkgs))
except SystemError as e:
self.log.printo("mark install devpkgs failed: %s" % str(e))
try:
self.get_rpcaptcache(env=self.sysrootenv).commit()
except SystemError as e:
self.log.printo("commiting changes failed: %s" % str(e))
raise AptCacheCommitError(str(e))
try:
self.sysrootenv.rfs.dump_elbeversion(self.xml)
except IOError:
self.log.printo("dump elbeversion into sysroot failed")
sysrootfilelist = os.path.join(self.builddir, "sysroot-filelist")
with self.sysrootenv.rfs:
self.log.do("chroot %s /usr/bin/symlinks -cr /usr/lib" %
self.sysrootpath)
paths = self.get_sysroot_paths()
self.log.do("rm %s" % sysrootfilelist, allow_fail=True)
os.chdir(self.sysrootpath)
for p in paths:
self.log.do('find -path "%s" >> %s' % (p, sysrootfilelist))
self.log.do("tar cfJ %s/sysroot.tar.xz -C %s -T %s" %
(self.builddir, self.sysrootpath, sysrootfilelist))
def build_host_sysroot(self, pkgs, hostsysrootpath):
self.log.do('rm -rf %s; mkdir "%s"' %
(hostsysrootpath, hostsysrootpath))
self.host_sysrootenv = BuildEnv(self.xml,
self.log,
hostsysrootpath,
clean=True,
arch="amd64")
# Import keyring
self.host_sysrootenv.import_keys()
self.log.printo("Keys imported")
with self.host_sysrootenv:
try:
cache = self.get_rpcaptcache(env=self.host_sysrootenv,
norecommend=True)
cache.update()
except Exception as e:
raise AptCacheUpdateError(e)
for p in pkgs:
try:
cache.mark_install(p, None)
except KeyError:
self.log.printo("No Package " + p)
except SystemError as e:
self.log.printo("Error: Unable to correct problems "
"in package %s (%s)" % (p, str(e)))
try:
cache.commit()
except SystemError as e:
self.log.printo("commiting changes failed: %s" % str(e))
raise AptCacheCommitError(str(e))
# This is just a sysroot, some directories
# need to be removed.
#
# This can move into finetuning in the
# second implementation step.
self.host_sysrootenv.rfs.rmtree('/boot')
self.host_sysrootenv.rfs.rmtree('/dev')
self.host_sysrootenv.rfs.rmtree('/etc')
self.host_sysrootenv.rfs.rmtree('/home')
self.host_sysrootenv.rfs.rmtree('/media')
self.host_sysrootenv.rfs.rmtree('/mnt')
self.host_sysrootenv.rfs.rmtree('/proc')
self.host_sysrootenv.rfs.rmtree('/root')
self.host_sysrootenv.rfs.rmtree('/run')
self.host_sysrootenv.rfs.rmtree('/sys')
self.host_sysrootenv.rfs.rmtree('/tmp')
self.host_sysrootenv.rfs.rmtree('/var')
def build_sdk(self):
triplet = self.xml.defs["triplet"]
elfcode = self.xml.defs["elfcode"]
host_pkglist = []
if self.xml.tgt.has('hostsdk-pkg-list'):
for p in self.xml.tgt.node('hostsdk-pkg-list'):
if p.tag == 'pkg':
host_pkglist.append(p.et.text.strip())
else:
try:
host_pkglist.append("g++-%s" % self.xml.defs["sdkarch"])
except KeyError:
raise UnsupportedSDKException(triplet)
host_pkglist.append('gdb-multiarch')
# build target sysroot including libs and headers for the target
self.build_sysroot()
sdktargetpath = os.path.join(self.sdkpath, "sysroots", "target")
self.log.do("mkdir -p %s" % sdktargetpath)
self.log.do("tar xJf %s/sysroot.tar.xz -C %s" %
(self.builddir, sdktargetpath))
# build host sysroot including cross compiler
hostsysrootpath = os.path.join(self.sdkpath, 'sysroots', 'host')
self.build_host_sysroot(host_pkglist, hostsysrootpath)
n = gen_sdk_scripts(triplet, elfcode, self.name,
self.xml.text("project/version"), self.builddir,
self.sdkpath)
# create sdk tar and append it to setup script
self.log.do("cd %s; tar cJf ../sdk.txz ." % self.sdkpath)
self.log.do("cd %s; rm -rf sdk" % self.builddir)
self.log.do("cd %s; cat sdk.txz >> %s" % (self.builddir, n))
self.log.do("cd %s; chmod +x %s" % (self.builddir, n))
self.log.do("cd %s; rm sdk.txz" % self.builddir)
def pbuild(self, p):
self.pdebuild_init()
src_path = os.path.join(self.builddir, "pdebuilder", "current")
src_uri = p.text('.').replace("LOCALMACHINE", "10.0.2.2").strip()
self.log.printo("retrieve pbuild sources: %s" % src_uri)
if p.tag == 'git':
self.log.do("git clone %s %s" % (src_uri, src_path))
try:
self.log.do("cd %s; git reset --hard %s" %
(src_path, p.et.attrib['revision']))
except IndexError:
pass
elif p.tag == 'svn':
self.log.do("svn co --non-interactive %s %s" % (src_uri, src_path))
else:
self.log.printo("unknown pbuild source vcs: %s" % p.tag)
# pdebuild_build(-1) means use all cpus
self.pdebuild_build(cpuset=-1, profile="")
def build_cdroms(self,
build_bin=True,
build_sources=False,
cdrom_size=None):
self.repo_images = []
elog = ASCIIDocLog(self.validationpath, True)
env = None
sysrootstr = ""
if os.path.exists(self.sysrootpath):
sysrootstr = "(including sysroot packages)"
env = BuildEnv(self.xml,
self.log,
self.sysrootpath,
build_sources=build_sources,
clean=False)
else:
env = BuildEnv(self.xml,
self.log,
self.chrootpath,
build_sources=build_sources,
clean=False)
# ensure the /etc/apt/sources.list is created according to
# buil_sources, # build_bin flag, ensure to reopen it with
# the new 'sources.list'
with env:
env.seed_etc()
self.drop_rpcaptcache(env=env)
with env:
init_codename = self.xml.get_initvm_codename()
if build_bin:
elog.h1("Binary CD %s" % sysrootstr)
self.repo_images += mk_binary_cdrom(env.rfs,
self.arch,
self.codename,
init_codename,
self.xml,
self.builddir,
self.log,
cdrom_size=cdrom_size)
if build_sources:
elog.h1("Source CD %s" % sysrootstr)
try:
self.repo_images += mk_source_cdrom(env.rfs,
self.arch,
self.codename,
init_codename,
self.builddir,
self.log,
cdrom_size=cdrom_size,
xml=self.xml)
except SystemError as e:
# e.g. no deb-src urls specified
elog.printo(str(e))
def build(self,
build_bin=False,
build_sources=False,
cdrom_size=None,
skip_pkglist=False,
skip_pbuild=False):
# pylint: disable=too-many-arguments
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
# Write the log header
self.write_log_header()
# Validate Apt Sources
if build_sources:
m = ValidationMode.CHECK_ALL
else:
m = ValidationMode.CHECK_BINARIES
self.xml.validate_apt_sources(m, self.arch)
if self.xml.has('target/pbuilder') and not skip_pbuild:
if not os.path.exists(os.path.join(self.builddir, "pbuilder")):
self.create_pbuilder()
for p in self.xml.node('target/pbuilder'):
self.pbuild(p)
# the package might be needed by a following pbuild, so update
# the project repo that it can be installed in as
# build-dependency
self.repo.finalize()
# To avoid update cache errors, the project repo needs to have
# Release and Packages files, even if it's empty. So don't do this
# in the if case above!
self.repo.finalize()
# Create the build environment, if it does not a valid one
# self.buildenv might be set when we come here.
# However, if its not a full_buildenv, we specify clean here,
# so it gets rebuilt properly.
if not self.has_full_buildenv():
self.log.do('mkdir -p "%s"' % self.chrootpath)
self.buildenv = BuildEnv(self.xml,
self.log,
self.chrootpath,
build_sources=build_sources,
clean=True)
skip_pkglist = False
# Import keyring
self.buildenv.import_keys()
self.log.printo("Keys imported")
# Install packages
if not skip_pkglist:
self.install_packages(self.buildenv)
try:
self.buildenv.rfs.dump_elbeversion(self.xml)
except IOError:
self.log.printo("dump elbeversion failed")
# Extract target FS. We always create a new instance here with
# clean=true, because we want a pristine directory.
self.targetfs = TargetFs(self.targetpath,
self.log,
self.buildenv.xml,
clean=True)
os.chdir(self.buildenv.rfs.fname(''))
extract_target(self.buildenv.rfs, self.xml, self.targetfs, self.log,
self.get_rpcaptcache())
# The validation file is created using check_full_pkgs() and
# elbe_report(), both opening the file in append mode. So if an
# old validation file already exists, it must be deleted first.
if os.path.isfile(self.validationpath):
os.unlink(self.validationpath)
# Package validation and package list
if not skip_pkglist:
pkgs = self.xml.xml.node("/target/pkg-list")
if self.xml.has("fullpkgs"):
check_full_pkgs(pkgs, self.xml.xml.node("/fullpkgs"),
self.validationpath, self.get_rpcaptcache())
else:
check_full_pkgs(pkgs, None, self.validationpath,
self.get_rpcaptcache())
dump_fullpkgs(self.xml, self.buildenv.rfs, self.get_rpcaptcache())
self.xml.dump_elbe_version()
self.targetfs.write_fstab(self.xml)
# Dump ELBE version
try:
self.targetfs.dump_elbeversion(self.xml)
except MemoryError:
self.log.printo("dump elbeversion failed")
# install packages for buildenv
if not skip_pkglist:
self.install_packages(self.buildenv, buildenv=True)
# Write source.xml
try:
sourcexmlpath = os.path.join(self.builddir, "source.xml")
self.xml.xml.write(sourcexmlpath)
except MemoryError:
self.log.printo("write source.xml failed (archive to huge?)")
# Elbe report
reportpath = os.path.join(self.builddir, "elbe-report.txt")
elbe_report(self.xml, self.buildenv, self.get_rpcaptcache(),
reportpath, self.validationpath, self.targetfs)
# the current license code raises an exception that interrupts the hole
# build if a licence can't be converted to utf-8. Exception handling
# can be removed as soon as the licence code is more stable
lic_err = False
try:
f = io.open(os.path.join(self.builddir, "licence.txt"),
"w+",
encoding='utf-8')
self.buildenv.rfs.write_licenses(
f, self.log, os.path.join(self.builddir, "licence.xml"))
except Exception:
self.log.printo("error during generating licence.txt/xml")
self.log.printo(sys.exc_info()[0])
lic_err = True
finally:
f.close()
if lic_err:
os.remove(os.path.join(self.builddir, "licence.txt"))
os.remove(os.path.join(self.builddir, "licence.xml"))
# Use some handwaving to determine grub version
# jessie and wheezy grubs are 2.0 but differ in behaviour
#
# We might also want support for legacy grub
if (self.get_rpcaptcache().is_installed('grub-pc')
and self.get_rpcaptcache().is_installed('grub-efi-amd64-bin')):
grub_version = 202
grub_fw_type = "hybrid"
elif self.get_rpcaptcache().is_installed('grub-pc'):
if self.codename == "wheezy":
grub_version = 199
else:
grub_version = 202
grub_fw_type = "bios"
elif self.get_rpcaptcache().is_installed('grub-efi-amd64'):
grub_version = 202
grub_fw_type = "efi"
elif self.get_rpcaptcache().is_installed('grub-legacy'):
self.log.printo("package grub-legacy is installed, "
"this is obsolete, skipping grub")
grub_version = 0
grub_fw_type = ""
else:
self.log.printo("package grub-pc is not installed, skipping grub")
# version 0 == skip_grub
grub_version = 0
grub_fw_type = ""
self.targetfs.part_target(self.builddir, grub_version, grub_fw_type)
self.build_cdroms(build_bin, build_sources, cdrom_size)
if self.postbuild_file:
self.log.h2("postbuild script:")
self.log.do(self.postbuild_file + ' "%s %s %s"' %
(self.builddir, self.xml.text("project/version"),
self.xml.text("project/name")),
allow_fail=True)
do_prj_finetuning(self.xml, self.log, self.buildenv, self.targetfs,
self.builddir)
self.targetfs.pack_images(self.builddir)
system('cat "%s"' % self.validationpath)
def pdebuild_init(self):
# Remove pdebuilder directory, containing last build results
self.log.do('rm -rf "%s"' % os.path.join(self.builddir, "pdebuilder"))
# Remove pbuilder/result directory
self.log.do('rm -rf "%s"' %
os.path.join(self.builddir, "pbuilder", "result"))
# Recreate the directories removed
self.log.do('mkdir -p "%s"' %
os.path.join(self.builddir, "pbuilder", "result"))
def pdebuild(self, cpuset, profile):
self.pdebuild_init()
pbdir = os.path.join(self.builddir, "pdebuilder", "current")
self.log.do('mkdir -p "%s"' % os.path.join(pbdir))
try:
for orig_fname in self.orig_files:
ofname = os.path.join(self.builddir, orig_fname)
self.log.do(
'mv "%s" "%s"' %
(ofname, os.path.join(self.builddir, "pdebuilder")))
finally:
self.orig_fname = None
self.orig_files = []
# Untar current_pdebuild.tar.gz into pdebuilder/current
self.log.do(
'tar xfz "%s" -C "%s"' %
(os.path.join(self.builddir, "current_pdebuild.tar.gz"), pbdir))
self.pdebuild_build(cpuset, profile)
self.repo.finalize()
def pdebuild_build(self, cpuset, profile):
# check whether we have to use taskset to run pdebuild
# this might be useful, when things like java dont
# work with multithreading
#
if cpuset != -1:
cpuset_cmd = 'taskset %d ' % cpuset
else:
# cpuset == -1 means empty cpuset_cmd
cpuset_cmd = ''
try:
self.log.do('cd "%s"; %s pdebuild --debbuildopts "-j%s -sa" '
'--configfile "%s" '
'--use-pdebuild-internal --buildresult "%s"' %
(os.path.join(self.builddir, "pdebuilder", "current"),
cpuset_cmd, cfg['pbuilder_jobs'],
os.path.join(self.builddir, "pbuilderrc"),
os.path.join(self.builddir, "pbuilder", "result")),
env_add={'DEB_BUILD_PROFILES': profile})
self.repo.remove(
os.path.join(self.builddir, "pdebuilder", "current", "debian",
"control"))
self.repo.include(
os.path.join(self.builddir, "pbuilder", "result", "*.changes"))
except CommandError:
self.log.printo('')
self.log.printo('Package fails to build.')
self.log.printo('Please make sure, that the submitted package '
'builds in pbuilder')
def update_pbuilder(self):
self.log.do('pbuilder --update --configfile "%s" --aptconfdir "%s"' %
(os.path.join(self.builddir, "pbuilderrc"),
os.path.join(self.builddir, "aptconfdir")))
def create_pbuilder(self):
# Remove old pbuilder directory, if it exists
self.log.do('rm -rf "%s"' % os.path.join(self.builddir, "pbuilder"))
# make hooks.d and pbuilder directory
self.log.do('mkdir -p "%s"' %
os.path.join(self.builddir, "pbuilder", "hooks.d"))
self.log.do('mkdir -p "%s"' %
os.path.join(self.builddir, "pbuilder", "aptcache"))
self.log.do('mkdir -p "%s"' %
os.path.join(self.builddir, "aptconfdir", "apt.conf.d"))
# write config files
pbuilder_write_config(self.builddir, self.xml, self.log)
pbuilder_write_apt_conf(self.builddir, self.xml)
pbuilder_write_repo_hook(self.builddir, self.xml)
self.log.do('chmod -R 755 "%s"' %
os.path.join(self.builddir, "pbuilder", "hooks.d"))
# Run pbuilder --create
self.log.do('pbuilder --create --configfile "%s" --aptconfdir "%s" '
'--debootstrapopts --include="git gnupg"' %
(os.path.join(self.builddir, "pbuilderrc"),
os.path.join(self.builddir, "aptconfdir")))
def sync_xml_to_disk(self):
try:
sourcexmlpath = os.path.join(self.builddir, "source.xml")
self.xml.xml.write(sourcexmlpath)
except MemoryError:
self.log.printo("write source.xml failed (archive to huge?)")
def get_rpcaptcache(self, env=None, norecommend=None):
if not env:
env = self.buildenv
if norecommend is None:
norecommend = not self.xml.prj.has('install-recommends'),
if env.arch == "default":
arch = self.arch
else:
arch = env.arch
if env.rpcaptcache is None:
env.rpcaptcache = get_rpcaptcache(env.rfs, self.log.fp.name, arch,
self.rpcaptcache_notifier,
norecommend,
self.xml.prj.has('noauth'))
return env.rpcaptcache
def drop_rpcaptcache(self, env=None):
if not env:
env = self.buildenv
env.rpcaptcache = None
def has_full_buildenv(self):
if os.path.exists(self.chrootpath):
elbeversionpath = os.path.join(self.chrootpath, "etc",
"elbe_version")
if os.path.isfile(elbeversionpath):
return True
self.log.printo("%s exists, but it does not have "
"an etc/elbe_version file." % self.chrootpath)
# Apparently we do not have a functional build environment
return False
return False
def set_xml(self, xmlpath):
# Use supplied XML file, if given, otherwise change to source.xml
if not xmlpath:
xmlpath = os.path.join(self.builddir, "source.xml")
newxml = ElbeXML(xmlpath,
buildtype=self.override_buildtype,
skip_validate=self.skip_validate,
url_validation=self.url_validation)
# New XML file has to have the same architecture
oldarch = self.xml.text("project/arch", key="arch")
newarch = newxml.text("project/arch", key="arch")
if newarch != oldarch:
raise IncompatibleArchitectureException(oldarch, newarch)
# Throw away old APT cache, targetfs and buildenv
self.targetfs = None
self.buildenv = None
# dont create sysroot instance, it should be build from scratch
# each time, because the pkglist including the -dev packages is
# tracked nowhere.
self.sysrootenv = None
self.log.do('rm -rf %s' % self.sysrootpath)
self.xml = newxml
# Create a new BuildEnv instance, if we have a build directory
if self.has_full_buildenv():
self.buildenv = BuildEnv(self.xml,
self.log,
self.chrootpath,
clean=False)
# Create TargetFs instance, if the target directory exists.
# We use the old content of the directory if no rebuild is done, so
# don't clean it (yet).
if os.path.exists(self.targetpath):
self.targetfs = TargetFs(self.targetpath,
self.log,
self.xml,
clean=False)
else:
self.targetfs = None
def write_log_header(self):
if self.name:
self.log.h1("ELBE Report for Project " + self.name)
else:
self.log.h1("ELBE Report")
self.log.printo("report timestamp: " +
datetime.datetime.now().strftime("%Y%m%d-%H%M%S"))
def install_packages(self, target, buildenv=False):
# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
with target:
# First update the apt cache
try:
self.get_rpcaptcache(env=target).update()
except Exception as e:
raise AptCacheUpdateError(e)
# Then dump the debootstrap packages
if target.fresh_debootstrap:
if target.need_dumpdebootstrap:
dump_debootstrappkgs(self.xml,
self.get_rpcaptcache(env=target))
dump_initvmpkgs(self.xml)
target.need_dumpdebootstrap = False
source = self.xml
try:
initxml = ElbeXML("/var/cache/elbe/source.xml",
skip_validate=self.skip_validate,
url_validation=ValidationMode.NO_CHECK)
self.xml.get_initvmnode_from(initxml)
except ValidationError as e:
self.log.printo(
"/var/cache/elbe/source.xml validation failed")
self.log.printo(str(e))
self.log.printo("will not copy initvm node")
except IOError:
self.log.printo("/var/cache/elbe/source.xml not available")
self.log.printo("can not copy initvm node")
except NoInitvmNode:
self.log.printo("/var/cache/elbe/source.xml is available")
self.log.printo("But it does not contain an initvm node")
else:
sourcepath = os.path.join(self.builddir, "source.xml")
source = ElbeXML(sourcepath,
buildtype=self.override_buildtype,
skip_validate=self.skip_validate,
url_validation=self.url_validation)
self.xml.get_debootstrappkgs_from(source)
try:
self.xml.get_initvmnode_from(source)
except NoInitvmNode:
self.log.printo("source.xml is available")
self.log.printo("But it does not contain an initvm node")
# Seed /etc, we need /etc/hosts for hostname -f to work correctly
if not buildenv:
target.seed_etc()
# remove all non-essential packages to ensure that on a incremental
# build packages can be removed
debootstrap_pkgs = []
for p in self.xml.node("debootstrappkgs"):
debootstrap_pkgs.append(p.et.text)
pkgs = target.xml.get_target_packages() + debootstrap_pkgs
if buildenv:
pkgs = pkgs + target.xml.get_buildenv_packages()
# Now install requested packages
for p in pkgs:
try:
self.get_rpcaptcache(env=target).mark_install(p, None)
except KeyError:
self.log.printo("No Package " + p)
except SystemError as e:
self.log.printo("Error: Unable to correct problems in "
"package %s (%s)" % (p, str(e)))
# temporary disabled because of
# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=776057
# the functions cleans up to much
# self.get_rpcaptcache().cleanup(debootstrap_pkgs + pkgs)
try:
self.get_rpcaptcache(env=target).commit()
except SystemError as e:
self.log.printo("commiting changes failed: %s" % str(e))
raise AptCacheCommitError(str(e))
def install_packages(self, target, buildenv=False):
# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
with target:
# First update the apt cache
try:
self.get_rpcaptcache(env=target).update()
except Exception as e:
raise AptCacheUpdateError(e)
# Then dump the debootstrap packages
if target.fresh_debootstrap:
if target.need_dumpdebootstrap:
dump_debootstrappkgs(self.xml,
self.get_rpcaptcache(env=target))
dump_initvmpkgs(self.xml)
target.need_dumpdebootstrap = False
source = self.xml
source_path = "/var/cache/elbe/source.xml"
try:
initxml = ElbeXML(source_path,
skip_validate=self.skip_validate,
url_validation=ValidationMode.NO_CHECK)
self.xml.get_initvmnode_from(initxml)
except ValidationError:
logging.exception(
"%s validation failed. "
"Will not copy initvm node", source_path)
except IOError:
logging.exception(
"%s not available. "
"Can not copy initvm node", source_path)
except NoInitvmNode:
logging.exception(
"%s is available. But it does not "
"contain an initvm node", source_path)
else:
sourcepath = os.path.join(self.builddir, "source.xml")
source = ElbeXML(sourcepath,
buildtype=self.override_buildtype,
skip_validate=self.skip_validate,
url_validation=self.url_validation)
self.xml.get_debootstrappkgs_from(source)
try:
self.xml.get_initvmnode_from(source)
except NoInitvmNode:
logging.exception("source.xml is available. "
"But it does not contain an initvm node")
# Seed /etc, we need /etc/hosts for hostname -f to work correctly
if not buildenv:
target.seed_etc()
# remove all non-essential packages to ensure that on a incremental
# build packages can be removed
debootstrap_pkgs = []
for p in self.xml.node("debootstrappkgs"):
debootstrap_pkgs.append(p.et.text)
pkgs = target.xml.get_target_packages() + debootstrap_pkgs
if buildenv:
pkgs = pkgs + target.xml.get_buildenv_packages()
# Now install requested packages
for p in pkgs:
try:
self.get_rpcaptcache(env=target).mark_install(p, None)
except KeyError:
logging.exception("No Package %s", p)
except SystemError:
logging.exception(
"Unable to correct problems "
"in package %s", p)
# temporary disabled because of
# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=776057
# the functions cleans up to much
# self.get_rpcaptcache().cleanup(debootstrap_pkgs + pkgs)
try:
self.get_rpcaptcache(env=target).commit()
except SystemError as e:
logging.exception("Commiting changes failed")
raise AptCacheCommitError(str(e))
def gen_update_pkg(project, xml_filename, upd_filename,
override_buildtype=None, skip_validate=False, debug=False,
cmd_dir=None, cfg_dir=None):
# pylint: disable=too-many-arguments
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
if xml_filename:
xml = ElbeXML(xml_filename, buildtype=override_buildtype,
skip_validate=skip_validate)
if not xml.has("fullpkgs"):
raise MissingData("Xml does not have fullpkgs list")
if not project.xml.has("fullpkgs"):
raise MissingData("Source Xml does not have fullpkgs list")
if not project.buildenv.rfs:
raise MissingData("Target does not have a build environment")
cache = project.get_rpcaptcache()
instpkgs = cache.get_installed_pkgs()
instindex = {}
for p in instpkgs:
instindex[p.name] = p
xmlpkgs = xml.node("/fullpkgs")
xmlindex = {}
fnamelist = []
for p in xmlpkgs:
name = p.et.text
ver = p.et.get('version')
md5 = p.et.get('md5')
xmlindex[name] = p
if name not in instindex:
print("package removed: %s" % name)
continue
ipkg = instindex[name]
comp = cache.compare_versions(ipkg.installed_version, ver)
pfname = ipkg.installed_deb
if comp == 0:
print("package ok: %s-%s" % (name, ipkg.installed_version))
if debug:
fnamelist.append(pfname)
continue
if comp > 0:
print("package upgrade: %s" % pfname)
fnamelist.append(pfname)
else:
print(
"package downgrade: %s-%s" %
(name, ipkg.installed_version))
for p in instpkgs:
if p.name in xmlindex:
continue
print("package %s newly installed" % p.name)
pfname = p.installed_deb
fnamelist.append(pfname)
update = os.path.join(project.builddir, "update")
if os.path.exists(update):
rmtree(update)
os.system('mkdir -p %s' % update)
if xml_filename:
repodir = os.path.join(update, "repo")
repo = UpdateRepo(xml, repodir, project.log)
for fname in fnamelist:
path = os.path.join(
project.chrootpath,
"var/cache/apt/archives",
fname)
repo.includedeb(path)
repo.finalize()
dump_fullpkgs(project.xml, project.buildenv.rfs, cache)
project.xml.xml.write(os.path.join(update, "new.xml"))
os.system(
"cp %s %s" %
(xml_filename,
os.path.join(
update,
"base.xml")))
else:
os.system("cp source.xml update/new.xml")
if project.presh_file:
copyfile(project.presh_file, update + '/pre.sh')
os.chmod(update + '/pre.sh', 0o755)
if project.postsh_file:
copyfile(project.postsh_file, update + '/post.sh')
os.chmod(update + '/post.sh', 0o755)
if cmd_dir:
inlucdedir(update, 'cmd', cmd_dir, mode=0o755)
if cfg_dir:
inlucdedir(update, 'conf', cfg_dir)
create_zip_archive(upd_filename, update, ".")
if project.postbuild_file:
project.log.h2("postbuild script")
project.log.do(project.postbuild_file + ' "%s %s %s"' % (
upd_filename,
project.xml.text("project/version"),
project.xml.text("project/name")),
allow_fail=True)
def gen_update_pkg (project, xml_filename, upd_filename,
override_buildtype = None, skip_validate = False, debug = False,
cmd_dir = None, cfg_dir=None):
if xml_filename:
xml = ElbeXML( xml_filename, buildtype=override_buildtype,
skip_validate=skip_validate )
if not xml.has("fullpkgs"):
raise MissingData("Xml does not have fullpkgs list")
if not project.xml.has("fullpkgs"):
raise MissingData("Source Xml does not have fullpkgs list")
if not project.buildenv.rfs:
raise MissingData("Target does not have a build environment")
cache = project.get_rpcaptcache()
instpkgs = cache.get_installed_pkgs()
instindex = {}
for p in instpkgs:
instindex[p.name] = p
xmlpkgs = xml.node("/fullpkgs")
xmlindex = {}
fnamelist = []
for p in xmlpkgs:
name = p.et.text
ver = p.et.get('version')
md5 = p.et.get('md5')
xmlindex[name] = p
if not name in instindex:
print "package removed: " + name
continue
ipkg = instindex[name]
comp = cache.compare_versions(ipkg.installed_version, ver)
pfname = ipkg.installed_deb
if comp == 0:
print "package ok: " + name + "-" + ipkg.installed_version
if debug:
fnamelist.append( pfname )
continue
if comp > 0:
print "package upgrade: " + pfname
fnamelist.append( pfname )
else:
print "package downgrade: " + name + "-" + ipkg.installed_version
for p in instpkgs:
if p.name in xmlindex:
continue
print "package new installed " + p.name
pfname = p.installed_deb
fnamelist.append( pfname )
update = os.path.join(project.builddir, "update")
if os.path.exists( update ):
rmtree( update )
os.system( 'mkdir -p %s' % update )
if xml_filename:
repodir = os.path.join(update, "repo" )
repo = UpdateRepo( xml, repodir, project.log )
for fname in fnamelist:
path = os.path.join( project.chrootpath, "var/cache/apt/archives", fname )
repo.includedeb( path )
repo.finalize ()
dump_fullpkgs(project.xml, project.buildenv.rfs, cache)
project.xml.xml.write( os.path.join( update, "new.xml" ) )
os.system( "cp %s %s" % (xml_filename, os.path.join( update, "base.xml" )) )
else:
os.system( "cp source.xml update/new.xml")
if project.presh_file:
copyfile (project.presh_file, update + '/pre.sh')
os.chmod (update + '/pre.sh', 0755)
if project.postsh_file:
copyfile (project.postsh_file, update + '/post.sh')
os.chmod (update + '/post.sh', 0755)
if cmd_dir:
inlucdedir (update, 'cmd', cmd_dir, mode=0755)
if cfg_dir:
inlucdedir (update, 'conf', cfg_dir)
create_zip_archive( upd_filename, update, "." )
if project.postbuild_file:
project.log.h2 ("postbuild script")
project.log.do (project.postbuild_file+' "%s %s %s"'%(
upd_filename,
project.xml.text ("project/version"),
project.xml.text ("project/name")),
allow_fail=True)
def execute(self, initvmdir, opt, args):
try:
have_session = os.system( "tmux has-session -t ElbeInitVMSession >/dev/null 2>&1" )
except CommandError as e:
print ("tmux execution failed, tmux version 1.9 or higher is required")
sys.exit(20)
if have_session == 0:
print ("ElbeInitVMSession already exists in tmux.", file=sys.stderr)
print ("", file=sys.stderr)
print ("There can only exist a single ElbeInitVMSession, and this session", file=sys.stderr)
print ("can also be used to make your build.", file=sys.stderr)
print ("See 'elbe initvm submit', 'elbe initvm attach' and 'elbe control'", file=sys.stderr)
sys.exit(20)
# Init cdrom to None, if we detect it, we set it
cdrom = None
if len(args) == 1:
if args[0].endswith ('.xml'):
# We have an xml file, use that for elbe init
exampl = args[0]
try:
xml = etree( exampl )
except ValidationError as e:
print ('XML file is inavlid: ' + str(e))
# Use default XML if no initvm was specified
if not xml.has( "initvm" ):
exampl = os.path.join (elbepack.__path__[0], "init/default-init.xml")
elif args[0].endswith ('.iso'):
# We have an iso image, extract xml from there.
tmp = TmpdirFilesystem ()
os.system ('7z x -o%s "%s" source.xml' % (tmp.path, args[0]))
if not tmp.isfile ('source.xml'):
print ('Iso image does not contain a source.xml file', file=sys.stderr)
print ('This is not supported by "elbe initvm"', file=sys.stderr)
print ('', file=sys.stderr)
print ('Exiting !!!', file=sys.stderr)
sys.exit (20)
try:
exml = ElbeXML (tmp.fname ('source.xml'), skip_urlcheck=True)
except ValidationError as e:
print ('Iso image does contain a source.xml file.', file=sys.stderr)
print ('But that xml does not validate correctly', file=sys.stderr)
print ('', file=sys.stderr)
print ('Exiting !!!', file=sys.stderr)
sys.exit (20)
print ('Iso Image with valid source.xml detected !')
print ('Image was generated using Elbe Version %s' % exml.get_elbe_version ())
exampl = tmp.fname ('source.xml')
cdrom = args[0]
else:
print ('Unknown file ending (use either xml or iso)', file=sys.stderr)
sys.exit (20)
else:
# No xml File was specified, build the default elbe-init-with-ssh
exampl = os.path.join (elbepack.__path__[0], "init/default-init.xml")
try:
if opt.devel:
devel = ' --devel'
else:
devel = ''
if cdrom:
system ('%s init %s --directory "%s" --cdrom "%s" "%s"' % (elbe_exe, devel, initvmdir, cdrom, exampl))
else:
system ('%s init %s --directory "%s" "%s"' % (elbe_exe, devel, initvmdir, exampl))
except CommandError:
print ("'elbe init' Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
try:
system ('cd "%s"; make' % (initvmdir))
except CommandError:
print ("Building the initvm Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
try:
system ('%s initvm start --directory "%s"' % (elbe_exe, initvmdir))
except CommandError:
print ("Starting the initvm Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
if len(args) == 1:
# if provided xml file has no initvm section exampl is set to a
# default initvm XML file. But we need the original file here
if args[0].endswith ('.xml'):
# stop here if no project node was specified
try:
x = ElbeXML (args[0])
except ValidationError as e:
print ('XML file is inavlid: ' + str(e))
sys.exit(20)
if not x.has('project'):
print ('elbe initvm ready: use "elbe initvm submit myproject.xml" to build a project');
sys.exit(0)
ret, prjdir, err = command_out_stderr ('%s control create_project "%s"' % (elbe_exe, args[0]))
else:
ret, prjdir, err = command_out_stderr ('%s control create_project "%s"' % (elbe_exe, exampl))
if ret != 0:
print ("elbe control create_project failed.", file=sys.stderr)
print (err, file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
prjdir = prjdir.strip()
if cdrom is not None:
print ("Uploading CDROM. This might take a while")
try:
system ('%s control set_cdrom "%s" "%s"' % (elbe_exe, prjdir, cdrom) )
except CommandError:
print ("elbe control set_cdrom Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
print ("Upload finished")
build_opts = ''
if opt.build_bin:
build_opts += '--build-bin '
if opt.build_sources:
build_opts += '--build-sources '
try:
system ('%s control build "%s" %s' % (elbe_exe, prjdir, build_opts) )
except CommandError:
print ("elbe control build Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
try:
system ('%s control wait_busy "%s"' % (elbe_exe, prjdir) )
except CommandError:
print ("elbe control wait_busy Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
print ("")
print ("Build finished !")
print ("")
try:
system ('%s control dump_file "%s" validation.txt' % (elbe_exe, prjdir) )
except CommandError:
print ("Project failed to generate validation.txt", file=sys.stderr)
print ("Getting log.txt", file=sys.stderr)
try:
system ('%s control dump_file "%s" log.txt' % (elbe_exe, prjdir) )
except CommandError:
print ("Failed to dump log.txt", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
if opt.skip_download:
print ("")
print ("Listing available files:")
print ("")
try:
system ('%s control get_files "%s"' % (elbe_exe, prjdir) )
except CommandError:
print ("elbe control Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
print ("")
print ('Get Files with: elbe control get_file "%s" <filename>' % prjdir)
else:
ensure_outdir (wdfs, opt)
try:
system ('%s control get_files --output "%s" "%s"' % (elbe_exe, opt.outdir, prjdir) )
except CommandError:
print ("elbe control get_files Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
