def extract_cdrom(cdrom):
""" Extract cdrom iso image
returns a TmpdirFilesystem() object containing
the source.xml, which is also validated.
"""
tmp = TmpdirFilesystem()
system('7z x -o%s "%s" source.xml' % (tmp.path, cdrom))
print("", file=sys.stderr)
if not tmp.isfile('source.xml'):
print("Iso image does not contain a source.xml file", file=sys.stderr)
print("This is not supported by 'elbe initvm'", file=sys.stderr)
print("", file=sys.stderr)
print("Exiting !!!", file=sys.stderr)
sys.exit(20)
try:
exml = ElbeXML(tmp.fname('source.xml'),
url_validation=ValidationMode.NO_CHECK)
except ValidationError as e:
print("Iso image does contain a source.xml file.", file=sys.stderr)
print("But that xml does not validate correctly", file=sys.stderr)
print("", file=sys.stderr)
print("Exiting !!!", file=sys.stderr)
print(e)
sys.exit(20)
print("Iso Image with valid source.xml detected !")
print("Image was generated using Elbe Version %s" %
exml.get_elbe_version())
return tmp
def execute(self, opt, args):
tmp = TmpdirFilesystem ()
if not opt.project:
print ('you need to specify --project option', file=sys.stderr)
sys.exit(20)
prjdir = opt.project
print ("Creating pbuilder")
try:
system ('%s control build_pbuilder "%s"' % (elbe_exe, prjdir))
except CommandError:
print ("elbe control build_pbuilder Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
try:
system ('%s control wait_busy "%s"' % (elbe_exe, prjdir) )
except CommandError:
print ("elbe control wait_busy Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
print ("")
print ("Building Pbuilder finished !")
print ("")
def copy_kinitrd(prj, target_dir):
suite = prj.text("suite")
try:
tmp = TmpdirFilesystem()
if prj.has("mirror/cdrom"):
system('7z x -o%s "%s" initrd-cdrom.gz vmlinuz' %
(tmp.fname('/'), prj.text("mirror/cdrom")))
# initrd.gz needs to be cdrom version !
copyfile(tmp.fname("initrd-cdrom.gz"),
os.path.join(target_dir, "initrd.gz"))
else:
mirror = get_primary_mirror(prj)
download_kinitrd(tmp, suite, mirror)
copyfile(tmp.fname("initrd.gz"),
os.path.join(target_dir, "initrd.gz"))
copyfile(tmp.fname("initrd-cdrom.gz"),
os.path.join(target_dir, "initrd-cdrom.gz"))
copyfile(tmp.fname("vmlinuz"), os.path.join(target_dir, "vmlinuz"))
except IOError as e:
raise NoKinitrdException('IoError %s' % str(e))
except InvalidSignature as e:
raise NoKinitrdException('InvalidSignature %s' % str(e))
except HashValidationFailed as e:
raise NoKinitrdException('HashValidationFailed %s' % str(e))
def extract_cdrom(cdrom):
""" Extract cdrom iso image
returns a TmpdirFilesystem() object containing
the source.xml, which is also validated.
"""
tmp = TmpdirFilesystem()
os.system('7z x -o%s "%s" source.xml' % (tmp.path, cdrom))
print("", file=sys.stderr)
if not tmp.isfile('source.xml'):
print(
"Iso image does not contain a source.xml file",
file=sys.stderr)
print(
"This is not supported by 'elbe initvm'",
file=sys.stderr)
print("", file=sys.stderr)
print("Exiting !!!", file=sys.stderr)
sys.exit(20)
try:
exml = ElbeXML(
tmp.fname('source.xml'),
url_validation=ValidationMode.NO_CHECK)
except ValidationError as e:
print(
"Iso image does contain a source.xml file.",
file=sys.stderr)
print(
"But that xml does not validate correctly",
file=sys.stderr)
print("", file=sys.stderr)
print("Exiting !!!", file=sys.stderr)
print(e)
sys.exit(20)
print("Iso Image with valid source.xml detected !")
print(
"Image was generated using Elbe Version %s" %
exml.get_elbe_version())
return tmp
def copy_kinitrd(prj, target_dir):
suite = prj.text("suite")
try:
tmp = TmpdirFilesystem()
if prj.has("mirror/cdrom"):
system('7z x -o%s "%s" initrd-cdrom.gz vmlinuz' %
(tmp.fname('/'), prj.text("mirror/cdrom")))
# initrd.gz needs to be cdrom version !
copyfile(tmp.fname("initrd-cdrom.gz"),
os.path.join(target_dir, "initrd.gz"))
else:
mirror = get_primary_mirror(prj)
download_kinitrd(tmp, suite, mirror)
copyfile(tmp.fname("initrd.gz"),
os.path.join(target_dir, "initrd.gz"))
copyfile(tmp.fname("initrd-cdrom.gz"),
os.path.join(target_dir, "initrd-cdrom.gz"))
copyfile(tmp.fname("vmlinuz"),
os.path.join(target_dir, "vmlinuz"))
except IOError as e:
raise NoKinitrdException('IoError %s' % e.message)
except InvalidSignature as e:
raise NoKinitrdException('InvalidSignature %s' % e.message)
except HashValidationFailed as e:
raise NoKinitrdException('HashValidationFailed %s' % e.message)
def execute(self, opt, args):
tmp = TmpdirFilesystem ()
if opt.xmlfile:
ret, prjdir, err = command_out_stderr ('%s control create_project' % (elbe_exe))
if ret != 0:
print ("elbe control create_project failed.", file=sys.stderr)
print (err, file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
prjdir = prjdir.strip()
ret, msg, err = command_out_stderr ('%s control set_xml "%s" "%s"' % (elbe_exe, prjdir, opt.xmlfile))
if ret != 0:
print ("elbe control set_xml failed.", file=sys.stderr)
print (err, file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
if opt.writeproject:
wpf = open (opt.writeproject, "w")
wpf.write (prjdir)
wpf.close()
elif opt.project:
prjdir = opt.project
else:
print ('you need to specify --project option', file=sys.stderr)
sys.exit(20)
print ("Creating pbuilder")
try:
system ('%s control build_pbuilder "%s"' % (elbe_exe, prjdir))
except CommandError:
print ("elbe control build_pbuilder Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
try:
system ('%s control wait_busy "%s"' % (elbe_exe, prjdir) )
except CommandError:
print ("elbe control wait_busy Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
print ("")
print ("Building Pbuilder finished !")
print ("")
def _combinearchivedir(xml, xpath, use_volume):
elbexml = etree(None)
elbexml.et = xml
tmp = TmpdirFilesystem()
for archivedir in elbexml.all(xpath):
try:
archiveurl = urljoin(archivedir.et.base, archivedir.et.text)
keep = archivedir.bool_attr("keep-attributes")
parent = archivedir.get_parent()
if use_volume:
volume_attr = archivedir.et.get('volume', default='all')
fname_suffix = volume_attr
arch = parent.node("archive[@volume='%s']" % volume_attr)
if arch is None:
arch = parent.append("archive")
arch.et.set("volume", volume_attr)
else:
arch = parent.ensure_child("archive")
fname_suffix = ''
get_and_append = get_and_append_method(archiveurl)
archname = tmp.fname('archive%s.tar.bz2' % fname_suffix)
get_and_append(archiveurl, archname, keep)
arch.set_text(enbase(archname, True))
parent.remove_child(archivedir)
except (CalledProcessError, OSError):
msg = "Failure while processing \"" + archivedir.text + "\":\n"
msg += str(sys.exc_info()[1])
raise ArchivedirError(msg)
def do_sdk(self, sdk):
with TmpdirFilesystem() as tmp:
# Make a copy of the installer
shutil.copyfile(sdk, tmp.fname(sdk))
# Let's work in our temp dir from now on
os.chdir(tmp.path)
# The script is self extracting; it needs to be executable
os.chmod(sdk, 0o744)
# Extract here with 'yes' to all answers
do("./%s -y -d ." % sdk)
# Get environment file
env = tmp.glob("environment-setup*")[0]
# NOTE! This script requires binfmt to be installed.
do("/bin/sh", stdin=self.script, env_add={"ELBE_SDK_ENV": env})
def execute(self, initvmdir, opt, args):
try:
have_session = os.system( "tmux has-session -t ElbeInitVMSession >/dev/null 2>&1" )
except CommandError as e:
print ("tmux execution failed, tmux version 1.9 or higher is required")
sys.exit(20)
if have_session == 256:
print ("ElbeInitVMSession does not exist in tmux.", file=sys.stderr)
print ("Try 'elbe initvm start' to start the session.", file=sys.stderr)
sys.exit(20)
try:
system ('%s initvm ensure --directory "%s"' % (elbe_exe, initvmdir))
except CommandError:
print ("Starting the initvm Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
# Init cdrom to None, if we detect it, we set it
cdrom = None
if len(args) == 1:
if args[0].endswith ('.xml'):
# We have an xml file, use that for elbe init
xmlfile = args[0]
elif args[0].endswith ('.iso'):
# We have an iso image, extract xml from there.
tmp = TmpdirFilesystem ()
os.system ('7z x -o%s "%s" source.xml' % (tmp.path, args[0]))
print ('', file=sys.stderr)
if not tmp.isfile ('source.xml'):
print ('Iso image does not contain a source.xml file', file=sys.stderr)
print ('This is not supported by "elbe initvm"', file=sys.stderr)
print ('', file=sys.stderr)
print ('Exiting !!!', file=sys.stderr)
sys.exit (20)
try:
exml = ElbeXML (tmp.fname ('source.xml'), skip_urlcheck=True)
except ValidationError as e:
print ('Iso image does contain a source.xml file.', file=sys.stderr)
print ('But that xml does not validate correctly', file=sys.stderr)
print ('', file=sys.stderr)
print ('Exiting !!!', file=sys.stderr)
sys.exit (20)
print ('Iso Image with valid source.xml detected !')
print ('Image was generated using Elbe Version %s' % exml.get_elbe_version ())
xmlfile = tmp.fname ('source.xml')
cdrom = args[0]
else:
print ('Unknown file ending (use either xml or iso)', file=sys.stderr)
sys.exit (20)
ret, prjdir, err = command_out_stderr ('%s control create_project --retries 60 "%s"' % (elbe_exe, xmlfile))
if ret != 0:
print ("elbe control create_project failed.", file=sys.stderr)
print (err, file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
prjdir = prjdir.strip()
if cdrom is not None:
print ("Uploading CDROM. This might take a while")
try:
system ('%s control set_cdrom "%s" "%s"' % (elbe_exe, prjdir, cdrom) )
except CommandError:
print ("elbe control set_cdrom Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
print ("Upload finished")
build_opts = ''
if opt.build_bin:
build_opts += '--build-bin '
if opt.build_sources:
build_opts += '--build-sources '
try:
system ('%s control build "%s" %s' % (elbe_exe, prjdir, build_opts) )
except CommandError:
print ("elbe control build Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
print ("Build started, waiting till it finishes")
try:
system ('%s control wait_busy "%s"' % (elbe_exe, prjdir) )
except CommandError:
print ("elbe control wait_busy Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
print ("")
print ("Build finished !")
print ("")
try:
system ('%s control dump_file "%s" validation.txt' % (elbe_exe, prjdir) )
except CommandError:
print ("Project failed to generate validation.txt", file=sys.stderr)
print ("Getting log.txt", file=sys.stderr)
try:
system ('%s control dump_file "%s" log.txt' % (elbe_exe, prjdir) )
except CommandError:
print ("Failed to dump log.txt", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
if opt.skip_download:
print ("")
print ("Listing available files:")
print ("")
try:
system ('%s control get_files "%s"' % (elbe_exe, prjdir) )
except CommandError:
print ("elbe control get_files Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
print ("")
print ('Get Files with: elbe control get_file "%s" <filename>' % prjdir)
else:
print ("")
print ("Getting generated Files")
print ("")
ensure_outdir (wdfs, opt)
try:
system ('%s control get_files --output "%s" "%s"' % (
elbe_exe, opt.outdir, prjdir ))
except CommandError:
print ("elbe control get_files Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
if not opt.keep_files:
try:
system ('%s control del_project "%s"' % (
elbe_exe, prjdir))
except CommandError:
print ("remove project from initvm failed",
file=sys.stderr)
sys.exit(20)
def execute(self, initvmdir, opt, args):
try:
have_session = os.system(
"tmux has-session -t ElbeInitVMSession >/dev/null 2>&1")
except CommandError as e:
print(
"tmux execution failed, tmux version 1.9 or higher is required"
)
sys.exit(20)
if have_session == 256:
print("ElbeInitVMSession does not exist in tmux.", file=sys.stderr)
print("Try 'elbe initvm start' to start the session.",
file=sys.stderr)
sys.exit(20)
try:
system('%s initvm ensure --directory "%s"' % (elbe_exe, initvmdir))
except CommandError:
print("Starting the initvm Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
# Init cdrom to None, if we detect it, we set it
cdrom = None
if len(args) == 1:
if args[0].endswith('.xml'):
# We have an xml file, use that for elbe init
xmlfile = args[0]
elif args[0].endswith('.iso'):
# We have an iso image, extract xml from there.
tmp = TmpdirFilesystem()
os.system('7z x -o%s "%s" source.xml' % (tmp.path, args[0]))
print('', file=sys.stderr)
if not tmp.isfile('source.xml'):
print('Iso image does not contain a source.xml file',
file=sys.stderr)
print('This is not supported by "elbe initvm"',
file=sys.stderr)
print('', file=sys.stderr)
print('Exiting !!!', file=sys.stderr)
sys.exit(20)
try:
exml = ElbeXML(tmp.fname('source.xml'), skip_urlcheck=True)
except ValidationError as e:
print('Iso image does contain a source.xml file.',
file=sys.stderr)
print('But that xml does not validate correctly',
file=sys.stderr)
print('', file=sys.stderr)
print('Exiting !!!', file=sys.stderr)
sys.exit(20)
print('Iso Image with valid source.xml detected !')
print('Image was generated using Elbe Version %s' %
exml.get_elbe_version())
xmlfile = tmp.fname('source.xml')
cdrom = args[0]
else:
print('Unknown file ending (use either xml or iso)',
file=sys.stderr)
sys.exit(20)
ret, prjdir, err = command_out_stderr(
'%s control create_project --retries 60 "%s"' %
(elbe_exe, xmlfile))
if ret != 0:
print("elbe control create_project failed.", file=sys.stderr)
print(err, file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
prjdir = prjdir.strip()
if cdrom is not None:
print("Uploading CDROM. This might take a while")
try:
system('%s control set_cdrom "%s" "%s"' %
(elbe_exe, prjdir, cdrom))
except CommandError:
print("elbe control set_cdrom Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("Upload finished")
build_opts = ''
if opt.build_bin:
build_opts += '--build-bin '
if opt.build_sources:
build_opts += '--build-sources '
try:
system('%s control build "%s" %s' %
(elbe_exe, prjdir, build_opts))
except CommandError:
print("elbe control build Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("Build started, waiting till it finishes")
try:
system('%s control wait_busy "%s"' % (elbe_exe, prjdir))
except CommandError:
print("elbe control wait_busy Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print("Build finished !")
print("")
try:
system('%s control dump_file "%s" validation.txt' %
(elbe_exe, prjdir))
except CommandError:
print("Project failed to generate validation.txt",
file=sys.stderr)
print("Getting log.txt", file=sys.stderr)
try:
system('%s control dump_file "%s" log.txt' %
(elbe_exe, prjdir))
except CommandError:
print("Failed to dump log.txt", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
if opt.skip_download:
print("")
print("Listing available files:")
print("")
try:
system('%s control get_files "%s"' % (elbe_exe, prjdir))
except CommandError:
print("elbe control get_files Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print('Get Files with: elbe control get_file "%s" <filename>' %
prjdir)
else:
print("")
print("Getting generated Files")
print("")
ensure_outdir(wdfs, opt)
try:
system('%s control get_files --output "%s" "%s"' %
(elbe_exe, opt.outdir, prjdir))
except CommandError:
print("elbe control get_files Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
if not opt.keep_files:
try:
system('%s control del_project "%s"' %
(elbe_exe, prjdir))
except CommandError:
print("remove project from initvm failed",
file=sys.stderr)
sys.exit(20)
def execute(self, initvmdir, opt, args):
try:
have_session = os.system(
"tmux has-session -t ElbeInitVMSession >/dev/null 2>&1")
except CommandError as e:
print(
"tmux execution failed, tmux version 1.9 or higher is required"
)
sys.exit(20)
if have_session == 0:
print("ElbeInitVMSession already exists in tmux.", file=sys.stderr)
print("", file=sys.stderr)
print(
"There can only exist a single ElbeInitVMSession, and this session",
file=sys.stderr)
print("can also be used to make your build.", file=sys.stderr)
print(
"See 'elbe initvm submit', 'elbe initvm attach' and 'elbe control'",
file=sys.stderr)
sys.exit(20)
# Init cdrom to None, if we detect it, we set it
cdrom = None
if len(args) == 1:
if args[0].endswith('.xml'):
# We have an xml file, use that for elbe init
exampl = args[0]
try:
xml = etree(exampl)
except ValidationError as e:
print('XML file is inavlid: ' + str(e))
# Use default XML if no initvm was specified
if not xml.has("initvm"):
exampl = os.path.join(elbepack.__path__[0],
"init/default-init.xml")
elif args[0].endswith('.iso'):
# We have an iso image, extract xml from there.
tmp = TmpdirFilesystem()
os.system('7z x -o%s "%s" source.xml' % (tmp.path, args[0]))
if not tmp.isfile('source.xml'):
print('Iso image does not contain a source.xml file',
file=sys.stderr)
print('This is not supported by "elbe initvm"',
file=sys.stderr)
print('', file=sys.stderr)
print('Exiting !!!', file=sys.stderr)
sys.exit(20)
try:
exml = ElbeXML(tmp.fname('source.xml'), skip_urlcheck=True)
except ValidationError as e:
print('Iso image does contain a source.xml file.',
file=sys.stderr)
print('But that xml does not validate correctly',
file=sys.stderr)
print('', file=sys.stderr)
print('Exiting !!!', file=sys.stderr)
sys.exit(20)
print('Iso Image with valid source.xml detected !')
print('Image was generated using Elbe Version %s' %
exml.get_elbe_version())
exampl = tmp.fname('source.xml')
cdrom = args[0]
else:
print('Unknown file ending (use either xml or iso)',
file=sys.stderr)
sys.exit(20)
else:
# No xml File was specified, build the default elbe-init-with-ssh
exampl = os.path.join(elbepack.__path__[0],
"init/default-init.xml")
try:
if opt.devel:
devel = ' --devel'
else:
devel = ''
if cdrom:
system('%s init %s --directory "%s" --cdrom "%s" "%s"' %
(elbe_exe, devel, initvmdir, cdrom, exampl))
else:
system('%s init %s --directory "%s" "%s"' %
(elbe_exe, devel, initvmdir, exampl))
except CommandError:
print("'elbe init' Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
try:
system('cd "%s"; make' % (initvmdir))
except CommandError:
print("Building the initvm Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
try:
system('%s initvm start --directory "%s"' % (elbe_exe, initvmdir))
except CommandError:
print("Starting the initvm Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
if len(args) == 1:
# if provided xml file has no initvm section exampl is set to a
# default initvm XML file. But we need the original file here
if args[0].endswith('.xml'):
# stop here if no project node was specified
try:
x = ElbeXML(args[0])
except ValidationError as e:
print('XML file is inavlid: ' + str(e))
sys.exit(20)
if not x.has('project'):
print(
'elbe initvm ready: use "elbe initvm submit myproject.xml" to build a project'
)
sys.exit(0)
ret, prjdir, err = command_out_stderr(
'%s control create_project "%s"' % (elbe_exe, args[0]))
else:
ret, prjdir, err = command_out_stderr(
'%s control create_project "%s"' % (elbe_exe, exampl))
if ret != 0:
print("elbe control create_project failed.", file=sys.stderr)
print(err, file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
prjdir = prjdir.strip()
if cdrom is not None:
print("Uploading CDROM. This might take a while")
try:
system('%s control set_cdrom "%s" "%s"' %
(elbe_exe, prjdir, cdrom))
except CommandError:
print("elbe control set_cdrom Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("Upload finished")
build_opts = ''
if opt.build_bin:
build_opts += '--build-bin '
if opt.build_sources:
build_opts += '--build-sources '
try:
system('%s control build "%s" %s' %
(elbe_exe, prjdir, build_opts))
except CommandError:
print("elbe control build Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
try:
system('%s control wait_busy "%s"' % (elbe_exe, prjdir))
except CommandError:
print("elbe control wait_busy Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print("Build finished !")
print("")
try:
system('%s control dump_file "%s" validation.txt' %
(elbe_exe, prjdir))
except CommandError:
print("Project failed to generate validation.txt",
file=sys.stderr)
print("Getting log.txt", file=sys.stderr)
try:
system('%s control dump_file "%s" log.txt' %
(elbe_exe, prjdir))
except CommandError:
print("Failed to dump log.txt", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
if opt.skip_download:
print("")
print("Listing available files:")
print("")
try:
system('%s control get_files "%s"' % (elbe_exe, prjdir))
except CommandError:
print("elbe control Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print('Get Files with: elbe control get_file "%s" <filename>' %
prjdir)
else:
ensure_outdir(wdfs, opt)
try:
system('%s control get_files --output "%s" "%s"' %
(elbe_exe, opt.outdir, prjdir))
except CommandError:
print("elbe control get_files Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
def execute(self, opt, _args):
# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
crossopt = ""
if opt.cross:
crossopt = "--cross"
tmp = TmpdirFilesystem()
if opt.xmlfile:
ret, prjdir, err = command_out_stderr(
'%s control create_project --retries 60 "%s"' %
(elbe_exe, opt.xmlfile))
if ret != 0:
print("elbe control create_project failed.", file=sys.stderr)
print(err, file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
prjdir = prjdir.strip()
try:
system('%s control build_pbuilder "%s"' % (elbe_exe, prjdir))
except CommandError:
print("elbe control build_pbuilder Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
try:
system('%s control wait_busy "%s"' % (elbe_exe, prjdir))
except CommandError:
print("elbe control wait_busy Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print("Building Pbuilder finished !")
print("")
elif opt.project:
prjdir = opt.project
system('%s control rm_log %s' % (elbe_exe, prjdir))
else:
print("you need to specify --project or --xmlfile option",
file=sys.stderr)
sys.exit(20)
print("")
print("Packing Source into tmp archive")
print("")
try:
system('tar cfz "%s" .' % (tmp.fname("pdebuild.tar.gz")))
except CommandError:
print("tar Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
for of in opt.origfile:
print("")
print("Pushing orig file '%s' into pbuilder" % of)
print("")
try:
system('%s control set_orig "%s" "%s"' %
(elbe_exe, prjdir, of))
except CommandError:
print("elbe control set_orig Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print("Pushing source into pbuilder")
print("")
try:
system('%s control set_pdebuild --cpuset "%d" --profile "%s" %s '
'"%s" "%s"' % (elbe_exe, opt.cpuset, opt.profile, crossopt,
prjdir, tmp.fname("pdebuild.tar.gz")))
except CommandError:
print("elbe control set_pdebuild Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
try:
system('%s control wait_busy "%s"' % (elbe_exe, prjdir))
except CommandError:
print("elbe control wait_busy Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print("Pdebuild finished !")
print("")
if opt.skip_download:
print("")
print("Listing available files:")
print("")
try:
system('%s control --pbuilder-only get_files "%s"' %
(elbe_exe, prjdir))
except CommandError:
print("elbe control get_files Failed", file=sys.stderr)
print("", file=sys.stderr)
print("dumping logfile", file=sys.stderr)
try:
system('%s control dump_file "%s" log.txt' %
(elbe_exe, prjdir))
except CommandError:
print("elbe control dump_file Failed", file=sys.stderr)
print("", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print("Get Files with: 'elbe control get_file %s <filename>'" %
prjdir)
else:
print("")
print("Getting generated Files")
print("")
ensure_outdir(opt)
try:
system(
'%s control --pbuilder-only get_files --output "%s" "%s"' %
(elbe_exe, opt.outdir, prjdir))
except CommandError:
print("elbe control get_files Failed", file=sys.stderr)
print("", file=sys.stderr)
print("dumping logfile", file=sys.stderr)
try:
system('%s control dump_file "%s" log.txt' %
(elbe_exe, prjdir))
except CommandError:
print("elbe control dump_file Failed", file=sys.stderr)
print("", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
def execute(self, initvmdir, opt, args):
try:
have_session = os.system( "tmux has-session -t ElbeInitVMSession >/dev/null 2>&1" )
except CommandError as e:
print ("tmux execution failed, tmux version 1.9 or higher is required")
sys.exit(20)
if have_session == 0:
print ("ElbeInitVMSession already exists in tmux.", file=sys.stderr)
print ("", file=sys.stderr)
print ("There can only exist a single ElbeInitVMSession, and this session", file=sys.stderr)
print ("can also be used to make your build.", file=sys.stderr)
print ("See 'elbe initvm submit', 'elbe initvm attach' and 'elbe control'", file=sys.stderr)
sys.exit(20)
# Init cdrom to None, if we detect it, we set it
cdrom = None
if len(args) == 1:
if args[0].endswith ('.xml'):
# We have an xml file, use that for elbe init
exampl = args[0]
try:
xml = etree( exampl )
except ValidationError as e:
print ('XML file is inavlid: ' + str(e))
# Use default XML if no initvm was specified
if not xml.has( "initvm" ):
exampl = os.path.join (elbepack.__path__[0], "init/default-init.xml")
elif args[0].endswith ('.iso'):
# We have an iso image, extract xml from there.
tmp = TmpdirFilesystem ()
os.system ('7z x -o%s "%s" source.xml' % (tmp.path, args[0]))
if not tmp.isfile ('source.xml'):
print ('Iso image does not contain a source.xml file', file=sys.stderr)
print ('This is not supported by "elbe initvm"', file=sys.stderr)
print ('', file=sys.stderr)
print ('Exiting !!!', file=sys.stderr)
sys.exit (20)
try:
exml = ElbeXML (tmp.fname ('source.xml'), skip_urlcheck=True)
except ValidationError as e:
print ('Iso image does contain a source.xml file.', file=sys.stderr)
print ('But that xml does not validate correctly', file=sys.stderr)
print ('', file=sys.stderr)
print ('Exiting !!!', file=sys.stderr)
sys.exit (20)
print ('Iso Image with valid source.xml detected !')
print ('Image was generated using Elbe Version %s' % exml.get_elbe_version ())
exampl = tmp.fname ('source.xml')
cdrom = args[0]
else:
print ('Unknown file ending (use either xml or iso)', file=sys.stderr)
sys.exit (20)
else:
# No xml File was specified, build the default elbe-init-with-ssh
exampl = os.path.join (elbepack.__path__[0], "init/default-init.xml")
try:
if opt.devel:
devel = ' --devel'
else:
devel = ''
if cdrom:
system ('%s init %s --directory "%s" --cdrom "%s" "%s"' % (elbe_exe, devel, initvmdir, cdrom, exampl))
else:
system ('%s init %s --directory "%s" "%s"' % (elbe_exe, devel, initvmdir, exampl))
except CommandError:
print ("'elbe init' Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
try:
system ('cd "%s"; make' % (initvmdir))
except CommandError:
print ("Building the initvm Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
try:
system ('%s initvm start --directory "%s"' % (elbe_exe, initvmdir))
except CommandError:
print ("Starting the initvm Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
if len(args) == 1:
# if provided xml file has no initvm section exampl is set to a
# default initvm XML file. But we need the original file here
if args[0].endswith ('.xml'):
# stop here if no project node was specified
try:
x = ElbeXML (args[0])
except ValidationError as e:
print ('XML file is inavlid: ' + str(e))
sys.exit(20)
if not x.has('project'):
print ('elbe initvm ready: use "elbe initvm submit myproject.xml" to build a project');
sys.exit(0)
ret, prjdir, err = command_out_stderr ('%s control create_project "%s"' % (elbe_exe, args[0]))
else:
ret, prjdir, err = command_out_stderr ('%s control create_project "%s"' % (elbe_exe, exampl))
if ret != 0:
print ("elbe control create_project failed.", file=sys.stderr)
print (err, file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
prjdir = prjdir.strip()
if cdrom is not None:
print ("Uploading CDROM. This might take a while")
try:
system ('%s control set_cdrom "%s" "%s"' % (elbe_exe, prjdir, cdrom) )
except CommandError:
print ("elbe control set_cdrom Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
print ("Upload finished")
build_opts = ''
if opt.build_bin:
build_opts += '--build-bin '
if opt.build_sources:
build_opts += '--build-sources '
try:
system ('%s control build "%s" %s' % (elbe_exe, prjdir, build_opts) )
except CommandError:
print ("elbe control build Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
try:
system ('%s control wait_busy "%s"' % (elbe_exe, prjdir) )
except CommandError:
print ("elbe control wait_busy Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
print ("")
print ("Build finished !")
print ("")
try:
system ('%s control dump_file "%s" validation.txt' % (elbe_exe, prjdir) )
except CommandError:
print ("Project failed to generate validation.txt", file=sys.stderr)
print ("Getting log.txt", file=sys.stderr)
try:
system ('%s control dump_file "%s" log.txt' % (elbe_exe, prjdir) )
except CommandError:
print ("Failed to dump log.txt", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
if opt.skip_download:
print ("")
print ("Listing available files:")
print ("")
try:
system ('%s control get_files "%s"' % (elbe_exe, prjdir) )
except CommandError:
print ("elbe control Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
print ("")
print ('Get Files with: elbe control get_file "%s" <filename>' % prjdir)
else:
ensure_outdir (wdfs, opt)
try:
system ('%s control get_files --output "%s" "%s"' % (elbe_exe, opt.outdir, prjdir) )
except CommandError:
print ("elbe control get_files Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
def __init__(self, xml):
# pylint: disable=too-many-statements
self.xml = xml
arch = xml.text("project/buildimage/arch", key="arch")
suite = xml.text("project/suite")
self.basefs = TmpdirFilesystem()
self.initialize_dirs()
create_apt_prefs(self.xml, self.basefs)
mirror = self.xml.create_apt_sources_list(build_sources=True,
initvm=False)
self.basefs.write_file("etc/apt/sources.list", 0o644, mirror)
self.setup_gpg()
self.import_keys()
apt_pkg.config.set("APT::Architecture", arch)
apt_pkg.config.set("APT::Architectures", arch)
apt_pkg.config.set("Acquire::http::Proxy::127.0.0.1", "DIRECT")
apt_pkg.config.set("APT::Install-Recommends", "0")
apt_pkg.config.set("Dir::Etc", self.basefs.fname('/'))
apt_pkg.config.set("Dir::Etc::Trusted",
self.basefs.fname('/etc/apt/trusted.gpg'))
apt_pkg.config.set("Dir::Etc::TrustedParts",
self.basefs.fname('/etc/apt/trusted.gpg.d'))
apt_pkg.config.set("APT::Cache-Limit", "0")
apt_pkg.config.set("APT::Cache-Start", "32505856")
apt_pkg.config.set("APT::Cache-Grow", "2097152")
apt_pkg.config.set("Dir::State", self.basefs.fname("state"))
apt_pkg.config.set("Dir::State::status",
self.basefs.fname("state/status"))
apt_pkg.config.set("Dir::Cache", self.basefs.fname("cache"))
apt_pkg.config.set("Dir::Cache::archives",
self.basefs.fname("cache/archives"))
apt_pkg.config.set("Dir::Etc", self.basefs.fname("etc/apt"))
apt_pkg.config.set("Dir::Log", self.basefs.fname("log"))
if self.xml.has('project/noauth'):
apt_pkg.config.set("APT::Get::AllowUnauthenticated", "1")
apt_pkg.config.set("Acquire::AllowInsecureRepositories", "1")
else:
apt_pkg.config.set("APT::Get::AllowUnauthenticated", "0")
apt_pkg.config.set("Acquire::AllowInsecureRepositories", "0")
apt_pkg.init_system()
self.source = apt_pkg.SourceList()
self.source.read_main_list()
self.cache = apt_pkg.Cache()
try:
self.cache.update(self, self.source)
except BaseException as e:
print(e)
apt_pkg.config.set("APT::Default-Release", suite)
self.cache = apt_pkg.Cache()
try:
self.cache.update(self, self.source)
except BaseException as e:
print(e)
try:
self.depcache = apt_pkg.DepCache(self.cache)
prefs_name = self.basefs.fname("/etc/apt/preferences")
self.depcache.read_pinfile(prefs_name)
except BaseException as e:
print(e)
self.downloads = {}
self.acquire = apt_pkg.Acquire(self)
class TestCopyFilelist(unittest.TestCase):
def setUp(self):
self.src = TmpdirFilesystem()
self.dst = TmpdirFilesystem()
def tearDown(self):
del self.src
del self.dst
def test_usrmerge_abs(self):
self.src.mkdir_p('/usr/bin')
# this will link to /usr/bin in the host RFS,
# when no special logic is applied.
self.src.symlink('/usr/bin', '/bin')
self.src.write_file('/bin/bla', 0o644, 'bla')
copy_filelist(self.src, ['/bin/bla'], self.dst)
# We should now have the same content from /SRC/usr/bin/bla in
# /DST/usr/bin/bla
self.assertEqual(self.src.read_file('/usr/bin/bla'),
self.dst.read_file('/usr/bin/bla'))
def test_usrmerge_rel(self):
self.src.mkdir_p('/usr/bin')
# create a proper relative path, that should
# work fine from inside.
self.src.symlink('usr/bin', '/bin')
self.src.write_file('/bin/bla', 0o644, 'bla')
copy_filelist(self.src, ['/bin/bla'], self.dst)
# We should now have the same content from /SRC/usr/bin/bla in
# /DST/usr/bin/bla
self.assertEqual(self.src.read_file('/usr/bin/bla'),
self.dst.read_file('/usr/bin/bla'))
def test_deeplinks(self):
self.src.mkdir_p('/a/b/c')
# c <- /a/b/d
self.src.symlink('c', '/a/b/d')
# This write into /a/b/c/bla (c instead of d)
self.src.write_file('/a/b/d/bla', 0o644, 'bla')
copy_filelist(self.src, ['/a/b/d/bla'], self.dst)
# We should now have the same content from /SRC/a/b/c/bla in
# /DST/a/b/c/bla
self.assertEqual(self.src.read_file('/a/b/c//bla'),
self.dst.read_file('/a/b/c/bla'))
def test_multilinks(self):
self.src.mkdir_p('/a')
# a <- b
# ../b <- /a/c
self.src.symlink('a', '/b')
self.src.symlink('../b', '/a/c')
# This write into /a/bla
self.src.write_file('a/c/bla', 0o644, 'bla')
copy_filelist(self.src, ['/a/c/bla'], self.dst)
# We should now have the content from /SRC/a/bla in /DST/a/bla
self.assertEqual(self.src.read_file('/a/bla'),
self.dst.read_file('/a/bla'))
@unittest.expectedFailure
def test_badfile(self):
# This should throw a CommandError
copy_filelist(self.src, ['/doesnt/exist'], self.dst)
def do_bin(self):
"""Check for binaries in bin-cdrom*.
Return a tuple of the form ({ "source-name" : [versions ..] }, src_cnt).
Put in other words, validating binary cdroms will give us back a
dictionnary where the keys are the source package's name, and the values
of these keys are lists of versions. It will also return a total count
of source that is in the dictionnary.
"""
# NOTE! - The binary table should be okay with the current
# bin-cdrom.iso. However, the way that the source table is genrated is
# not okay for now. Indeed, it's not aware of components. So for
# example, if two components have a binary in common, they will share
# the corresponding source in the source table. The solution is not as
# trivial as having a reference counter or appending to a list, because
# a component could have multiple binaries that match to the same source
# package. Thus, the only way is to make the source table component
# aware.
# pylint: disable=too-many-locals
# pylint: disable=too-many-branches
# pylint: disable=too-many-statements
# Every build has a source.xml where the list of binaries
# installed can be found
xml = etree("source.xml")
# Initial statistics fo the build
bin_cnt = 0
src_cnt = 0
bin_total = 0
binaries = {}
sources = {}
# Create a dictionnary of the form {"bin-name": [versions ..]}
# from the source.xml. We do this by iterating over all <pkg>
for tag in xml.all("./*/pkg"):
bin_pkg = tag.et.text
# Package already in the dictionnary? Add its version.
# Otherwise, add a new entry into the dictionnary
if bin_pkg in binaries:
binaries[bin_pkg].append(tag.et.attrib["version"])
else:
binaries[bin_pkg] = [tag.et.attrib["version"]]
bin_total += 1
# For every bin-cdrom, create a temporary directory where to
# extract it and find all *.deb files
for cdrom in glob.glob("bin-cdrom*"):
with TmpdirFilesystem() as tmp:
self.extract_cdrom(tmp.path, cdrom)
for _, realpath in tmp.walk_files():
if not realpath.endswith('.deb'):
continue
# Extract informations from .deb
infos = self.dpkg_get_infos(
realpath,
["Package", "Source", "Version", "Built-Using"])
src_name = None
src_version = None
bin_name = None
bin_version = None
for line in infos.split('\n'):
# Package: <PACKAGE>
if line.startswith("Package:"):
bin_name = line.split('Package:')[1].strip(' \t')
# Version: <VERSION>
#
# Skip PGP's version. The package version is
# supposed to be before the PGP signature. However,
# the PGP signature will put a 'Version' field.
# Thus, let's check if we already have found a
# binary version and don't overwrite it
elif line.startswith("Version:"):
if not bin_version:
bin_version = line.split('Version:')[1].strip(
' ')
# Source: <SOURCE> [(VERSION)]
#
# This field is optional. If it is not present, the
# source package default to the bin package
elif line.startswith("Source:"):
src_infos = line.split('Source:')[1].strip(
' ').split(' ')
src_name = src_infos[0]
if len(src_infos) > 1:
src_version = src_infos[1].strip("()")
# Built-Using: <SRC (=VERSION)>...
#
# Sources list in the built-using field are
# seperated by a comma
elif line.startswith("Built-Using:"):
built_using = line.split("Built-Using:")[1].strip(
' ').split(',')
for src in built_using:
name, version = src.strip(' ').split(' ', 1)
version = version.strip("(= )")
# TODO - This is not component aware!
if name in sources:
if version not in sources[name]:
sources[name].add(version)
src_cnt += 1
else:
src_cnt += 1
sources[name] = {version}
# No source was found
if src_name is None:
src_name = bin_name
src_version = bin_version
# No source version was found
elif src_version is None:
src_version = bin_version
# TODO - This is not component aware!
#
# Let's build a dictionnary of sources of the form
# {"source-name" : [versions ..]}. Same as the binary
# dictionnary before
if src_name in sources:
if src_version not in sources[src_name]:
sources[src_name].add(src_version)
src_cnt += 1
else:
sources[src_name] = {src_version}
src_cnt += 1
# Prune version of this binary
bin_cnt += 1
try:
binaries[bin_name].remove(bin_version)
logging.info("Validating binary %s_%s", bin_name,
bin_version)
logging.info("Adding source %s_%s", src_name,
src_version)
except KeyError:
logging.error("Foreign binary found %s_%s", bin_name,
bin_version)
self.ret = 1
# List all missing binaries
for bin_name in binaries:
for bin_version in binaries[bin_name]:
logging.error("Missing binary %s_%s", bin_name, bin_version)
logging.info("Succesfully validated %d binary packages out of %d",
bin_cnt, bin_total)
if bin_cnt != bin_total:
self.ret = 1
return sources, src_cnt
def do_src(self, sources, src_total):
"""Check for sources in src-cdrom*"""
# pylint: disable=too-many-locals
# pylint: disable=too-many-branches
iso_it = glob.iglob("src-cdrom*")
src_cnt = 0
# For every src-cdrom*, extract it to a temporary directory
# and find all *.dsc files
for cdrom in iso_it:
with TmpdirFilesystem() as tmp:
self.extract_cdrom(tmp.path, cdrom)
for _, realpath in tmp.walk_files():
if not realpath.endswith('.dsc'):
continue
infos = self.dpkg_get_infos(realpath,
["Source", "Version"])
src_name = None
src_version = None
for info in infos.split('\n'):
if info.startswith("Source:"):
src_name = info.split('Source:')[1].strip(' ')
# Same as for the binary version. The
# PGP's signature contains a version field
elif info.startswith("Version:"):
if not src_version:
src_version = info.split('Version:')[1].strip(
' ')
if src_name in sources:
match = False
for version in sources[src_name]:
# Found a matching version; prune it
if self.cmp_version(version, src_version) == 0:
logging.info("Validating source %s_%s",
src_name, version)
sources[src_name].remove(version)
src_cnt += 1
match = True
break
# NOTE! - Because the way the source table is
# generated, it's not possible to have multiple time
# the same version of a source (you have different
# versions only). However, this is totally possible
# for cdrom because of multiple components. Thus,
# whenever the source table can handle per component
# sources, this case should emit an error instead of
# a warning
if not match:
logging.warning(
"Can't find matching version for source %s_%s.\n"
"It might have already been validated",
src_name, src_version)
else:
logging.error("Extra source %s_%s found", src_name,
src_version)
self.ret = 1
# List missing sources
for src_name in sources:
for src_version in sources[src_name]:
logging.error("Missing source %s_%s", src_name, src_version)
logging.info("Succesfully validated %d source packages out of %d",
src_cnt, src_total)
if src_cnt != src_total:
self.ret = 1
def setUp(self):
self.src = TmpdirFilesystem()
self.dst = TmpdirFilesystem()
class VirtApt(object):
def __init__(self, xml):
# pylint: disable=too-many-statements
self.xml = xml
arch = xml.text("project/buildimage/arch", key="arch")
suite = xml.text("project/suite")
self.basefs = TmpdirFilesystem()
self.initialize_dirs()
create_apt_prefs(self.xml, self.basefs)
mirror = self.xml.create_apt_sources_list(build_sources=True,
initvm=False)
self.basefs.write_file("etc/apt/sources.list", 0o644, mirror)
self.setup_gpg()
self.import_keys()
apt_pkg.config.set("APT::Architecture", arch)
apt_pkg.config.set("APT::Architectures", arch)
apt_pkg.config.set("Acquire::http::Proxy::127.0.0.1", "DIRECT")
apt_pkg.config.set("APT::Install-Recommends", "0")
apt_pkg.config.set("Dir::Etc", self.basefs.fname('/'))
apt_pkg.config.set("Dir::Etc::Trusted",
self.basefs.fname('/etc/apt/trusted.gpg'))
apt_pkg.config.set("Dir::Etc::TrustedParts",
self.basefs.fname('/etc/apt/trusted.gpg.d'))
apt_pkg.config.set("APT::Cache-Limit", "0")
apt_pkg.config.set("APT::Cache-Start", "32505856")
apt_pkg.config.set("APT::Cache-Grow", "2097152")
apt_pkg.config.set("Dir::State", self.basefs.fname("state"))
apt_pkg.config.set("Dir::State::status",
self.basefs.fname("state/status"))
apt_pkg.config.set("Dir::Cache", self.basefs.fname("cache"))
apt_pkg.config.set("Dir::Cache::archives",
self.basefs.fname("cache/archives"))
apt_pkg.config.set("Dir::Etc", self.basefs.fname("etc/apt"))
apt_pkg.config.set("Dir::Log", self.basefs.fname("log"))
if self.xml.has('project/noauth'):
apt_pkg.config.set("APT::Get::AllowUnauthenticated", "1")
apt_pkg.config.set("Acquire::AllowInsecureRepositories", "1")
else:
apt_pkg.config.set("APT::Get::AllowUnauthenticated", "0")
apt_pkg.config.set("Acquire::AllowInsecureRepositories", "0")
apt_pkg.init_system()
self.source = apt_pkg.SourceList()
self.source.read_main_list()
self.cache = apt_pkg.Cache()
try:
self.cache.update(self, self.source)
except BaseException as e:
print(e)
apt_pkg.config.set("APT::Default-Release", suite)
self.cache = apt_pkg.Cache()
try:
self.cache.update(self, self.source)
except BaseException as e:
print(e)
try:
self.depcache = apt_pkg.DepCache(self.cache)
prefs_name = self.basefs.fname("/etc/apt/preferences")
self.depcache.read_pinfile(prefs_name)
except BaseException as e:
print(e)
self.downloads = {}
self.acquire = apt_pkg.Acquire(self)
def add_key(self, key):
cmd = 'echo "%s" > %s' % (key, self.basefs.fname("tmp/key.pub"))
clean = 'rm -f %s' % self.basefs.fname("tmp/key.pub")
system(cmd)
system('fakeroot apt-key --keyring "%s" add "%s"' %
(self.basefs.fname('/etc/apt/trusted.gpg'),
self.basefs.fname("tmp/key.pub")))
system(clean)
def import_keys(self):
if self.xml.has('project/mirror/url-list'):
# Should we use self.xml.prj.has("noauth")???
#
# If so, this is related to issue #220 -
# https://github.com/Linutronix/elbe/issues/220
#
# I could make a none global 'noauth' flag for mirrors
for url in self.xml.node('project/mirror/url-list'):
if url.has('raw-key'):
key = "\n".join(
line.strip(" \t")
for line in url.text('raw-key').splitlines()[1:-1])
self.add_key(key)
def start(self):
pass
def stop(self):
pass
@staticmethod
def pulse(_obj):
return True
def initialize_dirs(self):
self.basefs.mkdir_p("cache/archives/partial")
self.basefs.mkdir_p("etc/apt/preferences.d")
self.basefs.mkdir_p("etc/apt/trusted.gpg.d")
self.basefs.mkdir_p("db")
self.basefs.mkdir_p("log")
self.basefs.mkdir_p("state/lists/partial")
self.basefs.mkdir_p("tmp")
self.basefs.touch_file("state/status")
def setup_gpg(self):
ring_path = self.basefs.fname("etc/apt/trusted.gpg")
if not os.path.isdir("/etc/apt/trusted.gpg.d"):
print("/etc/apt/trusted.gpg.d doesn't exist")
print("apt-get install debian-archive-keyring may "
"fix this problem")
sys.exit(20)
if os.path.exists("/etc/apt/trusted.gpg"):
system('cp /etc/apt/trusted.gpg "%s"' % ring_path)
trustkeys = os.listdir("/etc/apt/trusted.gpg.d")
for key in trustkeys:
system('cp "/etc/apt/trusted.gpg.d/%s" "%s"' %
(key, ring_path + '.d'))
def mark_install(self, pkgname):
self.depcache.mark_install(self.cache[pkgname])
def marked_install(self, pkgname):
return self.depcache.marked_install(self.cache[pkgname])
def get_candidate_ver(self, pkgname):
return self.depcache.get_candidate_ver(self.cache[pkgname]).ver_str
def has_pkg(self, pkgname):
return pkgname in self.cache
def mark_pkg_download(self, pkgname):
pkg = self.cache[pkgname]
c = self.depcache.get_candidate_ver(pkg)
r = apt_pkg.PackageRecords(self.cache)
r.lookup(c.file_list[0])
x = self.source.find_index(c.file_list[0][0])
uri = x.archive_uri(r.filename)
hashval = str(r.hashes.find('SHA256'))
acq = apt_pkg.AcquireFile(self.acquire,
uri,
hash=hashval,
size=c.size,
descr=r.long_desc,
short_descr=r.short_desc,
destdir=self.basefs.fname('/cache/archives'))
self.downloads[pkgname] = acq
def do_downloads(self):
res = self.acquire.run()
print(res)
def get_downloaded_files(self):
ret = []
for _, d in self.downloads.items():
if d.complete:
ret.append(d.destfile)
else:
print('incomplete download "%s"' % d.desc_uri)
return ret
def get_downloaded_pkg(self, pkgname):
d = self.downloads[pkgname]
if not d.complete:
print('incomplete download "%s"' % d.desc_uri)
raise KeyError
return d.destfile
def get_uri(self, target_pkg, incl_deps=False):
d = apt_pkg.DepCache(self.cache)
if not incl_deps:
return [lookup_uri(self, d, target_pkg)]
deps = [lookup_uri(self, d, target_pkg)]
togo = [target_pkg]
while togo:
pp = togo.pop()
try:
pkg = self.cache[pp]
c = d.get_candidate_ver(pkg)
except KeyError:
pkg = None
c = None
if not c:
# pylint: disable=E1133
for p in self.cache.packages:
for x in p.provides_list:
if pp == x[0]:
pkg = self.cache[x[2].parent_pkg.name]
c = d.get_candidate_ver(pkg)
if not c:
print("couldnt get candidate: %s" % pkg)
else:
for p in getdeps(c):
if [y for y in deps if y[0] == p]:
continue
if p != target_pkg and p == pp:
continue
deps.append(lookup_uri(self, d, p))
togo.append(p)
return list(set(deps))
def execute(self, opt, _args):
# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
tmp = TmpdirFilesystem()
if opt.xmlfile:
ret, prjdir, err = command_out_stderr(
'%s control create_project --retries 60 "%s"' %
(elbe_exe, opt.xmlfile))
if ret != 0:
print("elbe control create_project failed.", file=sys.stderr)
print(err, file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
prjdir = prjdir.strip()
try:
system('%s control build_pbuilder "%s"' % (elbe_exe, prjdir))
except CommandError:
print("elbe control build_pbuilder Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
try:
system('%s control wait_busy "%s"' % (elbe_exe, prjdir))
except CommandError:
print("elbe control wait_busy Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print("Building Pbuilder finished !")
print("")
elif opt.project:
prjdir = opt.project
system('%s control rm_log %s' % (elbe_exe, prjdir))
else:
print(
"you need to specify --project or --xmlfile option",
file=sys.stderr)
sys.exit(20)
print("")
print("Packing Source into tmp archive")
print("")
try:
system('tar cfz "%s" .' % (tmp.fname("pdebuild.tar.gz")))
except CommandError:
print("tar Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
for of in opt.origfile:
print("")
print("Pushing orig file '%s' into pbuilder" % of)
print("")
try:
system(
'%s control set_orig "%s" "%s"' %
(elbe_exe, prjdir, of))
except CommandError:
print("elbe control set_orig Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print("Pushing source into pbuilder")
print("")
try:
system('%s control set_pdebuild --cpuset "%d" --profile "%s" '
'"%s" "%s"' %
(elbe_exe, opt.cpuset, opt.profile,
prjdir, tmp.fname("pdebuild.tar.gz")))
except CommandError:
print("elbe control set_pdebuild Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
try:
system('%s control wait_busy "%s"' % (elbe_exe, prjdir))
except CommandError:
print("elbe control wait_busy Failed", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print("Pdebuild finished !")
print("")
if opt.skip_download:
print("")
print("Listing available files:")
print("")
try:
system(
'%s control --pbuilder-only get_files "%s"' %
(elbe_exe, prjdir))
except CommandError:
print("elbe control get_files Failed", file=sys.stderr)
print("", file=sys.stderr)
print("dumping logfile", file=sys.stderr)
try:
system('%s control dump_file "%s" log.txt' % (
elbe_exe, prjdir))
except CommandError:
print("elbe control dump_file Failed", file=sys.stderr)
print("", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
print("")
print(
"Get Files with: 'elbe control get_file %s <filename>'" %
prjdir)
else:
print("")
print("Getting generated Files")
print("")
ensure_outdir(opt)
try:
system(
'%s control --pbuilder-only get_files --output "%s" "%s"' %
(elbe_exe, opt.outdir, prjdir))
except CommandError:
print("elbe control get_files Failed", file=sys.stderr)
print("", file=sys.stderr)
print("dumping logfile", file=sys.stderr)
try:
system('%s control dump_file "%s" log.txt' % (
elbe_exe, prjdir))
except CommandError:
print("elbe control dump_file Failed", file=sys.stderr)
print("", file=sys.stderr)
print("Giving up", file=sys.stderr)
sys.exit(20)
def extract_pkg_changelog(fname, extra_pkg=None):
pkgname = os.path.basename(fname)
m = re.match(re_pkgfilename, pkgname)
pkgname = m.group('name')
# pkgver = m.group('ver')
pkgarch = m.group('arch')
print('pkg: %s, arch: %s' % (pkgname, pkgarch))
fs = TmpdirFilesystem(debug=True)
if extra_pkg:
print('with extra ' + extra_pkg)
system('dpkg -x "%s" "%s"' % (extra_pkg, fs.fname('/')))
system('dpkg -x "%s" "%s"' % (fname, fs.fname('/')))
dch_dir = '/usr/share/doc/%s' % pkgname
if fs.islink(dch_dir) and not extra_pkg:
l = fs.readlink(dch_dir)
print(dch_dir, l)
raise ChangelogNeedsDependency(l)
dch_bin = '/usr/share/doc/%s/changelog.Debian.%s.gz' % (pkgname, pkgarch)
dch_src = '/usr/share/doc/%s/changelog.Debian.gz' % pkgname
print(fs.listdir('/usr/share/doc/'))
print(fs.listdir('/usr/share/doc/%s' % pkgname))
ret = ""
if fs.exists(dch_bin):
ret += fs.read_file(dch_bin, gz=True)
else:
print("no bin")
if fs.exists(dch_src):
ret += fs.read_file(dch_src, gz=True)
else:
print("no source")
return ret
def execute(self, opt, args):
tmp = TmpdirFilesystem ()
if opt.xmlfile:
ret, prjdir, err = command_out_stderr ('%s control create_project --retries 60 "%s"' % (elbe_exe, opt.xmlfile))
if ret != 0:
print ("elbe control create_project failed.", file=sys.stderr)
print (err, file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
prjdir = prjdir.strip()
try:
system ('%s control build "%s"' % (elbe_exe, prjdir) )
except CommandError:
print ("elbe control build Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
print ("Build started, waiting till it finishes")
try:
system ('%s control wait_busy "%s"' % (elbe_exe, prjdir) )
except CommandError:
print ("elbe control wait_busy Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
print ("")
print ("Build finished !")
print ("")
print ("Creating pbuilder")
try:
system ('%s control build_pbuilder "%s"' % (elbe_exe, prjdir))
except CommandError:
print ("elbe control build_pbuilder Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
try:
system ('%s control wait_busy "%s"' % (elbe_exe, prjdir) )
except CommandError:
print ("elbe control wait_busy Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
print ("")
print ("Building Pbuilder finished !")
print ("")
elif opt.project:
prjdir = opt.project
else:
print ('you need to specify --project or --xmlfile option', file=sys.stderr)
sys.exit(20)
print ("")
print ("Packing Source into tmp archive")
print ("")
try:
system ('tar cvfz "%s" .' % (tmp.fname ("pdebuild.tar.gz")))
except CommandError:
print ("tar Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
print ("")
print ("Pushing source into pbuilder")
print ("")
try:
system ('%s control set_pdebuild "%s" "%s"' % (elbe_exe, prjdir, tmp.fname ("pdebuild.tar.gz")))
except CommandError:
print ("elbe control set_pdebuild Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
try:
system ('%s control wait_busy "%s"' % (elbe_exe, prjdir) )
except CommandError:
print ("elbe control wait_busy Failed", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
print ("")
print ("Pdebuild finished !")
print ("")
if opt.skip_download:
print ("")
print ("Listing available files:")
print ("")
try:
system ('%s control --pbuilder-only get_files "%s"' % (elbe_exe, prjdir) )
except CommandError:
print ("elbe control get_files Failed", file=sys.stderr)
print ("", file=sys.stderr)
print ("dumping logfile", file=sys.stderr)
try:
system ('%s control dump_file "%s" log.txt' % (
elbe_exe, prjdir ))
except CommandError:
print ("elbe control dump_file Failed", file=sys.stderr)
print ("", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
print ("")
print ('Get Files with: elbe control get_file "%s" <filename>' % prjdir)
else:
print ("")
print ("Getting generated Files")
print ("")
ensure_outdir (wdfs, opt)
try:
system ('%s control --pbuilder-only get_files --output "%s" "%s"' % (
elbe_exe, opt.outdir, prjdir ))
except CommandError:
print ("elbe control get_files Failed", file=sys.stderr)
print ("", file=sys.stderr)
print ("dumping logfile", file=sys.stderr)
try:
system ('%s control dump_file "%s" log.txt' % (
elbe_exe, prjdir ))
except CommandError:
print ("elbe control dump_file Failed", file=sys.stderr)
print ("", file=sys.stderr)
print ("Giving up", file=sys.stderr)
sys.exit(20)
