def execute(self, opt, args): tmp = TmpdirFilesystem () if not opt.project: print ('you need to specify --project option', file=sys.stderr) sys.exit(20) prjdir = opt.project print ("Creating pbuilder") try: system ('%s control build_pbuilder "%s"' % (elbe_exe, prjdir)) except CommandError: print ("elbe control build_pbuilder Failed", file=sys.stderr) print ("Giving up", file=sys.stderr) sys.exit(20) try: system ('%s control wait_busy "%s"' % (elbe_exe, prjdir) ) except CommandError: print ("elbe control wait_busy Failed", file=sys.stderr) print ("Giving up", file=sys.stderr) sys.exit(20) print ("") print ("Building Pbuilder finished !") print ("")
def copy_kinitrd(prj, target_dir): suite = prj.text("suite") try: tmp = TmpdirFilesystem() if prj.has("mirror/cdrom"): system('7z x -o%s "%s" initrd-cdrom.gz vmlinuz' % (tmp.fname('/'), prj.text("mirror/cdrom"))) # initrd.gz needs to be cdrom version ! copyfile(tmp.fname("initrd-cdrom.gz"), os.path.join(target_dir, "initrd.gz")) else: mirror = get_primary_mirror(prj) download_kinitrd(tmp, suite, mirror) copyfile(tmp.fname("initrd.gz"), os.path.join(target_dir, "initrd.gz")) copyfile(tmp.fname("initrd-cdrom.gz"), os.path.join(target_dir, "initrd-cdrom.gz")) copyfile(tmp.fname("vmlinuz"), os.path.join(target_dir, "vmlinuz")) except IOError as e: raise NoKinitrdException('IoError %s' % str(e)) except InvalidSignature as e: raise NoKinitrdException('InvalidSignature %s' % str(e)) except HashValidationFailed as e: raise NoKinitrdException('HashValidationFailed %s' % str(e))
def extract_cdrom(cdrom): """ Extract cdrom iso image returns a TmpdirFilesystem() object containing the source.xml, which is also validated. """ tmp = TmpdirFilesystem() system('7z x -o%s "%s" source.xml' % (tmp.path, cdrom)) print("", file=sys.stderr) if not tmp.isfile('source.xml'): print("Iso image does not contain a source.xml file", file=sys.stderr) print("This is not supported by 'elbe initvm'", file=sys.stderr) print("", file=sys.stderr) print("Exiting !!!", file=sys.stderr) sys.exit(20) try: exml = ElbeXML(tmp.fname('source.xml'), url_validation=ValidationMode.NO_CHECK) except ValidationError as e: print("Iso image does contain a source.xml file.", file=sys.stderr) print("But that xml does not validate correctly", file=sys.stderr) print("", file=sys.stderr) print("Exiting !!!", file=sys.stderr) print(e) sys.exit(20) print("Iso Image with valid source.xml detected !") print("Image was generated using Elbe Version %s" % exml.get_elbe_version()) return tmp
def execute(self, opt, args): tmp = TmpdirFilesystem () if opt.xmlfile: ret, prjdir, err = command_out_stderr ('%s control create_project' % (elbe_exe)) if ret != 0: print ("elbe control create_project failed.", file=sys.stderr) print (err, file=sys.stderr) print ("Giving up", file=sys.stderr) sys.exit(20) prjdir = prjdir.strip() ret, msg, err = command_out_stderr ('%s control set_xml "%s" "%s"' % (elbe_exe, prjdir, opt.xmlfile)) if ret != 0: print ("elbe control set_xml failed.", file=sys.stderr) print (err, file=sys.stderr) print ("Giving up", file=sys.stderr) sys.exit(20) if opt.writeproject: wpf = open (opt.writeproject, "w") wpf.write (prjdir) wpf.close() elif opt.project: prjdir = opt.project else: print ('you need to specify --project option', file=sys.stderr) sys.exit(20) print ("Creating pbuilder") try: system ('%s control build_pbuilder "%s"' % (elbe_exe, prjdir)) except CommandError: print ("elbe control build_pbuilder Failed", file=sys.stderr) print ("Giving up", file=sys.stderr) sys.exit(20) try: system ('%s control wait_busy "%s"' % (elbe_exe, prjdir) ) except CommandError: print ("elbe control wait_busy Failed", file=sys.stderr) print ("Giving up", file=sys.stderr) sys.exit(20) print ("") print ("Building Pbuilder finished !") print ("")
def extract_pkg_changelog(fname, extra_pkg=None): pkgname = os.path.basename(fname) m = re.match(re_pkgfilename, pkgname) pkgname = m.group('name') # pkgver = m.group('ver') pkgarch = m.group('arch') print('pkg: %s, arch: %s' % (pkgname, pkgarch)) fs = TmpdirFilesystem(debug=True) if extra_pkg: print('with extra ' + extra_pkg) system('dpkg -x "%s" "%s"' % (extra_pkg, fs.fname('/'))) system('dpkg -x "%s" "%s"' % (fname, fs.fname('/'))) dch_dir = '/usr/share/doc/%s' % pkgname if fs.islink(dch_dir) and not extra_pkg: l = fs.readlink(dch_dir) print(dch_dir, l) raise ChangelogNeedsDependency(l) dch_bin = '/usr/share/doc/%s/changelog.Debian.%s.gz' % (pkgname, pkgarch) dch_src = '/usr/share/doc/%s/changelog.Debian.gz' % pkgname print(fs.listdir('/usr/share/doc/')) print(fs.listdir('/usr/share/doc/%s' % pkgname)) ret = "" if fs.exists(dch_bin): ret += fs.read_file(dch_bin, gz=True) else: print("no bin") if fs.exists(dch_src): ret += fs.read_file(dch_src, gz=True) else: print("no source") return ret
def do_sdk(self, sdk): with TmpdirFilesystem() as tmp: # Make a copy of the installer shutil.copyfile(sdk, tmp.fname(sdk)) # Let's work in our temp dir from now on os.chdir(tmp.path) # The script is self extracting; it needs to be executable os.chmod(sdk, 0o744) # Extract here with 'yes' to all answers do("./%s -y -d ." % sdk) # Get environment file env = tmp.glob("environment-setup*")[0] # NOTE! This script requires binfmt to be installed. do("/bin/sh", stdin=self.script, env_add={"ELBE_SDK_ENV": env})
def _combinearchivedir(xml, xpath, use_volume): elbexml = etree(None) elbexml.et = xml tmp = TmpdirFilesystem() for archivedir in elbexml.all(xpath): try: archiveurl = urljoin(archivedir.et.base, archivedir.et.text) keep = archivedir.bool_attr("keep-attributes") parent = archivedir.get_parent() if use_volume: volume_attr = archivedir.et.get('volume', default='all') fname_suffix = volume_attr arch = parent.node("archive[@volume='%s']" % volume_attr) if arch is None: arch = parent.append("archive") arch.et.set("volume", volume_attr) else: arch = parent.ensure_child("archive") fname_suffix = '' get_and_append = get_and_append_method(archiveurl) archname = tmp.fname('archive%s.tar.bz2' % fname_suffix) get_and_append(archiveurl, archname, keep) arch.set_text(enbase(archname, True)) parent.remove_child(archivedir) except (CalledProcessError, OSError): msg = "Failure while processing \"" + archivedir.text + "\":\n" msg += str(sys.exc_info()[1]) raise ArchivedirError(msg)
def __init__(self, xml): # pylint: disable=too-many-statements self.xml = xml arch = xml.text("project/buildimage/arch", key="arch") suite = xml.text("project/suite") self.basefs = TmpdirFilesystem() self.initialize_dirs() create_apt_prefs(self.xml, self.basefs) mirror = self.xml.create_apt_sources_list(build_sources=True, initvm=False) self.basefs.write_file("etc/apt/sources.list", 0o644, mirror) self.setup_gpg() self.import_keys() apt_pkg.config.set("APT::Architecture", arch) apt_pkg.config.set("APT::Architectures", arch) apt_pkg.config.set("Acquire::http::Proxy::127.0.0.1", "DIRECT") apt_pkg.config.set("APT::Install-Recommends", "0") apt_pkg.config.set("Dir::Etc", self.basefs.fname('/')) apt_pkg.config.set("Dir::Etc::Trusted", self.basefs.fname('/etc/apt/trusted.gpg')) apt_pkg.config.set("Dir::Etc::TrustedParts", self.basefs.fname('/etc/apt/trusted.gpg.d')) apt_pkg.config.set("APT::Cache-Limit", "0") apt_pkg.config.set("APT::Cache-Start", "32505856") apt_pkg.config.set("APT::Cache-Grow", "2097152") apt_pkg.config.set("Dir::State", self.basefs.fname("state")) apt_pkg.config.set("Dir::State::status", self.basefs.fname("state/status")) apt_pkg.config.set("Dir::Cache", self.basefs.fname("cache")) apt_pkg.config.set("Dir::Cache::archives", self.basefs.fname("cache/archives")) apt_pkg.config.set("Dir::Etc", self.basefs.fname("etc/apt")) apt_pkg.config.set("Dir::Log", self.basefs.fname("log")) if self.xml.has('project/noauth'): apt_pkg.config.set("APT::Get::AllowUnauthenticated", "1") apt_pkg.config.set("Acquire::AllowInsecureRepositories", "1") else: apt_pkg.config.set("APT::Get::AllowUnauthenticated", "0") apt_pkg.config.set("Acquire::AllowInsecureRepositories", "0") apt_pkg.init_system() self.source = apt_pkg.SourceList() self.source.read_main_list() self.cache = apt_pkg.Cache() try: self.cache.update(self, self.source) except BaseException as e: print(e) apt_pkg.config.set("APT::Default-Release", suite) self.cache = apt_pkg.Cache() try: self.cache.update(self, self.source) except BaseException as e: print(e) try: self.depcache = apt_pkg.DepCache(self.cache) prefs_name = self.basefs.fname("/etc/apt/preferences") self.depcache.read_pinfile(prefs_name) except BaseException as e: print(e) self.downloads = {} self.acquire = apt_pkg.Acquire(self)
def execute(self, initvmdir, opt, args): try: have_session = os.system( "tmux has-session -t ElbeInitVMSession >/dev/null 2>&1") except CommandError as e: print( "tmux execution failed, tmux version 1.9 or higher is required" ) sys.exit(20) if have_session == 256: print("ElbeInitVMSession does not exist in tmux.", file=sys.stderr) print("Try 'elbe initvm start' to start the session.", file=sys.stderr) sys.exit(20) try: system('%s initvm ensure --directory "%s"' % (elbe_exe, initvmdir)) except CommandError: print("Starting the initvm Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) # Init cdrom to None, if we detect it, we set it cdrom = None if len(args) == 1: if args[0].endswith('.xml'): # We have an xml file, use that for elbe init xmlfile = args[0] elif args[0].endswith('.iso'): # We have an iso image, extract xml from there. tmp = TmpdirFilesystem() os.system('7z x -o%s "%s" source.xml' % (tmp.path, args[0])) print('', file=sys.stderr) if not tmp.isfile('source.xml'): print('Iso image does not contain a source.xml file', file=sys.stderr) print('This is not supported by "elbe initvm"', file=sys.stderr) print('', file=sys.stderr) print('Exiting !!!', file=sys.stderr) sys.exit(20) try: exml = ElbeXML(tmp.fname('source.xml'), skip_urlcheck=True) except ValidationError as e: print('Iso image does contain a source.xml file.', file=sys.stderr) print('But that xml does not validate correctly', file=sys.stderr) print('', file=sys.stderr) print('Exiting !!!', file=sys.stderr) sys.exit(20) print('Iso Image with valid source.xml detected !') print('Image was generated using Elbe Version %s' % exml.get_elbe_version()) xmlfile = tmp.fname('source.xml') cdrom = args[0] else: print('Unknown file ending (use either xml or iso)', file=sys.stderr) sys.exit(20) ret, prjdir, err = command_out_stderr( '%s control create_project --retries 60 "%s"' % (elbe_exe, xmlfile)) if ret != 0: print("elbe control create_project failed.", file=sys.stderr) print(err, file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) prjdir = prjdir.strip() if cdrom is not None: print("Uploading CDROM. This might take a while") try: system('%s control set_cdrom "%s" "%s"' % (elbe_exe, prjdir, cdrom)) except CommandError: print("elbe control set_cdrom Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("Upload finished") build_opts = '' if opt.build_bin: build_opts += '--build-bin ' if opt.build_sources: build_opts += '--build-sources ' try: system('%s control build "%s" %s' % (elbe_exe, prjdir, build_opts)) except CommandError: print("elbe control build Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("Build started, waiting till it finishes") try: system('%s control wait_busy "%s"' % (elbe_exe, prjdir)) except CommandError: print("elbe control wait_busy Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("") print("Build finished !") print("") try: system('%s control dump_file "%s" validation.txt' % (elbe_exe, prjdir)) except CommandError: print("Project failed to generate validation.txt", file=sys.stderr) print("Getting log.txt", file=sys.stderr) try: system('%s control dump_file "%s" log.txt' % (elbe_exe, prjdir)) except CommandError: print("Failed to dump log.txt", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) if opt.skip_download: print("") print("Listing available files:") print("") try: system('%s control get_files "%s"' % (elbe_exe, prjdir)) except CommandError: print("elbe control get_files Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("") print('Get Files with: elbe control get_file "%s" <filename>' % prjdir) else: print("") print("Getting generated Files") print("") ensure_outdir(wdfs, opt) try: system('%s control get_files --output "%s" "%s"' % (elbe_exe, opt.outdir, prjdir)) except CommandError: print("elbe control get_files Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) if not opt.keep_files: try: system('%s control del_project "%s"' % (elbe_exe, prjdir)) except CommandError: print("remove project from initvm failed", file=sys.stderr) sys.exit(20)
def execute(self, initvmdir, opt, args): try: have_session = os.system( "tmux has-session -t ElbeInitVMSession >/dev/null 2>&1") except CommandError as e: print( "tmux execution failed, tmux version 1.9 or higher is required" ) sys.exit(20) if have_session == 0: print("ElbeInitVMSession already exists in tmux.", file=sys.stderr) print("", file=sys.stderr) print( "There can only exist a single ElbeInitVMSession, and this session", file=sys.stderr) print("can also be used to make your build.", file=sys.stderr) print( "See 'elbe initvm submit', 'elbe initvm attach' and 'elbe control'", file=sys.stderr) sys.exit(20) # Init cdrom to None, if we detect it, we set it cdrom = None if len(args) == 1: if args[0].endswith('.xml'): # We have an xml file, use that for elbe init exampl = args[0] try: xml = etree(exampl) except ValidationError as e: print('XML file is inavlid: ' + str(e)) # Use default XML if no initvm was specified if not xml.has("initvm"): exampl = os.path.join(elbepack.__path__[0], "init/default-init.xml") elif args[0].endswith('.iso'): # We have an iso image, extract xml from there. tmp = TmpdirFilesystem() os.system('7z x -o%s "%s" source.xml' % (tmp.path, args[0])) if not tmp.isfile('source.xml'): print('Iso image does not contain a source.xml file', file=sys.stderr) print('This is not supported by "elbe initvm"', file=sys.stderr) print('', file=sys.stderr) print('Exiting !!!', file=sys.stderr) sys.exit(20) try: exml = ElbeXML(tmp.fname('source.xml'), skip_urlcheck=True) except ValidationError as e: print('Iso image does contain a source.xml file.', file=sys.stderr) print('But that xml does not validate correctly', file=sys.stderr) print('', file=sys.stderr) print('Exiting !!!', file=sys.stderr) sys.exit(20) print('Iso Image with valid source.xml detected !') print('Image was generated using Elbe Version %s' % exml.get_elbe_version()) exampl = tmp.fname('source.xml') cdrom = args[0] else: print('Unknown file ending (use either xml or iso)', file=sys.stderr) sys.exit(20) else: # No xml File was specified, build the default elbe-init-with-ssh exampl = os.path.join(elbepack.__path__[0], "init/default-init.xml") try: if opt.devel: devel = ' --devel' else: devel = '' if cdrom: system('%s init %s --directory "%s" --cdrom "%s" "%s"' % (elbe_exe, devel, initvmdir, cdrom, exampl)) else: system('%s init %s --directory "%s" "%s"' % (elbe_exe, devel, initvmdir, exampl)) except CommandError: print("'elbe init' Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) try: system('cd "%s"; make' % (initvmdir)) except CommandError: print("Building the initvm Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) try: system('%s initvm start --directory "%s"' % (elbe_exe, initvmdir)) except CommandError: print("Starting the initvm Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) if len(args) == 1: # if provided xml file has no initvm section exampl is set to a # default initvm XML file. But we need the original file here if args[0].endswith('.xml'): # stop here if no project node was specified try: x = ElbeXML(args[0]) except ValidationError as e: print('XML file is inavlid: ' + str(e)) sys.exit(20) if not x.has('project'): print( 'elbe initvm ready: use "elbe initvm submit myproject.xml" to build a project' ) sys.exit(0) ret, prjdir, err = command_out_stderr( '%s control create_project "%s"' % (elbe_exe, args[0])) else: ret, prjdir, err = command_out_stderr( '%s control create_project "%s"' % (elbe_exe, exampl)) if ret != 0: print("elbe control create_project failed.", file=sys.stderr) print(err, file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) prjdir = prjdir.strip() if cdrom is not None: print("Uploading CDROM. This might take a while") try: system('%s control set_cdrom "%s" "%s"' % (elbe_exe, prjdir, cdrom)) except CommandError: print("elbe control set_cdrom Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("Upload finished") build_opts = '' if opt.build_bin: build_opts += '--build-bin ' if opt.build_sources: build_opts += '--build-sources ' try: system('%s control build "%s" %s' % (elbe_exe, prjdir, build_opts)) except CommandError: print("elbe control build Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) try: system('%s control wait_busy "%s"' % (elbe_exe, prjdir)) except CommandError: print("elbe control wait_busy Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("") print("Build finished !") print("") try: system('%s control dump_file "%s" validation.txt' % (elbe_exe, prjdir)) except CommandError: print("Project failed to generate validation.txt", file=sys.stderr) print("Getting log.txt", file=sys.stderr) try: system('%s control dump_file "%s" log.txt' % (elbe_exe, prjdir)) except CommandError: print("Failed to dump log.txt", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) if opt.skip_download: print("") print("Listing available files:") print("") try: system('%s control get_files "%s"' % (elbe_exe, prjdir)) except CommandError: print("elbe control Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("") print('Get Files with: elbe control get_file "%s" <filename>' % prjdir) else: ensure_outdir(wdfs, opt) try: system('%s control get_files --output "%s" "%s"' % (elbe_exe, opt.outdir, prjdir)) except CommandError: print("elbe control get_files Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20)
def execute(self, opt, _args): # pylint: disable=too-many-statements # pylint: disable=too-many-branches crossopt = "" if opt.cross: crossopt = "--cross" tmp = TmpdirFilesystem() if opt.xmlfile: ret, prjdir, err = command_out_stderr( '%s control create_project --retries 60 "%s"' % (elbe_exe, opt.xmlfile)) if ret != 0: print("elbe control create_project failed.", file=sys.stderr) print(err, file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) prjdir = prjdir.strip() try: system('%s control build_pbuilder "%s"' % (elbe_exe, prjdir)) except CommandError: print("elbe control build_pbuilder Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) try: system('%s control wait_busy "%s"' % (elbe_exe, prjdir)) except CommandError: print("elbe control wait_busy Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("") print("Building Pbuilder finished !") print("") elif opt.project: prjdir = opt.project system('%s control rm_log %s' % (elbe_exe, prjdir)) else: print("you need to specify --project or --xmlfile option", file=sys.stderr) sys.exit(20) print("") print("Packing Source into tmp archive") print("") try: system('tar cfz "%s" .' % (tmp.fname("pdebuild.tar.gz"))) except CommandError: print("tar Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) for of in opt.origfile: print("") print("Pushing orig file '%s' into pbuilder" % of) print("") try: system('%s control set_orig "%s" "%s"' % (elbe_exe, prjdir, of)) except CommandError: print("elbe control set_orig Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("") print("Pushing source into pbuilder") print("") try: system('%s control set_pdebuild --cpuset "%d" --profile "%s" %s ' '"%s" "%s"' % (elbe_exe, opt.cpuset, opt.profile, crossopt, prjdir, tmp.fname("pdebuild.tar.gz"))) except CommandError: print("elbe control set_pdebuild Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) try: system('%s control wait_busy "%s"' % (elbe_exe, prjdir)) except CommandError: print("elbe control wait_busy Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("") print("Pdebuild finished !") print("") if opt.skip_download: print("") print("Listing available files:") print("") try: system('%s control --pbuilder-only get_files "%s"' % (elbe_exe, prjdir)) except CommandError: print("elbe control get_files Failed", file=sys.stderr) print("", file=sys.stderr) print("dumping logfile", file=sys.stderr) try: system('%s control dump_file "%s" log.txt' % (elbe_exe, prjdir)) except CommandError: print("elbe control dump_file Failed", file=sys.stderr) print("", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("") print("Get Files with: 'elbe control get_file %s <filename>'" % prjdir) else: print("") print("Getting generated Files") print("") ensure_outdir(opt) try: system( '%s control --pbuilder-only get_files --output "%s" "%s"' % (elbe_exe, opt.outdir, prjdir)) except CommandError: print("elbe control get_files Failed", file=sys.stderr) print("", file=sys.stderr) print("dumping logfile", file=sys.stderr) try: system('%s control dump_file "%s" log.txt' % (elbe_exe, prjdir)) except CommandError: print("elbe control dump_file Failed", file=sys.stderr) print("", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20)
def execute(self, opt, args): tmp = TmpdirFilesystem () if opt.xmlfile: ret, prjdir, err = command_out_stderr ('%s control create_project --retries 60 "%s"' % (elbe_exe, opt.xmlfile)) if ret != 0: print ("elbe control create_project failed.", file=sys.stderr) print (err, file=sys.stderr) print ("Giving up", file=sys.stderr) sys.exit(20) prjdir = prjdir.strip() try: system ('%s control build "%s"' % (elbe_exe, prjdir) ) except CommandError: print ("elbe control build Failed", file=sys.stderr) print ("Giving up", file=sys.stderr) sys.exit(20) print ("Build started, waiting till it finishes") try: system ('%s control wait_busy "%s"' % (elbe_exe, prjdir) ) except CommandError: print ("elbe control wait_busy Failed", file=sys.stderr) print ("Giving up", file=sys.stderr) sys.exit(20) print ("") print ("Build finished !") print ("") print ("Creating pbuilder") try: system ('%s control build_pbuilder "%s"' % (elbe_exe, prjdir)) except CommandError: print ("elbe control build_pbuilder Failed", file=sys.stderr) print ("Giving up", file=sys.stderr) sys.exit(20) try: system ('%s control wait_busy "%s"' % (elbe_exe, prjdir) ) except CommandError: print ("elbe control wait_busy Failed", file=sys.stderr) print ("Giving up", file=sys.stderr) sys.exit(20) print ("") print ("Building Pbuilder finished !") print ("") elif opt.project: prjdir = opt.project else: print ('you need to specify --project or --xmlfile option', file=sys.stderr) sys.exit(20) print ("") print ("Packing Source into tmp archive") print ("") try: system ('tar cvfz "%s" .' % (tmp.fname ("pdebuild.tar.gz"))) except CommandError: print ("tar Failed", file=sys.stderr) print ("Giving up", file=sys.stderr) sys.exit(20) print ("") print ("Pushing source into pbuilder") print ("") try: system ('%s control set_pdebuild "%s" "%s"' % (elbe_exe, prjdir, tmp.fname ("pdebuild.tar.gz"))) except CommandError: print ("elbe control set_pdebuild Failed", file=sys.stderr) print ("Giving up", file=sys.stderr) sys.exit(20) try: system ('%s control wait_busy "%s"' % (elbe_exe, prjdir) ) except CommandError: print ("elbe control wait_busy Failed", file=sys.stderr) print ("Giving up", file=sys.stderr) sys.exit(20) print ("") print ("Pdebuild finished !") print ("") if opt.skip_download: print ("") print ("Listing available files:") print ("") try: system ('%s control --pbuilder-only get_files "%s"' % (elbe_exe, prjdir) ) except CommandError: print ("elbe control get_files Failed", file=sys.stderr) print ("", file=sys.stderr) print ("dumping logfile", file=sys.stderr) try: system ('%s control dump_file "%s" log.txt' % ( elbe_exe, prjdir )) except CommandError: print ("elbe control dump_file Failed", file=sys.stderr) print ("", file=sys.stderr) print ("Giving up", file=sys.stderr) sys.exit(20) print ("") print ('Get Files with: elbe control get_file "%s" <filename>' % prjdir) else: print ("") print ("Getting generated Files") print ("") ensure_outdir (wdfs, opt) try: system ('%s control --pbuilder-only get_files --output "%s" "%s"' % ( elbe_exe, opt.outdir, prjdir )) except CommandError: print ("elbe control get_files Failed", file=sys.stderr) print ("", file=sys.stderr) print ("dumping logfile", file=sys.stderr) try: system ('%s control dump_file "%s" log.txt' % ( elbe_exe, prjdir )) except CommandError: print ("elbe control dump_file Failed", file=sys.stderr) print ("", file=sys.stderr) print ("Giving up", file=sys.stderr) sys.exit(20)
def do_bin(self): """Check for binaries in bin-cdrom*. Return a tuple of the form ({ "source-name" : [versions ..] }, src_cnt). Put in other words, validating binary cdroms will give us back a dictionnary where the keys are the source package's name, and the values of these keys are lists of versions. It will also return a total count of source that is in the dictionnary. """ # NOTE! - The binary table should be okay with the current # bin-cdrom.iso. However, the way that the source table is genrated is # not okay for now. Indeed, it's not aware of components. So for # example, if two components have a binary in common, they will share # the corresponding source in the source table. The solution is not as # trivial as having a reference counter or appending to a list, because # a component could have multiple binaries that match to the same source # package. Thus, the only way is to make the source table component # aware. # pylint: disable=too-many-locals # pylint: disable=too-many-branches # pylint: disable=too-many-statements # Every build has a source.xml where the list of binaries # installed can be found xml = etree("source.xml") # Initial statistics fo the build bin_cnt = 0 src_cnt = 0 bin_total = 0 binaries = {} sources = {} # Create a dictionnary of the form {"bin-name": [versions ..]} # from the source.xml. We do this by iterating over all <pkg> for tag in xml.all("./*/pkg"): bin_pkg = tag.et.text # Package already in the dictionnary? Add its version. # Otherwise, add a new entry into the dictionnary if bin_pkg in binaries: binaries[bin_pkg].append(tag.et.attrib["version"]) else: binaries[bin_pkg] = [tag.et.attrib["version"]] bin_total += 1 # For every bin-cdrom, create a temporary directory where to # extract it and find all *.deb files for cdrom in glob.glob("bin-cdrom*"): with TmpdirFilesystem() as tmp: self.extract_cdrom(tmp.path, cdrom) for _, realpath in tmp.walk_files(): if not realpath.endswith('.deb'): continue # Extract informations from .deb infos = self.dpkg_get_infos( realpath, ["Package", "Source", "Version", "Built-Using"]) src_name = None src_version = None bin_name = None bin_version = None for line in infos.split('\n'): # Package: <PACKAGE> if line.startswith("Package:"): bin_name = line.split('Package:')[1].strip(' \t') # Version: <VERSION> # # Skip PGP's version. The package version is # supposed to be before the PGP signature. However, # the PGP signature will put a 'Version' field. # Thus, let's check if we already have found a # binary version and don't overwrite it elif line.startswith("Version:"): if not bin_version: bin_version = line.split('Version:')[1].strip( ' ') # Source: <SOURCE> [(VERSION)] # # This field is optional. If it is not present, the # source package default to the bin package elif line.startswith("Source:"): src_infos = line.split('Source:')[1].strip( ' ').split(' ') src_name = src_infos[0] if len(src_infos) > 1: src_version = src_infos[1].strip("()") # Built-Using: <SRC (=VERSION)>... # # Sources list in the built-using field are # seperated by a comma elif line.startswith("Built-Using:"): built_using = line.split("Built-Using:")[1].strip( ' ').split(',') for src in built_using: name, version = src.strip(' ').split(' ', 1) version = version.strip("(= )") # TODO - This is not component aware! if name in sources: if version not in sources[name]: sources[name].add(version) src_cnt += 1 else: src_cnt += 1 sources[name] = {version} # No source was found if src_name is None: src_name = bin_name src_version = bin_version # No source version was found elif src_version is None: src_version = bin_version # TODO - This is not component aware! # # Let's build a dictionnary of sources of the form # {"source-name" : [versions ..]}. Same as the binary # dictionnary before if src_name in sources: if src_version not in sources[src_name]: sources[src_name].add(src_version) src_cnt += 1 else: sources[src_name] = {src_version} src_cnt += 1 # Prune version of this binary bin_cnt += 1 try: binaries[bin_name].remove(bin_version) logging.info("Validating binary %s_%s", bin_name, bin_version) logging.info("Adding source %s_%s", src_name, src_version) except KeyError: logging.error("Foreign binary found %s_%s", bin_name, bin_version) self.ret = 1 # List all missing binaries for bin_name in binaries: for bin_version in binaries[bin_name]: logging.error("Missing binary %s_%s", bin_name, bin_version) logging.info("Succesfully validated %d binary packages out of %d", bin_cnt, bin_total) if bin_cnt != bin_total: self.ret = 1 return sources, src_cnt
def do_src(self, sources, src_total): """Check for sources in src-cdrom*""" # pylint: disable=too-many-locals # pylint: disable=too-many-branches iso_it = glob.iglob("src-cdrom*") src_cnt = 0 # For every src-cdrom*, extract it to a temporary directory # and find all *.dsc files for cdrom in iso_it: with TmpdirFilesystem() as tmp: self.extract_cdrom(tmp.path, cdrom) for _, realpath in tmp.walk_files(): if not realpath.endswith('.dsc'): continue infos = self.dpkg_get_infos(realpath, ["Source", "Version"]) src_name = None src_version = None for info in infos.split('\n'): if info.startswith("Source:"): src_name = info.split('Source:')[1].strip(' ') # Same as for the binary version. The # PGP's signature contains a version field elif info.startswith("Version:"): if not src_version: src_version = info.split('Version:')[1].strip( ' ') if src_name in sources: match = False for version in sources[src_name]: # Found a matching version; prune it if self.cmp_version(version, src_version) == 0: logging.info("Validating source %s_%s", src_name, version) sources[src_name].remove(version) src_cnt += 1 match = True break # NOTE! - Because the way the source table is # generated, it's not possible to have multiple time # the same version of a source (you have different # versions only). However, this is totally possible # for cdrom because of multiple components. Thus, # whenever the source table can handle per component # sources, this case should emit an error instead of # a warning if not match: logging.warning( "Can't find matching version for source %s_%s.\n" "It might have already been validated", src_name, src_version) else: logging.error("Extra source %s_%s found", src_name, src_version) self.ret = 1 # List missing sources for src_name in sources: for src_version in sources[src_name]: logging.error("Missing source %s_%s", src_name, src_version) logging.info("Succesfully validated %d source packages out of %d", src_cnt, src_total) if src_cnt != src_total: self.ret = 1
def setUp(self): self.src = TmpdirFilesystem() self.dst = TmpdirFilesystem()