def __init__(self, input=None): if type(input) == types.StringType: self._filename = input self.load(input) else: self.tagfile = TagFile(input)
def _getArchiveSourcePackageInfo(self, dist, component): source_path = self._archivePath + "/dists/%s/%s/source/Sources.gz" % ( dist, component) f = gzip.open(source_path, 'rb') tagf = TagFile(f) packageList = [] for section in tagf: # don't even try to build source-only packages if section.get('Extra-Source-Only', 'no') == 'yes': continue pkgname = section['Package'] archs_str = section['Architecture'] binaries = section['Binary'] pkgversion = section['Version'] pkg = PackageInfo(pkgname, pkgversion, dist, component) pkg.info = ( "Maintainer: <i>%s</i>\n<br>Co-Maintainers: <i>%s</i><br>\nVCS-Browser: %s" % (section['Maintainer'], section.get( 'Uploaders', 'Nobody'), section.get('Vcs-Browser', '#'))) packageList += [pkg] return packageList
def _get_packages_for(self, suite, component): if self.useMOMCache: source_path = self._archive_path + "/dists/%s-%s/%s/source/Sources" % ( self._distroName, suite, component) else: aroot = self._archive_path if suite.startswith("buildq"): aroot = self._bqueue_path source_path = aroot + "/%s/dists/%s/%s/source/Sources.gz" % ( self._distroName, suite, component) f = gzip.open(source_path, 'rb') tagf = TagFile(f) packageList = [] for section in tagf: archs = section['Architecture'] pkgversion = section['Version'] pkgname = section['Package'] directory = section['Directory'] dsc = find_dsc(section['Files']) pkg = PackageInfo(pkgname, pkgversion, suite, component, archs, directory, dsc) pkg.maintainer = section['Maintainer'] pkg.comaintainers = section.get('Uploaders', '') pkg.homepage = section.get('Homepage', None) if section.get('Extra-Source-Only', 'no') == 'yes': pkg.extra_source_only = True packageList.append(pkg) if self.extra_suite: packageList.extend( self._get_packages_for(self.extra_suite, component)) return packageList
def _build_installed_pkgs_cache(self, dist, component): for arch in self._supportedArchs: source_path = self._archivePath + "/dists/%s/%s/binary-%s/Packages.gz" % (dist, component, arch) f = gzip.open(source_path, 'rb') tagf = TagFile (f) for section in tagf: # make sure we have the right arch (closes bug in installed-detection) if section['Architecture'] != arch: continue pkgversion = section['Version'] pkgname = section['Package'] pkgsource = section.get('Source', '') # if source has different version, we cheat and set the binary pkg version # to the source package version if "(" in pkgsource: m = re.search(r"\((.*)\)", pkgsource) s = m.group(1).strip() if s != "": pkgversion = s pkid = "%s_%s" % (pkgname, arch) if pkid in self._installedPkgs: regVersion = self._installedPkgs[pkid] compare = version_compare(regVersion, pkgversion) if compare >= 0: continue self._installedPkgs[pkid] = pkgversion
def get_corresponding_source_packages(self, pkg_lst=None): if pkg_lst is None: pkg_lst = {p.name for p in self.cache if p.is_installed} src_set = set() with TagFile('/var/lib/dpkg/status') as tagfile: for section in tagfile: pkg = section['Package'] if pkg not in pkg_lst: continue tmp = self.cache[pkg].installed or self.cache[pkg].candidate src_set.add((tmp.source_name, tmp.source_version)) if "Built-Using" not in section: continue built_using_lst = section["Built-Using"].split(', ') for built_using in built_using_lst: name, version = built_using.split(' ', 1) version = version.strip('(= )') src_set.add((name, version)) return list(src_set)
def get_removed_sources(self): f = tempfile.TemporaryFile() f.write(self._removalsRFC822) f.seek(0) tagf = TagFile(f) resultsList = [] for section in tagf: suite = section.get('Suite', '').strip() sources_raw = section.get('Sources', '') # check if we have a source removal - the only thing of interest for us, at time if sources_raw == '' or suite == '': continue source_ids = [x.strip() for x in sources_raw.split('\n')] reason = section['Reason'] # NVITs at Debian are no reason to delete anything from Tanglu ;-) if "NVIT" in reason: continue for source_id in source_ids: if not "_" in source_id: continue version = self._get_version_from_pkid(source_id) source = self._get_pkgname_from_pkid(source_id) pkgrm = PackageRemovalItem(suite, source, version, reason) resultsList.append(pkgrm) return resultsList
def get_dsc_size(fname): tf = TagFile(fname) sz = os.path.getsize(fname) for sect in tf: if 'Files' in sect: files = sect['Files'].split('\n') files = [f.strip().split(' ') for f in files] for f in files: sz += int(f[1]) return sz
def get_cruft(self): n_cruft = 0 tagf = TagFile(open(self.status)) while tagf.step(): statusline = tagf.section.get("Status") (want, flag, status) = statusline.split() if want == "purge" and flag == "ok" and status == "not-installed": n_cruft += 1 logging.debug("DpkgStatusPlugin found %s cruft items" % n_cruft) if n_cruft: return [DpkgStatusCruft(n_cruft)] return [] # pragma: no cover
def load(self, input): """ Load control file """ plain_file = open(input) tagfile = TagFile(plain_file) # Loop to skip PGP signature while tagfile.step(): if tagfile.section.keys()[0][0] != '-': break self.tagfile = tagfile
def _parse_tagfile(filename, function): pdict = {} if filename[-3:] == '.gz': tagfile = gunzip(filename) elif filename[-4:] == '.bz2': tagfile = bunzip(filename) else: tagfile = file(filename) parser = TagFile(tagfile) while parser.Step(): k, v = function(parser.Section) pdict[k] = v return pdict
def get_packages_for(self, dist, component): # create a cache of all installed packages on the different architectures self._build_installed_pkgs_cache(dist, component) source_path = self._archivePath + "/dists/%s/%s/source/Sources.gz" % (dist, component) f = gzip.open(source_path, 'rb') tagf = TagFile (f) packageList = [] for section in tagf: # don't even try to build source-only packages if section.get('Extra-Source-Only', 'no') == 'yes': continue pkgname = section['Package'] if not pkgname in self._activePackages: continue archs_str = section['Architecture'] binaries = section['Binary'] pkgversion = section['Version'] if ' ' in archs_str: archs = archs_str.split(' ') else: archs = [archs_str] # remove duplicate archs from list # this is very important, because we otherwise will add duplicate build requests in Jenkins archs = list(set(archs)) pkg = PackageInfo(pkgname, pkgversion, dist, component, archs) pkg.info = ("Package: %s\nBinary Packages: %s\nMaintainer: %s\nCo-Maintainers: %s\nVCS-Browser: %s" % (pkgname, binaries, section['Maintainer'], section.get('Uploaders', 'Nobody'), section.get('Vcs-Browser', 'None set'))) # values needed for build-dependency solving pkg.build_depends = section.get('Build-Depends', '') pkg.build_conflicts = section.get('Build-Conflicts', '') pkg.archs_str = archs_str # we check if one of the arch-binaries exists. if it does, we consider the package built for this architecture # FIXME: This does not work well for binNMUed packages! Implement a possible solution later. # (at time, a version-check prevents packages from being built twice) if "," in binaries: binaryPkgs = binaries.split(', ') else: binaryPkgs = [binaries] for binaryName in binaryPkgs: self._set_pkg_installed_for_arch(section["Directory"], pkg, binaryName) #if (pkg.installedArchs != ["all"]) or (len(binaryPkgs) <= 0: packageList += [pkg] return packageList
def get_dsc_size(fname): if not virtapt_imported: return 0 tf = TagFile(fname) sz = os.path.getsize(fname) for sect in tf: if sect.has_key('Files'): files = sect['Files'].split('\n') files = [f.strip().split(' ') for f in files] for f in files: sz += int(f[1]) return sz
def get_cruft(self): n_cruft = 0 with open(self.status) as fp: tagf = TagFile(fp) while tagf.step(): statusline = tagf.section.get('Status') (want, flag, status) = statusline.split() if (want == 'purge' and flag == 'ok' and status == 'not-installed'): # Then... n_cruft += 1 logging.debug('DpkgStatusPlugin found {} cruft items'.format(n_cruft)) if n_cruft: return [DpkgStatusCruft(n_cruft)] return []
def _get_package_list(self, suite, component, is_build_queue=False): source_path = None if is_build_queue: source_path = self._bqueue_path + "/dists/%s/%s/source/Sources.gz" % ( suite, component) else: source_path = self._archive_path + "/dists/%s/%s/source/Sources.gz" % ( suite, component) f = gzip.open(source_path, 'rb') tagf = TagFile(f) packageList = [] for section in tagf: # don't even try to build source-only packages if section.get('Extra-Source-Only', 'no') == 'yes': continue pkgname = section['Package'] pkgversion = section['Version'] archs = list(set(section['Architecture'].split(None))) directory = section['Directory'] dsc = find_dsc(section['Files']) pkg = PackageInfo(pkgname, pkgversion, suite, component, archs, directory, dsc) if section.get('Extra-Source-Only', 'no') == 'yes': pkg.extra_source_only = True # values needed for build-dependency solving pkg.build_depends = section.get('Build-Depends', '') pkg.build_conflicts = section.get('Build-Conflicts', '') pkg.maintainer = section['Maintainer'] pkg.comaintainers = section.get('Uploaders', '') packageList += [pkg] bqueue = self._conf.get_build_queue(suite) if bqueue: packageList.extend( self._get_package_list(bqueue, component, is_build_queue=True)) return packageList
def _add_binaries_to_dict(self, pkg_dict, suite, component, arch, udeb=False): aroot = self._archive_path if suite.startswith("buildq"): aroot = self._bqueue_path if udeb: source_path = aroot + "/dists/%s/%s/debian-installer/binary-%s/Packages.gz" % ( suite, component, arch) else: source_path = aroot + "/dists/%s/%s/binary-%s/Packages.gz" % ( suite, component, arch) f = gzip.open(source_path, 'rb') tagf = TagFile(f) for section in tagf: # make sure we have the right arch (closes bug in installed-detection) if section['Architecture'] != arch: continue pkgversion = section['Version'] pkgname = section['Package'] pkgsource = section.get('Source', pkgname) # if source has different version, we cheat and set the binary pkg version # to the source package version if "(" in pkgsource: m = re.search(r"^(.*)\((.*)\)$", pkgsource) pkgsource = m.group(1).strip() pkgversion = m.group(2).strip() pkg = pkg_dict.get(pkgsource, None) if pkg is not None and pkg.version == pkgversion: if arch not in pkg.installed_archs: pkg.installed_archs += [arch] pkg.binaries += [(pkgname, arch, section['Filename'])] pkg_dict[pkgsource] = pkg return pkg_dict
def _get_packages_for(self, suite, component): index_path = os.path.join(self.get_cache_path(), suite, component, "Packages-amd64.gz") f = gzip.open(index_path, 'rb') tagf = TagFile(f) package_list = list() for section in tagf: pkgversion = section.get('Version') if not pkgversion: print("Debian: Bad package data found!") pkgname = section['Package'] pkg = PackageInfo(pkgname, pkgversion, self._get_upstream_version(pkgversion), suite, component) pkg.url = "https://packages.debian.org/%s/%s" % (suite, pkgname) package_list.append(pkg) packages_dict = package_list_to_dict(package_list) return packages_dict
def binary_packages(self, suite, component, arch): ''' Get a list of binary package information for the given repository suite, component and architecture. ''' assert type(suite) is ArchiveSuite assert type(component) is ArchiveComponent assert type(arch) is ArchiveArchitecture index_fname = self.index_file(suite.name, os.path.join(component.name, 'binary-{}'.format(arch.name), 'Packages.xz')) if not index_fname: return [] with TagFile(index_fname) as tf: return self._read_binary_packages_from_tf(tf, index_fname, suite, component, arch, DebType.DEB)
def installer_packages(self, suite, component, arch): ''' Get a list of binary installer packages for the given repository suite, component and architecture. These binary packages are typically udebs used by the debian-installer, and should not be installed on an user's system. ''' assert type(suite) is ArchiveSuite assert type(component) is ArchiveComponent assert type(arch) is ArchiveArchitecture index_fname = self.index_file(suite.name, os.path.join(component.name, 'debian-installer', 'binary-{}'.format(arch.name), 'Packages.xz')) if not index_fname: return [] with TagFile(index_fname) as tf: return self._read_binary_packages_from_tf(tf, index_fname, suite, component, arch, DebType.UDEB)
def batch_rebuild_packages(self, component, bad_depends, build_note, dry_run=True): source_path = self._archivePath + "/%s/dists/%s/%s/binary-i386/Packages.gz" % ( "tanglu", self._suite, component) f = gzip.open(source_path, 'rb') tagf = TagFile(f) rebuildSources = [] bad_depends = bad_depends.strip() for section in tagf: pkgname = section['Package'] source_pkg = section.get('Source', '') if source_pkg == '': source_pkg = pkgname if "(" in source_pkg: source_pkg = source_pkg.split("(")[0].strip() if source_pkg in rebuildSources: continue # we already handled a rebuild for that # old binary packages are not interesting for us if source_pkg in self._pkgs_tanglu: compare = version_compare( self._pkgs_tanglu[source_pkg].version, section['Version']) if compare > 0: continue depends = section.get('Depends', '') if depends == '': continue # we ignore pre-depends: Pre-depending stuff is much safer with a manual rebuild dep_chunks = depends.split(',') for dep in dep_chunks: dep = dep.strip() if dep.startswith(bad_depends): if dep == bad_depends: rebuildSources.append(source_pkg) continue if '(' not in dep: continue depid_parts = dep.split('(') if bad_depends == depid_parts[0].strip(): rebuildSources.append(source_pkg) continue print("Packages planned for rebuild:") if len(rebuildSources) == 0: print("No matching packages found.") return print("\n".join(rebuildSources)) if dry_run: return True # dry-run - nothing to do res = True for pkg in rebuildSources: if not self.trigger_package_rebuild(component, pkg, build_note, False): res = False print("\n".join(self._todo_cmds)) return res
def readReleaseFile(self, filename): """Read a Release file, return as a keyword/value dict.""" sections = list(TagFile(file(filename))) self.assertEqual(1, len(sections)) return dict(sections[0])
def _create_faux_packages(self, session, mi_wspace: str, suites_source: List[ArchiveSuite], suite_target: ArchiveSuite): ''' If we have a partial source and target suite, we need to let Britney know about the parent packages somehow. At the moment, we simply abuse the FauxPackages system for that. ''' # we don't support more than one source suite for this feature at the moment if len(suites_source) > 1: log.info( 'Not auto-generating faux packages: Multiple suites set as sources.' ) return suite_source = suites_source[0] if suite_source.parent and suite_target.parent: log.info( 'Creating faux-packages to aid resolving of partial suites.') else: log.info( 'No auto-generating faux packages: No source and target suite parents, generation is unnecessary.' ) return existing_pkg_arch_set = set() log.debug( 'Creating index of valid packages that do not need a faux package.' ) # we need repository information to only generate faux packages if a package doesn't exist # in our source suite(s) already repo = self._get_local_repo(session) for suite in suites_source: esuite = session.query(ArchiveSuite) \ .options(joinedload(ArchiveSuite.components)) \ .options(joinedload(ArchiveSuite.architectures)) \ .filter(ArchiveSuite.id == suite.id).one() session.expunge( esuite ) # we don't want packages accidentally added to the database here for component in esuite.components: for arch in esuite.architectures: aname = arch.name for bpkg in repo.binary_packages(esuite, component, arch): existing_pkg_arch_set.add(aname + ':' + bpkg.name) for spkg in repo.source_packages(esuite, component): existing_pkg_arch_set.add(aname + ':' + spkg.name) archive_root_dir = self._lconf.archive_root_dir fauxpkg_fname = os.path.join(mi_wspace, 'input', 'faux-packages') log.debug('Generating faux packages list') fauxpkg_data = {} for component in suite_target.parent.components: for installer_dir in ['', 'debian-installer']: for arch in suite_target.parent.architectures: pfile = os.path.join(archive_root_dir, 'dists', suite_target.parent.name, component.name, installer_dir, 'binary-{}'.format(arch.name), 'Packages.xz') if not os.path.isfile(pfile): continue log.debug('Reading data for faux packages list: {}'.format( pfile)) with TagFile(pfile) as tf: for e in tf: pkgname = e['Package'] pkgversion = e['Version'] pkgarch = e['Architecture'] pkid = '{}-{}-{}'.format(pkgname, pkgversion, pkgarch) if pkid in fauxpkg_data: continue pkgname_arch = pkgarch + ':' + pkgname if pkgname_arch in existing_pkg_arch_set: continue provides = e.get('Provides', '') data = 'Package: {}\nVersion: {}'.format( pkgname, pkgversion) if pkgarch and pkgarch != 'all': data = data + '\nArchitecture: {}'.format( pkgarch) if provides: data = data + '\nProvides: {}'.format(provides) if component.name != 'main': data = data + '\nComponent: {}'.format( component.name) fauxpkg_data[pkid] = data # FIXME: We shouldn't have to special-case this :any case, # rather Britney should do the right thing and recognize this # notation for faux-packages. But until that is fixed # properly and since a dependency on python3:any is so common, we # will work around this issue if pkgname == 'python3': pkid = '{}-{}-{}'.format( 'python3:any', pkgversion, pkgarch) if pkid in fauxpkg_data: continue fauxpkg_data[pkid] = data.replace( 'Package: python3\n', 'Package: python3:any\n') with open(fauxpkg_fname, 'w') as f: for segment in fauxpkg_data.values(): f.write(segment + '\n\n')
def source_packages(self, suite, component): ''' Return a list of all source packages in the given suite and component. ''' assert type(suite) is ArchiveSuite assert type(component) is ArchiveComponent index_fname = self.index_file(suite.name, os.path.join(component.name, 'source', 'Sources.xz')) if not index_fname: return [] pkgs = [] with TagFile(index_fname) as tf: for e in tf: pkgname = e['Package'] pkgversion = e['Version'] if not pkgname or not pkgversion: raise Exception('Found invalid block (no Package and Version fields) in Sources file "{}".'.format(index_fname)) break pkg = SourcePackage() pkg.repo = self._repo_entity pkg.name = pkgname pkg.component = component if suite not in pkg.suites: pkg.suites.append(suite) pkg.version = pkgversion pkg.architectures = split_strip(e['Architecture'], ' ') pkg.standards_version = e.get('Standards-Version', '0~notset') pkg.format_version = e['Format'] pkg.vcs_browser = e.get('Vcs-Browser') pkg.homepage = e.get('Homepage') pkg.maintainer = e['Maintainer'] pkg.uploaders = split_strip(e.get('Uploaders', ''), ',') # FIXME: Careful! Splitting just by comma isn't enough! We need to parse this properly. pkg.build_depends = split_strip(e.get('Build-Depends', ''), ',') pkg.directory = e['Directory'] pkg.files = parse_checksums_list(e.get('Checksums-Sha256'), pkg.directory) binaries = [] raw_pkg_list = e.get('Package-List', None) if not raw_pkg_list: for bpname in e.get('Binary', '').split(','): if not bpname: continue bpname = bpname.strip() pi = PackageInfo() pi.deb_type = DebType.DEB pi.name = bpname pi.ver = pkg.version binaries.append(pi) else: binaries = parse_package_list_str(raw_pkg_list, pkg.version) pkg.binaries = binaries # do some issue-reporting if not pkg.files and pkg.format_version != '1.0': log.warning('Source package {}/{} seems to have no files (in {}).'.format(pkg.name, pkg.version, self.location)) # add package to results set pkg.update_uuid() pkgs.append(pkg) return pkgs