def validate_sources(suite, component): """ Ensure files mentioned in Sources exist """ filename = "%s/dists/%s/%s/source/Sources" % (Cnf["Dir::Root"], suite, component) filename = utils.find_possibly_compressed_file(filename) print("Processing %s..." % (filename)) with apt_pkg.TagFile(filename) as Sources: while Sources.step(): source = Sources.section.find('Package') directory = Sources.section.find('Directory') files = Sources.section.find('Files') for i in files.split('\n'): (md5, size, name) = i.split() filename = "%s/%s/%s" % (Cnf["Dir::Root"], directory, name) if not os.path.exists(filename): if directory.find("potato") == -1: print("W: %s missing." % (filename)) else: pool_location = utils.poolify(source) pool_filename = "%s/%s/%s" % (Cnf["Dir::Pool"], pool_location, name) if not os.path.exists(pool_filename): print("E: %s missing (%s)." % (filename, pool_filename)) else: # Create symlink pool_filename = os.path.normpath(pool_filename) filename = os.path.normpath(filename) src = utils.clean_symlink(pool_filename, filename, Cnf["Dir::Root"]) print("Symlinking: %s -> %s" % (filename, src))
def validate_packages(suite, component, architecture): """ Ensure files mentioned in Packages exist """ filename = "%s/dists/%s/%s/binary-%s/Packages" \ % (Cnf["Dir::Root"], suite, component, architecture) filename = utils.find_possibly_compressed_file(filename) print("Processing %s..." % (filename)) with apt_pkg.TagFile(filename) as Packages: while Packages.step(): filename = "%s/%s" % (Cnf["Dir::Root"], Packages.section.find('Filename')) if not os.path.exists(filename): print("W: %s missing." % (filename))
def main(): global suite, suite_id, source_binaries, source_versions cnf = Config() Arguments = [('h', "help", "Cruft-Report::Options::Help"), ('m', "mode", "Cruft-Report::Options::Mode", "HasArg"), ('R', "rdep-check", "Cruft-Report::Options::Rdep-Check"), ('s', "suite", "Cruft-Report::Options::Suite", "HasArg"), ('w', "wanna-build-dump", "Cruft-Report::Options::Wanna-Build-Dump", "HasArg")] for i in ["help", "Rdep-Check"]: key = "Cruft-Report::Options::%s" % i if key not in cnf: cnf[key] = "" cnf["Cruft-Report::Options::Suite"] = cnf.get("Dinstall::DefaultSuite", "unstable") if "Cruft-Report::Options::Mode" not in cnf: cnf["Cruft-Report::Options::Mode"] = "daily" if "Cruft-Report::Options::Wanna-Build-Dump" not in cnf: cnf["Cruft-Report::Options::Wanna-Build-Dump"] = "/srv/ftp-master.debian.org/scripts/nfu" apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) Options = cnf.subtree("Cruft-Report::Options") if Options["Help"]: usage() if Options["Rdep-Check"]: rdeps = True else: rdeps = False # Set up checks based on mode if Options["Mode"] == "daily": checks = [ "nbs", "nviu", "nvit", "obsolete source", "outdated non-free", "nfu", "nbs metadata" ] elif Options["Mode"] == "full": checks = [ "nbs", "nviu", "nvit", "obsolete source", "outdated non-free", "nfu", "nbs metadata", "dubious nbs", "bnb", "bms", "anais" ] elif Options["Mode"] == "bdo": checks = ["nbs", "obsolete source"] else: utils.warn( "%s is not a recognised mode - only 'full', 'daily' or 'bdo' are understood." % (Options["Mode"])) usage(1) session = DBConn().session() bin_pkgs = {} src_pkgs = {} bin2source = {} bins_in_suite = {} nbs = {} source_versions = {} anais_output = "" nfu_packages = {} suite = get_suite(Options["Suite"].lower(), session) if not suite: utils.fubar("Cannot find suite %s" % Options["Suite"].lower()) suite_id = suite.suite_id suite_name = suite.suite_name.lower() if "obsolete source" in checks: report_obsolete_source(suite_name, session) if "nbs" in checks: reportAllNBS(suite_name, suite_id, session, rdeps) if "nbs metadata" in checks: reportNBSMetadata(suite_name, suite_id, session, rdeps) if "outdated non-free" in checks: report_outdated_nonfree(suite_name, session, rdeps) bin_not_built = {} if "bnb" in checks: bins_in_suite = get_suite_binaries(suite, session) # Checks based on the Sources files components = get_component_names(session) for component in components: filename = "%s/dists/%s/%s/source/Sources" % (suite.archive.path, suite_name, component) filename = utils.find_possibly_compressed_file(filename) with apt_pkg.TagFile(filename) as Sources: while Sources.step(): source = Sources.section.find('Package') source_version = Sources.section.find('Version') architecture = Sources.section.find('Architecture') binaries = Sources.section.find('Binary') binaries_list = [i.strip() for i in binaries.split(',')] if "bnb" in checks: # Check for binaries not built on any architecture. for binary in binaries_list: if binary not in bins_in_suite: bin_not_built.setdefault(source, {}) bin_not_built[source][binary] = "" if "anais" in checks: anais_output += do_anais(architecture, binaries_list, source, session) # build indices for checking "no source" later source_index = component + '/' + source src_pkgs[source] = source_index for binary in binaries_list: bin_pkgs[binary] = source source_binaries[source] = binaries source_versions[source] = source_version # Checks based on the Packages files check_components = components[:] if suite_name != "experimental": check_components.append('main/debian-installer') for component in check_components: architectures = [ a.arch_string for a in get_suite_architectures( suite_name, skipsrc=True, skipall=True, session=session) ] for architecture in architectures: if component == 'main/debian-installer' and re.match( "kfreebsd", architecture): continue if "nfu" in checks: nfu_packages.setdefault(architecture, []) nfu_entries = parse_nfu(architecture) filename = "%s/dists/%s/%s/binary-%s/Packages" % ( suite.archive.path, suite_name, component, architecture) filename = utils.find_possibly_compressed_file(filename) with apt_pkg.TagFile(filename) as Packages: while Packages.step(): package = Packages.section.find('Package') source = Packages.section.find('Source', "") version = Packages.section.find('Version') if source == "": source = package if package in bin2source and \ apt_pkg.version_compare(version, bin2source[package]["version"]) > 0: bin2source[package]["version"] = version bin2source[package]["source"] = source else: bin2source[package] = {} bin2source[package]["version"] = version bin2source[package]["source"] = source if source.find("(") != -1: m = re_extract_src_version.match(source) source = m.group(1) version = m.group(2) if package not in bin_pkgs: nbs.setdefault(source, {}) nbs[source].setdefault(package, {}) nbs[source][package][version] = "" else: if "nfu" in checks: if package in nfu_entries and \ version != source_versions[source]: # only suggest to remove out-of-date packages nfu_packages[architecture].append( (package, version, source_versions[source])) # Distinguish dubious (version numbers match) and 'real' NBS (they don't) dubious_nbs = {} version_sort_key = functools.cmp_to_key(apt_pkg.version_compare) for source in nbs: for package in nbs[source]: latest_version = max(nbs[source][package], key=version_sort_key) source_version = source_versions.get(source, "0") if apt_pkg.version_compare(latest_version, source_version) == 0: add_nbs(dubious_nbs, source, latest_version, package, suite_id, session) if "nviu" in checks: do_newer_version('unstable', 'experimental', 'NVIU', session) if "nvit" in checks: do_newer_version('testing', 'testing-proposed-updates', 'NVIT', session) ### if Options["Mode"] == "full": print("=" * 75) print() if "nfu" in checks: do_nfu(nfu_packages) if "bnb" in checks: print("Unbuilt binary packages") print("-----------------------") print() for source in sorted(bin_not_built): binaries = sorted(bin_not_built[source]) print(" o %s: %s" % (source, ", ".join(binaries))) print() if "bms" in checks: report_multiple_source(suite) if "anais" in checks: print("Architecture Not Allowed In Source") print("----------------------------------") print(anais_output) print() if "dubious nbs" in checks: do_dubious_nbs(dubious_nbs)
def main(): global suite, suite_id, source_binaries, source_versions cnf = Config() Arguments = [('h', "help", "Cruft-Report::Options::Help"), ('m', "mode", "Cruft-Report::Options::Mode", "HasArg"), ('R', "rdep-check", "Cruft-Report::Options::Rdep-Check"), ('s', "suite", "Cruft-Report::Options::Suite", "HasArg"), ('w', "wanna-build-dump", "Cruft-Report::Options::Wanna-Build-Dump", "HasArg")] for i in ["help", "Rdep-Check"]: key = "Cruft-Report::Options::%s" % i if key not in cnf: cnf[key] = "" cnf["Cruft-Report::Options::Suite"] = cnf.get("Dinstall::DefaultSuite", "unstable") if "Cruft-Report::Options::Mode" not in cnf: cnf["Cruft-Report::Options::Mode"] = "daily" if "Cruft-Report::Options::Wanna-Build-Dump" not in cnf: cnf["Cruft-Report::Options::Wanna-Build-Dump"] = "/srv/ftp-master.debian.org/scripts/nfu" apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) Options = cnf.subtree("Cruft-Report::Options") if Options["Help"]: usage() if Options["Rdep-Check"]: rdeps = True else: rdeps = False # Set up checks based on mode if Options["Mode"] == "daily": checks = ["nbs", "nviu", "nvit", "obsolete source", "outdated non-free", "nfu"] elif Options["Mode"] == "full": checks = ["nbs", "nviu", "nvit", "obsolete source", "outdated non-free", "nfu", "dubious nbs", "bnb", "bms", "anais"] elif Options["Mode"] == "bdo": checks = ["nbs", "obsolete source"] else: utils.warn("%s is not a recognised mode - only 'full', 'daily' or 'bdo' are understood." % (Options["Mode"])) usage(1) session = DBConn().session() bin_pkgs = {} src_pkgs = {} bin2source = {} bins_in_suite = {} nbs = {} source_versions = {} anais_output = "" nfu_packages = {} suite = get_suite(Options["Suite"].lower(), session) if not suite: utils.fubar("Cannot find suite %s" % Options["Suite"].lower()) suite_id = suite.suite_id suite_name = suite.suite_name.lower() if "obsolete source" in checks: report_obsolete_source(suite_name, session) if "nbs" in checks: reportAllNBS(suite_name, suite_id, session, rdeps) if "outdated non-free" in checks: report_outdated_nonfree(suite_name, session, rdeps) bin_not_built = {} if "bnb" in checks: bins_in_suite = get_suite_binaries(suite, session) # Checks based on the Sources files components = get_component_names(session) for component in components: filename = "%s/dists/%s/%s/source/Sources" % (suite.archive.path, suite_name, component) filename = utils.find_possibly_compressed_file(filename) with apt_pkg.TagFile(filename) as Sources: while Sources.step(): source = Sources.section.find('Package') source_version = Sources.section.find('Version') architecture = Sources.section.find('Architecture') binaries = Sources.section.find('Binary') binaries_list = [i.strip() for i in binaries.split(',')] if "bnb" in checks: # Check for binaries not built on any architecture. for binary in binaries_list: if binary not in bins_in_suite: bin_not_built.setdefault(source, {}) bin_not_built[source][binary] = "" if "anais" in checks: anais_output += do_anais(architecture, binaries_list, source, session) # build indices for checking "no source" later source_index = component + '/' + source src_pkgs[source] = source_index for binary in binaries_list: bin_pkgs[binary] = source source_binaries[source] = binaries source_versions[source] = source_version # Checks based on the Packages files check_components = components[:] if suite_name != "experimental": check_components.append('main/debian-installer') for component in check_components: architectures = [a.arch_string for a in get_suite_architectures(suite_name, skipsrc=True, skipall=True, session=session)] for architecture in architectures: if component == 'main/debian-installer' and re.match("kfreebsd", architecture): continue if "nfu" in checks: nfu_packages.setdefault(architecture, []) nfu_entries = parse_nfu(architecture) filename = "%s/dists/%s/%s/binary-%s/Packages" % (suite.archive.path, suite_name, component, architecture) filename = utils.find_possibly_compressed_file(filename) with apt_pkg.TagFile(filename) as Packages: while Packages.step(): package = Packages.section.find('Package') source = Packages.section.find('Source', "") version = Packages.section.find('Version') if source == "": source = package if package in bin2source and \ apt_pkg.version_compare(version, bin2source[package]["version"]) > 0: bin2source[package]["version"] = version bin2source[package]["source"] = source else: bin2source[package] = {} bin2source[package]["version"] = version bin2source[package]["source"] = source if source.find("(") != -1: m = re_extract_src_version.match(source) source = m.group(1) version = m.group(2) if package not in bin_pkgs: nbs.setdefault(source, {}) nbs[source].setdefault(package, {}) nbs[source][package][version] = "" else: if "nfu" in checks: if package in nfu_entries and \ version != source_versions[source]: # only suggest to remove out-of-date packages nfu_packages[architecture].append((package, version, source_versions[source])) # Distinguish dubious (version numbers match) and 'real' NBS (they don't) dubious_nbs = {} version_sort_key = functools.cmp_to_key(apt_pkg.version_compare) for source in nbs: for package in nbs[source]: latest_version = max(nbs[source][package], key=version_sort_key) source_version = source_versions.get(source, "0") if apt_pkg.version_compare(latest_version, source_version) == 0: add_nbs(dubious_nbs, source, latest_version, package, suite_id, session) if "nviu" in checks: do_newer_version('unstable', 'experimental', 'NVIU', session) if "nvit" in checks: do_newer_version('testing', 'testing-proposed-updates', 'NVIT', session) ### if Options["Mode"] == "full": print("=" * 75) print() if "nfu" in checks: do_nfu(nfu_packages) if "bnb" in checks: print("Unbuilt binary packages") print("-----------------------") print() for source in sorted(bin_not_built): binaries = sorted(bin_not_built[source]) print(" o %s: %s" % (source, ", ".join(binaries))) print() if "bms" in checks: report_multiple_source(suite) if "anais" in checks: print("Architecture Not Allowed In Source") print("----------------------------------") print(anais_output) print() if "dubious nbs" in checks: do_dubious_nbs(dubious_nbs)