def validate_sources(suite, component):
"""
Ensure files mentioned in Sources exist
"""
filename = "%s/dists/%s/%s/source/Sources" % (Cnf["Dir::Root"], suite,
component)
filename = utils.find_possibly_compressed_file(filename)
print("Processing %s..." % (filename))
with apt_pkg.TagFile(filename) as Sources:
while Sources.step():
source = Sources.section.find('Package')
directory = Sources.section.find('Directory')
files = Sources.section.find('Files')
for i in files.split('\n'):
(md5, size, name) = i.split()
filename = "%s/%s/%s" % (Cnf["Dir::Root"], directory, name)
if not os.path.exists(filename):
if directory.find("potato") == -1:
print("W: %s missing." % (filename))
else:
pool_location = utils.poolify(source)
pool_filename = "%s/%s/%s" % (Cnf["Dir::Pool"],
pool_location, name)
if not os.path.exists(pool_filename):
print("E: %s missing (%s)." %
(filename, pool_filename))
else:
# Create symlink
pool_filename = os.path.normpath(pool_filename)
filename = os.path.normpath(filename)
src = utils.clean_symlink(pool_filename, filename,
Cnf["Dir::Root"])
print("Symlinking: %s -> %s" % (filename, src))
def validate_sources(suite, component):
"""
Ensure files mentioned in Sources exist
"""
filename = "%s/dists/%s/%s/source/Sources" % (Cnf["Dir::Root"], suite, component)
filename = utils.find_possibly_compressed_file(filename)
print("Processing %s..." % (filename))
with apt_pkg.TagFile(filename) as Sources:
while Sources.step():
source = Sources.section.find('Package')
directory = Sources.section.find('Directory')
files = Sources.section.find('Files')
for i in files.split('\n'):
(md5, size, name) = i.split()
filename = "%s/%s/%s" % (Cnf["Dir::Root"], directory, name)
if not os.path.exists(filename):
if directory.find("potato") == -1:
print("W: %s missing." % (filename))
else:
pool_location = utils.poolify(source)
pool_filename = "%s/%s/%s" % (Cnf["Dir::Pool"], pool_location, name)
if not os.path.exists(pool_filename):
print("E: %s missing (%s)." % (filename, pool_filename))
else:
# Create symlink
pool_filename = os.path.normpath(pool_filename)
filename = os.path.normpath(filename)
src = utils.clean_symlink(pool_filename, filename, Cnf["Dir::Root"])
print("Symlinking: %s -> %s" % (filename, src))
def validate_packages(suite, component, architecture):
"""
Ensure files mentioned in Packages exist
"""
filename = "%s/dists/%s/%s/binary-%s/Packages" \
% (Cnf["Dir::Root"], suite, component, architecture)
filename = utils.find_possibly_compressed_file(filename)
print("Processing %s..." % (filename))
with apt_pkg.TagFile(filename) as Packages:
while Packages.step():
filename = "%s/%s" % (Cnf["Dir::Root"], Packages.section.find('Filename'))
if not os.path.exists(filename):
print("W: %s missing." % (filename))
def validate_packages(suite, component, architecture):
"""
Ensure files mentioned in Packages exist
"""
filename = "%s/dists/%s/%s/binary-%s/Packages" \
% (Cnf["Dir::Root"], suite, component, architecture)
filename = utils.find_possibly_compressed_file(filename)
print("Processing %s..." % (filename))
with apt_pkg.TagFile(filename) as Packages:
while Packages.step():
filename = "%s/%s" % (Cnf["Dir::Root"],
Packages.section.find('Filename'))
if not os.path.exists(filename):
print("W: %s missing." % (filename))
def main():
global suite, suite_id, source_binaries, source_versions
cnf = Config()
Arguments = [('h', "help", "Cruft-Report::Options::Help"),
('m', "mode", "Cruft-Report::Options::Mode", "HasArg"),
('R', "rdep-check", "Cruft-Report::Options::Rdep-Check"),
('s', "suite", "Cruft-Report::Options::Suite", "HasArg"),
('w', "wanna-build-dump",
"Cruft-Report::Options::Wanna-Build-Dump", "HasArg")]
for i in ["help", "Rdep-Check"]:
key = "Cruft-Report::Options::%s" % i
if key not in cnf:
cnf[key] = ""
cnf["Cruft-Report::Options::Suite"] = cnf.get("Dinstall::DefaultSuite",
"unstable")
if "Cruft-Report::Options::Mode" not in cnf:
cnf["Cruft-Report::Options::Mode"] = "daily"
if "Cruft-Report::Options::Wanna-Build-Dump" not in cnf:
cnf["Cruft-Report::Options::Wanna-Build-Dump"] = "/srv/ftp-master.debian.org/scripts/nfu"
apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Cruft-Report::Options")
if Options["Help"]:
usage()
if Options["Rdep-Check"]:
rdeps = True
else:
rdeps = False
# Set up checks based on mode
if Options["Mode"] == "daily":
checks = [
"nbs", "nviu", "nvit", "obsolete source", "outdated non-free",
"nfu", "nbs metadata"
]
elif Options["Mode"] == "full":
checks = [
"nbs", "nviu", "nvit", "obsolete source", "outdated non-free",
"nfu", "nbs metadata", "dubious nbs", "bnb", "bms", "anais"
]
elif Options["Mode"] == "bdo":
checks = ["nbs", "obsolete source"]
else:
utils.warn(
"%s is not a recognised mode - only 'full', 'daily' or 'bdo' are understood."
% (Options["Mode"]))
usage(1)
session = DBConn().session()
bin_pkgs = {}
src_pkgs = {}
bin2source = {}
bins_in_suite = {}
nbs = {}
source_versions = {}
anais_output = ""
nfu_packages = {}
suite = get_suite(Options["Suite"].lower(), session)
if not suite:
utils.fubar("Cannot find suite %s" % Options["Suite"].lower())
suite_id = suite.suite_id
suite_name = suite.suite_name.lower()
if "obsolete source" in checks:
report_obsolete_source(suite_name, session)
if "nbs" in checks:
reportAllNBS(suite_name, suite_id, session, rdeps)
if "nbs metadata" in checks:
reportNBSMetadata(suite_name, suite_id, session, rdeps)
if "outdated non-free" in checks:
report_outdated_nonfree(suite_name, session, rdeps)
bin_not_built = {}
if "bnb" in checks:
bins_in_suite = get_suite_binaries(suite, session)
# Checks based on the Sources files
components = get_component_names(session)
for component in components:
filename = "%s/dists/%s/%s/source/Sources" % (suite.archive.path,
suite_name, component)
filename = utils.find_possibly_compressed_file(filename)
with apt_pkg.TagFile(filename) as Sources:
while Sources.step():
source = Sources.section.find('Package')
source_version = Sources.section.find('Version')
architecture = Sources.section.find('Architecture')
binaries = Sources.section.find('Binary')
binaries_list = [i.strip() for i in binaries.split(',')]
if "bnb" in checks:
# Check for binaries not built on any architecture.
for binary in binaries_list:
if binary not in bins_in_suite:
bin_not_built.setdefault(source, {})
bin_not_built[source][binary] = ""
if "anais" in checks:
anais_output += do_anais(architecture, binaries_list,
source, session)
# build indices for checking "no source" later
source_index = component + '/' + source
src_pkgs[source] = source_index
for binary in binaries_list:
bin_pkgs[binary] = source
source_binaries[source] = binaries
source_versions[source] = source_version
# Checks based on the Packages files
check_components = components[:]
if suite_name != "experimental":
check_components.append('main/debian-installer')
for component in check_components:
architectures = [
a.arch_string for a in get_suite_architectures(
suite_name, skipsrc=True, skipall=True, session=session)
]
for architecture in architectures:
if component == 'main/debian-installer' and re.match(
"kfreebsd", architecture):
continue
if "nfu" in checks:
nfu_packages.setdefault(architecture, [])
nfu_entries = parse_nfu(architecture)
filename = "%s/dists/%s/%s/binary-%s/Packages" % (
suite.archive.path, suite_name, component, architecture)
filename = utils.find_possibly_compressed_file(filename)
with apt_pkg.TagFile(filename) as Packages:
while Packages.step():
package = Packages.section.find('Package')
source = Packages.section.find('Source', "")
version = Packages.section.find('Version')
if source == "":
source = package
if package in bin2source and \
apt_pkg.version_compare(version, bin2source[package]["version"]) > 0:
bin2source[package]["version"] = version
bin2source[package]["source"] = source
else:
bin2source[package] = {}
bin2source[package]["version"] = version
bin2source[package]["source"] = source
if source.find("(") != -1:
m = re_extract_src_version.match(source)
source = m.group(1)
version = m.group(2)
if package not in bin_pkgs:
nbs.setdefault(source, {})
nbs[source].setdefault(package, {})
nbs[source][package][version] = ""
else:
if "nfu" in checks:
if package in nfu_entries and \
version != source_versions[source]: # only suggest to remove out-of-date packages
nfu_packages[architecture].append(
(package, version,
source_versions[source]))
# Distinguish dubious (version numbers match) and 'real' NBS (they don't)
dubious_nbs = {}
version_sort_key = functools.cmp_to_key(apt_pkg.version_compare)
for source in nbs:
for package in nbs[source]:
latest_version = max(nbs[source][package], key=version_sort_key)
source_version = source_versions.get(source, "0")
if apt_pkg.version_compare(latest_version, source_version) == 0:
add_nbs(dubious_nbs, source, latest_version, package, suite_id,
session)
if "nviu" in checks:
do_newer_version('unstable', 'experimental', 'NVIU', session)
if "nvit" in checks:
do_newer_version('testing', 'testing-proposed-updates', 'NVIT',
session)
###
if Options["Mode"] == "full":
print("=" * 75)
print()
if "nfu" in checks:
do_nfu(nfu_packages)
if "bnb" in checks:
print("Unbuilt binary packages")
print("-----------------------")
print()
for source in sorted(bin_not_built):
binaries = sorted(bin_not_built[source])
print(" o %s: %s" % (source, ", ".join(binaries)))
print()
if "bms" in checks:
report_multiple_source(suite)
if "anais" in checks:
print("Architecture Not Allowed In Source")
print("----------------------------------")
print(anais_output)
print()
if "dubious nbs" in checks:
do_dubious_nbs(dubious_nbs)
def main():
global suite, suite_id, source_binaries, source_versions
cnf = Config()
Arguments = [('h', "help", "Cruft-Report::Options::Help"),
('m', "mode", "Cruft-Report::Options::Mode", "HasArg"),
('R', "rdep-check", "Cruft-Report::Options::Rdep-Check"),
('s', "suite", "Cruft-Report::Options::Suite", "HasArg"),
('w', "wanna-build-dump", "Cruft-Report::Options::Wanna-Build-Dump", "HasArg")]
for i in ["help", "Rdep-Check"]:
key = "Cruft-Report::Options::%s" % i
if key not in cnf:
cnf[key] = ""
cnf["Cruft-Report::Options::Suite"] = cnf.get("Dinstall::DefaultSuite", "unstable")
if "Cruft-Report::Options::Mode" not in cnf:
cnf["Cruft-Report::Options::Mode"] = "daily"
if "Cruft-Report::Options::Wanna-Build-Dump" not in cnf:
cnf["Cruft-Report::Options::Wanna-Build-Dump"] = "/srv/ftp-master.debian.org/scripts/nfu"
apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Cruft-Report::Options")
if Options["Help"]:
usage()
if Options["Rdep-Check"]:
rdeps = True
else:
rdeps = False
# Set up checks based on mode
if Options["Mode"] == "daily":
checks = ["nbs", "nviu", "nvit", "obsolete source", "outdated non-free", "nfu"]
elif Options["Mode"] == "full":
checks = ["nbs", "nviu", "nvit", "obsolete source", "outdated non-free", "nfu", "dubious nbs", "bnb", "bms", "anais"]
elif Options["Mode"] == "bdo":
checks = ["nbs", "obsolete source"]
else:
utils.warn("%s is not a recognised mode - only 'full', 'daily' or 'bdo' are understood." % (Options["Mode"]))
usage(1)
session = DBConn().session()
bin_pkgs = {}
src_pkgs = {}
bin2source = {}
bins_in_suite = {}
nbs = {}
source_versions = {}
anais_output = ""
nfu_packages = {}
suite = get_suite(Options["Suite"].lower(), session)
if not suite:
utils.fubar("Cannot find suite %s" % Options["Suite"].lower())
suite_id = suite.suite_id
suite_name = suite.suite_name.lower()
if "obsolete source" in checks:
report_obsolete_source(suite_name, session)
if "nbs" in checks:
reportAllNBS(suite_name, suite_id, session, rdeps)
if "outdated non-free" in checks:
report_outdated_nonfree(suite_name, session, rdeps)
bin_not_built = {}
if "bnb" in checks:
bins_in_suite = get_suite_binaries(suite, session)
# Checks based on the Sources files
components = get_component_names(session)
for component in components:
filename = "%s/dists/%s/%s/source/Sources" % (suite.archive.path, suite_name, component)
filename = utils.find_possibly_compressed_file(filename)
with apt_pkg.TagFile(filename) as Sources:
while Sources.step():
source = Sources.section.find('Package')
source_version = Sources.section.find('Version')
architecture = Sources.section.find('Architecture')
binaries = Sources.section.find('Binary')
binaries_list = [i.strip() for i in binaries.split(',')]
if "bnb" in checks:
# Check for binaries not built on any architecture.
for binary in binaries_list:
if binary not in bins_in_suite:
bin_not_built.setdefault(source, {})
bin_not_built[source][binary] = ""
if "anais" in checks:
anais_output += do_anais(architecture, binaries_list, source, session)
# build indices for checking "no source" later
source_index = component + '/' + source
src_pkgs[source] = source_index
for binary in binaries_list:
bin_pkgs[binary] = source
source_binaries[source] = binaries
source_versions[source] = source_version
# Checks based on the Packages files
check_components = components[:]
if suite_name != "experimental":
check_components.append('main/debian-installer')
for component in check_components:
architectures = [a.arch_string for a in get_suite_architectures(suite_name,
skipsrc=True, skipall=True,
session=session)]
for architecture in architectures:
if component == 'main/debian-installer' and re.match("kfreebsd", architecture):
continue
if "nfu" in checks:
nfu_packages.setdefault(architecture, [])
nfu_entries = parse_nfu(architecture)
filename = "%s/dists/%s/%s/binary-%s/Packages" % (suite.archive.path, suite_name, component, architecture)
filename = utils.find_possibly_compressed_file(filename)
with apt_pkg.TagFile(filename) as Packages:
while Packages.step():
package = Packages.section.find('Package')
source = Packages.section.find('Source', "")
version = Packages.section.find('Version')
if source == "":
source = package
if package in bin2source and \
apt_pkg.version_compare(version, bin2source[package]["version"]) > 0:
bin2source[package]["version"] = version
bin2source[package]["source"] = source
else:
bin2source[package] = {}
bin2source[package]["version"] = version
bin2source[package]["source"] = source
if source.find("(") != -1:
m = re_extract_src_version.match(source)
source = m.group(1)
version = m.group(2)
if package not in bin_pkgs:
nbs.setdefault(source, {})
nbs[source].setdefault(package, {})
nbs[source][package][version] = ""
else:
if "nfu" in checks:
if package in nfu_entries and \
version != source_versions[source]: # only suggest to remove out-of-date packages
nfu_packages[architecture].append((package, version, source_versions[source]))
# Distinguish dubious (version numbers match) and 'real' NBS (they don't)
dubious_nbs = {}
version_sort_key = functools.cmp_to_key(apt_pkg.version_compare)
for source in nbs:
for package in nbs[source]:
latest_version = max(nbs[source][package], key=version_sort_key)
source_version = source_versions.get(source, "0")
if apt_pkg.version_compare(latest_version, source_version) == 0:
add_nbs(dubious_nbs, source, latest_version, package, suite_id, session)
if "nviu" in checks:
do_newer_version('unstable', 'experimental', 'NVIU', session)
if "nvit" in checks:
do_newer_version('testing', 'testing-proposed-updates', 'NVIT', session)
###
if Options["Mode"] == "full":
print("=" * 75)
print()
if "nfu" in checks:
do_nfu(nfu_packages)
if "bnb" in checks:
print("Unbuilt binary packages")
print("-----------------------")
print()
for source in sorted(bin_not_built):
binaries = sorted(bin_not_built[source])
print(" o %s: %s" % (source, ", ".join(binaries)))
print()
if "bms" in checks:
report_multiple_source(suite)
if "anais" in checks:
print("Architecture Not Allowed In Source")
print("----------------------------------")
print(anais_output)
print()
if "dubious nbs" in checks:
do_dubious_nbs(dubious_nbs)