def main():
global suite, suite_id, source_binaries, source_versions
cnf = Config()
Arguments = [('h', "help", "Cruft-Report::Options::Help"),
('m', "mode", "Cruft-Report::Options::Mode", "HasArg"),
('R', "rdep-check", "Cruft-Report::Options::Rdep-Check"),
('s', "suite", "Cruft-Report::Options::Suite", "HasArg"),
('w', "wanna-build-dump",
"Cruft-Report::Options::Wanna-Build-Dump", "HasArg")]
for i in ["help", "Rdep-Check"]:
key = "Cruft-Report::Options::%s" % i
if key not in cnf:
cnf[key] = ""
cnf["Cruft-Report::Options::Suite"] = cnf.get("Dinstall::DefaultSuite",
"unstable")
if "Cruft-Report::Options::Mode" not in cnf:
cnf["Cruft-Report::Options::Mode"] = "daily"
if "Cruft-Report::Options::Wanna-Build-Dump" not in cnf:
cnf["Cruft-Report::Options::Wanna-Build-Dump"] = "/srv/ftp-master.debian.org/scripts/nfu"
apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Cruft-Report::Options")
if Options["Help"]:
usage()
if Options["Rdep-Check"]:
rdeps = True
else:
rdeps = False
# Set up checks based on mode
if Options["Mode"] == "daily":
checks = [
"nbs", "nviu", "nvit", "obsolete source", "outdated non-free",
"nfu", "nbs metadata"
]
elif Options["Mode"] == "full":
checks = [
"nbs", "nviu", "nvit", "obsolete source", "outdated non-free",
"nfu", "nbs metadata", "dubious nbs", "bnb", "bms", "anais"
]
elif Options["Mode"] == "bdo":
checks = ["nbs", "obsolete source"]
else:
utils.warn(
"%s is not a recognised mode - only 'full', 'daily' or 'bdo' are understood."
% (Options["Mode"]))
usage(1)
session = DBConn().session()
bin_pkgs = {}
src_pkgs = {}
bin2source = {}
bins_in_suite = {}
nbs = {}
source_versions = {}
anais_output = ""
nfu_packages = {}
suite = get_suite(Options["Suite"].lower(), session)
if not suite:
utils.fubar("Cannot find suite %s" % Options["Suite"].lower())
suite_id = suite.suite_id
suite_name = suite.suite_name.lower()
if "obsolete source" in checks:
report_obsolete_source(suite_name, session)
if "nbs" in checks:
reportAllNBS(suite_name, suite_id, session, rdeps)
if "nbs metadata" in checks:
reportNBSMetadata(suite_name, suite_id, session, rdeps)
if "outdated non-free" in checks:
report_outdated_nonfree(suite_name, session, rdeps)
bin_not_built = {}
if "bnb" in checks:
bins_in_suite = get_suite_binaries(suite, session)
# Checks based on the Sources files
components = get_component_names(session)
for component in components:
filename = "%s/dists/%s/%s/source/Sources" % (suite.archive.path,
suite_name, component)
filename = utils.find_possibly_compressed_file(filename)
with apt_pkg.TagFile(filename) as Sources:
while Sources.step():
source = Sources.section.find('Package')
source_version = Sources.section.find('Version')
architecture = Sources.section.find('Architecture')
binaries = Sources.section.find('Binary')
binaries_list = [i.strip() for i in binaries.split(',')]
if "bnb" in checks:
# Check for binaries not built on any architecture.
for binary in binaries_list:
if binary not in bins_in_suite:
bin_not_built.setdefault(source, {})
bin_not_built[source][binary] = ""
if "anais" in checks:
anais_output += do_anais(architecture, binaries_list,
source, session)
# build indices for checking "no source" later
source_index = component + '/' + source
src_pkgs[source] = source_index
for binary in binaries_list:
bin_pkgs[binary] = source
source_binaries[source] = binaries
source_versions[source] = source_version
# Checks based on the Packages files
check_components = components[:]
if suite_name != "experimental":
check_components.append('main/debian-installer')
for component in check_components:
architectures = [
a.arch_string for a in get_suite_architectures(
suite_name, skipsrc=True, skipall=True, session=session)
]
for architecture in architectures:
if component == 'main/debian-installer' and re.match(
"kfreebsd", architecture):
continue
if "nfu" in checks:
nfu_packages.setdefault(architecture, [])
nfu_entries = parse_nfu(architecture)
filename = "%s/dists/%s/%s/binary-%s/Packages" % (
suite.archive.path, suite_name, component, architecture)
filename = utils.find_possibly_compressed_file(filename)
with apt_pkg.TagFile(filename) as Packages:
while Packages.step():
package = Packages.section.find('Package')
source = Packages.section.find('Source', "")
version = Packages.section.find('Version')
if source == "":
source = package
if package in bin2source and \
apt_pkg.version_compare(version, bin2source[package]["version"]) > 0:
bin2source[package]["version"] = version
bin2source[package]["source"] = source
else:
bin2source[package] = {}
bin2source[package]["version"] = version
bin2source[package]["source"] = source
if source.find("(") != -1:
m = re_extract_src_version.match(source)
source = m.group(1)
version = m.group(2)
if package not in bin_pkgs:
nbs.setdefault(source, {})
nbs[source].setdefault(package, {})
nbs[source][package][version] = ""
else:
if "nfu" in checks:
if package in nfu_entries and \
version != source_versions[source]: # only suggest to remove out-of-date packages
nfu_packages[architecture].append(
(package, version,
source_versions[source]))
# Distinguish dubious (version numbers match) and 'real' NBS (they don't)
dubious_nbs = {}
version_sort_key = functools.cmp_to_key(apt_pkg.version_compare)
for source in nbs:
for package in nbs[source]:
latest_version = max(nbs[source][package], key=version_sort_key)
source_version = source_versions.get(source, "0")
if apt_pkg.version_compare(latest_version, source_version) == 0:
add_nbs(dubious_nbs, source, latest_version, package, suite_id,
session)
if "nviu" in checks:
do_newer_version('unstable', 'experimental', 'NVIU', session)
if "nvit" in checks:
do_newer_version('testing', 'testing-proposed-updates', 'NVIT',
session)
###
if Options["Mode"] == "full":
print("=" * 75)
print()
if "nfu" in checks:
do_nfu(nfu_packages)
if "bnb" in checks:
print("Unbuilt binary packages")
print("-----------------------")
print()
for source in sorted(bin_not_built):
binaries = sorted(bin_not_built[source])
print(" o %s: %s" % (source, ", ".join(binaries)))
print()
if "bms" in checks:
report_multiple_source(suite)
if "anais" in checks:
print("Architecture Not Allowed In Source")
print("----------------------------------")
print(anais_output)
print()
if "dubious nbs" in checks:
do_dubious_nbs(dubious_nbs)
def main ():
global suite, suite_id, source_binaries, source_versions
cnf = Config()
Arguments = [('h',"help","Cruft-Report::Options::Help"),
('m',"mode","Cruft-Report::Options::Mode", "HasArg"),
('R',"rdep-check", "Cruft-Report::Options::Rdep-Check"),
('s',"suite","Cruft-Report::Options::Suite","HasArg"),
('w',"wanna-build-dump","Cruft-Report::Options::Wanna-Build-Dump","HasArg")]
for i in [ "help", "Rdep-Check" ]:
if not cnf.has_key("Cruft-Report::Options::%s" % (i)):
cnf["Cruft-Report::Options::%s" % (i)] = ""
cnf["Cruft-Report::Options::Suite"] = cnf.get("Dinstall::DefaultSuite", "unstable")
if not cnf.has_key("Cruft-Report::Options::Mode"):
cnf["Cruft-Report::Options::Mode"] = "daily"
if not cnf.has_key("Cruft-Report::Options::Wanna-Build-Dump"):
cnf["Cruft-Report::Options::Wanna-Build-Dump"] = "/srv/ftp-master.debian.org/scripts/nfu"
apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Cruft-Report::Options")
if Options["Help"]:
usage()
if Options["Rdep-Check"]:
rdeps = True
else:
rdeps = False
# Set up checks based on mode
if Options["Mode"] == "daily":
checks = [ "nbs", "nviu", "nvit", "obsolete source", "outdated non-free", "nfu" ]
elif Options["Mode"] == "full":
checks = [ "nbs", "nviu", "nvit", "obsolete source", "outdated non-free", "nfu", "dubious nbs", "bnb", "bms", "anais" ]
elif Options["Mode"] == "bdo":
checks = [ "nbs", "obsolete source" ]
else:
utils.warn("%s is not a recognised mode - only 'full', 'daily' or 'bdo' are understood." % (Options["Mode"]))
usage(1)
session = DBConn().session()
bin_pkgs = {}
src_pkgs = {}
bin2source = {}
bins_in_suite = {}
nbs = {}
source_versions = {}
anais_output = ""
nfu_packages = {}
suite = get_suite(Options["Suite"].lower(), session)
if not suite:
utils.fubar("Cannot find suite %s" % Options["Suite"].lower())
suite_id = suite.suite_id
suite_name = suite.suite_name.lower()
if "obsolete source" in checks:
report_obsolete_source(suite_name, session)
if "nbs" in checks:
reportAllNBS(suite_name, suite_id, session, rdeps)
if "outdated non-free" in checks:
report_outdated_nonfree(suite_name, session, rdeps)
bin_not_built = {}
if "bnb" in checks:
bins_in_suite = get_suite_binaries(suite, session)
# Checks based on the Sources files
components = get_component_names(session)
for component in components:
filename = "%s/dists/%s/%s/source/Sources.gz" % (suite.archive.path, suite_name, component)
# apt_pkg.TagFile needs a real file handle and can't handle a GzipFile instance...
(fd, temp_filename) = utils.temp_filename()
(result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename))
if (result != 0):
sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output))
sys.exit(result)
sources = utils.open_file(temp_filename)
Sources = apt_pkg.TagFile(sources)
while Sources.step():
source = Sources.section.find('Package')
source_version = Sources.section.find('Version')
architecture = Sources.section.find('Architecture')
binaries = Sources.section.find('Binary')
binaries_list = [ i.strip() for i in binaries.split(',') ]
if "bnb" in checks:
# Check for binaries not built on any architecture.
for binary in binaries_list:
if not bins_in_suite.has_key(binary):
bin_not_built.setdefault(source, {})
bin_not_built[source][binary] = ""
if "anais" in checks:
anais_output += do_anais(architecture, binaries_list, source, session)
# build indices for checking "no source" later
source_index = component + '/' + source
src_pkgs[source] = source_index
for binary in binaries_list:
bin_pkgs[binary] = source
source_binaries[source] = binaries
source_versions[source] = source_version
sources.close()
os.unlink(temp_filename)
# Checks based on the Packages files
check_components = components[:]
if suite_name != "experimental":
check_components.append('main/debian-installer');
for component in check_components:
architectures = [ a.arch_string for a in get_suite_architectures(suite_name,
skipsrc=True, skipall=True,
session=session) ]
for architecture in architectures:
if component == 'main/debian-installer' and re.match("kfreebsd", architecture):
continue
filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (suite.archive.path, suite_name, component, architecture)
# apt_pkg.TagFile needs a real file handle
(fd, temp_filename) = utils.temp_filename()
(result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename))
if (result != 0):
sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output))
sys.exit(result)
if "nfu" in checks:
nfu_packages.setdefault(architecture,[])
nfu_entries = parse_nfu(architecture)
packages = utils.open_file(temp_filename)
Packages = apt_pkg.TagFile(packages)
while Packages.step():
package = Packages.section.find('Package')
source = Packages.section.find('Source', "")
version = Packages.section.find('Version')
if source == "":
source = package
if bin2source.has_key(package) and \
apt_pkg.version_compare(version, bin2source[package]["version"]) > 0:
bin2source[package]["version"] = version
bin2source[package]["source"] = source
else:
bin2source[package] = {}
bin2source[package]["version"] = version
bin2source[package]["source"] = source
if source.find("(") != -1:
m = re_extract_src_version.match(source)
source = m.group(1)
version = m.group(2)
if not bin_pkgs.has_key(package):
nbs.setdefault(source,{})
nbs[source].setdefault(package, {})
nbs[source][package][version] = ""
else:
if "nfu" in checks:
if package in nfu_entries and \
version != source_versions[source]: # only suggest to remove out-of-date packages
nfu_packages[architecture].append((package,version,source_versions[source]))
packages.close()
os.unlink(temp_filename)
# Distinguish dubious (version numbers match) and 'real' NBS (they don't)
dubious_nbs = {}
for source in nbs.keys():
for package in nbs[source].keys():
versions = nbs[source][package].keys()
versions.sort(apt_pkg.version_compare)
latest_version = versions.pop()
source_version = source_versions.get(source,"0")
if apt_pkg.version_compare(latest_version, source_version) == 0:
add_nbs(dubious_nbs, source, latest_version, package, suite_id, session)
if "nviu" in checks:
do_newer_version('unstable', 'experimental', 'NVIU', session)
if "nvit" in checks:
do_newer_version('testing', 'testing-proposed-updates', 'NVIT', session)
###
if Options["Mode"] == "full":
print "="*75
print
if "nfu" in checks:
do_nfu(nfu_packages)
if "bnb" in checks:
print "Unbuilt binary packages"
print "-----------------------"
print
keys = bin_not_built.keys()
keys.sort()
for source in keys:
binaries = bin_not_built[source].keys()
binaries.sort()
print " o %s: %s" % (source, ", ".join(binaries))
print
if "bms" in checks:
report_multiple_source(suite)
if "anais" in checks:
print "Architecture Not Allowed In Source"
print "----------------------------------"
print anais_output
print
if "dubious nbs" in checks:
do_dubious_nbs(dubious_nbs)
def main():
global suite, suite_id, source_binaries, source_versions
cnf = Config()
Arguments = [('h', "help", "Cruft-Report::Options::Help"),
('m', "mode", "Cruft-Report::Options::Mode", "HasArg"),
('R', "rdep-check", "Cruft-Report::Options::Rdep-Check"),
('s', "suite", "Cruft-Report::Options::Suite", "HasArg"),
('w', "wanna-build-dump",
"Cruft-Report::Options::Wanna-Build-Dump", "HasArg")]
for i in ["help", "Rdep-Check"]:
if not cnf.has_key("Cruft-Report::Options::%s" % (i)):
cnf["Cruft-Report::Options::%s" % (i)] = ""
cnf["Cruft-Report::Options::Suite"] = cnf.get("Dinstall::DefaultSuite",
"unstable")
if not cnf.has_key("Cruft-Report::Options::Mode"):
cnf["Cruft-Report::Options::Mode"] = "daily"
if not cnf.has_key("Cruft-Report::Options::Wanna-Build-Dump"):
cnf["Cruft-Report::Options::Wanna-Build-Dump"] = "/srv/ftp-master.debian.org/scripts/nfu"
apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv)
Options = cnf.subtree("Cruft-Report::Options")
if Options["Help"]:
usage()
if Options["Rdep-Check"]:
rdeps = True
else:
rdeps = False
# Set up checks based on mode
if Options["Mode"] == "daily":
checks = [
"nbs", "nviu", "nvit", "obsolete source", "outdated non-free",
"nfu"
]
elif Options["Mode"] == "full":
checks = [
"nbs", "nviu", "nvit", "obsolete source", "outdated non-free",
"nfu", "dubious nbs", "bnb", "bms", "anais"
]
elif Options["Mode"] == "bdo":
checks = ["nbs", "obsolete source"]
else:
utils.warn(
"%s is not a recognised mode - only 'full', 'daily' or 'bdo' are understood."
% (Options["Mode"]))
usage(1)
session = DBConn().session()
bin_pkgs = {}
src_pkgs = {}
bin2source = {}
bins_in_suite = {}
nbs = {}
source_versions = {}
anais_output = ""
nfu_packages = {}
suite = get_suite(Options["Suite"].lower(), session)
if not suite:
utils.fubar("Cannot find suite %s" % Options["Suite"].lower())
suite_id = suite.suite_id
suite_name = suite.suite_name.lower()
if "obsolete source" in checks:
report_obsolete_source(suite_name, session)
if "nbs" in checks:
reportAllNBS(suite_name, suite_id, session, rdeps)
if "outdated non-free" in checks:
report_outdated_nonfree(suite_name, session, rdeps)
bin_not_built = {}
if "bnb" in checks:
bins_in_suite = get_suite_binaries(suite, session)
# Checks based on the Sources files
components = get_component_names(session)
for component in components:
filename = "%s/dists/%s/%s/source/Sources.gz" % (suite.archive.path,
suite_name, component)
# apt_pkg.TagFile needs a real file handle and can't handle a GzipFile instance...
(fd, temp_filename) = utils.temp_filename()
(result, output) = commands.getstatusoutput("gunzip -c %s > %s" %
(filename, temp_filename))
if (result != 0):
sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output))
sys.exit(result)
sources = utils.open_file(temp_filename)
Sources = apt_pkg.TagFile(sources)
while Sources.step():
source = Sources.section.find('Package')
source_version = Sources.section.find('Version')
architecture = Sources.section.find('Architecture')
binaries = Sources.section.find('Binary')
binaries_list = [i.strip() for i in binaries.split(',')]
if "bnb" in checks:
# Check for binaries not built on any architecture.
for binary in binaries_list:
if not bins_in_suite.has_key(binary):
bin_not_built.setdefault(source, {})
bin_not_built[source][binary] = ""
if "anais" in checks:
anais_output += do_anais(architecture, binaries_list, source,
session)
# build indices for checking "no source" later
source_index = component + '/' + source
src_pkgs[source] = source_index
for binary in binaries_list:
bin_pkgs[binary] = source
source_binaries[source] = binaries
source_versions[source] = source_version
sources.close()
os.unlink(temp_filename)
# Checks based on the Packages files
check_components = components[:]
if suite_name != "staging":
check_components.append('main/debian-installer')
for component in check_components:
architectures = [
a.arch_string for a in get_suite_architectures(
suite_name, skipsrc=True, skipall=True, session=session)
]
for architecture in architectures:
if component == 'main/debian-installer' and re.match(
"kfreebsd", architecture):
continue
filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (
suite.archive.path, suite_name, component, architecture)
# apt_pkg.TagFile needs a real file handle
(fd, temp_filename) = utils.temp_filename()
(result, output) = commands.getstatusoutput(
"gunzip -c %s > %s" % (filename, temp_filename))
if (result != 0):
sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output))
sys.exit(result)
if "nfu" in checks:
nfu_packages.setdefault(architecture, [])
nfu_entries = parse_nfu(architecture)
packages = utils.open_file(temp_filename)
Packages = apt_pkg.TagFile(packages)
while Packages.step():
package = Packages.section.find('Package')
source = Packages.section.find('Source', "")
version = Packages.section.find('Version')
if source == "":
source = package
if bin2source.has_key(package) and \
apt_pkg.version_compare(version, bin2source[package]["version"]) > 0:
bin2source[package]["version"] = version
bin2source[package]["source"] = source
else:
bin2source[package] = {}
bin2source[package]["version"] = version
bin2source[package]["source"] = source
if source.find("(") != -1:
m = re_extract_src_version.match(source)
source = m.group(1)
version = m.group(2)
if not bin_pkgs.has_key(package):
nbs.setdefault(source, {})
nbs[source].setdefault(package, {})
nbs[source][package][version] = ""
else:
if "nfu" in checks:
if package in nfu_entries and \
version != source_versions[source]: # only suggest to remove out-of-date packages
nfu_packages[architecture].append(
(package, version, source_versions[source]))
packages.close()
os.unlink(temp_filename)
# Distinguish dubious (version numbers match) and 'real' NBS (they don't)
dubious_nbs = {}
for source in nbs.keys():
for package in nbs[source].keys():
versions = nbs[source][package].keys()
versions.sort(apt_pkg.version_compare)
latest_version = versions.pop()
source_version = source_versions.get(source, "0")
if apt_pkg.version_compare(latest_version, source_version) == 0:
add_nbs(dubious_nbs, source, latest_version, package, suite_id,
session)
if "nviu" in checks:
do_newer_version('chromodoris', 'staging', 'NVIU', session)
# FIXME: Not used in Tanglu
#if "nvit" in checks:
# do_newer_version('testing', 'testing-proposed-updates', 'NVIT', session)
###
if Options["Mode"] == "full":
print "=" * 75
print
if "nfu" in checks:
do_nfu(nfu_packages)
if "bnb" in checks:
print "Unbuilt binary packages"
print "-----------------------"
print
keys = bin_not_built.keys()
keys.sort()
for source in keys:
binaries = bin_not_built[source].keys()
binaries.sort()
print " o %s: %s" % (source, ", ".join(binaries))
print
if "bms" in checks:
report_multiple_source(suite)
if "anais" in checks:
print "Architecture Not Allowed In Source"
print "----------------------------------"
print anais_output
print
if "dubious nbs" in checks:
do_dubious_nbs(dubious_nbs)