def check_reverse_depends(removals, suite, arches=None, session=None, cruft=False, quiet=False):
dbsuite = get_suite(suite, session)
overridesuite = dbsuite
if dbsuite.overridesuite is not None:
overridesuite = get_suite(dbsuite.overridesuite, session)
dep_problem = 0
p2c = {}
all_broken = defaultdict(lambda: defaultdict(set))
if arches:
all_arches = set(arches)
else:
all_arches = set(x.arch_string for x in get_suite_architectures(suite))
all_arches -= set(["source", "all"])
removal_set = set(removals)
metakey_d = get_or_set_metadatakey("Depends", session)
metakey_p = get_or_set_metadatakey("Provides", session)
params = {
'suite_id': dbsuite.suite_id,
'metakey_d_id': metakey_d.key_id,
'metakey_p_id': metakey_p.key_id,
}
for architecture in all_arches | set(['all']):
deps = {}
sources = {}
virtual_packages = {}
params['arch_id'] = get_architecture(architecture, session).arch_id
statement = '''
SELECT b.package, s.source, c.name as component,
(SELECT bmd.value FROM binaries_metadata bmd WHERE bmd.bin_id = b.id AND bmd.key_id = :metakey_d_id) AS depends,
(SELECT bmp.value FROM binaries_metadata bmp WHERE bmp.bin_id = b.id AND bmp.key_id = :metakey_p_id) AS provides
FROM binaries b
JOIN bin_associations ba ON b.id = ba.bin AND ba.suite = :suite_id
JOIN source s ON b.source = s.id
JOIN files_archive_map af ON b.file = af.file_id
JOIN component c ON af.component_id = c.id
WHERE b.architecture = :arch_id'''
query = session.query('package', 'source', 'component', 'depends', 'provides'). \
from_statement(statement).params(params)
for package, source, component, depends, provides in query:
sources[package] = source
p2c[package] = component
if depends is not None:
deps[package] = depends
# Maintain a counter for each virtual package. If a
# Provides: exists, set the counter to 0 and count all
# provides by a package not in the list for removal.
# If the counter stays 0 at the end, we know that only
# the to-be-removed packages provided this virtual
# package.
if provides is not None:
for virtual_pkg in provides.split(","):
virtual_pkg = virtual_pkg.strip()
if virtual_pkg == package: continue
if not virtual_packages.has_key(virtual_pkg):
virtual_packages[virtual_pkg] = 0
if package not in removals:
virtual_packages[virtual_pkg] += 1
# If a virtual package is only provided by the to-be-removed
# packages, treat the virtual package as to-be-removed too.
removal_set.update(virtual_pkg for virtual_pkg in virtual_packages if not virtual_packages[virtual_pkg])
# Check binary dependencies (Depends)
for package in deps:
if package in removals: continue
try:
parsed_dep = apt_pkg.parse_depends(deps[package])
except ValueError as e:
print "Error for package %s: %s" % (package, e)
parsed_dep = []
for dep in parsed_dep:
# Check for partial breakage. If a package has a ORed
# dependency, there is only a dependency problem if all
# packages in the ORed depends will be removed.
unsat = 0
for dep_package, _, _ in dep:
if dep_package in removals:
unsat += 1
if unsat == len(dep):
component = p2c[package]
source = sources[package]
if component != "main":
source = "%s/%s" % (source, component)
all_broken[source][package].add(architecture)
dep_problem = 1
if all_broken and not quiet:
if cruft:
print " - broken Depends:"
else:
print "# Broken Depends:"
for source, bindict in sorted(all_broken.items()):
lines = []
for binary, arches in sorted(bindict.items()):
if arches == all_arches or 'all' in arches:
lines.append(binary)
else:
lines.append('%s [%s]' % (binary, ' '.join(sorted(arches))))
if cruft:
print ' %s: %s' % (source, lines[0])
else:
print '%s: %s' % (source, lines[0])
for line in lines[1:]:
if cruft:
print ' ' + ' ' * (len(source) + 2) + line
else:
print ' ' * (len(source) + 2) + line
if not cruft:
print
# Check source dependencies (Build-Depends and Build-Depends-Indep)
all_broken = defaultdict(set)
metakey_bd = get_or_set_metadatakey("Build-Depends", session)
metakey_bdi = get_or_set_metadatakey("Build-Depends-Indep", session)
params = {
'suite_id': dbsuite.suite_id,
'metakey_ids': (metakey_bd.key_id, metakey_bdi.key_id),
}
statement = '''
SELECT s.source, string_agg(sm.value, ', ') as build_dep
FROM source s
JOIN source_metadata sm ON s.id = sm.src_id
WHERE s.id in
(SELECT source FROM src_associations
WHERE suite = :suite_id)
AND sm.key_id in :metakey_ids
GROUP BY s.id, s.source'''
query = session.query('source', 'build_dep').from_statement(statement). \
params(params)
for source, build_dep in query:
if source in removals: continue
parsed_dep = []
if build_dep is not None:
# Remove [arch] information since we want to see breakage on all arches
build_dep = re_build_dep_arch.sub("", build_dep)
try:
parsed_dep = apt_pkg.parse_src_depends(build_dep)
except ValueError as e:
print "Error for source %s: %s" % (source, e)
for dep in parsed_dep:
unsat = 0
for dep_package, _, _ in dep:
if dep_package in removals:
unsat += 1
if unsat == len(dep):
component, = session.query(Component.component_name) \
.join(Component.overrides) \
.filter(Override.suite == overridesuite) \
.filter(Override.package == re.sub('/(contrib|non-free)$', '', source)) \
.join(Override.overridetype).filter(OverrideType.overridetype == 'dsc') \
.first()
key = source
if component != "main":
key = "%s/%s" % (source, component)
all_broken[key].add(pp_deps(dep))
dep_problem = 1
if all_broken and not quiet:
if cruft:
print " - broken Build-Depends:"
else:
print "# Broken Build-Depends:"
for source, bdeps in sorted(all_broken.items()):
bdeps = sorted(bdeps)
if cruft:
print ' %s: %s' % (source, bdeps[0])
else:
print '%s: %s' % (source, bdeps[0])
for bdep in bdeps[1:]:
if cruft:
print ' ' + ' ' * (len(source) + 2) + bdep
else:
print ' ' * (len(source) + 2) + bdep
if not cruft:
print
return dep_problem
def _load_package_information(session, suite_id, suite_archs2id):
package_dependencies = defaultdict(lambda: defaultdict(set))
arch_providers_of = defaultdict(lambda: defaultdict(set))
arch_provided_by = defaultdict(lambda: defaultdict(set))
source_deps = defaultdict(set)
metakey_d = get_or_set_metadatakey("Depends", session)
metakey_p = get_or_set_metadatakey("Provides", session)
params = {
'suite_id': suite_id,
'arch_all_id': suite_archs2id['all'],
'metakey_d_id': metakey_d.key_id,
'metakey_p_id': metakey_p.key_id,
}
all_arches = set(suite_archs2id)
all_arches.discard('source')
package_dependencies['source'] = source_deps
for architecture in all_arches:
deps = defaultdict(set)
providers_of = defaultdict(set)
provided_by = defaultdict(set)
arch_providers_of[architecture] = providers_of
arch_provided_by[architecture] = provided_by
package_dependencies[architecture] = deps
params['arch_id'] = suite_archs2id[architecture]
statement = '''
SELECT b.package,
(SELECT bmd.value FROM binaries_metadata bmd WHERE bmd.bin_id = b.id AND bmd.key_id = :metakey_d_id) AS depends,
(SELECT bmp.value FROM binaries_metadata bmp WHERE bmp.bin_id = b.id AND bmp.key_id = :metakey_p_id) AS provides
FROM binaries b
JOIN bin_associations ba ON b.id = ba.bin AND ba.suite = :suite_id
WHERE b.architecture = :arch_id OR b.architecture = :arch_all_id'''
query = session.query('package', 'depends', 'provides'). \
from_statement(statement).params(params)
for package, depends, provides in query:
if depends is not None:
try:
parsed_dep = []
for dep in apt_pkg.parse_depends(depends):
parsed_dep.append(frozenset(d[0] for d in dep))
deps[package].update(parsed_dep)
except ValueError as e:
print "Error for package %s: %s" % (package, e)
# Maintain a counter for each virtual package. If a
# Provides: exists, set the counter to 0 and count all
# provides by a package not in the list for removal.
# If the counter stays 0 at the end, we know that only
# the to-be-removed packages provided this virtual
# package.
if provides is not None:
for virtual_pkg in provides.split(","):
virtual_pkg = virtual_pkg.strip()
if virtual_pkg == package:
continue
provided_by[virtual_pkg].add(package)
providers_of[package].add(virtual_pkg)
# Check source dependencies (Build-Depends and Build-Depends-Indep)
metakey_bd = get_or_set_metadatakey("Build-Depends", session)
metakey_bdi = get_or_set_metadatakey("Build-Depends-Indep", session)
params = {
'suite_id': suite_id,
'metakey_ids': (metakey_bd.key_id, metakey_bdi.key_id),
}
statement = '''
SELECT s.source, string_agg(sm.value, ', ') as build_dep
FROM source s
JOIN source_metadata sm ON s.id = sm.src_id
WHERE s.id in
(SELECT source FROM src_associations
WHERE suite = :suite_id)
AND sm.key_id in :metakey_ids
GROUP BY s.id, s.source'''
query = session.query('source', 'build_dep').from_statement(statement). \
params(params)
for source, build_dep in query:
if build_dep is not None:
# Remove [arch] information since we want to see breakage on all arches
build_dep = re_build_dep_arch.sub("", build_dep)
try:
parsed_dep = []
for dep in apt_pkg.parse_src_depends(build_dep):
parsed_dep.append(frozenset(d[0] for d in dep))
source_deps[source].update(parsed_dep)
except ValueError as e:
print "Error for package %s: %s" % (source, e)
return package_dependencies, arch_providers_of, arch_provided_by
def check_reverse_depends(removals, suite, arches=None, session=None, cruft=False, quiet=False, include_arch_all=True):
dbsuite = get_suite(suite, session)
overridesuite = dbsuite
if dbsuite.overridesuite is not None:
overridesuite = get_suite(dbsuite.overridesuite, session)
dep_problem = 0
p2c = {}
all_broken = defaultdict(lambda: defaultdict(set))
if arches:
all_arches = set(arches)
else:
all_arches = set(x.arch_string for x in get_suite_architectures(suite))
all_arches -= set(["source", "all"])
removal_set = set(removals)
metakey_d = get_or_set_metadatakey("Depends", session)
metakey_p = get_or_set_metadatakey("Provides", session)
params = {
'suite_id': dbsuite.suite_id,
'metakey_d_id': metakey_d.key_id,
'metakey_p_id': metakey_p.key_id,
}
if include_arch_all:
rdep_architectures = all_arches | set(['all'])
else:
rdep_architectures = all_arches
for architecture in rdep_architectures:
deps = {}
sources = {}
virtual_packages = {}
try:
params['arch_id'] = get_architecture(architecture, session).arch_id
except AttributeError:
continue
statement = sql.text('''
SELECT b.package, s.source, c.name as component,
(SELECT bmd.value FROM binaries_metadata bmd WHERE bmd.bin_id = b.id AND bmd.key_id = :metakey_d_id) AS depends,
(SELECT bmp.value FROM binaries_metadata bmp WHERE bmp.bin_id = b.id AND bmp.key_id = :metakey_p_id) AS provides
FROM binaries b
JOIN bin_associations ba ON b.id = ba.bin AND ba.suite = :suite_id
JOIN source s ON b.source = s.id
JOIN files_archive_map af ON b.file = af.file_id
JOIN component c ON af.component_id = c.id
WHERE b.architecture = :arch_id''')
query = session.query('package', 'source', 'component', 'depends', 'provides'). \
from_statement(statement).params(params)
for package, source, component, depends, provides in query:
sources[package] = source
p2c[package] = component
if depends is not None:
deps[package] = depends
# Maintain a counter for each virtual package. If a
# Provides: exists, set the counter to 0 and count all
# provides by a package not in the list for removal.
# If the counter stays 0 at the end, we know that only
# the to-be-removed packages provided this virtual
# package.
if provides is not None:
for virtual_pkg in provides.split(","):
virtual_pkg = virtual_pkg.strip()
if virtual_pkg == package: continue
if virtual_pkg not in virtual_packages:
virtual_packages[virtual_pkg] = 0
if package not in removals:
virtual_packages[virtual_pkg] += 1
# If a virtual package is only provided by the to-be-removed
# packages, treat the virtual package as to-be-removed too.
removal_set.update(virtual_pkg for virtual_pkg in virtual_packages if not virtual_packages[virtual_pkg])
# Check binary dependencies (Depends)
for package in deps:
if package in removals: continue
try:
parsed_dep = apt_pkg.parse_depends(deps[package])
except ValueError as e:
print "Error for package %s: %s" % (package, e)
parsed_dep = []
for dep in parsed_dep:
# Check for partial breakage. If a package has a ORed
# dependency, there is only a dependency problem if all
# packages in the ORed depends will be removed.
unsat = 0
for dep_package, _, _ in dep:
if dep_package in removals:
unsat += 1
if unsat == len(dep):
component = p2c[package]
source = sources[package]
if component != "main":
source = "%s/%s" % (source, component)
all_broken[source][package].add(architecture)
dep_problem = 1
if all_broken and not quiet:
if cruft:
print " - broken Depends:"
else:
print "# Broken Depends:"
for source, bindict in sorted(all_broken.items()):
lines = []
for binary, arches in sorted(bindict.items()):
if arches == all_arches or 'all' in arches:
lines.append(binary)
else:
lines.append('%s [%s]' % (binary, ' '.join(sorted(arches))))
if cruft:
print ' %s: %s' % (source, lines[0])
else:
print '%s: %s' % (source, lines[0])
for line in lines[1:]:
if cruft:
print ' ' + ' ' * (len(source) + 2) + line
else:
print ' ' * (len(source) + 2) + line
if not cruft:
print
# Check source dependencies (Build-Depends and Build-Depends-Indep)
all_broken = defaultdict(set)
metakey_bd = get_or_set_metadatakey("Build-Depends", session)
metakey_bdi = get_or_set_metadatakey("Build-Depends-Indep", session)
if include_arch_all:
metakey_ids = (metakey_bd.key_id, metakey_bdi.key_id)
else:
metakey_ids = (metakey_bd.key_id,)
params = {
'suite_id': dbsuite.suite_id,
'metakey_ids': metakey_ids,
}
statement = sql.text('''
SELECT s.source, string_agg(sm.value, ', ') as build_dep
FROM source s
JOIN source_metadata sm ON s.id = sm.src_id
WHERE s.id in
(SELECT src FROM newest_src_association
WHERE suite = :suite_id)
AND sm.key_id in :metakey_ids
GROUP BY s.id, s.source''')
query = session.query('source', 'build_dep').from_statement(statement). \
params(params)
for source, build_dep in query:
if source in removals: continue
parsed_dep = []
if build_dep is not None:
# Remove [arch] information since we want to see breakage on all arches
build_dep = re_build_dep_arch.sub("", build_dep)
try:
parsed_dep = apt_pkg.parse_src_depends(build_dep)
except ValueError as e:
print "Error for source %s: %s" % (source, e)
for dep in parsed_dep:
unsat = 0
for dep_package, _, _ in dep:
if dep_package in removals:
unsat += 1
if unsat == len(dep):
component, = session.query(Component.component_name) \
.join(Component.overrides) \
.filter(Override.suite == overridesuite) \
.filter(Override.package == re.sub('/(contrib|non-free)$', '', source)) \
.join(Override.overridetype).filter(OverrideType.overridetype == 'dsc') \
.first()
key = source
if component != "main":
key = "%s/%s" % (source, component)
all_broken[key].add(pp_deps(dep))
dep_problem = 1
if all_broken and not quiet:
if cruft:
print " - broken Build-Depends:"
else:
print "# Broken Build-Depends:"
for source, bdeps in sorted(all_broken.items()):
bdeps = sorted(bdeps)
if cruft:
print ' %s: %s' % (source, bdeps[0])
else:
print '%s: %s' % (source, bdeps[0])
for bdep in bdeps[1:]:
if cruft:
print ' ' + ' ' * (len(source) + 2) + bdep
else:
print ' ' * (len(source) + 2) + bdep
if not cruft:
print
return dep_problem
def _load_package_information(session, suite_id, suite_archs2id):
package_dependencies = defaultdict(lambda: defaultdict(set))
arch_providers_of = defaultdict(lambda: defaultdict(set))
arch_provided_by = defaultdict(lambda: defaultdict(set))
source_deps = defaultdict(set)
metakey_d = get_or_set_metadatakey("Depends", session)
metakey_p = get_or_set_metadatakey("Provides", session)
params = {
'suite_id': suite_id,
'arch_all_id': suite_archs2id['all'],
'metakey_d_id': metakey_d.key_id,
'metakey_p_id': metakey_p.key_id,
}
all_arches = set(suite_archs2id)
all_arches.discard('source')
package_dependencies['source'] = source_deps
for architecture in all_arches:
deps = defaultdict(set)
providers_of = defaultdict(set)
provided_by = defaultdict(set)
arch_providers_of[architecture] = providers_of
arch_provided_by[architecture] = provided_by
package_dependencies[architecture] = deps
params['arch_id'] = suite_archs2id[architecture]
statement = '''
SELECT b.package,
(SELECT bmd.value FROM binaries_metadata bmd WHERE bmd.bin_id = b.id AND bmd.key_id = :metakey_d_id) AS depends,
(SELECT bmp.value FROM binaries_metadata bmp WHERE bmp.bin_id = b.id AND bmp.key_id = :metakey_p_id) AS provides
FROM binaries b
JOIN bin_associations ba ON b.id = ba.bin AND ba.suite = :suite_id
WHERE b.architecture = :arch_id OR b.architecture = :arch_all_id'''
query = session.query('package', 'depends', 'provides'). \
from_statement(statement).params(params)
for package, depends, provides in query:
if depends is not None:
try:
parsed_dep = []
for dep in apt_pkg.parse_depends(depends):
parsed_dep.append(frozenset(d[0] for d in dep))
deps[package].update(parsed_dep)
except ValueError as e:
print "Error for package %s: %s" % (package, e)
# Maintain a counter for each virtual package. If a
# Provides: exists, set the counter to 0 and count all
# provides by a package not in the list for removal.
# If the counter stays 0 at the end, we know that only
# the to-be-removed packages provided this virtual
# package.
if provides is not None:
for virtual_pkg in provides.split(","):
virtual_pkg = virtual_pkg.strip()
if virtual_pkg == package:
continue
provided_by[virtual_pkg].add(package)
providers_of[package].add(virtual_pkg)
# Check source dependencies (Build-Depends and Build-Depends-Indep)
metakey_bd = get_or_set_metadatakey("Build-Depends", session)
metakey_bdi = get_or_set_metadatakey("Build-Depends-Indep", session)
params = {
'suite_id': suite_id,
'metakey_ids': (metakey_bd.key_id, metakey_bdi.key_id),
}
statement = '''
SELECT s.source, string_agg(sm.value, ', ') as build_dep
FROM source s
JOIN source_metadata sm ON s.id = sm.src_id
WHERE s.id in
(SELECT source FROM src_associations
WHERE suite = :suite_id)
AND sm.key_id in :metakey_ids
GROUP BY s.id, s.source'''
query = session.query('source', 'build_dep').from_statement(statement). \
params(params)
for source, build_dep in query:
if build_dep is not None:
# Remove [arch] information since we want to see breakage on all arches
build_dep = re_build_dep_arch.sub("", build_dep)
try:
parsed_dep = []
for dep in apt_pkg.parse_src_depends(build_dep):
parsed_dep.append(frozenset(d[0] for d in dep))
source_deps[source].update(parsed_dep)
except ValueError as e:
print "Error for package %s: %s" % (source, e)
return package_dependencies, arch_providers_of, arch_provided_by
