def open(self, file):
"""Open the package."""
depends_tags = ["Build-Depends", "Build-Depends-Indep"]
conflicts_tags = ["Build-Conflicts", "Build-Conflicts-Indep"]
fd = apt_pkg.open_maybe_clear_signed_file(file)
fobj = os.fdopen(fd)
tagfile = apt_pkg.TagFile(fobj)
try:
for sec in tagfile:
for tag in depends_tags:
if tag not in sec:
continue
self._depends.extend(apt_pkg.parse_src_depends(sec[tag]))
for tag in conflicts_tags:
if tag not in sec:
continue
self._conflicts.extend(apt_pkg.parse_src_depends(sec[tag]))
if 'Source' in sec:
self.pkgname = sec['Source']
if 'Binary' in sec:
self.binaries = [
b.strip() for b in sec['Binary'].split(',')
]
for tag in sec.keys():
if tag in sec:
self._sections[tag] = sec[tag]
finally:
del tagfile
fobj.close()
s = _("Install Build-Dependencies for "
"source package '%s' that builds %s\n") % (
self.pkgname, " ".join(self.binaries))
self._sections["Description"] = s
self._check_was_run = False
def open(self, file):
# type: (str) -> None
"""Open the package."""
depends_tags = ["Build-Depends", "Build-Depends-Indep"]
conflicts_tags = ["Build-Conflicts", "Build-Conflicts-Indep"]
fd = apt_pkg.open_maybe_clear_signed_file(file)
fobj = os.fdopen(fd)
tagfile = apt_pkg.TagFile(fobj)
try:
for sec in tagfile:
for tag in depends_tags:
if tag not in sec:
continue
self._depends.extend(apt_pkg.parse_src_depends(sec[tag]))
for tag in conflicts_tags:
if tag not in sec:
continue
self._conflicts.extend(apt_pkg.parse_src_depends(sec[tag]))
if 'Source' in sec:
self.pkgname = sec['Source']
if 'Binary' in sec:
self.binaries = [b.strip() for b in
sec['Binary'].split(',')]
for tag in sec.keys(): # type: ignore
if tag in sec:
self._sections[tag] = sec[tag]
finally:
del tagfile
fobj.close()
s = _("Install Build-Dependencies for "
"source package '%s' that builds %s\n") % (self.pkgname,
" ".join(self.binaries))
self._sections["Description"] = s
self._check_was_run = False
def open(self, file):
"""Open the package."""
depends_tags = ["Build-Depends", "Build-Depends-Indep"]
conflicts_tags = ["Build-Conflicts", "Build-Conflicts-Indep"]
fobj = open(file)
tagfile = apt_pkg.TagFile(fobj)
try:
for sec in tagfile:
for tag in depends_tags:
if tag not in sec:
continue
self._depends.extend(apt_pkg.parse_src_depends(sec[tag]))
for tag in conflicts_tags:
if tag not in sec:
continue
self._conflicts.extend(apt_pkg.parse_src_depends(sec[tag]))
if 'Source' in sec:
self.pkgname = sec['Source']
if 'Binary' in sec:
self.binaries = sec['Binary'].split(', ')
if 'Version' in sec:
self._sections['Version'] = sec['Version']
finally:
del tagfile
fobj.close()
s = _("Install Build-Dependencies for "
"source package '%s' that builds %s\n") % (self.pkgname,
" ".join(self.binaries))
self._sections["Description"] = s
self._check_was_run = False
def open(self, file):
"""Open the package."""
depends_tags = ["Build-Depends", "Build-Depends-Indep"]
conflicts_tags = ["Build-Conflicts", "Build-Conflicts-Indep"]
fobj = open(file)
tagfile = apt_pkg.TagFile(fobj)
try:
for sec in tagfile:
for tag in depends_tags:
if not tag in sec:
continue
self._depends.extend(apt_pkg.parse_src_depends(sec[tag]))
for tag in conflicts_tags:
if not tag in sec:
continue
self._conflicts.extend(apt_pkg.parse_src_depends(sec[tag]))
if 'Source' in sec:
self.pkgname = sec['Source']
if 'Binary' in sec:
self.binaries = sec['Binary'].split(', ')
if 'Version' in sec:
self._sections['Version'] = sec['Version']
finally:
del tagfile
fobj.close()
s = _("Install Build-Dependencies for "
"source package '%s' that builds %s\n") % (
self.pkgname, " ".join(self.binaries))
self._sections["Description"] = s
def check(self, upload):
if upload.changes.source is None:
if upload.changes.sourceful:
raise Reject(
"{}: Architecture field includes source, but no source package is included in the upload"
.format(upload.changes.filename))
return True
if not upload.changes.sourceful:
raise Reject(
"{}: Architecture field does not include source, but a source package is included in the upload"
.format(upload.changes.filename))
changes = upload.changes.changes
source = upload.changes.source
control = source.dsc
dsc_fn = source._dsc_file.filename
check_fields_for_valid_utf8(dsc_fn, control)
# check fields
if not re_field_package.match(control['Source']):
raise Reject('{0}: Invalid Source field'.format(dsc_fn))
if control['Source'] != changes['Source']:
raise Reject(
'{0}: Source field does not match Source field in changes'.
format(dsc_fn))
if control['Version'] != changes['Version']:
raise Reject(
'{0}: Version field does not match Version field in changes'.
format(dsc_fn))
# check filenames
self.check_filename(control, dsc_fn, re_file_dsc)
for f in six.itervalues(source.files):
self.check_filename(control, f.filename, re_file_source)
# check dependency field syntax
for field in ('Build-Conflicts', 'Build-Conflicts-Indep',
'Build-Depends', 'Build-Depends-Arch',
'Build-Depends-Indep'):
value = control.get(field)
if value is not None:
if value.strip() == '':
raise Reject('{0}: empty {1} field'.format(dsc_fn, field))
try:
apt_pkg.parse_src_depends(value)
except Exception as e:
raise Reject(
'{0}: APT could not parse {1} field: {2}'.format(
dsc_fn, field, e))
rejects = utils.check_dsc_files(dsc_fn, control,
list(source.files.keys()))
if len(rejects) > 0:
raise Reject("\n".join(rejects))
return True
def chk_bd_process_dir (unused, dirname, filenames):
for name in filenames:
if not name.endswith(".dsc"):
continue
filename = os.path.abspath(dirname+'/'+name)
dsc = utils.parse_changes(filename, dsc_file=1)
for field_name in [ "build-depends", "build-depends-indep" ]:
field = dsc.get(field_name)
if field:
try:
apt_pkg.parse_src_depends(field)
except:
print "E: [%s] %s: %s" % (filename, field_name, field)
pass
def chk_bd_process_dir(unused, dirname, filenames):
for name in filenames:
if not name.endswith(".dsc"):
continue
filename = os.path.abspath(dirname + '/' + name)
dsc = utils.parse_changes(filename, dsc_file=1)
for field_name in ["build-depends", "build-depends-indep"]:
field = dsc.get(field_name)
if field:
try:
apt_pkg.parse_src_depends(field)
except:
print "E: [%s] %s: %s" % (filename, field_name, field)
pass
def test_parse_src_depends(self):
"""dependencies: Test apt_pkg.parse_src_depends()."""
# Check that architecture exclusion works
# depends_this: Current architecture is included
# depends_this_too: Another architecture is excluded
# depends_other: The current architecture is excluded
# depends_other: Another architecture is requested.
architecture = apt_pkg.config["APT::Architecture"]
depends_this = apt_pkg.parse_src_depends("p [%s]" % architecture)
depends_this_too = apt_pkg.parse_src_depends("p [!not-existing-arch]")
depends_other = apt_pkg.parse_src_depends("p [!%s]" % architecture)
depends_other_too = apt_pkg.parse_src_depends("p [not-existing-arch]")
self.assertEqual(len(depends_this), len(depends_this_too), 1)
self.assertEqual(len(depends_other), len(depends_other_too), 0)
def test_parse_src_depends(self):
"""dependencies: Test apt_pkg.parse_src_depends()."""
# Check that architecture exclusion works
# depends_this: Current architecture is included
# depends_this_too: Another architecture is excluded
# depends_other: The current architecture is excluded
# depends_other: Another architecture is requested.
architecture = apt_pkg.config["APT::Architecture"]
depends_this = apt_pkg.parse_src_depends("p [%s]" % architecture)
depends_this_too = apt_pkg.parse_src_depends("p [!not-existing-arch]")
depends_other = apt_pkg.parse_src_depends("p [!%s]" % architecture)
depends_other_too = apt_pkg.parse_src_depends("p [not-existing-arch]")
self.assertEqual(len(depends_this), len(depends_this_too), 1)
self.assertEqual(len(depends_other), len(depends_other_too), 0)
def check(self, upload):
if upload.changes.source is None:
if "source" in upload.changes.architectures:
raise Reject("{}: Architecture field includes source, but no source package is included in the upload".format(upload.changes.filename))
return True
if "source" not in upload.changes.architectures:
raise Reject("{}: Architecture field does not include source, but a source package is included in the upload".format(upload.changes.filename))
changes = upload.changes.changes
source = upload.changes.source
control = source.dsc
dsc_fn = source._dsc_file.filename
check_fields_for_valid_utf8(dsc_fn, control)
# check fields
if not re_field_package.match(control['Source']):
raise Reject('{0}: Invalid Source field'.format(dsc_fn))
if control['Source'] != changes['Source']:
raise Reject('{0}: Source field does not match Source field in changes'.format(dsc_fn))
if control['Version'] != changes['Version']:
raise Reject('{0}: Version field does not match Version field in changes'.format(dsc_fn))
# check filenames
self.check_filename(control, dsc_fn, re_file_dsc)
for f in source.files.itervalues():
self.check_filename(control, f.filename, re_file_source)
# check dependency field syntax
for field in ('Build-Conflicts', 'Build-Conflicts-Indep', 'Build-Depends', 'Build-Depends-Arch', 'Build-Depends-Indep'):
value = control.get(field)
if value is not None:
if value.strip() == '':
raise Reject('{0}: empty {1} field'.format(dsc_fn, field))
try:
apt_pkg.parse_src_depends(value)
except Exception as e:
raise Reject('{0}: APT could not parse {1} field: {2}'.format(dsc_fn, field, e))
rejects = utils.check_dsc_files(dsc_fn, control, source.files.keys())
if len(rejects) > 0:
raise Reject("\n".join(rejects))
return True
def open(self, file):
"""Open the package."""
depends_tags = ["Build-Depends", "Build-Depends-Indep"]
conflicts_tags = ["Build-Conflicts", "Build-Conflicts-Indep"]
fobj = open(file)
tagfile = apt_pkg.TagFile(fobj)
try:
for sec in tagfile:
# we only care about the stanza with the "Format:" tag, the
# rest is gpg signature noise. we should probably have
# bindings for apts OpenMaybeClearsignedFile()
if "Format" not in sec:
continue
for tag in depends_tags:
if tag not in sec:
continue
self._depends.extend(apt_pkg.parse_src_depends(sec[tag]))
for tag in conflicts_tags:
if tag not in sec:
continue
self._conflicts.extend(apt_pkg.parse_src_depends(sec[tag]))
if 'Source' in sec:
self.pkgname = sec['Source']
if 'Binary' in sec:
self.binaries = [b.strip() for b in
sec['Binary'].split(',')]
for tag in sec.keys():
if tag in sec:
self._sections[tag] = sec[tag]
finally:
del tagfile
fobj.close()
s = _("Install Build-Dependencies for "
"source package '%s' that builds %s\n") % (self.pkgname,
" ".join(self.binaries))
self._sections["Description"] = s
self._check_was_run = False
def _parseSource(self, section):
"""Parse a section from a Sources file."""
src = section["Package"]
ver = section["Version"]
# If we have already seen an equal or newer version of this source,
# then skip this section.
if src in self.sources:
last_ver = self.sources[src]["Version"]
if apt_pkg.version_compare(last_ver, ver) >= 0:
return
self.sources[src] = {}
self.sources[src]["Maintainer"] = \
unicode(section.get("Maintainer", ""), "utf8", "replace")
self.sources[src]["Version"] = ver
for field in "Build-Depends", "Build-Depends-Indep":
value = section.get(field, "")
self.sources[src][field] = apt_pkg.parse_src_depends(value)
binaries = apt_pkg.parse_depends(section.get("Binary", src))
self.sources[src]["Binaries"] = [ b[0][0] for b in binaries ]
def check_binary(self, upload, binary):
fn = binary.hashed_file.filename
control = binary.control
for field in ('Package', 'Architecture', 'Version', 'Description'):
if field not in control:
raise Reject('{0}: Missing mandatory field {0}.'.format(
fn, field))
check_fields_for_valid_utf8(fn, control)
# check fields
package = control['Package']
if not re_field_package.match(package):
raise Reject('{0}: Invalid Package field'.format(fn))
version = control['Version']
version_match = re_field_version.match(version)
if not version_match:
raise Reject('{0}: Invalid Version field'.format(fn))
version_without_epoch = version_match.group('without_epoch')
architecture = control['Architecture']
if architecture not in upload.changes.architectures:
raise Reject(
'{0}: Architecture not in Architecture field in changes file'.
format(fn))
if architecture == 'source':
raise Reject(
'{0}: Architecture "source" invalid for binary packages'.
format(fn))
source = control.get('Source')
if source is not None and not re_field_source.match(source):
raise Reject('{0}: Invalid Source field'.format(fn))
# check filename
match = re_file_binary.match(fn)
if package != match.group('package'):
raise Reject(
'{0}: filename does not match Package field'.format(fn))
if version_without_epoch != match.group('version'):
raise Reject(
'{0}: filename does not match Version field'.format(fn))
if architecture != match.group('architecture'):
raise Reject(
'{0}: filename does not match Architecture field'.format(fn))
# check dependency field syntax
for field in ('Breaks', 'Conflicts', 'Depends', 'Enhances',
'Pre-Depends', 'Provides', 'Recommends', 'Replaces',
'Suggests'):
value = control.get(field)
if value is not None:
if value.strip() == '':
raise Reject('{0}: empty {1} field'.format(fn, field))
try:
apt_pkg.parse_depends(value)
except:
raise Reject('{0}: APT could not parse {1} field'.format(
fn, field))
for field in ('Built-Using', ):
value = control.get(field)
if value is not None:
if value.strip() == '':
raise Reject('{0}: empty {1} field'.format(fn, field))
try:
apt_pkg.parse_src_depends(value)
except:
raise Reject('{0}: APT could not parse {1} field'.format(
fn, field))
def check_reverse_depends(removals, suite, arches=None, session=None, cruft=False, quiet=False):
dbsuite = get_suite(suite, session)
overridesuite = dbsuite
if dbsuite.overridesuite is not None:
overridesuite = get_suite(dbsuite.overridesuite, session)
dep_problem = 0
p2c = {}
all_broken = defaultdict(lambda: defaultdict(set))
if arches:
all_arches = set(arches)
else:
all_arches = set(x.arch_string for x in get_suite_architectures(suite))
all_arches -= set(["source", "all"])
removal_set = set(removals)
metakey_d = get_or_set_metadatakey("Depends", session)
metakey_p = get_or_set_metadatakey("Provides", session)
params = {
'suite_id': dbsuite.suite_id,
'metakey_d_id': metakey_d.key_id,
'metakey_p_id': metakey_p.key_id,
}
for architecture in all_arches | set(['all']):
deps = {}
sources = {}
virtual_packages = {}
params['arch_id'] = get_architecture(architecture, session).arch_id
statement = '''
SELECT b.package, s.source, c.name as component,
(SELECT bmd.value FROM binaries_metadata bmd WHERE bmd.bin_id = b.id AND bmd.key_id = :metakey_d_id) AS depends,
(SELECT bmp.value FROM binaries_metadata bmp WHERE bmp.bin_id = b.id AND bmp.key_id = :metakey_p_id) AS provides
FROM binaries b
JOIN bin_associations ba ON b.id = ba.bin AND ba.suite = :suite_id
JOIN source s ON b.source = s.id
JOIN files_archive_map af ON b.file = af.file_id
JOIN component c ON af.component_id = c.id
WHERE b.architecture = :arch_id'''
query = session.query('package', 'source', 'component', 'depends', 'provides'). \
from_statement(statement).params(params)
for package, source, component, depends, provides in query:
sources[package] = source
p2c[package] = component
if depends is not None:
deps[package] = depends
# Maintain a counter for each virtual package. If a
# Provides: exists, set the counter to 0 and count all
# provides by a package not in the list for removal.
# If the counter stays 0 at the end, we know that only
# the to-be-removed packages provided this virtual
# package.
if provides is not None:
for virtual_pkg in provides.split(","):
virtual_pkg = virtual_pkg.strip()
if virtual_pkg == package: continue
if not virtual_packages.has_key(virtual_pkg):
virtual_packages[virtual_pkg] = 0
if package not in removals:
virtual_packages[virtual_pkg] += 1
# If a virtual package is only provided by the to-be-removed
# packages, treat the virtual package as to-be-removed too.
removal_set.update(virtual_pkg for virtual_pkg in virtual_packages if not virtual_packages[virtual_pkg])
# Check binary dependencies (Depends)
for package in deps:
if package in removals: continue
try:
parsed_dep = apt_pkg.parse_depends(deps[package])
except ValueError as e:
print "Error for package %s: %s" % (package, e)
parsed_dep = []
for dep in parsed_dep:
# Check for partial breakage. If a package has a ORed
# dependency, there is only a dependency problem if all
# packages in the ORed depends will be removed.
unsat = 0
for dep_package, _, _ in dep:
if dep_package in removals:
unsat += 1
if unsat == len(dep):
component = p2c[package]
source = sources[package]
if component != "main":
source = "%s/%s" % (source, component)
all_broken[source][package].add(architecture)
dep_problem = 1
if all_broken and not quiet:
if cruft:
print " - broken Depends:"
else:
print "# Broken Depends:"
for source, bindict in sorted(all_broken.items()):
lines = []
for binary, arches in sorted(bindict.items()):
if arches == all_arches or 'all' in arches:
lines.append(binary)
else:
lines.append('%s [%s]' % (binary, ' '.join(sorted(arches))))
if cruft:
print ' %s: %s' % (source, lines[0])
else:
print '%s: %s' % (source, lines[0])
for line in lines[1:]:
if cruft:
print ' ' + ' ' * (len(source) + 2) + line
else:
print ' ' * (len(source) + 2) + line
if not cruft:
print
# Check source dependencies (Build-Depends and Build-Depends-Indep)
all_broken = defaultdict(set)
metakey_bd = get_or_set_metadatakey("Build-Depends", session)
metakey_bdi = get_or_set_metadatakey("Build-Depends-Indep", session)
params = {
'suite_id': dbsuite.suite_id,
'metakey_ids': (metakey_bd.key_id, metakey_bdi.key_id),
}
statement = '''
SELECT s.source, string_agg(sm.value, ', ') as build_dep
FROM source s
JOIN source_metadata sm ON s.id = sm.src_id
WHERE s.id in
(SELECT source FROM src_associations
WHERE suite = :suite_id)
AND sm.key_id in :metakey_ids
GROUP BY s.id, s.source'''
query = session.query('source', 'build_dep').from_statement(statement). \
params(params)
for source, build_dep in query:
if source in removals: continue
parsed_dep = []
if build_dep is not None:
# Remove [arch] information since we want to see breakage on all arches
build_dep = re_build_dep_arch.sub("", build_dep)
try:
parsed_dep = apt_pkg.parse_src_depends(build_dep)
except ValueError as e:
print "Error for source %s: %s" % (source, e)
for dep in parsed_dep:
unsat = 0
for dep_package, _, _ in dep:
if dep_package in removals:
unsat += 1
if unsat == len(dep):
component, = session.query(Component.component_name) \
.join(Component.overrides) \
.filter(Override.suite == overridesuite) \
.filter(Override.package == re.sub('/(contrib|non-free)$', '', source)) \
.join(Override.overridetype).filter(OverrideType.overridetype == 'dsc') \
.first()
key = source
if component != "main":
key = "%s/%s" % (source, component)
all_broken[key].add(pp_deps(dep))
dep_problem = 1
if all_broken and not quiet:
if cruft:
print " - broken Build-Depends:"
else:
print "# Broken Build-Depends:"
for source, bdeps in sorted(all_broken.items()):
bdeps = sorted(bdeps)
if cruft:
print ' %s: %s' % (source, bdeps[0])
else:
print '%s: %s' % (source, bdeps[0])
for bdep in bdeps[1:]:
if cruft:
print ' ' + ' ' * (len(source) + 2) + bdep
else:
print ' ' * (len(source) + 2) + bdep
if not cruft:
print
return dep_problem
def verify(self):
"""Verify the uploaded .dsc file.
This method is an error generator, i.e, it returns an iterator over
all exceptions that are generated while processing DSC file checks.
"""
for error in SourceUploadFile.verify(self):
yield error
# Check size and checksum of the DSC file itself
try:
self.checkSizeAndCheckSum()
except UploadError as error:
yield error
try:
raw_files = parse_and_merge_file_lists(self._dict, changes=False)
except UploadError as e:
yield e
return
files = []
for attr in raw_files:
filename, hashes, size = attr
if not re_issource.match(filename):
# DSC files only really hold on references to source
# files; they are essentially a description of a source
# package. Anything else is crack.
yield UploadError("%s: File %s does not look sourceful." %
(self.filename, filename))
continue
filepath = os.path.join(self.dirname, filename)
try:
file_instance = DSCUploadedFile(filepath, hashes, size,
self.policy, self.logger)
except UploadError as error:
yield error
else:
files.append(file_instance)
self.files = files
if not re_valid_pkg_name.match(self.source):
yield UploadError("%s: invalid source name %s" %
(self.filename, self.source))
if not re_valid_version.match(self.dsc_version):
yield UploadError("%s: invalid version %s" %
(self.filename, self.dsc_version))
if not self.policy.distroseries.isSourcePackageFormatPermitted(
self.format):
yield UploadError(
"%s: format '%s' is not permitted in %s." %
(self.filename, self.format, self.policy.distroseries.name))
# Validate the build dependencies
for field_name in ['Build-Depends', 'Build-Depends-Indep']:
field = self._dict.get(field_name, None)
if field is not None:
if field.startswith("ARRAY"):
yield UploadError(
"%s: invalid %s field produced by a broken version "
"of dpkg-dev (1.10.11)" % (self.filename, field_name))
try:
apt_pkg.parse_src_depends(field)
except (SystemExit, KeyboardInterrupt):
raise
except Exception as error:
# Swallow everything apt_pkg throws at us because
# it is not desperately pythonic and can raise odd
# or confusing exceptions at times and is out of
# our control.
yield UploadError(
"%s: invalid %s field; cannot be parsed by apt: %s" %
(self.filename, field_name, error))
# Verify if version declared in changesfile is the same than that
# in DSC (including epochs).
if self.dsc_version != self.version:
yield UploadError(
"%s: version ('%s') in .dsc does not match version "
"('%s') in .changes." %
(self.filename, self.dsc_version, self.version))
for error in self.checkFiles():
yield error
def verify(self):
"""Verify the uploaded .dsc file.
This method is an error generator, i.e, it returns an iterator over
all exceptions that are generated while processing DSC file checks.
"""
for error in SourceUploadFile.verify(self):
yield error
# Check size and checksum of the DSC file itself
try:
self.checkSizeAndCheckSum()
except UploadError as error:
yield error
try:
raw_files = parse_and_merge_file_lists(self._dict, changes=False)
except UploadError as e:
yield e
return
files = []
for attr in raw_files:
filename, hashes, size = attr
if not re_issource.match(filename):
# DSC files only really hold on references to source
# files; they are essentially a description of a source
# package. Anything else is crack.
yield UploadError("%s: File %s does not look sourceful." % (
self.filename, filename))
continue
filepath = os.path.join(self.dirname, filename)
try:
file_instance = DSCUploadedFile(
filepath, hashes, size, self.policy, self.logger)
except UploadError as error:
yield error
else:
files.append(file_instance)
self.files = files
if not re_valid_pkg_name.match(self.source):
yield UploadError(
"%s: invalid source name %s" % (self.filename, self.source))
if not re_valid_version.match(self.dsc_version):
yield UploadError(
"%s: invalid version %s" % (self.filename, self.dsc_version))
if not self.policy.distroseries.isSourcePackageFormatPermitted(
self.format):
yield UploadError(
"%s: format '%s' is not permitted in %s." %
(self.filename, self.format, self.policy.distroseries.name))
# Validate the build dependencies
for field_name in ['Build-Depends', 'Build-Depends-Indep']:
field = self._dict.get(field_name, None)
if field is not None:
if field.startswith("ARRAY"):
yield UploadError(
"%s: invalid %s field produced by a broken version "
"of dpkg-dev (1.10.11)" % (self.filename, field_name))
try:
apt_pkg.parse_src_depends(field)
except (SystemExit, KeyboardInterrupt):
raise
except Exception as error:
# Swallow everything apt_pkg throws at us because
# it is not desperately pythonic and can raise odd
# or confusing exceptions at times and is out of
# our control.
yield UploadError(
"%s: invalid %s field; cannot be parsed by apt: %s"
% (self.filename, field_name, error))
# Verify if version declared in changesfile is the same than that
# in DSC (including epochs).
if self.dsc_version != self.version:
yield UploadError(
"%s: version ('%s') in .dsc does not match version "
"('%s') in .changes."
% (self.filename, self.dsc_version, self.version))
for error in self.checkFiles():
yield error
def check_binary(self, upload, binary):
fn = binary.hashed_file.filename
control = binary.control
for field in ('Package', 'Architecture', 'Version', 'Description', 'Section'):
if field not in control:
raise Reject('{0}: Missing mandatory field {0}.'.format(fn, field))
check_fields_for_valid_utf8(fn, control)
# check fields
package = control['Package']
if not re_field_package.match(package):
raise Reject('{0}: Invalid Package field'.format(fn))
version = control['Version']
version_match = re_field_version.match(version)
if not version_match:
raise Reject('{0}: Invalid Version field'.format(fn))
version_without_epoch = version_match.group('without_epoch')
architecture = control['Architecture']
if architecture not in upload.changes.architectures:
raise Reject('{0}: Architecture not in Architecture field in changes file'.format(fn))
if architecture == 'source':
raise Reject('{0}: Architecture "source" invalid for binary packages'.format(fn))
source = control.get('Source')
if source is not None and not re_field_source.match(source):
raise Reject('{0}: Invalid Source field'.format(fn))
# check filename
match = re_file_binary.match(fn)
if package != match.group('package'):
raise Reject('{0}: filename does not match Package field'.format(fn))
if version_without_epoch != match.group('version'):
raise Reject('{0}: filename does not match Version field'.format(fn))
if architecture != match.group('architecture'):
raise Reject('{0}: filename does not match Architecture field'.format(fn))
# check dependency field syntax
for field in ('Breaks', 'Conflicts', 'Depends', 'Enhances', 'Pre-Depends',
'Provides', 'Recommends', 'Replaces', 'Suggests'):
value = control.get(field)
if value is not None:
if value.strip() == '':
raise Reject('{0}: empty {1} field'.format(fn, field))
try:
apt_pkg.parse_depends(value)
except:
raise Reject('{0}: APT could not parse {1} field'.format(fn, field))
for field in ('Built-Using',):
value = control.get(field)
if value is not None:
if value.strip() == '':
raise Reject('{0}: empty {1} field'.format(fn, field))
try:
apt_pkg.parse_src_depends(value)
except:
raise Reject('{0}: APT could not parse {1} field'.format(fn, field))
# "Multi-Arch: no" breaks wanna-build, #768353
multi_arch = control.get("Multi-Arch")
if multi_arch == 'no':
raise Reject('{0}: Multi-Arch: no support in Debian is broken (#768353)'.format(fn))
def check_reverse_depends(removals, suite, arches=None, session=None, cruft=False, quiet=False, include_arch_all=True):
dbsuite = get_suite(suite, session)
overridesuite = dbsuite
if dbsuite.overridesuite is not None:
overridesuite = get_suite(dbsuite.overridesuite, session)
dep_problem = 0
p2c = {}
all_broken = defaultdict(lambda: defaultdict(set))
if arches:
all_arches = set(arches)
else:
all_arches = set(x.arch_string for x in get_suite_architectures(suite))
all_arches -= set(["source", "all"])
removal_set = set(removals)
metakey_d = get_or_set_metadatakey("Depends", session)
metakey_p = get_or_set_metadatakey("Provides", session)
params = {
'suite_id': dbsuite.suite_id,
'metakey_d_id': metakey_d.key_id,
'metakey_p_id': metakey_p.key_id,
}
if include_arch_all:
rdep_architectures = all_arches | set(['all'])
else:
rdep_architectures = all_arches
for architecture in rdep_architectures:
deps = {}
sources = {}
virtual_packages = {}
try:
params['arch_id'] = get_architecture(architecture, session).arch_id
except AttributeError:
continue
statement = sql.text('''
SELECT b.package, s.source, c.name as component,
(SELECT bmd.value FROM binaries_metadata bmd WHERE bmd.bin_id = b.id AND bmd.key_id = :metakey_d_id) AS depends,
(SELECT bmp.value FROM binaries_metadata bmp WHERE bmp.bin_id = b.id AND bmp.key_id = :metakey_p_id) AS provides
FROM binaries b
JOIN bin_associations ba ON b.id = ba.bin AND ba.suite = :suite_id
JOIN source s ON b.source = s.id
JOIN files_archive_map af ON b.file = af.file_id
JOIN component c ON af.component_id = c.id
WHERE b.architecture = :arch_id''')
query = session.query('package', 'source', 'component', 'depends', 'provides'). \
from_statement(statement).params(params)
for package, source, component, depends, provides in query:
sources[package] = source
p2c[package] = component
if depends is not None:
deps[package] = depends
# Maintain a counter for each virtual package. If a
# Provides: exists, set the counter to 0 and count all
# provides by a package not in the list for removal.
# If the counter stays 0 at the end, we know that only
# the to-be-removed packages provided this virtual
# package.
if provides is not None:
for virtual_pkg in provides.split(","):
virtual_pkg = virtual_pkg.strip()
if virtual_pkg == package: continue
if virtual_pkg not in virtual_packages:
virtual_packages[virtual_pkg] = 0
if package not in removals:
virtual_packages[virtual_pkg] += 1
# If a virtual package is only provided by the to-be-removed
# packages, treat the virtual package as to-be-removed too.
removal_set.update(virtual_pkg for virtual_pkg in virtual_packages if not virtual_packages[virtual_pkg])
# Check binary dependencies (Depends)
for package in deps:
if package in removals: continue
try:
parsed_dep = apt_pkg.parse_depends(deps[package])
except ValueError as e:
print "Error for package %s: %s" % (package, e)
parsed_dep = []
for dep in parsed_dep:
# Check for partial breakage. If a package has a ORed
# dependency, there is only a dependency problem if all
# packages in the ORed depends will be removed.
unsat = 0
for dep_package, _, _ in dep:
if dep_package in removals:
unsat += 1
if unsat == len(dep):
component = p2c[package]
source = sources[package]
if component != "main":
source = "%s/%s" % (source, component)
all_broken[source][package].add(architecture)
dep_problem = 1
if all_broken and not quiet:
if cruft:
print " - broken Depends:"
else:
print "# Broken Depends:"
for source, bindict in sorted(all_broken.items()):
lines = []
for binary, arches in sorted(bindict.items()):
if arches == all_arches or 'all' in arches:
lines.append(binary)
else:
lines.append('%s [%s]' % (binary, ' '.join(sorted(arches))))
if cruft:
print ' %s: %s' % (source, lines[0])
else:
print '%s: %s' % (source, lines[0])
for line in lines[1:]:
if cruft:
print ' ' + ' ' * (len(source) + 2) + line
else:
print ' ' * (len(source) + 2) + line
if not cruft:
print
# Check source dependencies (Build-Depends and Build-Depends-Indep)
all_broken = defaultdict(set)
metakey_bd = get_or_set_metadatakey("Build-Depends", session)
metakey_bdi = get_or_set_metadatakey("Build-Depends-Indep", session)
if include_arch_all:
metakey_ids = (metakey_bd.key_id, metakey_bdi.key_id)
else:
metakey_ids = (metakey_bd.key_id,)
params = {
'suite_id': dbsuite.suite_id,
'metakey_ids': metakey_ids,
}
statement = sql.text('''
SELECT s.source, string_agg(sm.value, ', ') as build_dep
FROM source s
JOIN source_metadata sm ON s.id = sm.src_id
WHERE s.id in
(SELECT src FROM newest_src_association
WHERE suite = :suite_id)
AND sm.key_id in :metakey_ids
GROUP BY s.id, s.source''')
query = session.query('source', 'build_dep').from_statement(statement). \
params(params)
for source, build_dep in query:
if source in removals: continue
parsed_dep = []
if build_dep is not None:
# Remove [arch] information since we want to see breakage on all arches
build_dep = re_build_dep_arch.sub("", build_dep)
try:
parsed_dep = apt_pkg.parse_src_depends(build_dep)
except ValueError as e:
print "Error for source %s: %s" % (source, e)
for dep in parsed_dep:
unsat = 0
for dep_package, _, _ in dep:
if dep_package in removals:
unsat += 1
if unsat == len(dep):
component, = session.query(Component.component_name) \
.join(Component.overrides) \
.filter(Override.suite == overridesuite) \
.filter(Override.package == re.sub('/(contrib|non-free)$', '', source)) \
.join(Override.overridetype).filter(OverrideType.overridetype == 'dsc') \
.first()
key = source
if component != "main":
key = "%s/%s" % (source, component)
all_broken[key].add(pp_deps(dep))
dep_problem = 1
if all_broken and not quiet:
if cruft:
print " - broken Build-Depends:"
else:
print "# Broken Build-Depends:"
for source, bdeps in sorted(all_broken.items()):
bdeps = sorted(bdeps)
if cruft:
print ' %s: %s' % (source, bdeps[0])
else:
print '%s: %s' % (source, bdeps[0])
for bdep in bdeps[1:]:
if cruft:
print ' ' + ' ' * (len(source) + 2) + bdep
else:
print ' ' * (len(source) + 2) + bdep
if not cruft:
print
return dep_problem
def _load_package_information(session, suite_id, suite_archs2id):
package_dependencies = defaultdict(lambda: defaultdict(set))
arch_providers_of = defaultdict(lambda: defaultdict(set))
arch_provided_by = defaultdict(lambda: defaultdict(set))
source_deps = defaultdict(set)
metakey_d = get_or_set_metadatakey("Depends", session)
metakey_p = get_or_set_metadatakey("Provides", session)
params = {
'suite_id': suite_id,
'arch_all_id': suite_archs2id['all'],
'metakey_d_id': metakey_d.key_id,
'metakey_p_id': metakey_p.key_id,
}
all_arches = set(suite_archs2id)
all_arches.discard('source')
package_dependencies['source'] = source_deps
for architecture in all_arches:
deps = defaultdict(set)
providers_of = defaultdict(set)
provided_by = defaultdict(set)
arch_providers_of[architecture] = providers_of
arch_provided_by[architecture] = provided_by
package_dependencies[architecture] = deps
params['arch_id'] = suite_archs2id[architecture]
statement = '''
SELECT b.package,
(SELECT bmd.value FROM binaries_metadata bmd WHERE bmd.bin_id = b.id AND bmd.key_id = :metakey_d_id) AS depends,
(SELECT bmp.value FROM binaries_metadata bmp WHERE bmp.bin_id = b.id AND bmp.key_id = :metakey_p_id) AS provides
FROM binaries b
JOIN bin_associations ba ON b.id = ba.bin AND ba.suite = :suite_id
WHERE b.architecture = :arch_id OR b.architecture = :arch_all_id'''
query = session.query('package', 'depends', 'provides'). \
from_statement(statement).params(params)
for package, depends, provides in query:
if depends is not None:
try:
parsed_dep = []
for dep in apt_pkg.parse_depends(depends):
parsed_dep.append(frozenset(d[0] for d in dep))
deps[package].update(parsed_dep)
except ValueError as e:
print "Error for package %s: %s" % (package, e)
# Maintain a counter for each virtual package. If a
# Provides: exists, set the counter to 0 and count all
# provides by a package not in the list for removal.
# If the counter stays 0 at the end, we know that only
# the to-be-removed packages provided this virtual
# package.
if provides is not None:
for virtual_pkg in provides.split(","):
virtual_pkg = virtual_pkg.strip()
if virtual_pkg == package:
continue
provided_by[virtual_pkg].add(package)
providers_of[package].add(virtual_pkg)
# Check source dependencies (Build-Depends and Build-Depends-Indep)
metakey_bd = get_or_set_metadatakey("Build-Depends", session)
metakey_bdi = get_or_set_metadatakey("Build-Depends-Indep", session)
params = {
'suite_id': suite_id,
'metakey_ids': (metakey_bd.key_id, metakey_bdi.key_id),
}
statement = '''
SELECT s.source, string_agg(sm.value, ', ') as build_dep
FROM source s
JOIN source_metadata sm ON s.id = sm.src_id
WHERE s.id in
(SELECT source FROM src_associations
WHERE suite = :suite_id)
AND sm.key_id in :metakey_ids
GROUP BY s.id, s.source'''
query = session.query('source', 'build_dep').from_statement(statement). \
params(params)
for source, build_dep in query:
if build_dep is not None:
# Remove [arch] information since we want to see breakage on all arches
build_dep = re_build_dep_arch.sub("", build_dep)
try:
parsed_dep = []
for dep in apt_pkg.parse_src_depends(build_dep):
parsed_dep.append(frozenset(d[0] for d in dep))
source_deps[source].update(parsed_dep)
except ValueError as e:
print "Error for package %s: %s" % (source, e)
return package_dependencies, arch_providers_of, arch_provided_by
def _load_package_information(session, suite_id, suite_archs2id):
package_dependencies = defaultdict(lambda: defaultdict(set))
arch_providers_of = defaultdict(lambda: defaultdict(set))
arch_provided_by = defaultdict(lambda: defaultdict(set))
source_deps = defaultdict(set)
metakey_d = get_or_set_metadatakey("Depends", session)
metakey_p = get_or_set_metadatakey("Provides", session)
params = {
'suite_id': suite_id,
'arch_all_id': suite_archs2id['all'],
'metakey_d_id': metakey_d.key_id,
'metakey_p_id': metakey_p.key_id,
}
all_arches = set(suite_archs2id)
all_arches.discard('source')
package_dependencies['source'] = source_deps
for architecture in all_arches:
deps = defaultdict(set)
providers_of = defaultdict(set)
provided_by = defaultdict(set)
arch_providers_of[architecture] = providers_of
arch_provided_by[architecture] = provided_by
package_dependencies[architecture] = deps
params['arch_id'] = suite_archs2id[architecture]
statement = '''
SELECT b.package,
(SELECT bmd.value FROM binaries_metadata bmd WHERE bmd.bin_id = b.id AND bmd.key_id = :metakey_d_id) AS depends,
(SELECT bmp.value FROM binaries_metadata bmp WHERE bmp.bin_id = b.id AND bmp.key_id = :metakey_p_id) AS provides
FROM binaries b
JOIN bin_associations ba ON b.id = ba.bin AND ba.suite = :suite_id
WHERE b.architecture = :arch_id OR b.architecture = :arch_all_id'''
query = session.query('package', 'depends', 'provides'). \
from_statement(statement).params(params)
for package, depends, provides in query:
if depends is not None:
try:
parsed_dep = []
for dep in apt_pkg.parse_depends(depends):
parsed_dep.append(frozenset(d[0] for d in dep))
deps[package].update(parsed_dep)
except ValueError as e:
print "Error for package %s: %s" % (package, e)
# Maintain a counter for each virtual package. If a
# Provides: exists, set the counter to 0 and count all
# provides by a package not in the list for removal.
# If the counter stays 0 at the end, we know that only
# the to-be-removed packages provided this virtual
# package.
if provides is not None:
for virtual_pkg in provides.split(","):
virtual_pkg = virtual_pkg.strip()
if virtual_pkg == package:
continue
provided_by[virtual_pkg].add(package)
providers_of[package].add(virtual_pkg)
# Check source dependencies (Build-Depends and Build-Depends-Indep)
metakey_bd = get_or_set_metadatakey("Build-Depends", session)
metakey_bdi = get_or_set_metadatakey("Build-Depends-Indep", session)
params = {
'suite_id': suite_id,
'metakey_ids': (metakey_bd.key_id, metakey_bdi.key_id),
}
statement = '''
SELECT s.source, string_agg(sm.value, ', ') as build_dep
FROM source s
JOIN source_metadata sm ON s.id = sm.src_id
WHERE s.id in
(SELECT source FROM src_associations
WHERE suite = :suite_id)
AND sm.key_id in :metakey_ids
GROUP BY s.id, s.source'''
query = session.query('source', 'build_dep').from_statement(statement). \
params(params)
for source, build_dep in query:
if build_dep is not None:
# Remove [arch] information since we want to see breakage on all arches
build_dep = re_build_dep_arch.sub("", build_dep)
try:
parsed_dep = []
for dep in apt_pkg.parse_src_depends(build_dep):
parsed_dep.append(frozenset(d[0] for d in dep))
source_deps[source].update(parsed_dep)
except ValueError as e:
print "Error for package %s: %s" % (source, e)
return package_dependencies, arch_providers_of, arch_provided_by
