def update_control():
"""Update the debian control file.
"""
from debian.deb822 import Deb822, PkgRelation
f = open('debian/control', 'r')
iter = Deb822.iter_paragraphs(f)
source = iter.next()
def update_deps(control, field, package, min_version):
bdi = PkgRelation.parse_relations(control[field])
update_relation(bdi, package, ">=", "%d.%d.%d~" % min_version)
control[field] = PkgRelation.str(bdi)
talloc_version = find_version(os.path.join(tree, "lib/talloc/wscript"))
eq_config = LibraryEquivalents('debian/library-equivalents')
min_talloc_version = eq_config.find_oldest_compatible("talloc", talloc_version)
update_deps(source, "Build-Depends", "libtalloc-dev", min_talloc_version)
update_deps(source, "Build-Depends", "python-talloc-dev", min_talloc_version)
o = open("debian/control", "w+")
source.dump(o)
for binary in iter:
o.write("\n")
binary.dump(o)
o.close()
def handle(self, *args, **options):
if not args:
raise CommandError("Require a tag, e.g. sid, wheezy, squeeze, etc...")
tag = args[0]
filenames = args[1:]
if not tag.isalpha() or not tag.islower():
raise CommandError("First argument must be tag, e.g. sid, wheezy, squeeze, etc...")
self.session = db.get_db_session()
# self.session.bind.echo=False
self.descr_text_map = {}
self.descr_map = {}
self.stats = defaultdict(int)
for file in filenames:
self.stderr.write('Processing %s\n' % file)
f = self._open(file)
for para in Deb822.iter_paragraphs(f):
if 'Description-en' in para:
self._handle_translation_en(para)
if 'Version' in para:
try:
self._handle_packages(tag, para)
except Exception, e:
self.stdout.write("Problem processing %r\n%s\n" % (para, e))
self.session.commit()
def update_control():
"""Update the debian control file.
"""
from debian.deb822 import Deb822, PkgRelation
f = open('debian/control', 'rb')
iter = Deb822.iter_paragraphs(f.readlines())
for i in iter:
source = i
break
def update_deps(control, field, package, min_version, epoch=None):
bdi = PkgRelation.parse_relations(control[field])
if epoch is not None:
epoch = "%d:" % epoch
else:
epoch = ""
update_relation(bdi, package, ">=", epoch + "%d.%d.%d~" % min_version)
control[field] = PkgRelation.str(bdi)
tdb_version = find_version(os.path.join(tree, "lib/tdb/wscript"))
talloc_version = find_version(os.path.join(tree, "lib/talloc/wscript"))
ldb_version = find_version(os.path.join(tree, "lib/ldb/wscript"))
tevent_version = find_version(os.path.join(tree, "lib/tevent/wscript"))
eq_config = LibraryEquivalents('debian/library-equivalents')
min_tdb_version = eq_config.find_oldest_compatible("tdb", tdb_version)
min_talloc_version = eq_config.find_oldest_compatible("talloc", talloc_version)
min_ldb_version = eq_config.find_oldest_compatible("ldb", ldb_version)
min_tevent_version = eq_config.find_oldest_compatible("tevent", tevent_version)
update_deps(source, "Build-Depends", "libtdb-dev", min_tdb_version)
update_deps(source, "Build-Depends", "python-tdb", min_tdb_version)
update_deps(source, "Build-Depends", "libtalloc-dev", min_talloc_version)
update_deps(source, "Build-Depends", "python-talloc-dev", min_talloc_version)
update_deps(source, "Build-Depends", "libldb-dev", min_ldb_version, 1)
update_deps(source, "Build-Depends", "python-ldb-dev", min_ldb_version, 1)
update_deps(source, "Build-Depends", "python-ldb", min_ldb_version, 1)
update_deps(source, "Build-Depends", "libtevent-dev", min_tevent_version)
with open("debian/control", "wb+") as o:
for i in source.keys():
if i == "Build-Depends":
value=",\n ".join(source[i].split(', '))
else:
value=source[i]
o.write(("%s: %s\n" % (i, value)).encode('UTF-8'))
for binary in iter:
o.write(b"\n")
binary.dump(o)
def update_control():
"""Update the debian control file.
"""
from debian.deb822 import Deb822, PkgRelation
f = open('debian/control', 'r')
iter = Deb822.iter_paragraphs(f)
source = iter.next()
def update_deps(control, field, package, min_version, epoch=None):
bdi = PkgRelation.parse_relations(control[field])
if epoch is not None:
epoch = "%d:" % epoch
else:
epoch = ""
update_relation(bdi, package, ">=", epoch + "%d.%d.%d~" % min_version)
control[field] = PkgRelation.str(bdi)
tdb_version = find_version(os.path.join(tree, "lib/tdb/wscript"))
talloc_version = find_version(os.path.join(tree, "lib/talloc/wscript"))
ldb_version = find_version(os.path.join(tree, "lib/ldb/wscript"))
tevent_version = find_version(os.path.join(tree, "lib/tevent/wscript"))
eq_config = LibraryEquivalents('debian/library-equivalents')
min_tdb_version = eq_config.find_oldest_compatible("tdb", tdb_version)
min_talloc_version = eq_config.find_oldest_compatible("talloc", talloc_version)
min_ldb_version = eq_config.find_oldest_compatible("ldb", ldb_version)
min_tevent_version = eq_config.find_oldest_compatible("tevent", tevent_version)
update_deps(source, "Build-Depends", "libtdb-dev", min_tdb_version)
update_deps(source, "Build-Depends", "python-tdb", min_tdb_version)
update_deps(source, "Build-Depends", "libtalloc-dev", min_talloc_version)
update_deps(source, "Build-Depends", "python-talloc-dev", min_talloc_version)
update_deps(source, "Build-Depends", "libldb-dev", min_ldb_version, 1)
update_deps(source, "Build-Depends", "python-ldb-dev", min_ldb_version, 1)
update_deps(source, "Build-Depends", "python-ldb", min_ldb_version, 1)
update_deps(source, "Build-Depends", "libtevent-dev", min_tevent_version)
o = open("debian/control", "w+")
try:
source.dump(o)
for binary in iter:
o.write("\n")
binary.dump(o)
finally:
o.close()
def _get_description(name):
for package in Deb822.iter_paragraphs(
io.open('debian/control', 'r', encoding='utf-8')):
if package.get('Package') == name:
description = package['Description']
return description.split('\n .\n')[0]
os.path.join(os.path.dirname(__file__), "..", "renamed-tags.json")
)
if not os.path.isfile(path):
import pkg_resources
path = pkg_resources.resource_filename(
__name__, "lintian-brush/renamed-tags.json"
)
if not os.path.isfile(path):
# Urgh.
path = "/usr/share/lintian-brush/renamed-tags.json"
with open(path, "rb") as f:
return json.load(f)
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser()
parser.add_argument(
"--remove-unused", action="store_true", help="Remove unused overrides."
)
args = parser.parse_args()
if args.remove_unused:
from debian.deb822 import Deb822
with open("debian/control", "r") as f:
removed = remove_unused(Deb822.iter_paragraphs(f))
print("Removed %d unused overrides" % len(removed))
else:
parser.print_usage()
def __init__(self,
fn,
review_type,
peer_hooks=None,
overrides=None,
peer_hooks_link=None):
Review.__init__(self, fn, review_type, overrides=overrides)
# The cr_* scripts only support 15.04 snaps (v1). Use sr_* scripts for
# 16.04 (v2) or higher
if not self.is_click and not self.is_snap1:
return
if not self.pkg_filename.endswith(".click") and \
not self.pkg_filename.endswith(".snap"):
if self.pkg_filename.endswith(".deb"):
error("filename does not end with '.click', but '.deb' "
"instead. See http://askubuntu.com/a/485544/94326 for "
"how click packages are different.")
error("filename does not end with '.click'")
self.manifest = None
self.click_pkgname = None
self.click_version = None
self.pkg_arch = []
self.is_snap_oem = False
self.peer_hooks = peer_hooks
self.peer_hooks_link = peer_hooks_link
if self.is_snap1:
pkg_yaml = self._extract_package_yaml()
if pkg_yaml:
try:
self.pkg_yaml = yaml.safe_load(pkg_yaml)
except Exception:
error(
"Could not load package.yaml. Is it properly formatted?"
)
self._verify_package_yaml_structure()
else:
error("Could not load package.yaml.")
# default to 'app'
if 'type' not in self.pkg_yaml:
self.pkg_yaml['type'] = 'app'
if 'architectures' in self.pkg_yaml:
self.pkg_arch = self.pkg_yaml['architectures']
elif 'architecture' in self.pkg_yaml:
if isinstance(self.pkg_yaml['architecture'], str):
self.pkg_arch = [self.pkg_yaml['architecture']]
elif isinstance(self.pkg_yaml['architecture'], list):
self.pkg_arch = self.pkg_yaml['architecture']
else:
error(
"Could not load package.yaml: invalid 'architecture'")
else:
self.pkg_arch = ['all']
if 'type' in self.pkg_yaml and self.pkg_yaml['type'] == 'oem':
self.is_snap_oem = True
if self.is_click or self.is_snap1:
# Get some basic information from the control file
control_file = self._extract_control_file()
tmp = list(Deb822.iter_paragraphs(control_file))
if len(tmp) != 1:
error("malformed control file: too many paragraphs")
control = tmp[0]
self.click_pkgname = control['Package']
self.click_version = control['Version']
if self.is_click:
if control['Architecture'] not in self.pkg_arch:
self.pkg_arch.append(control['Architecture'])
self.pkgfmt["version"] = str(control['Click-Version'])
# Parse and store the manifest
manifest_json = self._extract_manifest_file()
try:
self.manifest = json.load(manifest_json)
except Exception:
error(
"Could not load manifest file. Is it properly formatted?")
self._verify_manifest_structure()
self.valid_frameworks = self._extract_click_frameworks()
def _parse_deb_archi(control='debian/control'):
return filter(None, [
p.get('Architecture').strip()
for p in Deb822.iter_paragraphs(file(control))
])
#!/usr/bin/env python3
import sys
import gzip
import requests
from debian.deb822 import Deb822
URL = 'https://archive.raspbian.org/raspbian/dists/buster/main/binary-armhf/Packages.gz'
r = requests.get(URL, stream=True)
r.raw.decode_content = True
gzip_file = gzip.GzipFile(fileobj=r.raw)
package = 'kodi'
last_known_version = '2:17.6+dfsg1-4+b1'
for p in Deb822.iter_paragraphs(gzip_file):
if p['Package'] == package:
print('Current version: %s' % p['Version'])
if p['Version'] != last_known_version:
print('Version has changed from %s to %s' % (last_known_version, p['Version']))
sys.exit(1)
break
def __init__(self,
buildable,
*,
binary_version_suffix='',
link_builds=(),
orig_dirs=('..', ),
output_dir=None,
output_parent,
vendor):
self.buildable = buildable
self._product_prefix = None
self._source_version = None
self.arch_wildcards = set()
self.archs = []
self.autopkgtest_failures = []
self.binary_packages = []
self.binary_version_suffix = binary_version_suffix
self.changes_produced = {}
self.dirname = None
self.dsc = None
self.dsc_name = None
self.indep = False
self.indep_together_with = None
self.link_builds = link_builds
self.logs = {}
self.merged_changes = OrderedDict()
self.nominal_suite = None
self.orig_dirs = orig_dirs
self.output_dir = output_dir
self.piuparts_failures = []
self.source_from_archive = False
self.source_package = None
self.source_together_with = None
self.sourceful_changes_name = None
self.suite = None
self.vendor = vendor
if os.path.exists(self.buildable):
if os.path.isdir(self.buildable):
changelog = os.path.join(self.buildable, 'debian', 'changelog')
changelog = Changelog(open(changelog))
self.source_package = changelog.get_package()
self.nominal_suite = changelog.distributions
self._source_version = Version(changelog.version)
control = os.path.join(self.buildable, 'debian', 'control')
if len(changelog.distributions.split()) != 1:
raise ArgumentError(
'Cannot build for multiple distributions at once')
for paragraph in Deb822.iter_paragraphs(open(control)):
self.arch_wildcards |= set(
paragraph.get('architecture', '').split())
binary = paragraph.get('package')
if binary is not None:
self.binary_packages.append(binary)
elif self.buildable.endswith('.changes'):
self.dirname = os.path.dirname(self.buildable) or os.curdir
self.sourceful_changes_name = self.buildable
sourceful_changes = Changes(open(self.buildable))
if 'source' not in sourceful_changes['architecture'].split():
raise ArgumentError(
'Changes file {!r} must be sourceful'.format(
self.buildable))
self.nominal_suite = sourceful_changes['distribution']
for f in sourceful_changes['files']:
if f['name'].endswith('.dsc'):
if self.dsc_name is not None:
raise ArgumentError(
'Changes file {!r} contained more than one '
'.dsc file'.format(self.buildable))
self.dsc_name = os.path.join(self.dirname, f['name'])
if self.dsc_name is None:
raise ArgumentError(
'Changes file {!r} did not contain a .dsc file'.format(
self.buildable))
self.dsc = Dsc(open(self.dsc_name))
elif self.buildable.endswith('.dsc'):
self.dirname = os.path.dirname(self.buildable) or os.curdir
self.dsc_name = self.buildable
self.dsc = Dsc(open(self.dsc_name))
else:
raise ArgumentError(
'buildable must be .changes, .dsc or directory, not '
'{!r}'.format(self.buildable))
else:
self.source_from_archive = True
if '_' in self.buildable:
source, version = self.buildable.split('_', 1)
else:
source = self.buildable
version = None
self.source_package = source
if version is not None:
self._source_version = Version(version)
if self.dsc is not None:
self.source_package = self.dsc['source']
self._source_version = Version(self.dsc['version'])
self.arch_wildcards = set(self.dsc['architecture'].split())
self.binary_packages = [
p.strip() for p in self.dsc['binary'].split(',')
]
if self._source_version is not None:
self._binary_version = Version(
str(self._source_version) + self.binary_version_suffix)
timestamp = time.strftime('%Y%m%dt%H%M%S', time.gmtime())
if self.output_dir is None:
if self._binary_version is None:
dirname = '{}_{}'.format(self.source_package, timestamp)
else:
dirname = '{}_{}_{}'.format(self.source_package,
self._binary_version, timestamp)
self.output_dir = os.path.join(output_parent, dirname)
# For convenience, create a symbolic link for the latest build of
# each source package: hello_latest -> hello_2.10-1_20170319t102623
unversioned_symlink = os.path.join(output_parent,
self.source_package + '_latest')
with suppress(FileNotFoundError):
os.unlink(unversioned_symlink)
os.symlink(dirname, unversioned_symlink)
# If we know the version, also create a symbolic link for the
# latest build of each source/version pair:
# hello_2.10-1 -> hello_2.10-1_20170319t102623
if self._binary_version is not None:
versioned_symlink = os.path.join(
output_parent, '{}_{}'.format(self.source_package,
self._binary_version))
with suppress(FileNotFoundError):
os.unlink(versioned_symlink)
os.symlink(dirname, versioned_symlink)
# It's OK if the output directory exists but is empty.
with suppress(FileNotFoundError):
os.rmdir(self.output_dir)
# Otherwise, if someone already created this, we'll just crash out.
os.mkdir(self.output_dir)
if self.dsc is not None:
abs_file = os.path.abspath(self.dsc_name)
abs_dir, base = os.path.split(abs_file)
os.symlink(abs_file, os.path.join(self.output_dir, base))
for l in self.link_builds:
symlink = os.path.join(l, base)
with suppress(FileNotFoundError):
os.unlink(symlink)
os.symlink(abs_file, symlink)
for f in self.dsc['files']:
abs_file = os.path.join(abs_dir, f['name'])
os.symlink(abs_file, os.path.join(self.output_dir, f['name']))
for l in self.link_builds:
symlink = os.path.join(l, f['name'])
with suppress(FileNotFoundError):
os.unlink(symlink)
os.symlink(abs_file, symlink)
if dists is None:
dists = ('unstable (new queue)', )
try:
page = open_url(NEWQUEUE_URL, http_proxy, timeout)
except NoNetwork:
return {}
except urllib2.HTTPError, x:
print >> sys.stderr, "Warning:", x
return {}
if not page:
return {}
versions = {}
# iter over the entries, one paragraph at a time
for para in Deb822.iter_paragraphs(page):
if para['Source'] == package:
k = para['Distribution'] + ' (' + para['Queue'] + ')'
versions[k] = para['Version']
return versions
def get_incoming_version(package, timeout, http_proxy=None, arch='i386'):
try:
page = open_url(INCOMING_URL, http_proxy, timeout)
except NoNetwork:
return None
except urllib2.HTTPError, x:
print >> sys.stderr, "Warning:", x
return None
if not page:
def __init__(self, buildable, *, vendor):
self.buildable = buildable
self._product_prefix = None
self.arch_wildcards = set()
self.archs = []
self.binary_packages = []
self.changes_produced = {}
self.dirname = None
self.dsc = None
self.dsc_name = None
self.indep = False
self.logs = {}
self.merged_changes = OrderedDict()
self.nominal_suite = None
self.source_from_archive = False
self.source_package = None
self.sourceful_changes_name = None
self.suite = None
self.together_with = None
self.vendor = vendor
self._version = None
if os.path.exists(self.buildable):
if os.path.isdir(self.buildable):
changelog = os.path.join(self.buildable, 'debian', 'changelog')
changelog = Changelog(open(changelog))
self.source_package = changelog.get_package()
self.nominal_suite = changelog.distributions
self._version = Version(changelog.version)
control = os.path.join(self.buildable, 'debian', 'control')
if len(changelog.distributions.split()) != 1:
raise ArgumentError('Cannot build for multiple '
'distributions at once')
for paragraph in Deb822.iter_paragraphs(open(control)):
self.arch_wildcards |= set(
paragraph.get('architecture', '').split())
binary = paragraph.get('package')
if binary is not None:
self.binary_packages.append(binary)
elif self.buildable.endswith('.changes'):
self.dirname = os.path.dirname(self.buildable)
self.sourceful_changes_name = self.buildable
sourceful_changes = Changes(open(self.buildable))
if 'source' not in sourceful_changes['architecture']:
raise ArgumentError('Changes file {!r} must be '
'sourceful'.format(self.buildable))
self.nominal_suite = sourceful_changes['distribution']
for f in sourceful_changes['files']:
if f['name'].endswith('.dsc'):
if self.dsc_name is not None:
raise ArgumentError('Changes file {!r} contained '
'more than one .dsc '
'file'.format(self.buildable))
self.dsc_name = os.path.join(self.dirname, f['name'])
if self.dsc_name is None:
raise ArgumentError('Changes file {!r} did not contain a '
'.dsc file'.format(self.buildable))
self.dsc = Dsc(open(self.dsc_name))
elif self.buildable.endswith('.dsc'):
self.dirname = os.path.dirname(self.buildable)
self.dsc_name = self.buildable
self.dsc = Dsc(open(self.dsc_name))
else:
raise ArgumentError('buildable must be .changes, .dsc or '
'directory, not {!r}'.format(
self.buildable))
else:
self.source_from_archive = True
if '_' in self.buildable:
source, version = self.buildable.split('_', 1)
else:
source = self.buildable
version = None
self.source_package = source
if version is not None:
self._version = Version(version)
if self.dsc is not None:
self.source_package = self.dsc['source']
self._version = Version(self.dsc['version'])
self.arch_wildcards = set(self.dsc['architecture'].split())
self.binary_packages = [
p.strip() for p in self.dsc['binary'].split(',')
]
def get_build_deps(self):
with open(self.tree.abspath("debian/control"), "r") as f:
return next(Deb822.iter_paragraphs(f)).get("Build-Depends", "")
def _remove(self, path, codename, component):
for p in Deb822.iter_paragraphs(file(path)):
if 'Source' in p:
self._removesrc(p['Source'], codename)
elif 'Package' in p:
self._removedeb(p['Package'], codename)
def removesrc(self, path, component="main"):
for p in Deb822.iter_paragraphs(file(path)):
if 'Source' in p:
self._removesrc(p['Source'], self.repo_attr.codename)
if dists is None:
dists = ('unstable (new queue)', )
try:
page = open_url(NEWQUEUE_URL, http_proxy, timeout)
except NoNetwork:
return {}
except urllib2.HTTPError, x:
print >> sys.stderr, "Warning:", x
return {}
if not page:
return {}
versions = {}
# iter over the entries, one paragraph at a time
for para in Deb822.iter_paragraphs(page):
if para['Source'] == package:
k = para['Distribution'] + ' (' + para['Queue'] + ')'
versions[k] = para['Version']
return versions
def get_incoming_version(package, timeout, http_proxy=None, arch='i386'):
try:
page = open_url(INCOMING_URL, http_proxy, timeout)
except NoNetwork:
return None
except urllib2.HTTPError, x:
print >> sys.stderr, "Warning:", x
return None
if not page:
def read(self):
with open(self.path, 'r') as f:
for repo822 in Deb822.iter_paragraphs(f, shared_storage=False):
self.repos822.append(repo822)
def removesrc(self, path, component="main"):
for p in Deb822.iter_paragraphs(file(path)):
if 'Source' in p:
self._removesrc(p['Source'], self.repo_attr.codename,
component)
def read(self):
with open(self.path, 'r') as f:
for repo822 in Deb822.iter_paragraphs(f, shared_storage=False):
self.repos822.append(repo822)
def __init__(self, buildinfo_file):
self.orig_file = buildinfo_file
self.source = None
self.architecture = None
self.binary = None
self.version = None
self.release = None
self.epoch = None
self.build_path = None
self.build_arch = None
self.build_date = None
self.host_arch = None
self.checksums = {}
self.build_depends = []
self.env = {}
self.required_timestamps = []
self.fedora_release = None
if not os.path.exists(self.orig_file):
raise BuildInfoException(
"Cannot find buildinfo file: {}".format(self.orig_file))
with open(self.orig_file) as fd:
for paragraph in Deb822.iter_paragraphs(fd.read()):
for item in paragraph.items():
if item[0] == 'Source':
self.source = item[1]
if item[0] == 'Architecture':
self.architecture = item[1].split()
# Currently not in 1.0-rpm format
if item[0] == 'Binary':
self.binary = item[1].split()
if item[0] == 'Version':
self.version = item[1]
if item[0] == 'Release':
self.release = item[1]
if item[0] == 'Epoch':
# TODO: should we let 0 prefixing almost every package?
if item[1] != "0":
self.epoch = item[1]
if item[0] == 'Build-Path':
self.build_path = item[1]
if item[0] == 'Build-Architecture':
self.build_arch = item[1]
# Currently not in 1.0-rpm format because that would
# break buildinfo package reproducibility
if item[0] == 'Build-Date':
self.build_date = item[1]
if item[0] == 'Host-Architecture':
self.host_arch = item[1]
# Currently not in 1.0-rpm format
if item[0].startswith('Checksums-'):
alg = item[0].replace('Checksums-', '').lower()
for line in item[1].lstrip('\n').split('\n'):
parsed_line = line.split()
if not self.checksums.get(parsed_line[2], {}):
self.checksums[parsed_line[2]] = {}
self.checksums[parsed_line[2]].update({
"size": parsed_line[1],
alg: parsed_line[0],
})
if item[0] == 'Installed-Build-Depends':
for pkg in item[1].lstrip('\n').split('\n'):
if pkg.endswith('.(none)'):
continue
parsed_pkg = parsePkg(pkg)
if not parsed_pkg:
raise BuildInfoException(
"Cannot parse package: %s" % pkg)
self.build_depends.append(parsed_pkg)
if item[0] == 'Environment':
for line in item[1].lstrip('\n').split('\n'):
parsed_line = re.match(r'^[^=](.*)="(.*)"', line)
if parsed_line:
self.env[parsed_line.group(1).strip()] = \
parsed_line.group(2).strip()
self.build_source = len(
[arch for arch in self.architecture if arch == "src"]) == 1
self.build_archall = len(
[arch for arch in self.architecture if arch == "noarch"]) == 1
self.architecture = [
arch for arch in self.architecture if arch != "noarch"]
if len(self.architecture) > 1:
raise BuildInfoException(
"More than one architecture in Architecture field")
if not self.build_arch:
raise BuildInfoException("Need Build-Architecture field")
if not self.host_arch:
self.host_arch = self.build_arch
if not self.build_path:
self.build_path = "/build/{}-{}".format(self.source, randomName())
src_package = f'{self.source}-{self.version}-{self.release}.src'
if self.epoch:
src_package = f'{self.epoch}:{src_package}'
self.package = parsePkg(src_package)
