def make_unpacked_source(self):
"""Create an unpacked source tree in a branch. Return the working tree"""
tree = self.make_branch_and_tree('.')
cl_file = 'debian/changelog'
source_files = ['debian/'] + [cl_file]
self.build_tree(source_files)
c = self.make_changelog()
with open(cl_file, 'w') as f:
c.write_to_open_file(f)
tree.add(source_files)
source = Deb822()
source['Source'] = self.package_name
binary = Deb822()
binary['Package'] = self.package_name
binary['Architecture'] = 'all'
with open('debian/control', 'wb') as f:
source.dump(f)
f.write(b'\n')
binary.dump(f)
tree.add('debian/control')
self.build_tree(['debian/test-file'])
tree.add('debian/test-file')
return tree
def generate_control_file(self):
dsc = Dsc()
build_deps = []
packages = OrderedDict()
self.installs = {}
self.symlinks = {}
for item in self.generate_control_content():
log.debug(repr(item))
if isinstance(item, SourceControl):
for key, value in item.items():
key = '-'.join([k.capitalize() for k in key.split('_')])
if key == 'Description':
value = value.strip() \
.replace('\n\n', '\n.\n') \
.replace('\n', '\n ')
dsc[key] = value
elif isinstance(item, BuildDependency):
build_deps.append(item.dependency)
elif isinstance(item, Package):
control = Deb822()
for key, value in item._asdict().items():
key = '-'.join([k.capitalize() for k in key.split('_')])
if key == 'Description':
value = value.strip() \
.replace('\n\n', '\n.\n') \
.replace('\n', '\n ')
control[key] = value
packages[item.package] = (control, [], [])
self.installs[item.package] = []
self.symlinks[item.package] = []
elif isinstance(item, Dependency):
packages[item.package][1].append(item.dependency)
elif isinstance(item, Provide):
packages[item.package][2].append(item.provide)
elif isinstance(item, Symlink):
self.symlinks[item.package].append((item.dest, item.src))
elif isinstance(item, FastBuild):
if self.fast_build is None:
self.fast_build = item.possible
else:
self.fast_build = self.fast_build and item.possible
else:
raise NotImplementedError(
'Got unexcepted action item'
' on control generation {!r}'.format(item))
if self.installs:
build_deps.append('dh-exec')
with open(self.debian_file('control'), 'wb') as control_file:
dsc['Build-Depends'] = ', '.join(build_deps)
dsc.dump(control_file)
for control, deps, provides in packages.values():
if deps:
control['Depends'] = ', '.join(deps)
if provides:
control['Provides'] = ', '.join(provides)
control_file.write(b'\n')
control.dump(control_file)
def getBinFromDb(db, pckid):
"""
Get a binary package from a database
"""
pdict = db.getbinary(pckid)
pdict['cdict'] = Deb822(pdict['control'])
return BinPackageDb(**pdict)
def update(self, repo):
repo_dict = dict([(str(k), str(v)) for (k, v) in repo.items()])
repo_dict['id'] = repo.id
self.repos822[:] = [
repo822 if repo822['id'] != repo.id else Deb822(repo_dict)
for repo822 in self.repos822
]
async def add_to_cache(url: URLObject):
package = app.registry.get_or_create(url)
if package.status == Status.UNSEEN:
package_archive = await app.http.get(url)
if not package_archive.status == 200:
return
fileobj = io.BytesIO(await package_archive.read())
fileobj.seek(0)
untarred = tarfile.open(fileobj=fileobj, mode="r:*")
meta = {}
try:
for f in untarred.getmembers():
if f.name.endswith("DESCRIPTION"):
logger.debug(f"add_to_cache:found {f.name}")
meta = Deb822(
untarred.extractfile(f).read().decode("utf8"))
logger.debug(meta)
package.name = meta.get("Package")
package.version = meta.get("Version")
if meta.get("NeedsCompilation") == "yes":
temp_dir = tempfile.mkdtemp()
untarred.extractall(temp_dir)
package.fs_path = pathlib.Path(temp_dir) / meta.get("Package")
package.status = Status.TOBUILD
await app.compile_queue.put(package)
else:
package.status = Status.NONBINARY
finally:
untarred.close()
fileobj.close()
def test_control_field_parsing(self):
"""Test the parsing of control file fields."""
deb822_package = Deb822([
'Package: python-py2deb',
'Depends: python-deb-pkg-tools, python-pip, python-pip-accel',
'Installed-Size: 42'
])
parsed_info = parse_control_fields(deb822_package)
assert parsed_info == {
'Package':
'python-py2deb',
'Depends':
RelationshipSet(Relationship(name=u'python-deb-pkg-tools'),
Relationship(name=u'python-pip'),
Relationship(name=u'python-pip-accel')),
'Installed-Size':
42
}
# Test backwards compatibility with the old interface where `Depends'
# like fields were represented as a list of strings (shallow parsed).
parsed_info['Depends'] = [text_type(r) for r in parsed_info['Depends']]
assert unparse_control_fields(parsed_info) == deb822_package
# Test compatibility with fields like `Depends' containing a string.
parsed_info['Depends'] = deb822_package['Depends']
assert unparse_control_fields(parsed_info) == deb822_package
def listBin(self, arch, component, idrel, name):
"""
Return an iterator over packages.
arch, component and relese can all be either None, meaning
no restriction, a string or number (in case of idrel),
or an iterable.
"""
(sqlj, sqlw, sqlparams) = makeQuery(arch, component, idrel, name)
sql = 'SELECT b.*,rel.Codename AS release, r.component, r.Filename ' \
+ sqlj \
+ ' JOIN releases rel on r.idrel = rel.id ' \
+ sqlw \
+ ' ORDER BY rel.Codename, r.component, b.name, b.Architecture'
logger.debug("Query for list = %s", sql)
self.dbc.execute(sql, sqlparams)
while True:
r = self.dbc.fetchone()
if r is None: return
# make a dict from the query in the usual way
p = collections.defaultdict(lambda: '', zip(r.keys(), r))
# update with keys from control file
p.update(Deb822(p['control']))
del p['control']
p['shortdesc'] = (p['Description'].partition('\n'))[0]
yield p
def apt_get_source_package(name: str) -> Deb822:
"""Get source package metadata.
Args:
name: Name of the source package
Returns:
A `Deb822` object
"""
import apt_pkg
apt_pkg.init()
try:
sources = apt_pkg.SourceRecords()
except apt_pkg.Error as e:
if e.args[0] == (
"E:You must put some 'deb-src' URIs in your sources.list"):
raise NoAptSources()
raise
by_version: Dict[str, Deb822] = {}
while sources.lookup(name):
by_version[sources.version] = sources.record # type: ignore
if len(by_version) == 0:
raise NoSuchPackage(name)
# Try the latest version
version = sorted(by_version, key=Version)[-1]
return Deb822(by_version[version])
def test_control_field_parsing(self):
deb822_package = Deb822([
'Package: python-py2deb',
'Depends: python-deb-pkg-tools, python-pip, python-pip-accel',
'Installed-Size: 42'
])
parsed_info = parse_control_fields(deb822_package)
self.assertEqual(
parsed_info, {
'Package':
'python-py2deb',
'Depends':
RelationshipSet(Relationship(name=u'python-deb-pkg-tools'),
Relationship(name=u'python-pip'),
Relationship(name=u'python-pip-accel')),
'Installed-Size':
42
})
# Test backwards compatibility with the old interface where `Depends'
# like fields were represented as a list of strings (shallow parsed).
parsed_info['Depends'] = [unicode(r) for r in parsed_info['Depends']]
self.assertEqual(unparse_control_fields(parsed_info), deb822_package)
# Test compatibility with fields like `Depends' containing a string.
parsed_info['Depends'] = deb822_package['Depends']
self.assertEqual(unparse_control_fields(parsed_info), deb822_package)
def test_repository_creation(self, preserve=False):
"""Test the creation of trivial repositories."""
if SKIP_SLOW_TESTS:
return self.skipTest("skipping slow tests")
with Context() as finalizers:
config_dir = tempfile.mkdtemp()
repo_dir = tempfile.mkdtemp()
if not preserve:
finalizers.register(shutil.rmtree, config_dir)
finalizers.register(shutil.rmtree, repo_dir)
from deb_pkg_tools import config
config.user_config_directory = config_dir
with open(os.path.join(config_dir, config.repo_config_file),
'w') as handle:
handle.write('[test]\n')
handle.write('directory = %s\n' % repo_dir)
handle.write('release-origin = %s\n' % TEST_REPO_ORIGIN)
self.test_package_building(repo_dir)
update_repository(
repo_dir,
release_fields=dict(description=TEST_REPO_DESCRIPTION),
cache=self.package_cache)
assert os.path.isfile(os.path.join(repo_dir, 'Packages'))
assert os.path.isfile(os.path.join(repo_dir, 'Packages.gz'))
assert os.path.isfile(os.path.join(repo_dir, 'Release'))
with open(os.path.join(repo_dir, 'Release')) as handle:
fields = Deb822(handle)
assert fields['Origin'] == TEST_REPO_ORIGIN
assert fields['Description'] == TEST_REPO_DESCRIPTION
if not apt_supports_trusted_option():
assert os.path.isfile(os.path.join(repo_dir, 'Release.gpg'))
return repo_dir
def test_control_field_merging(self):
"""Test the merging of control file fields."""
defaults = Deb822(['Package: python-py2deb',
'Depends: python-deb-pkg-tools',
'Architecture: all'])
# The field names of the dictionary with overrides are lower case on
# purpose; control file merging should work properly regardless of
# field name casing.
overrides = Deb822(dict(version='1.0',
depends='python-pip, python-pip-accel',
architecture='amd64'))
assert merge_control_fields(defaults, overrides) == \
Deb822(['Package: python-py2deb',
'Version: 1.0',
'Depends: python-deb-pkg-tools, python-pip, python-pip-accel',
'Architecture: amd64'])
def handle(self, *args, **options):
if len(args) != 3:
raise CommandError(
"Requires a language, a tag and an output filename")
lang = args[0]
tag = args[1]
filename = args[2]
f = open(filename, "w")
session = db.get_db_session()
session.bind.echo = False
output = False
for trans, descr, descr_tag in session.query(ddtp.Translation, ddtp.Description, ddtp.DescriptionTag). \
filter(ddtp.Translation.description_id == ddtp.Description.description_id). \
filter(ddtp.Description.description_id == ddtp.DescriptionTag.description_id). \
filter(ddtp.Translation.language == lang). \
filter(ddtp.DescriptionTag.date_end >= date.today()-timedelta(days=7)). \
filter(ddtp.DescriptionTag.tag == tag). \
order_by(ddtp.Description.package). \
yield_per(100):
trans_para = Deb822()
trans_para['Package'] = descr.package
trans_para['Description-md5'] = descr.description_md5
trans_para['Description-%s' % lang] = trans.translation
# Minor nagic here: the translation has an extra newline here,
# which we use to seperate the paragraphs
f.write(trans_para.dump().encode('utf-8'))
output = True
if not output:
self.stderr.write("WARNING: No output for tag %r, language %r\n" %
(tag, lang))
def debcontrol(self):
""" Return the debian/control as a Deb822 (a Debian-specific dict-like
class) object.
For a string representation of debian/control try
.get_content('control') """
return Deb822(self.get_content(CONTROL_FILE))
def deb822_from_string(string):
"""
Create a :class:`debian.deb822.Deb822` object from a string.
:param string: The string containing the control fields to parse.
:returns: A :class:`debian.deb822.Deb822` object.
"""
return Deb822(StringIO(textwrap.dedent(string).strip()))
def _read_module_attributes_from_source_package(module):
# type: (BaseModule) -> umc.UMC_Module
umc_module_definition_file = os.path.join(module['abs_path_to_src_pkg'], 'debian', '{}{}'.format(module['module_name'], UMC_MODULES))
with open(umc_module_definition_file, 'r') as fd:
def_file = fd.read()
attributes = Deb822(def_file)
attributes = dict((k, [v]) for k, v in attributes.items()) # simulate dh_ucs.parseRfc822 behaviour
return attributes
def load_control_file(control_file):
"""
Load a control file and return the parsed control fields.
:param control_file: The filename of the control file to load (a string).
:returns: A dictionary created by :func:`parse_control_fields()`.
"""
with open(control_file) as handle:
return parse_control_fields(Deb822(handle))
def read_desc_files(path):
for entry in os.scandir(path):
if entry.is_dir():
read_desc_files(entry.path)
elif entry.name.endswith('.tag'):
with open(entry.path, 'r') as f:
desc = Deb822(f)
for renamed_from in desc.get('Renamed-From', '').splitlines():
if renamed_from.strip():
renames[renamed_from.strip()] = desc['Tag']
def get_changes_file(changes_file_url):
if changes_file_url is not None:
changes_text = urllib.request.urlopen(changes_file_url).read()
changes_dict = Deb822(changes_text)
else:
return
if 'Launchpad-Bugs-Fixed' in changes_dict.keys():
bugnum = changes_dict['Launchpad-Bugs-Fixed']
return bugnum.split(' ') # This can be a string which has many entries
def assertControlFile(control_filename):
control = None
with open(control_filename) as f:
control = Deb822(f)
defined_deps = str(
control.get('Depends', '') + control.get('Build-Depends', ''))
if defined_deps == '':
return
deps = set(d.strip() for d in defined_deps.split(','))
assert deps.issubset(IMAGES_NAME), 'control dependencies must exist'
def _expand_build_depends(src, dest, arch):
with open(src, 'r') as fd:
control = Deb822(fd)
deps = []
if 'XCS-Needs-Cross-Toolchain' not in control or \
control['XCS-Needs-Cross-Toolchain'] != 'no':
deps += [arch + '-cross-toolchain']
if 'XCS-Cross-Host-Build-Depends' in control:
deps += control['XCS-Cross-Host-Build-Depends'].split(',')
if 'XCS-Cross-Build-Depends' in control:
deps += _cross_dep(control['XCS-Cross-Build-Depends'], arch)
control['Build-Depends'] = ', '.join(deps)
with open(dest, 'w') as ofd:
control.dump(ofd)
def test_control_file_patching_and_loading(self):
deb822_package = Deb822(
['Package: unpatched-example', 'Depends: some-dependency'])
with Context() as finalizers:
control_file = tempfile.mktemp()
finalizers.register(os.unlink, control_file)
with open(control_file, 'wb') as handle:
deb822_package.dump(handle)
call('--patch=%s' % control_file, '--set=Package: patched-example',
'--set=Depends: another-dependency')
patched_fields = load_control_file(control_file)
self.assertEqual(patched_fields['Package'], 'patched-example')
self.assertEqual(str(patched_fields['Depends']),
'another-dependency, some-dependency')
def get_debian_name():
"""obtain the debian package name
The information is found in debian/control withe the 'Source:' field
"""
try:
control = osp.join('debian', 'control')
deb822 = Deb822(open(control), fields='Source')
return deb822['Source']
except IOError as err:
raise LGPException('a Debian control file should exist in "%s"' %
control)
except KeyError as err:
raise LGPException("No 'Source' field in '%s'" % control)
def targeted_python_versions(tree: Tree, subpath: str) -> List[str]:
with tree.get_file(os.path.join(subpath, "debian/control")) as f:
control = Deb822(f)
build_depends = PkgRelation.parse_relations(
control.get("Build-Depends", ""))
all_build_deps: Set[str] = set()
for or_deps in build_depends:
all_build_deps.update(or_dep["name"] for or_dep in or_deps)
targeted = []
if any(x.startswith("python3-") for x in all_build_deps):
targeted.append("python3")
if any(x.startswith("pypy") for x in all_build_deps):
targeted.append("pypy")
if any(x.startswith("python-") for x in all_build_deps):
targeted.append("python")
return targeted
def install_built_package(local_tree, subpath, build_target_dir):
import re
import subprocess
with open(local_tree.abspath(os.path.join(subpath, 'debian/changelog')), 'r') as f:
cl = Changelog(f)
non_epoch_version = cl[0].version.upstream_version
if cl[0].version.debian_version is not None:
non_epoch_version += "-%s" % cl[0].version.debian_version
c = re.compile('%s_%s_(.*).changes' % (re.escape(cl[0].package), re.escape(non_epoch_version))) # type: ignore
for entry in os.scandir(build_target_dir):
if not c.match(entry.name):
continue
with open(entry.path, 'rb') as g:
changes = Deb822(g)
if changes.get('Binary'):
subprocess.check_call(['debi', entry.path])
def query_cache(pkgs):
stdout, _, _ = proc.run([config['cmd_apt_cache'], 'show'] + list(pkgs))
pkgs = OrderedDict()
for dump in stdout.split('\n\n'):
# skip empty lines
if not dump or dump.isspace():
continue
try:
pkg_info = Deb822(dump)
except ValueError:
log.debug(dump)
raise RemoteFailureError('Error parsing Deb822 info.')
pkgs[pkg_info['Package']] = pkg_info
return Unchanged(pkgs)
def test_repository_activation(self):
if not SKIP_SLOW_TESTS and os.getuid() == 0:
repository = self.test_repository_creation(preserve=True)
call('--activate-repo=%s' % repository)
try:
handle = os.popen('apt-cache show %s' % TEST_PACKAGE_NAME)
fields = Deb822(handle)
self.assertEqual(fields['Package'], TEST_PACKAGE_NAME)
finally:
call('--deactivate-repo=%s' % repository)
# XXX If we skipped the GPG key handling because apt supports the
# [trusted=yes] option, re-run the test *including* GPG key
# handling (we want this to be tested...).
import deb_pkg_tools
if deb_pkg_tools.repo.apt_supports_trusted_option():
deb_pkg_tools.repo.trusted_option_supported = False
self.test_repository_activation()
def get_control_content(control_dict, fd=None):
"""
:param fd: If fd is None, returns a unicode object. Otherwise, fd is assumed to
be a file-like object, and this method will write the data to it
instead of returning a unicode object.
:type control_dict: Dictionary
:rtype: str
:return: Control in RFC822 Format
"""
# Using Deb822 to format control
if "Description" in control_dict.keys():
control_dict["Description"] = format_multiline(control_dict["Description"])
rfc822 = Deb822(control_dict)
if fd is not None:
rfc822.dump(fd, "utf-8")
else:
return rfc822.dump(encoding="utf-8")
def look_up(self, name, url):
if "/" in name:
(name, version) = name.split("/", 1)
else:
version = None
try:
import apt_pkg
except ImportError as e:
raise DependencyNotPresent('apt_pkg', e)
apt_pkg.init()
sources = apt_pkg.SourceRecords()
urls = {}
while sources.lookup(name):
control = Deb822(sources.record)
pkg_version = control["Version"]
try:
urls[pkg_version] = control["Dgit"].split(' ')
except KeyError:
pass
if len(urls) == 0:
raise urlutils.InvalidURL(path=url, extra='no URLs found')
if version is None:
# Try the latest version
version = sorted(urls, key=Version)[-1]
if version not in urls:
raise urlutils.InvalidURL(path=url,
extra='version %s not found' % version)
if len(urls[version]) < 3:
raise urlutils.InvalidURL(
path=url,
extra='dgit header does not have location information')
url = urlutils.join_segment_parameters(
urls[version][3], {"tag": urlutils.quote(urls[version][2], '')})
note("Resolved package URL from Debian package %s/%s: %s", name,
version, url)
return url
def determine_package_archive(directory):
"""
Determine the name of the ``*.deb`` package archive that will be generated
from a directory tree suitable for packaging with ``dpkg-deb --build``. See
also :py:func:`parse_filename()`.
:param source_directory: The pathname of a directory tree suitable for
packaging with ``dpkg-deb --build``.
:returns: The filename of the ``*.deb`` archive to be built.
"""
with open(os.path.join(directory, 'DEBIAN', 'control')) as control_file:
fields = Deb822(control_file)
components = [fields['Package'], fields['Version']]
architecture = fields.get('Architecture', '').strip()
if architecture:
components.append(architecture)
return '%s.deb' % '_'.join(components)
def parse(cls, obj):
# `obj` can be anything accepted by Deb822: `str`, `bytes`, or a
# sequence of lines (including file-like objects)
fields = {
k.lower(): v for k,v in Deb822(obj).items()
}
files = {}
for h in Hash:
hashlist = fields.pop(h.index_field, '')
for line in hashlist.splitlines():
if line.strip() != '':
hashsum, size, filename = line.strip().split(None, 2)
size = int(size)
if filename not in files:
files[filename] = IndexEntry(filename)
files[filename].add_checksum(h, hashsum, size)
return cls(files, fields)
