def generate_control_file(self):
dsc = Dsc()
build_deps = []
packages = OrderedDict()
self.installs = {}
self.symlinks = {}
for item in self.generate_control_content():
log.debug(repr(item))
if isinstance(item, SourceControl):
for key, value in item.items():
key = '-'.join([k.capitalize() for k in key.split('_')])
if key == 'Description':
value = value.strip() \
.replace('\n\n', '\n.\n') \
.replace('\n', '\n ')
dsc[key] = value
elif isinstance(item, BuildDependency):
build_deps.append(item.dependency)
elif isinstance(item, Package):
control = Deb822()
for key, value in item._asdict().items():
key = '-'.join([k.capitalize() for k in key.split('_')])
if key == 'Description':
value = value.strip() \
.replace('\n\n', '\n.\n') \
.replace('\n', '\n ')
control[key] = value
packages[item.package] = (control, [], [])
self.installs[item.package] = []
self.symlinks[item.package] = []
elif isinstance(item, Dependency):
packages[item.package][1].append(item.dependency)
elif isinstance(item, Provide):
packages[item.package][2].append(item.provide)
elif isinstance(item, Symlink):
self.symlinks[item.package].append((item.dest, item.src))
elif isinstance(item, FastBuild):
if self.fast_build is None:
self.fast_build = item.possible
else:
self.fast_build = self.fast_build and item.possible
else:
raise NotImplementedError(
'Got unexcepted action item'
' on control generation {!r}'.format(item))
if self.installs:
build_deps.append('dh-exec')
with open(self.debian_file('control'), 'wb') as control_file:
dsc['Build-Depends'] = ', '.join(build_deps)
dsc.dump(control_file)
for control, deps, provides in packages.values():
if deps:
control['Depends'] = ', '.join(deps)
if provides:
control['Provides'] = ', '.join(provides)
control_file.write(b'\n')
control.dump(control_file)
def recognizes(cls, file):
if not super().recognizes(file):
return False
with open(file.path, 'rb') as f:
dsc = Dsc(f)
for d in dsc.get('Files'):
md5 = hashlib.md5()
# XXX: this will not work for containers
in_dsc_path = os.path.join(
os.path.dirname(file.path),
d['Name'],
)
if not os.path.exists(in_dsc_path):
return False
with open(in_dsc_path, 'rb') as f:
for buf in iter(functools.partial(f.read, 32768), b''):
md5.update(buf)
if md5.hexdigest() != d['md5sum']:
return False
file._deb822 = dsc
return True
def recognizes(cls, file):
if not super().recognizes(file):
return False
with open(file.path, 'rb') as f:
# We can parse .buildinfo files just like .dsc
buildinfo = Dsc(f)
if 'Checksums-Sha256' not in buildinfo:
return False
for d in buildinfo.get('Checksums-Sha256'):
sha256 = hashlib.sha256()
# XXX: this will not work for containers
in_buildinfo_path = os.path.join(
os.path.dirname(file.path),
d['Name'],
)
if not os.path.exists(in_buildinfo_path):
return False
with open(in_buildinfo_path, 'rb') as f:
for buf in iter(functools.partial(f.read, 32768), b''):
sha256.update(buf)
if sha256.hexdigest() != d['sha256']:
return False
file._deb822 = buildinfo
return True
def recognizes(file):
if not DotDscFile.RE_FILE_EXTENSION.search(file.name):
return False
with open(file.path, 'rb') as f:
dsc = Dsc(f)
for d in dsc.get('Files'):
md5 = hashlib.md5()
# XXX: this will not work for containers
in_dsc_path = os.path.join(os.path.dirname(file.path), d['Name'])
if not os.path.exists(in_dsc_path):
return False
with open(in_dsc_path, 'rb') as f:
for buf in iter(partial(f.read, 32768), b''):
md5.update(buf)
if md5.hexdigest() != d['md5sum']:
return False
file._deb822 = dsc
return True
async def put_dsc_package(host: str, user: str, password: str, dist: str,
file: str):
with open(file) as fh:
dsc = Dsc(fh)
dir: str = os.path.dirname(file)
files: List[str] = [dir + "/" + file["name"] for file in dsc["files"]]
await post_package_multipart(f"{host}/includedsc/{dist}", "dsc", file,
files, user, password)
def recognizes(file):
if not DotBuildinfoFile.RE_FILE_EXTENSION.search(file.name):
return False
with open(file.path, 'rb') as f:
# We can parse .buildinfo just like .dsc
buildinfo = Dsc(f)
if not 'Checksums-Sha256' in buildinfo:
return False
for d in buildinfo.get('Checksums-Sha256'):
sha256 = hashlib.sha256()
# XXX: this will not work for containers
in_buildinfo_path = os.path.join(os.path.dirname(file.path), d['Name'])
if not os.path.exists(in_buildinfo_path):
return False
with open(in_buildinfo_path, 'rb') as f:
for buf in iter(partial(f.read, 32768), b''):
sha256.update(buf)
if sha256.hexdigest() != d['sha256']:
return False
file._deb822 = buildinfo
return True
def recognizes(file):
if not DotBuildinfoFile.RE_FILE_EXTENSION.search(file.name):
return False
with open(file.path, 'rb') as f:
# We can parse .buildinfo just like .dsc
buildinfo = Dsc(f)
if not 'Checksums-Sha256' in buildinfo:
return False
for d in buildinfo.get('Checksums-Sha256'):
sha256 = hashlib.sha256()
# XXX: this will not work for containers
in_buildinfo_path = os.path.join(os.path.dirname(file.path),
d['Name'])
if not os.path.exists(in_buildinfo_path):
return False
with open(in_buildinfo_path, 'rb') as f:
for buf in iter(partial(f.read, 32768), b''):
sha256.update(buf)
if sha256.hexdigest() != d['sha256']:
return False
file._deb822 = buildinfo
return True
def getBaseDsc(self):
dsc = Dsc()
dsc["Architecture"] = "all"
dsc["Version"] = "0.42"
dsc["Source"] = "dulwich"
dsc["Binary"] = "python-dulwich"
dsc["Standards-Version"] = "0.2.2"
dsc["Maintainer"] = "Jelmer Vernooij <*****@*****.**>"
dsc["Files"] = [{
"md5sum": "5e8ba79b4074e2f305ddeaf2543afe83",
"size": "182280",
"name": "dulwich_0.42.tar.gz"
}]
return dsc
def make_dsc_file_available(self, filename, owner=None):
d = os.path.dirname(filename) or os.curdir
with open(filename) as reader:
dsc = Dsc(reader)
to = self.new_directory()
self.argv.append('--copy={}:{}'.format(
filename, '{}/{}'.format(to, os.path.basename(filename))))
for f in dsc['files']:
self.argv.append('--copy={}:{}'.format(
os.path.join(d, f['name']), '{}/{}'.format(to, f['name'])))
return to, os.path.basename(filename)
def make_dsc_file_available(self, filename, owner=None):
d = os.path.dirname(filename) or os.curdir
with open(filename) as reader:
dsc = Dsc(reader)
to = self.new_directory()
files = [to, '{}/{}'.format(to, os.path.basename(filename))]
self.copy_to_guest(filename, files[-1])
for f in dsc['files']:
files.append('{}/{}'.format(to, f['name']))
self.copy_to_guest(os.path.join(d, f['name']), files[-1])
if owner is not None:
self.check_call(['chown', owner] + files)
return files[0], os.path.basename(filename)
def get_package_metadata(package, extracted_path, keyrings, log=None):
"""Get the package metadata from the source package at dsc_path,
extracted in extracted_path.
Args:
package: the package dict (with a dsc_path key)
extracted_path: the path where the package got extracted
keyrings: a list of keyrings to use for gpg actions
log: a logging.Logger object
Returns: a dict with the following keys
history: list of (package_name, package_version) tuples parsed from
the package changelog
source_files: information about all the files in the source package
"""
ret = {}
# Parse the dsc file to retrieve all the original artifact files
dsc_path = package['dsc']
with open(dsc_path, 'rb') as dsc:
parsed_dsc = Dsc(dsc)
source_files = [get_file_info(dsc_path)]
dsc_dir = os.path.dirname(dsc_path)
for file in parsed_dsc['files']:
file_path = os.path.join(dsc_dir, file['name'])
file_info = get_file_info(file_path)
source_files.append(file_info)
ret['original_artifact'] = source_files
# Parse the changelog to retrieve the rest of the package information
changelog_path = os.path.join(extracted_path, b'debian/changelog')
with open(changelog_path, 'rb') as changelog:
try:
parsed_changelog = Changelog(changelog)
except UnicodeDecodeError:
if log:
log.warn('Unknown encoding for changelog %s,'
' falling back to iso' %
changelog_path.decode('utf-8'), extra={
'swh_type': 'deb_changelog_encoding',
'swh_name': package['name'],
'swh_version': str(package['version']),
'swh_changelog': changelog_path.decode('utf-8'),
})
# need to reset as Changelog scrolls to the end of the file
changelog.seek(0)
parsed_changelog = Changelog(changelog, encoding='iso-8859-15')
package_info = {
'name': package['name'],
'version': str(package['version']),
'lister_metadata': {
'lister': 'snapshot.debian.org',
'id': package['id'],
},
'changelog': {
'person': converters.uid_to_person(parsed_changelog.author),
'date': parse_date(parsed_changelog.date),
'history': [(block.package, str(block.version))
for block in parsed_changelog][1:],
}
}
try:
gpg_info = parsed_dsc.get_gpg_info(keyrings=keyrings)
package_info['pgp_signature'] = get_gpg_info_signature(gpg_info)
except ValueError:
if log:
log.info('Could not get PGP signature on package %s_%s' %
(package['name'], package['version']),
extra={
'swh_type': 'deb_missing_signature',
'swh_name': package['name'],
'swh_version': str(package['version']),
})
package_info['pgp_signature'] = None
maintainers = [
converters.uid_to_person(parsed_dsc['Maintainer'], encode=False),
]
maintainers.extend(
converters.uid_to_person(person, encode=False)
for person in UPLOADERS_SPLIT.split(parsed_dsc.get('Uploaders', ''))
)
package_info['maintainers'] = maintainers
ret['package_info'] = package_info
return ret
def _autopkgtest(things,
*,
architecture,
built_binaries,
lxc_24bit_subnet,
lxc_worker,
mirrors,
modes,
qemu_ram_size,
schroot_worker,
storage,
suite,
vendor,
worker,
extra_repositories=()):
binaries = []
sources = []
for thing in things:
if os.path.exists(thing):
if thing.endswith('.changes'):
with open(thing) as reader:
c = Changes(reader)
for f in c['files']:
n = os.path.join(
os.path.dirname(thing) or os.curdir, f['name'])
if f['name'].endswith('.deb'):
binaries.append(n)
elif f['name'].endswith('.dsc'):
sources.append(Source(n, dsc=Dsc(open(n))))
elif thing.endswith('.dsc'):
sources.append(Source(thing, dsc=Dsc(open(thing))))
elif thing.endswith('.deb'):
binaries.append(thing)
else:
sources.append(Source(thing))
failures = set()
for source in sources:
source_dsc = None
source_package = None
if source.dsc is not None:
source_dsc = source.name
else:
source_package = source.name
if built_binaries is None:
built_binaries = not binaries
for failure in run_autopkgtest(
architecture=architecture,
binaries=binaries,
built_binaries=built_binaries,
components=(),
extra_repositories=extra_repositories,
lxc_24bit_subnet=lxc_24bit_subnet,
lxc_worker=lxc_worker,
mirrors=mirrors,
modes=modes,
qemu_ram_size=qemu_ram_size,
schroot_worker=schroot_worker,
source_dsc=source_dsc,
source_package=source_package,
storage=storage,
suite=suite,
vendor=vendor,
worker=worker,
):
source.failures.append(failure)
failures.add(source)
return failures
def sbuild(self):
self.worker.check_call([
'install', '-d', '-m755', '-osbuild', '-gsbuild',
'{}/out'.format(self.worker.scratch)
])
sbuild_version = Version(
self.worker.check_output(
['dpkg-query', '-W', '-f${Version}', 'sbuild'],
universal_newlines=True).rstrip('\n'))
logger.info('Building architecture: %s', self.arch)
if self.arch in ('all', 'source'):
logger.info('(on %s)', self.worker.dpkg_architecture)
use_arch = self.worker.dpkg_architecture
else:
use_arch = self.arch
hierarchy = self.suite.hierarchy
sbuild_tarball = ('sbuild-{vendor}-{base}-{arch}.tar.gz'.format(
arch=use_arch,
vendor=self.buildable.vendor,
base=hierarchy[-1],
))
self.worker.copy_to_guest(os.path.join(self.storage, sbuild_tarball),
'{}/in/{}'.format(self.worker.scratch,
sbuild_tarball),
cache=True)
chroot = '{base}-{arch}-sbuild'.format(base=hierarchy[-1],
arch=use_arch)
with TemporaryDirectory() as tmp:
with AtomicWriter(os.path.join(tmp, 'sbuild.conf')) as writer:
writer.write(
textwrap.dedent('''
[{chroot}]
type=file
description=An autobuilder
file={scratch}/in/{sbuild_tarball}
groups=root,sbuild
root-groups=root,sbuild
profile=sbuild
''').format(chroot=chroot,
sbuild_tarball=sbuild_tarball,
scratch=self.worker.scratch))
self.worker.copy_to_guest(
os.path.join(tmp, 'sbuild.conf'),
'/etc/schroot/chroot.d/{}'.format(chroot))
# Backwards compatibility goo for Debian jessie buildd backport:
# it can't do "sbuild hello", only "sbuild hello_2.10-1"
if (self.buildable.source_from_archive
and self.buildable.version is None
and sbuild_version < Version('0.69.0')):
lines = self.worker.check_output(
[
'schroot',
'-c',
chroot,
'--',
'sh',
'-c',
'apt-get update >&2 && '
'apt-cache showsrc --only-source "$1" | '
'sed -ne "s/^Version: *//p"',
'sh', # argv[0]
self.buildable.source_package
],
universal_newlines=True).strip().splitlines()
self.buildable.version = sorted(map(Version, lines))[-1]
self.buildable.buildable = '{}_{}'.format(
self.buildable.source_package,
self.buildable.version,
)
argv = [
self.worker.command_wrapper,
'--chdir',
'{}/out'.format(self.worker.scratch),
'--',
'runuser',
'-u',
'sbuild',
'--',
'sbuild',
'-c',
chroot,
'-d',
str(self.buildable.nominal_suite),
'--no-run-lintian',
]
for x in self.dpkg_buildpackage_options:
argv.append('--debbuildopt=' + x)
for x in self.dpkg_source_options:
argv.append('--dpkg-source-opt=' + x)
for child in hierarchy[:-1]:
argv.append('--extra-repository')
argv.append('deb {} {} {}'.format(
child.mirror, child.apt_suite, ' '.join(
set(self.components or child.components)
& child.all_components)))
if child.sbuild_resolver:
argv.extend(child.sbuild_resolver)
for x in self.extra_repositories:
argv.append('--extra-repository')
argv.append(x)
if self.arch == 'all':
logger.info('Architecture: all')
argv.append('-A')
# Backwards compatibility goo for Debian jessie buildd backport
if sbuild_version < Version('0.69.0'):
argv.append('--arch-all-only')
else:
argv.append('--no-arch-any')
elif self.arch == self.buildable.together_with:
logger.info('Architecture: %s + all', self.arch)
argv.append('-A')
argv.append('--arch')
argv.append(self.arch)
elif self.arch == 'source':
logger.info('Source-only')
# Backwards compatibility goo for Debian jessie buildd backport
if sbuild_version < Version('0.69.0'):
# If we only build 'all', and we don't build 'all',
# then logically we build nothing (except source).
argv.append('--arch-all-only')
argv.append('--no-arch-all')
# Urgh. This sbuild expects to find foo_1_amd64.changes
# even for a source-only build (because it doesn't really
# support source-only builds), so we have to cheat.
# sbuild splits the command on spaces so we need to have
# a one-liner that doesn't contain embedded whitespace.
# Luckily, Perl can be written as line-noise.
argv.append('--finished-build-commands=perl -e ' +
'$arch=qx(dpkg\\x20--print-architecture);' +
'chomp($arch);' + 'chdir(shift);' +
'foreach(glob("../*_source.changes")){' +
'$orig=$_;' +
's/_source\\.changes$/_${arch}.changes/;' +
'print("Renaming\\x20$orig\\x20to\\x20$_\\n");' +
'rename($orig,$_)||die("$!");' + '}' + ' %p')
else:
argv.append('--no-arch-any')
argv.append('--source')
else:
logger.info('Architecture: %s only', self.arch)
argv.append('--arch')
argv.append(self.arch)
if self.buildable.dsc_name is not None:
if 'source' in self.buildable.changes_produced:
argv.append('{}/out/{}'.format(
self.worker.scratch,
os.path.basename(self.buildable.dsc_name)))
else:
argv.append('{}/in/{}'.format(
self.worker.scratch,
os.path.basename(self.buildable.dsc_name)))
elif self.buildable.source_from_archive:
argv.append(self.buildable.buildable)
else:
# Build a clean source package as a side-effect of the first
# build (in practice this will be the 'source' build).
if '--source' not in argv:
argv.append('--source')
# jessie sbuild doesn't support --no-clean-source so build
# the temporary source package ourselves.
self.worker.check_call([
self.worker.command_wrapper, '--chdir',
'{}/in/{}_source'.format(self.worker.scratch,
self.buildable.product_prefix), '--',
'dpkg-source', '-b', '.'
])
argv.append('{}/in/{}.dsc'.format(self.worker.scratch,
self.buildable.product_prefix))
logger.info('Running %r', argv)
try:
self.worker.check_call(argv)
finally:
# Note that we mix use_arch and arch here: an Architecture: all
# build produces foo_1.2_amd64.build, which we rename.
# We also check for foo_amd64.build because
# that's what comes out if we do "vectis sbuild --suite=sid hello".
for prefix in (self.buildable.source_package,
self.buildable.product_prefix):
product = '{}/out/{}_{}.build'.format(self.worker.scratch,
prefix, use_arch)
product = self.worker.check_output(
['readlink', '-f', product],
universal_newlines=True).rstrip('\n')
if (self.worker.call(['test', '-e', product]) == 0
and self.output_builds is not None):
logger.info('Copying %s back to host as %s_%s.build...',
product, self.buildable.product_prefix,
self.arch)
copied_back = os.path.join(
self.output_builds, '{}_{}_{}.build'.format(
self.buildable.product_prefix, self.arch,
time.strftime('%Y%m%dt%H%M%S', time.gmtime())))
self.worker.copy_to_host(product, copied_back)
self.buildable.logs[self.arch] = copied_back
symlink = os.path.join(
self.output_builds,
'{}_{}.build'.format(self.buildable.product_prefix,
self.arch))
try:
os.remove(symlink)
except FileNotFoundError:
pass
os.symlink(os.path.abspath(copied_back), symlink)
break
else:
logger.warning('Did not find build log at %s', product)
logger.warning(
'Possible build logs:\n%s',
self.worker.check_call([
'sh',
'-c',
'cd "$1"; ls -l *.build || :',
'sh', # argv[0]
self.worker.scratch
]))
if self.arch == 'source' and self.buildable.source_from_archive:
dscs = self.worker.check_output(
[
'sh',
'-c',
'exec ls "$1"/out/*.dsc',
'sh', # argv[0]
self.worker.scratch
],
universal_newlines=True)
dscs = dscs.splitlines()
if len(dscs) != 1:
raise CannotHappen('sbuild --source produced more than one '
'.dsc file from {!r}'.format(
self.buildable))
product = dscs[0]
with TemporaryDirectory() as tmp:
copied_back = os.path.join(
tmp, '{}.dsc'.format(self.buildable.buildable))
self.worker.copy_to_host(product, copied_back)
self.buildable.dsc = Dsc(open(copied_back))
self.buildable.source_package = self.buildable.dsc['source']
self.buildable.version = Version(self.buildable.dsc['version'])
self.buildable.arch_wildcards = set(
self.buildable.dsc['architecture'].split())
self.buildable.binary_packages = [
p.strip() for p in self.buildable.dsc['binary'].split(',')
]
if self.arch == 'source' and self.output_builds is not None:
# Make sure the orig.tar.* are in the out directory, because
# we will be building from the rebuilt source in future
self.worker.check_call([
'sh',
'-c',
'ln -nsf "$1"/in/*.orig.tar.* "$1"/out/',
'sh', # argv[0]
self.worker.scratch
])
if self.output_builds is None:
return
for product_arch in (self.arch, self.worker.dpkg_architecture):
product = '{}/out/{}_{}.changes'.format(
self.worker.scratch, self.buildable.product_prefix,
product_arch)
if self.worker.call(['test', '-e', product]) == 0:
break
else:
raise CannotHappen('sbuild produced no .changes file from '
'{!r}'.format(self.buildable))
logger.info('Copying %s back to host...', product)
copied_back = os.path.join(
self.output_builds,
'{}_{}.changes'.format(self.buildable.product_prefix, self.arch))
self.worker.copy_to_host(product, copied_back)
self.buildable.changes_produced[self.arch] = copied_back
changes_out = Changes(open(copied_back))
if self.arch == 'source':
self.buildable.dsc_name = None
self.buildable.sourceful_changes_name = copied_back
for f in changes_out['files']:
if f['name'].endswith('.dsc'):
# expect to find exactly one .dsc file
assert self.buildable.dsc_name is None
self.buildable.dsc_name = os.path.join(
self.output_builds, f['name'])
assert self.buildable.dsc_name is not None
# Save some space
self.worker.check_call([
'rm', '-fr',
'{}/in/{}_source/'.format(self.worker.scratch,
self.buildable.product_prefix)
])
for f in changes_out['files']:
assert '/' not in f['name']
assert not f['name'].startswith('.')
logger.info('Additionally copying %s back to host...', f['name'])
product = '{}/out/{}'.format(self.worker.scratch, f['name'])
copied_back = os.path.join(self.output_builds, f['name'])
self.worker.copy_to_host(product, copied_back)
def _sbuild(self, chroot, sbuild_options=()):
sbuild_version = self.worker.dpkg_version('sbuild')
# Backwards compatibility goo for Debian jessie buildd backport:
# it can't do "sbuild hello", only "sbuild hello_2.10-1".
if (self.buildable.source_from_archive
and self.buildable.source_version is None
and sbuild_version < Version('0.69.0')):
lines = chroot.check_output(
[
'sh',
'-c',
'apt-get update >&2 && '
'( apt-cache showsrc --only-source "$1" || '
' apt-cache showsrc "$1" ) | '
'sed -ne "s/^Version: *//p"',
'sh', # argv[0]
self.buildable.source_package,
],
universal_newlines=True).strip().splitlines()
self.buildable.source_version = sorted(map(Version, lines))[-1]
self.buildable.buildable = '{}_{}'.format(
self.buildable.source_package,
self.buildable.source_version,
)
argv = [
self.worker.command_wrapper,
'--chdir',
'{}/out'.format(self.worker.scratch),
'--',
'runuser',
'-u',
'sbuild',
'--',
'env',
]
for k, v in sorted(self.environ.items()):
argv.append('{}={}'.format(k, v))
argv.extend((
'sbuild',
'-c',
chroot.chroot,
'-d',
str(self.buildable.nominal_suite),
'--no-run-lintian',
))
if self.profiles:
argv.append('--profiles={}'.format(','.join(self.profiles)))
for x in self.dpkg_buildpackage_options:
argv.append('--debbuildopt=' + x)
for child in chroot.suite.hierarchy[:-1]:
# The schroot already has the apt sources, we just need the
# resolver
if child.sbuild_resolver:
argv.extend(child.sbuild_resolver)
break
if self.arch == 'all':
logger.info('Architecture: all')
argv.append('-A')
# Backwards compatibility goo for Debian jessie buildd backport
if sbuild_version < Version('0.69.0'):
argv.append('--arch-all-only')
else:
argv.append('--no-arch-any')
elif self.arch == self.buildable.indep_together_with:
logger.info('Architecture: %s + all', self.arch)
argv.append('-A')
argv.append('--arch')
argv.append(self.arch)
elif self.arch == 'source':
logger.info('Source-only')
argv.append('--no-arch-any')
if sbuild_version < Version('0.69.0'):
# Backwards compatibility for Debian jessie buildd backport,
# and for sbuild in Ubuntu xenial.
# sbuild < 0.69.0 expects to find foo_1_amd64.changes
# even for a source-only build (because it doesn't really
# support source-only builds), so we have to cheat.
perl = ("'" + '$arch = qx(dpkg\\x20--print-architecture);\n' +
'chomp($arch);\n' + 'chdir(shift);\n' +
'foreach(glob("../*_source.changes")) {\n' +
' $orig = $_;\n' +
' s/_source\\.changes$/_${arch}.changes/;\n' +
' print("Renaming\\x20$orig\\x20to\\x20$_\\n");\n' +
' rename($orig,$_) || die("$!");\n' + '}\n' + "'")
argv.append(
'--finished-build-commands=perl -e {} %p'.format(perl))
else:
logger.info('Architecture: %s only', self.arch)
argv.append('--arch')
argv.append(self.arch)
if self.arch in ('source', self.buildable.source_together_with):
# Build a clean source package as a side-effect of one
# build.
argv.append('--source')
for x in self.dpkg_source_options:
argv.append('--debbuildopt=--source-option={}'.format(x))
if self.buildable.binary_version_suffix:
argv.append('--append-to-version={}'.format(
self.buildable.binary_version_suffix))
for x in sbuild_options:
argv.append(x)
if self.buildable.dsc_name is not None:
if 'source' in self.buildable.changes_produced:
# We rebuilt the source already. Use the rebuilt version
# for all subsequent builds.
argv.append('{}/out/{}'.format(
self.worker.scratch,
os.path.basename(self.buildable.dsc_name)))
else:
# We got a .dsc from outside Vectis and are not
# rebuilding it.
argv.append('{}/in/{}'.format(
self.worker.scratch,
os.path.basename(self.buildable.dsc_name)))
elif self.buildable.source_from_archive:
argv.append(self.buildable.buildable)
else:
# jessie sbuild doesn't support --no-clean-source so build
# the temporary source package ourselves.
ds_argv = [
self.worker.command_wrapper,
'--chdir',
'{}/in/{}_source'.format(self.worker.scratch,
self.buildable.product_prefix),
'--',
'dpkg-source',
]
for x in self.dpkg_source_options:
ds_argv.append(x)
ds_argv.extend(('-b', '.'))
self.worker.check_call(ds_argv)
argv.append('{}/in/{}.dsc'.format(self.worker.scratch,
self.buildable.product_prefix))
logger.info('Running %r', argv)
try:
self.worker.check_call(argv)
finally:
# Note that we mix chroot.dpkg_architecture and arch here: an
# Architecture: all build produces foo_1.2_amd64.build, which we
# rename.
# We also check for foo_amd64.build because
# that's what comes out if we do "vectis sbuild --suite=sid hello".
for prefix in (self.buildable.source_package,
self.buildable.product_prefix):
product = '{}/out/{}_{}.build'.format(self.worker.scratch,
prefix,
chroot.dpkg_architecture)
product = self.worker.check_output(
['readlink', '-f', product],
universal_newlines=True).rstrip('\n')
if (self.worker.call(['test', '-e', product]) == 0
and self.output_dir is not None):
logger.info('Copying %s back to host as %s_%s.build...',
product, self.buildable.product_prefix,
self.arch)
copied_back = os.path.join(
self.output_dir, '{}_{}_{}.build'.format(
self.buildable.product_prefix, self.arch,
time.strftime('%Y%m%dt%H%M%S', time.gmtime())))
self.worker.copy_to_host(product, copied_back)
self.buildable.logs[self.arch] = copied_back
symlink = os.path.join(
self.output_dir,
'{}_{}.build'.format(self.buildable.product_prefix,
self.arch))
try:
os.remove(symlink)
except FileNotFoundError:
pass
os.symlink(os.path.abspath(copied_back), symlink)
break
else:
logger.warning('Did not find build log at %s', product)
logger.warning(
'Possible build logs:\n%s',
self.worker.check_call([
'sh',
'-c',
'cd "$1"; ls -l *.build || :',
'sh', # argv[0]
self.worker.scratch
]))
if self.arch == 'source' and self.buildable.source_from_archive:
dscs = self.worker.check_output(
[
'sh',
'-c',
'exec ls "$1"/out/*.dsc',
'sh', # argv[0]
self.worker.scratch
],
universal_newlines=True)
dscs = dscs.splitlines()
if len(dscs) != 1:
raise CannotHappen('sbuild --source produced more than one '
'.dsc file from {!r}'.format(
self.buildable))
product = dscs[0]
with TemporaryDirectory(prefix='vectis-sbuild-') as tmp:
copied_back = os.path.join(
tmp, '{}.dsc'.format(self.buildable.buildable))
self.worker.copy_to_host(product, copied_back)
self.buildable.dsc = Dsc(open(copied_back))
self.buildable.source_package = self.buildable.dsc['source']
self.buildable.source_version = Version(
self.buildable.dsc['version'])
self.buildable.arch_wildcards = set(
self.buildable.dsc['architecture'].split())
self.buildable.binary_packages = [
p.strip() for p in self.buildable.dsc['binary'].split(',')
]
if self.arch == 'source' and self.output_dir is not None:
# Make sure the orig.tar.* are in the out directory, because
# we will be building from the rebuilt source in future
self.worker.check_call([
'sh',
'-c',
'ln -nsf "$1"/in/*.orig.tar.* "$1"/out/',
'sh', # argv[0]
self.worker.scratch
])
if self.output_dir is None:
return
product_arch = None
for candidate in (self.arch, self.worker.dpkg_architecture):
product = '{}/out/{}_{}.changes'.format(
self.worker.scratch, self.buildable.product_prefix, candidate)
if self.worker.call(['test', '-e', product]) == 0:
product_arch = candidate
break
else:
raise CannotHappen(
'sbuild produced no .changes file from {!r}'.format(
self.buildable))
copied_back = self.copy_back_product(
'{}_{}.changes'.format(self.buildable.product_prefix,
product_arch),
'{}_{}.changes'.format(self.buildable.product_prefix, self.arch))
if copied_back is not None:
self.buildable.changes_produced[self.arch] = copied_back
changes_out = Changes(open(copied_back))
if 'source' in changes_out['architecture'].split():
self.buildable.dsc_name = None
self.buildable.sourceful_changes_name = copied_back
for f in changes_out['files']:
if f['name'].endswith('.dsc'):
# expect to find exactly one .dsc file
assert self.buildable.dsc_name is None
self.buildable.dsc_name = os.path.join(
self.output_dir, f['name'])
assert self.buildable.dsc_name is not None
# Save some space
self.worker.check_call([
'rm', '-fr',
'{}/in/{}_source/'.format(self.worker.scratch,
self.buildable.product_prefix)
])
dsc = None
for f in changes_out['files']:
copied_back = self.copy_back_product(f['name'])
if copied_back is not None and f['name'].endswith('.dsc'):
dsc = Dsc(open(copied_back))
if dsc is not None:
if self.buildable.dsc is None:
self.buildable.dsc = dsc
for f in dsc['files']:
# The orig.tar.* might not have come back. Copy that too,
# if necessary.
self.copy_back_product(f['name'], skip_if_exists=True)
def __init__(self,
buildable,
*,
binary_version_suffix='',
link_builds=(),
orig_dirs=('..', ),
output_dir=None,
output_parent,
vendor):
self.buildable = buildable
self._product_prefix = None
self._source_version = None
self.arch_wildcards = set()
self.archs = []
self.autopkgtest_failures = []
self.binary_packages = []
self.binary_version_suffix = binary_version_suffix
self.changes_produced = {}
self.dirname = None
self.dsc = None
self.dsc_name = None
self.indep = False
self.indep_together_with = None
self.link_builds = link_builds
self.logs = {}
self.merged_changes = OrderedDict()
self.nominal_suite = None
self.orig_dirs = orig_dirs
self.output_dir = output_dir
self.piuparts_failures = []
self.source_from_archive = False
self.source_package = None
self.source_together_with = None
self.sourceful_changes_name = None
self.suite = None
self.vendor = vendor
if os.path.exists(self.buildable):
if os.path.isdir(self.buildable):
changelog = os.path.join(self.buildable, 'debian', 'changelog')
changelog = Changelog(open(changelog))
self.source_package = changelog.get_package()
self.nominal_suite = changelog.distributions
self._source_version = Version(changelog.version)
control = os.path.join(self.buildable, 'debian', 'control')
if len(changelog.distributions.split()) != 1:
raise ArgumentError(
'Cannot build for multiple distributions at once')
for paragraph in Deb822.iter_paragraphs(open(control)):
self.arch_wildcards |= set(
paragraph.get('architecture', '').split())
binary = paragraph.get('package')
if binary is not None:
self.binary_packages.append(binary)
elif self.buildable.endswith('.changes'):
self.dirname = os.path.dirname(self.buildable) or os.curdir
self.sourceful_changes_name = self.buildable
sourceful_changes = Changes(open(self.buildable))
if 'source' not in sourceful_changes['architecture'].split():
raise ArgumentError(
'Changes file {!r} must be sourceful'.format(
self.buildable))
self.nominal_suite = sourceful_changes['distribution']
for f in sourceful_changes['files']:
if f['name'].endswith('.dsc'):
if self.dsc_name is not None:
raise ArgumentError(
'Changes file {!r} contained more than one '
'.dsc file'.format(self.buildable))
self.dsc_name = os.path.join(self.dirname, f['name'])
if self.dsc_name is None:
raise ArgumentError(
'Changes file {!r} did not contain a .dsc file'.format(
self.buildable))
self.dsc = Dsc(open(self.dsc_name))
elif self.buildable.endswith('.dsc'):
self.dirname = os.path.dirname(self.buildable) or os.curdir
self.dsc_name = self.buildable
self.dsc = Dsc(open(self.dsc_name))
else:
raise ArgumentError(
'buildable must be .changes, .dsc or directory, not '
'{!r}'.format(self.buildable))
else:
self.source_from_archive = True
if '_' in self.buildable:
source, version = self.buildable.split('_', 1)
else:
source = self.buildable
version = None
self.source_package = source
if version is not None:
self._source_version = Version(version)
if self.dsc is not None:
self.source_package = self.dsc['source']
self._source_version = Version(self.dsc['version'])
self.arch_wildcards = set(self.dsc['architecture'].split())
self.binary_packages = [
p.strip() for p in self.dsc['binary'].split(',')
]
if self._source_version is not None:
self._binary_version = Version(
str(self._source_version) + self.binary_version_suffix)
timestamp = time.strftime('%Y%m%dt%H%M%S', time.gmtime())
if self.output_dir is None:
if self._binary_version is None:
dirname = '{}_{}'.format(self.source_package, timestamp)
else:
dirname = '{}_{}_{}'.format(self.source_package,
self._binary_version, timestamp)
self.output_dir = os.path.join(output_parent, dirname)
# For convenience, create a symbolic link for the latest build of
# each source package: hello_latest -> hello_2.10-1_20170319t102623
unversioned_symlink = os.path.join(output_parent,
self.source_package + '_latest')
with suppress(FileNotFoundError):
os.unlink(unversioned_symlink)
os.symlink(dirname, unversioned_symlink)
# If we know the version, also create a symbolic link for the
# latest build of each source/version pair:
# hello_2.10-1 -> hello_2.10-1_20170319t102623
if self._binary_version is not None:
versioned_symlink = os.path.join(
output_parent, '{}_{}'.format(self.source_package,
self._binary_version))
with suppress(FileNotFoundError):
os.unlink(versioned_symlink)
os.symlink(dirname, versioned_symlink)
# It's OK if the output directory exists but is empty.
with suppress(FileNotFoundError):
os.rmdir(self.output_dir)
# Otherwise, if someone already created this, we'll just crash out.
os.mkdir(self.output_dir)
if self.dsc is not None:
abs_file = os.path.abspath(self.dsc_name)
abs_dir, base = os.path.split(abs_file)
os.symlink(abs_file, os.path.join(self.output_dir, base))
for l in self.link_builds:
symlink = os.path.join(l, base)
with suppress(FileNotFoundError):
os.unlink(symlink)
os.symlink(abs_file, symlink)
for f in self.dsc['files']:
abs_file = os.path.join(abs_dir, f['name'])
os.symlink(abs_file, os.path.join(self.output_dir, f['name']))
for l in self.link_builds:
symlink = os.path.join(l, f['name'])
with suppress(FileNotFoundError):
os.unlink(symlink)
os.symlink(abs_file, symlink)
def get_package_metadata(package, dsc_path, extracted_path):
"""Get the package metadata from the source package at dsc_path,
extracted in extracted_path.
Args:
package: the package dict (with a dsc_path key)
dsc_path: path to the package's dsc file
extracted_path: the path where the package got extracted
Returns:
dict: a dictionary with the following keys:
- history: list of (package_name, package_version) tuples parsed from
the package changelog
- source_files: information about all the files in the source package
"""
ret = {}
with open(dsc_path, 'rb') as dsc:
parsed_dsc = Dsc(dsc)
source_files = [get_file_info(dsc_path)]
dsc_dir = os.path.dirname(dsc_path)
for filename in package['files']:
file_path = os.path.join(dsc_dir, filename)
file_info = get_file_info(file_path)
source_files.append(file_info)
ret['original_artifact'] = source_files
# Parse the changelog to retrieve the rest of the package information
changelog_path = os.path.join(extracted_path, 'debian/changelog')
with open(changelog_path, 'rb') as changelog:
try:
parsed_changelog = Changelog(changelog)
except UnicodeDecodeError:
log.warn('Unknown encoding for changelog %s,'
' falling back to iso' %
changelog_path.decode('utf-8'), extra={
'swh_type': 'deb_changelog_encoding',
'swh_name': package['name'],
'swh_version': str(package['version']),
'swh_changelog': changelog_path.decode('utf-8'),
})
# need to reset as Changelog scrolls to the end of the file
changelog.seek(0)
parsed_changelog = Changelog(changelog, encoding='iso-8859-15')
package_info = {
'name': package['name'],
'version': str(package['version']),
'changelog': {
'person': converters.uid_to_person(parsed_changelog.author),
'date': parse_date(parsed_changelog.date),
'history': [(block.package, str(block.version))
for block in parsed_changelog][1:],
}
}
maintainers = [
converters.uid_to_person(parsed_dsc['Maintainer'], encode=False),
]
maintainers.extend(
converters.uid_to_person(person, encode=False)
for person in UPLOADERS_SPLIT.split(parsed_dsc.get('Uploaders', ''))
)
package_info['maintainers'] = maintainers
ret['package_info'] = package_info
return ret
def __init__(self, buildable, *, vendor):
self.buildable = buildable
self._product_prefix = None
self.arch_wildcards = set()
self.archs = []
self.binary_packages = []
self.changes_produced = {}
self.dirname = None
self.dsc = None
self.dsc_name = None
self.indep = False
self.logs = {}
self.merged_changes = OrderedDict()
self.nominal_suite = None
self.source_from_archive = False
self.source_package = None
self.sourceful_changes_name = None
self.suite = None
self.together_with = None
self.vendor = vendor
self._version = None
if os.path.exists(self.buildable):
if os.path.isdir(self.buildable):
changelog = os.path.join(self.buildable, 'debian', 'changelog')
changelog = Changelog(open(changelog))
self.source_package = changelog.get_package()
self.nominal_suite = changelog.distributions
self._version = Version(changelog.version)
control = os.path.join(self.buildable, 'debian', 'control')
if len(changelog.distributions.split()) != 1:
raise ArgumentError('Cannot build for multiple '
'distributions at once')
for paragraph in Deb822.iter_paragraphs(open(control)):
self.arch_wildcards |= set(
paragraph.get('architecture', '').split())
binary = paragraph.get('package')
if binary is not None:
self.binary_packages.append(binary)
elif self.buildable.endswith('.changes'):
self.dirname = os.path.dirname(self.buildable)
self.sourceful_changes_name = self.buildable
sourceful_changes = Changes(open(self.buildable))
if 'source' not in sourceful_changes['architecture']:
raise ArgumentError('Changes file {!r} must be '
'sourceful'.format(self.buildable))
self.nominal_suite = sourceful_changes['distribution']
for f in sourceful_changes['files']:
if f['name'].endswith('.dsc'):
if self.dsc_name is not None:
raise ArgumentError('Changes file {!r} contained '
'more than one .dsc '
'file'.format(self.buildable))
self.dsc_name = os.path.join(self.dirname, f['name'])
if self.dsc_name is None:
raise ArgumentError('Changes file {!r} did not contain a '
'.dsc file'.format(self.buildable))
self.dsc = Dsc(open(self.dsc_name))
elif self.buildable.endswith('.dsc'):
self.dirname = os.path.dirname(self.buildable)
self.dsc_name = self.buildable
self.dsc = Dsc(open(self.dsc_name))
else:
raise ArgumentError('buildable must be .changes, .dsc or '
'directory, not {!r}'.format(
self.buildable))
else:
self.source_from_archive = True
if '_' in self.buildable:
source, version = self.buildable.split('_', 1)
else:
source = self.buildable
version = None
self.source_package = source
if version is not None:
self._version = Version(version)
if self.dsc is not None:
self.source_package = self.dsc['source']
self._version = Version(self.dsc['version'])
self.arch_wildcards = set(self.dsc['architecture'].split())
self.binary_packages = [
p.strip() for p in self.dsc['binary'].split(',')
]
try:
dsc_name = [name for name in filelist if name.endswith(".dsc")][0]
except IndexError:
# raise SourceError("No dsc file found")
return []
try:
dsc = self.obs.getFile(action["sourceproject"],
action["sourcepackage"], dsc_name,
action["sourcerevision"])
except Exception, exobj:
raise SourceError(
"Failed to fetch dsc file %s/%s/%s rev %s: %s" %
(action["sourceproject"], action["sourcepackage"], dsc_name,
action["sourcerevision"], exobj))
try:
dsc = Dsc(dsc)
sources = [fentry["name"] for fentry in dsc["files"]]
except Exception, exobj:
raise SourceError("Failed to parse dsc file: %s" % exobj)
return sources
@CheckActionProcessor("check_package_is_complete_sources")
def check_source_files(self, action, _wid, filelist):
"""Check that filelist and spec sources match"""
sources = set()
msg = ""
try:
sources.update(self.get_rpm_sources(action, filelist))
except SourceError, exobj:
msg += str(exobj)
try:
def get_intrinsic_package_metadata(
p_info: DebianPackageInfo, dsc_path: str,
extracted_path: str) -> IntrinsicPackageMetadata:
"""Get the package metadata from the source package at dsc_path,
extracted in extracted_path.
Args:
p_info: the package information
dsc_path: path to the package's dsc file
extracted_path: the path where the package got extracted
Returns:
dict: a dictionary with the following keys:
- history: list of (package_name, package_version) tuples parsed from
the package changelog
"""
with open(dsc_path, "rb") as dsc:
parsed_dsc = Dsc(dsc)
# Parse the changelog to retrieve the rest of the package information
changelog_path = path.join(extracted_path, "debian/changelog")
with open(changelog_path, "rb") as changelog_file:
try:
parsed_changelog = Changelog(changelog_file)
except UnicodeDecodeError:
logger.warning(
"Unknown encoding for changelog %s,"
" falling back to iso" % changelog_path,
extra={
"swh_type": "deb_changelog_encoding",
"swh_name": p_info.name,
"swh_version": str(p_info.version),
"swh_changelog": changelog_path,
},
)
# need to reset as Changelog scrolls to the end of the file
changelog_file.seek(0)
parsed_changelog = Changelog(changelog_file,
encoding="iso-8859-15")
history: List[Tuple[str, str]] = []
for block in parsed_changelog:
assert block.package is not None
history.append((block.package, str(block.version)))
changelog = DebianPackageChangelog(
person=uid_to_person(parsed_changelog.author),
date=parse_date(parsed_changelog.date).isoformat(),
history=history[1:],
)
maintainers = [
uid_to_person(parsed_dsc["Maintainer"]),
]
maintainers.extend(
uid_to_person(person)
for person in UPLOADERS_SPLIT.split(parsed_dsc.get("Uploaders", "")))
return IntrinsicPackageMetadata(
name=p_info.name,
version=str(p_info.intrinsic_version),
changelog=changelog,
maintainers=maintainers,
)
