def check_build_product(self, base):
"""
Check whether base is a safe filename to copy back to the host.
If we don't trust the build system, we don't want to create symbolic
links with arbitrary names under its control.
"""
if os.path.basename(base) != base:
raise ArgumentError('Contains a path separator')
if base.startswith('.'):
raise ArgumentError('Is a hidden file')
if base.endswith(('.deb', '.udeb')):
return
if not base.startswith(self.source_package + '_'):
raise ArgumentError('Unexpected prefix')
if base.endswith(('.changes', '.dsc', '.buildinfo', '.diff.gz')):
return
if base.startswith(self.source_package + '_') and '.tar.' in base:
return
raise ArgumentError('Unexpected filename')
def run(args):
if args.suite is None:
if args.default_suite is not None:
args.suite = args.default_suite
else:
raise ArgumentError('--suite must be specified')
worker = VirtWorker(
args.piuparts_worker,
mirrors=args.get_mirrors(),
storage=args.storage,
suite=args.piuparts_worker_suite,
)
failures = _piuparts(
args._things,
architecture=args.architecture,
extra_repositories=args._extra_repository,
mirrors=args.get_mirrors(),
storage=args.storage,
suite=args.suite,
tarballs=args.get_piuparts_tarballs(
architecture=args.architecture,
suite=args.suite,
vendor=args.vendor,
),
vendor=args.vendor,
worker=worker,
)
for failure in sorted(failures):
logger.error('%s failed testing', failure)
def select_suite(self, factory, override):
suite_name = override
if suite_name is None:
suite_name = self.nominal_suite
if suite_name is None:
raise ArgumentError(
'Must specify --suite when building from {!r}'.format(
self.buildable))
if isinstance(suite_name, Suite):
self.suite = suite_name
else:
if suite_name == 'UNRELEASED':
logger.info('Replacing UNRELEASED with %s',
self.vendor.default_suite)
suite_name = self.vendor.default_suite
if suite_name.endswith('-UNRELEASED'):
suite_name = suite_name[:-len('-UNRELEASED')]
logger.info('Replacing %s-UNRELEASED with %s', suite_name,
suite_name)
self.suite = factory.get_suite(self.vendor, suite_name)
if self.nominal_suite is None:
self.nominal_suite = str(self.suite)
def select_suite(self, suite):
self.suite = self.nominal_suite
if suite is not None:
self.suite = suite
if self.nominal_suite is None:
self.nominal_suite = suite
if self.suite is None:
raise ArgumentError('Must specify --suite when building from '
'{!r}'.format(self.buildable))
def run(args):
if args._shell_command is None and not args._argv:
raise ArgumentError(
'Usage: vectis run -- PROGRAM [$1 [$2...]] '
'or vectis run -c "shell one-liner" [$0 [$1 [$2...]]]')
with Worker(['qemu', args.qemu_image]) as worker:
worker.set_up_apt(args.suite)
if args._shell_command is not None:
worker.check_call(['sh', '-c', args._shell_command] + args._argv)
else:
worker.check_call(args._argv)
def run(args, really=True):
if args.suite is None:
if args.default_suite is not None:
args.suite = args.default_suite
else:
raise ArgumentError('--suite must be specified')
mirrors = args.get_mirrors()
worker = VirtWorker(
args.worker,
mirrors=mirrors,
storage=args.storage,
suite=args.worker_suite,
)
if (args.lxc_worker == args.worker
and args.lxc_worker_suite == args.worker_suite):
lxc_worker = worker
else:
lxc_worker = VirtWorker(
args.lxc_worker,
mirrors=mirrors,
storage=args.storage,
suite=args.lxc_worker_suite,
)
failures = _autopkgtest(
args._things,
architecture=args.architecture,
built_binaries=args._built_binaries,
extra_repositories=args._extra_repository,
lxc_24bit_subnet=args.lxc_24bit_subnet,
lxc_worker=lxc_worker,
worker=worker,
mirrors=mirrors,
modes=args.autopkgtest,
qemu_ram_size=args.qemu_ram_size,
# use the misc worker instead of a specific schroot worker
schroot_worker=None,
storage=args.storage,
suite=args.suite,
vendor=args.vendor,
)
for failure in sorted(failures):
logger.error('%s failed testing: %s', failure, failure.failures)
def run(args):
if args.suite is None:
if args.worker_suite is not None:
args.suite = args.worker_suite
else:
raise ArgumentError('--suite must be specified')
with TemporaryDirectory() as scratch:
subprocess.check_call(
vmdebootstrap_argv(
args, '/usr/share/autopkgtest/setup-commands/setup-testbed') +
['--image={}/output.raw'.format(scratch)])
subprocess.check_call([
'qemu-img', 'convert', '-f', 'raw', '-O', 'qcow2', '-c', '-p',
'{}/output.raw'.format(scratch), '{}/output.qcow2'.format(scratch)
])
out = args.write_qemu_image
shutil.move('{}/output.qcow2'.format(scratch), out + '.new')
try:
with Worker(['qemu', '{}.new'.format(out)]) as worker:
worker.set_up_apt(args.suite)
worker.check_call([
'apt-get',
'-y',
'--no-install-recommends',
'install',
'python3',
'sbuild',
'schroot',
])
except:
if not args._keep:
os.remove(out + '.new')
raise
else:
os.rename(out + '.new', out)
def run(args):
if args.suite is None:
if args.worker_suite is not None:
args.suite = args.worker_suite
else:
raise ArgumentError('--suite must be specified')
architecture = args.architecture
keep = args._keep
kernel_package = args.get_kernel_package(architecture)
mirrors = args.get_mirrors()
out = args.write_qemu_image
qemu_image_size = args.qemu_image_size
storage = args.storage
suite = args.suite
uri = args._uri
vmdebootstrap_options = args.vmdebootstrap_options
if uri is None:
uri = mirrors.lookup_suite(suite)
try:
version = subprocess.check_output(
['dpkg-query', '-W', '-f${Version}', 'vmdebootstrap'],
universal_newlines=True).rstrip('\n')
except:
# non-dpkg host, guess a recent version
version = Version('1.7')
else:
version = Version(version)
with TemporaryDirectory(prefix='vectis-bootstrap-') as scratch:
argv = [
'sudo',
os.path.join(
os.path.dirname(__file__), os.pardir,
'vectis-command-wrapper'),
'--',
]
argv.extend(
vmdebootstrap_argv(
version,
architecture=architecture,
kernel_package=kernel_package,
qemu_image_size=qemu_image_size,
suite=suite,
uri=uri,
),
)
argv.extend(vmdebootstrap_options)
argv.append(
'--customize={}'.format(os.path.join(
os.path.dirname(__file__), os.pardir, 'setup-testbed')))
argv.append('--owner={}'.format(pwd.getpwuid(os.getuid())[0]))
argv.append('--image={}/output.raw'.format(scratch))
subprocess.check_call(argv)
subprocess.check_call([
'qemu-img', 'convert', '-f', 'raw',
'-O', 'qcow2', '-c', '-p',
'{}/output.raw'.format(scratch),
'{}/output.qcow2'.format(scratch),
])
os.makedirs(os.path.dirname(out) or os.curdir, exist_ok=True)
shutil.move('{}/output.qcow2'.format(scratch), out + '.new')
try:
with VirtWorker(
['qemu', '{}.new'.format(out)],
storage=storage,
suite=suite,
mirrors=mirrors) as worker:
worker.check_call([
'env',
'DEBIAN_FRONTEND=noninteractive',
'apt-get',
'-y',
'--no-install-recommends',
'-t', suite.apt_suite,
'install',
'python3',
'sbuild',
'schroot',
])
except:
if not keep:
os.remove(out + '.new')
raise
else:
os.rename(out + '.new', out)
def run(args):
if args.suite is None:
if args.default_suite is not None:
args.suite = args.default_suite
else:
raise ArgumentError('--suite must be specified')
apt_update = args._apt_update
argv = args._argv
chdir = args._chdir
mirrors = args.get_mirrors()
input_ = args._input
output_dir = args.output_dir
output_parent = args.output_parent
qemu_image = args.qemu_image
qemu_ram_size = args.qemu_ram_size
shell_command = args._shell_command
storage = args.storage
suite = args.suite
timestamp = time.strftime('%Y%m%dt%H%M%S', time.gmtime())
if output_dir is None:
output_dir = os.path.join(output_parent,
'vectis-run_{}'.format(timestamp))
with suppress(FileNotFoundError):
os.rmdir(output_dir)
os.mkdir(output_dir)
if shell_command is None and not argv:
raise ArgumentError(
'Usage: vectis run -- PROGRAM [$1 [$2...]] or vectis run '
'-c "shell one-liner" [$0 [$1 [$2...]]]')
for suite in (suite, ):
for ancestor in suite.hierarchy:
mirror = mirrors.lookup_suite(ancestor)
if mirror is None:
raise ArgumentError(
'No mirror configured for {}'.format(ancestor))
virt = ['qemu']
if qemu_ram_size is not None:
virt.append('--ram-size={}'.format(qemu_ram_size // _1M))
virt.append(qemu_image)
with VirtWorker(
virt,
apt_update=apt_update,
mirrors=mirrors,
storage=storage,
suite=suite,
) as worker:
worker_input = worker.scratch + '/in'
temp = worker.scratch + '/tmp'
artifacts = worker.scratch + '/out'
worker.check_call(['mkdir', artifacts, worker_input, temp])
if chdir == 'in':
chdir = worker_input
elif chdir == 'out':
chdir = artifacts
elif chdir == 'tmp':
chdir = temp
elif chdir[0] != '/' and chdir != '.':
raise ArgumentError(
"Argument to --chdir must be 'in', 'out', 'tmp', "
"'.' or absolute")
wrapper = [
'sh',
'-c',
'cd "$1" && shift && exec "$@"',
'sh',
chdir,
'env',
'AUTOPKGTEST_ARTIFACTS={}'.format(artifacts),
'ADT_ARTIFACTS={}'.format(artifacts),
'VECTIS_OUT={}'.format(artifacts),
'VECTIS_TMP={}'.format(temp),
'AUTOPKGTEST_TMP={}'.format(temp),
'ADTTMP={}'.format(temp),
]
if input_ is not None:
if os.path.isdir(input_):
worker.copy_to_guest(os.path.join(input_, ''),
worker_input + '/')
else:
worker_input = worker_input + '/' + os.path.basename(input_)
worker.copy_to_guest(input_, worker_input)
wrapper.append('VECTIS_IN={}'.format(worker_input))
try:
if shell_command is not None:
worker.check_call(wrapper + ['sh', '-c', shell_command] + argv)
else:
worker.check_call(wrapper + argv)
finally:
if worker.call(['rmdir', artifacts],
stderr=subprocess.DEVNULL) == 0:
logger.info('Command produced no artifacts')
os.rmdir(output_dir)
else:
worker.copy_to_host(artifacts + '/',
os.path.join(output_dir, ''))
logger.info('Artifacts produced by command are in %s',
output_dir)
def run(args):
if args.suite is None:
if args.default_suite is not None:
args.suite = args.default_suite
else:
raise ArgumentError('--suite must be specified')
# From argv or configuration
architecture = args.architecture
components = args.components
debootstrap_script = args.debootstrap_script
keep = args._keep
mirrors = args.get_mirrors()
storage = args.storage
suite = args.suite
test_package = args._test_package
uri = args._uri
vendor = args.vendor
worker_argv = args.worker
worker_suite = args.worker_suite
# From configuration
apt_key = args.apt_key
apt_key_package = args.apt_key_package
os.makedirs(storage, exist_ok=True)
for suite in (worker_suite, suite):
for ancestor in suite.hierarchy:
mirror = mirrors.lookup_suite(ancestor)
if mirror is None:
raise ArgumentError(
'No mirror configured for {}'.format(ancestor))
if uri is None:
uri = mirrors.lookup_suite(suite)
sbuild_tarball = '{arch}/{vendor}/{suite}/sbuild.tar.gz'.format(
arch=architecture,
vendor=vendor,
suite=suite,
)
logger.info('Creating tarball %s...', sbuild_tarball)
with VirtWorker(
worker_argv,
mirrors=mirrors,
storage=storage,
suite=worker_suite,
) as worker:
logger.info('Installing debootstrap and sbuild')
worker.check_call([
'env',
'DEBIAN_FRONTEND=noninteractive',
'apt-get',
'-y',
'--no-install-recommends',
'-t',
worker_suite.apt_suite,
'install',
'debootstrap',
'python3',
'sbuild',
'schroot',
])
keyring = apt_key_package
if keyring is not None:
worker.call([
'env',
'DEBIAN_FRONTEND=noninteractive',
'apt-get',
'-y',
'-t',
worker_suite.apt_suite,
'--no-install-recommends',
'install',
keyring,
])
debootstrap_args = []
if worker.call(['test', '-f', apt_key]) == 0:
logger.info('Found apt key worker:{}'.format(apt_key))
debootstrap_args.append('--keyring={}'.format(apt_key))
elif os.path.exists(apt_key):
logger.info('Found apt key host:{}, copying to worker:{}'.format(
apt_key, '{}/apt-key.gpg'.format(worker.scratch)))
worker.copy_to_guest(apt_key,
'{}/apt-key.gpg'.format(worker.scratch))
debootstrap_args.append('--keyring={}/apt-key.gpg'.format(
worker.scratch))
else:
logger.warning(
'Apt key host:{} not found; leaving it out and hoping '
'for the best'.format(apt_key))
debootstrap_args.append('--components={}'.format(','.join(components)))
worker.check_call([
'env',
'DEBIAN_FRONTEND=noninteractive',
worker.command_wrapper,
'--',
'sbuild-createchroot',
'--arch={}'.format(architecture),
'--include=fakeroot,sudo,vim',
'--components={}'.format(','.join(components)),
'--make-sbuild-tarball={}/output.tar.gz'.format(worker.scratch),
] + debootstrap_args + [
str(suite),
'{}/chroot'.format(worker.scratch),
uri,
'/usr/share/debootstrap/scripts/{}'.format(debootstrap_script),
])
out = os.path.join(storage, sbuild_tarball)
os.makedirs(os.path.dirname(out) or os.curdir, exist_ok=True)
# Smoke-test the new tarball before being prepared to use it.
if test_package:
try:
lines = worker.check_output(
[
'schroot',
'-c',
'{}-{}-sbuild'.format(suite, architecture),
'--',
'sh',
'-c',
'apt-get update >&2 && '
'( apt-cache showsrc --only-source "$1" || '
' apt-cache showsrc "$1" ) | '
'sed -ne "s/^Version: *//p"',
'sh', # argv[0]
test_package,
],
universal_newlines=True).strip().splitlines()
version = sorted(map(Version, lines))[-1]
buildable = '{}_{}'.format(test_package, version)
worker.check_call([
worker.command_wrapper,
'--chdir',
worker.scratch,
'--',
'runuser',
'-u',
'sbuild',
'--',
'sbuild',
'--arch',
architecture,
'-c',
'{}-{}-sbuild'.format(suite, architecture),
'-d',
'whatever',
'--no-run-lintian',
buildable,
])
except Exception:
if keep:
worker.copy_to_host(
'{}/output.tar.gz'.format(worker.scratch),
out + '.new')
raise
worker.copy_to_host('{}/output.tar.gz'.format(worker.scratch),
out + '.new')
os.rename(out + '.new', out)
logger.info('Created tarball %s', sbuild_tarball)
def run(args):
if args.suite is None:
if args.default_suite is not None:
args.suite = args.default_suite
else:
raise ArgumentError('--suite must be specified')
apt_key = args.apt_key
apt_key_package = args.apt_key_package
architecture = args.architecture
components = args.components
keep = args._keep
kernel_package = args.get_kernel_package(architecture)
include = args._include
mirrors = args.get_mirrors()
out = args.write_qemu_image
qemu_image_size = args.qemu_image_size
storage = args.storage
uri = args._uri
vendor = args.vendor
suite = args.get_suite(vendor, args.suite)
vmdebootstrap_options = args.vmdebootstrap_options
vmdebootstrap_worker = args.vmdebootstrap_worker
vmdebootstrap_worker_suite = args.vmdebootstrap_worker_suite
default_dir = os.path.join(storage, architecture, str(vendor), str(suite))
if uri is None:
uri = mirrors.lookup_suite(suite)
if False:
created, out = new_ubuntu_cloud(
architecture=architecture,
default_dir=default_dir,
out=out,
qemu_image_size=qemu_image_size,
suite=suite,
uri=uri,
vendor=vendor,
)
else:
created, out = new(
apt_key=apt_key,
apt_key_package=apt_key_package,
architecture=architecture,
components=components,
default_dir=default_dir,
kernel_package=kernel_package,
include=include,
merged_usr=args._merged_usr,
mirrors=mirrors,
out=out,
qemu_image_size=qemu_image_size,
storage=storage,
suite=suite,
uri=uri,
vmdebootstrap_options=vmdebootstrap_options,
vmdebootstrap_worker=vmdebootstrap_worker,
vmdebootstrap_worker_suite=vmdebootstrap_worker_suite,
)
try:
with VirtWorker(
['qemu', created],
mirrors=mirrors,
storage=storage,
suite=suite,
) as worker:
worker.set_up_apt()
worker.check_call(['apt-get', '-y', 'update'])
worker.check_call([
'env',
'DEBIAN_FRONTEND=noninteractive',
'apt-get',
'-y',
'--no-install-recommends',
'install',
'python3',
'sbuild',
'schroot',
])
except Exception:
if keep:
if created != out + '.new':
os.rename(created, out + '.new')
else:
os.remove(created)
raise
else:
os.rename(created, out)
def run(args):
deb_build_options = set()
if 'DEB_BUILD_OPTIONS' in os.environ:
for arg in os.environ['DEB_BUILD_OPTIONS'].split():
deb_build_options.add(arg)
for arg in args._add_deb_build_option:
deb_build_options.add(arg)
for arg in deb_build_options:
if arg == 'parallel' or arg.startswith('parallel='):
break
else:
deb_build_options.add('parallel={}'.format(args.parallel))
profiles = set()
if args._build_profiles is not None:
for arg in args._build_profiles.split(','):
profiles.add(arg)
elif 'DEB_BUILD_PROFILES' in os.environ:
for arg in os.environ['DEB_BUILD_PROFILES'].split():
profiles.add(arg)
for arg in args._add_build_profile:
profiles.add(arg)
db_options = []
if args._versions_since:
db_options.append('-v{}'.format(args._versions_since))
if args._include_orig_source is not None:
MAP = {
'yes': 'a',
'always': 'a',
'force': 'a',
'a': 'a',
'auto': 'i',
'maybe': 'i',
'i': 'i',
'no': 'd',
'never': 'd',
'd': 'd',
}
db_options.append('-s{}'.format(MAP[args._include_orig_source]))
ds_options = []
if args.dpkg_source_diff_ignore is ...:
ds_options.append('-i')
elif args.dpkg_source_diff_ignore is not None:
ds_options.append('-i{}'.format(
args.dpkg_source_diff_ignore))
for pattern in args.dpkg_source_tar_ignore:
if pattern is ...:
ds_options.append('-I')
else:
ds_options.append('-I{}'.format(pattern))
for pattern in args.dpkg_source_extend_diff_ignore:
ds_options.append('--extend-diff-ignore={}'.format(pattern))
group = BuildGroup(
binary_version_suffix=args._append_to_version,
buildables=(args._buildables or '.'),
components=args.components,
deb_build_options=deb_build_options,
dpkg_buildpackage_options=db_options,
dpkg_source_options=ds_options,
extra_repositories=args._extra_repository,
link_builds=args.link_builds,
orig_dirs=args.orig_dirs,
output_dir=args.output_dir,
output_parent=args.output_parent,
mirrors=args.get_mirrors(),
profiles=profiles,
sbuild_options=args._sbuild_options,
storage=args.storage,
suite=args.suite,
vendor=args.vendor,
)
group.select_suites(args)
for b in group.buildables:
for suite in (b.suite, args.sbuild_worker_suite):
assert isinstance(suite, Suite)
for ancestor in suite.hierarchy:
mirror = group.mirrors.lookup_suite(ancestor)
if mirror is None:
raise ArgumentError(
'No mirror configured for {}'.format(ancestor))
sbuild_worker = group.get_worker(
args.sbuild_worker,
args.sbuild_worker_suite,
)
group.sbuild(
sbuild_worker,
archs=args._archs,
build_source=args._build_source,
indep=args._indep,
indep_together=args.build_indep_together,
source_only=args._source_only,
source_together=args.sbuild_source_together,
)
misc_worker = group.get_worker(args.worker, args.worker_suite)
piuparts_worker = group.get_worker(
args.piuparts_worker,
args.piuparts_worker_suite,
)
lxc_worker = group.get_worker(
args.lxc_worker,
args.lxc_worker_suite,
)
lxd_worker = group.get_worker(
args.lxd_worker,
args.lxd_worker_suite,
)
interrupted = False
try:
group.autopkgtest(
default_architecture=sbuild_worker.dpkg_architecture,
lxc_24bit_subnet=args.lxc_24bit_subnet,
lxc_worker=lxc_worker,
lxd_worker=lxd_worker,
modes=args.autopkgtest,
qemu_ram_size=args.qemu_ram_size,
schroot_worker=sbuild_worker,
worker=misc_worker,
)
except KeyboardInterrupt:
interrupted = True
if args.piuparts_tarballs and not interrupted:
try:
group.piuparts(
default_architecture=sbuild_worker.dpkg_architecture,
tarballs=args.piuparts_tarballs,
worker=piuparts_worker,
)
except KeyboardInterrupt:
interrupted = True
_summarize(group.buildables)
if not interrupted:
try:
_lintian(group.buildables)
except KeyboardInterrupt:
logger.warning('lintian interrupted')
interrupted = True
if args._reprepro_dir and not interrupted:
_publish(group.buildables, args._reprepro_dir, args._reprepro_suite)
# We print these separately, right at the end, so that if you built more
# than one thing, the last screenful of information is the really
# important bit for testing/signing/upload
for buildable in group.buildables:
logger.info(
'Merged changes files from %s:\n\t%s',
buildable,
'\n\t'.join(buildable.merged_changes.values()),
)
if buildable.autopkgtest_failures:
logger.error('Autopkgtest failures for %s:', buildable)
for x in buildable.autopkgtest_failures:
logger.error('- %s', x)
if buildable.piuparts_failures:
logger.error('Piuparts failures for %s:', buildable)
for x in buildable.piuparts_failures:
logger.error('- %s', x)
for buildable in group.buildables:
logger.info(
'Output directory for %s: %s',
buildable,
buildable.output_dir,
)
def new(*,
apt_key,
apt_key_package,
architecture,
components,
default_dir,
include=(),
kernel_package,
merged_usr,
mirrors,
out,
qemu_image_size,
storage,
suite,
uri,
vmdebootstrap_options,
vmdebootstrap_worker,
vmdebootstrap_worker_suite):
for suite in (vmdebootstrap_worker_suite, suite):
for ancestor in suite.hierarchy:
mirror = mirrors.lookup_suite(ancestor)
if mirror is None:
raise ArgumentError(
'No mirror configured for {}'.format(ancestor))
with VirtWorker(
vmdebootstrap_worker,
mirrors=mirrors,
storage=storage,
suite=vmdebootstrap_worker_suite,
) as worker:
worker.check_call([
'env',
'DEBIAN_FRONTEND=noninteractive',
'apt-get',
'-y',
'upgrade',
])
worker_packages = [
'autopkgtest',
'grub2-common',
'python3',
'qemu-utils',
'vmdebootstrap',
]
# Optional (x86 only, but necessary for wheezy)
optional_worker_packages = [
'extlinux',
'mbr',
]
keyring = apt_key_package
if keyring is not None:
optional_worker_packages.append(keyring)
worker.check_call([
'env',
'DEBIAN_FRONTEND=noninteractive',
'apt-get',
'-y',
'-t',
vmdebootstrap_worker_suite.apt_suite,
'--no-install-recommends',
'install',
] + worker_packages)
# Failure is ignored for these non-critical packages
for p in optional_worker_packages:
worker.call([
'env', 'DEBIAN_FRONTEND=noninteractive', 'apt-get', '-y', '-t',
vmdebootstrap_worker_suite.apt_suite,
'--no-install-recommends', 'install', p
])
version = worker.dpkg_version('vmdebootstrap')
debootstrap_version = worker.dpkg_version('debootstrap')
argv, debootstrap_args, default_name = vmdebootstrap_argv(
version,
architecture=architecture,
components=components,
debootstrap_version=debootstrap_version,
include=include,
kernel_package=kernel_package,
qemu_image_size=qemu_image_size,
suite=suite,
uri=uri,
merged_usr=merged_usr,
)
argv.extend(vmdebootstrap_options)
if worker.call(['test', '-f', apt_key]) == 0:
logger.info('Found apt key worker:{}'.format(apt_key))
debootstrap_args.append('keyring={}'.format(apt_key))
elif os.path.exists(apt_key):
logger.info('Found apt key host:{}, copying to worker:{}'.format(
apt_key, '{}/apt-key.gpg'.format(worker.scratch)))
worker.copy_to_guest(apt_key,
'{}/apt-key.gpg'.format(worker.scratch))
debootstrap_args.append('keyring={}/apt-key.gpg'.format(
worker.scratch))
else:
logger.warning('Apt key host:{} not found; leaving it out and '
'hoping for the best'.format(apt_key))
if debootstrap_args:
argv.append('--debootstrapopts={}'.format(
' '.join(debootstrap_args)))
worker.copy_to_guest(
os.path.join(os.path.dirname(__file__), '..', 'setup-testbed'),
'{}/setup-testbed'.format(worker.scratch))
worker.check_call(
['chmod', '0755', '{}/setup-testbed'.format(worker.scratch)])
worker.check_call([
'env',
'DEBIAN_FRONTEND=noninteractive',
worker.command_wrapper,
'--',
] + argv + [
'--customize={}/setup-testbed'.format(worker.scratch),
'--image={}/output.raw'.format(worker.scratch),
])
worker.check_call([
'qemu-img',
'convert',
'-f',
'raw',
'-O',
'qcow2',
'-c',
'-p',
'{}/output.raw'.format(worker.scratch),
'{}/output.qcow2'.format(worker.scratch),
])
if out is None:
out = os.path.join(default_dir, default_name)
os.makedirs(os.path.dirname(out) or os.curdir, exist_ok=True)
worker.copy_to_host('{}/output.qcow2'.format(worker.scratch),
out + '.new')
return out + '.new', out
def run(args):
if args.suite is None:
if args.default_suite is not None:
args.suite = args.default_suite
else:
raise ArgumentError('--suite must be specified')
# From argv or configuration
architecture = args.architecture
components = args.components
keep = args._keep
mirrors = args.get_mirrors()
storage = args.storage
suite = args.suite
test_package = args._test_package
uri = args._uri
vendor = args.vendor
worker_argv = args.worker
worker_suite = args.worker_suite
# From configuration
apt_key = args.apt_key
apt_key_package = args.apt_key_package
os.makedirs(storage, exist_ok=True)
for suite in (worker_suite, suite):
for ancestor in suite.hierarchy:
mirror = mirrors.lookup_suite(ancestor)
if mirror is None:
raise ArgumentError(
'No mirror configured for {}'.format(ancestor))
if uri is None:
uri = mirrors.lookup_suite(suite)
tarball = '{arch}/{vendor}/{suite}/pbuilder.tar.gz'.format(
arch=architecture,
vendor=vendor,
suite=suite,
)
logger.info('Creating tarball %s...', tarball)
with VirtWorker(
worker_argv,
mirrors=mirrors,
storage=storage,
suite=worker_suite,
) as worker:
logger.info('Installing debootstrap and pbuilder')
worker.check_call([
'env',
'DEBIAN_FRONTEND=noninteractive',
'apt-get',
'-y',
'--no-install-recommends',
'-t',
worker_suite.apt_suite,
'install',
'debootstrap',
'python3',
'pbuilder',
])
keyring = apt_key_package
if keyring is not None:
worker.call([
'env',
'DEBIAN_FRONTEND=noninteractive',
'apt-get',
'-y',
'-t',
worker_suite.apt_suite,
'--no-install-recommends',
'install',
keyring,
])
pbuilder_args = [
'create',
'--aptcache',
'',
'--architecture',
architecture,
'--components',
' '.join(components),
'--basetgz',
'{}/output.tar.gz'.format(worker.scratch),
'--mirror',
uri,
'--distribution',
str(suite),
]
debootstrap_args = []
if worker.call(['test', '-f', apt_key]) == 0:
logger.info('Found apt key worker:{}'.format(apt_key))
pbuilder_args.append('--keyring')
pbuilder_args.append(apt_key)
debootstrap_args.append('--keyring={}'.format(apt_key))
elif os.path.exists(apt_key):
logger.info('Found apt key host:{}, copying to worker:{}'.format(
apt_key, '{}/apt-key.gpg'.format(worker.scratch)))
worker.copy_to_guest(apt_key,
'{}/apt-key.gpg'.format(worker.scratch))
pbuilder_args.append('--keyring')
pbuilder_args.append('{}/apt-key.gpg'.format(worker.scratch))
debootstrap_args.append('--keyring={}/apt-key.gpg'.format(
worker.scratch))
else:
logger.warning(
'Apt key host:{} not found; leaving it out and hoping '
'for the best'.format(apt_key))
for arg in debootstrap_args:
pbuilder_args.append('--debootstrapopts')
pbuilder_args.append(arg)
worker.check_call([
'touch',
'/root/.pbuilderrc',
])
logger.info('pbuilder %r', pbuilder_args)
worker.check_call([
'env',
'DEBIAN_FRONTEND=noninteractive',
worker.command_wrapper,
'--',
'pbuilder',
] + pbuilder_args)
out = os.path.join(storage, tarball)
os.makedirs(os.path.dirname(out) or os.curdir, exist_ok=True)
# Smoke-test the new tarball before being prepared to use it.
if test_package:
with TemporaryDirectory(prefix='vectis-pbuilder-') as tmp:
with AtomicWriter(os.path.join(tmp, 'script')) as writer:
writer.write(
textwrap.dedent('''\
#!/bin/sh
set -e
set -u
cd "$2"
perl -ne 'print $_; if (s/^deb\s/deb-src /) { print $_ }' \
< /etc/apt/sources.list \
> /etc/apt/sources.list.new
mv /etc/apt/sources.list.new /etc/apt/sources.list
apt-get update >&2
apt-get --download-only source "$1" >&2
mv *.dsc "$1.dsc"
'''))
worker.copy_to_guest(os.path.join(tmp, 'script'),
'{}/script'.format(worker.scratch))
worker.check_call([
'chmod',
'0755',
'{}/script'.format(worker.scratch),
])
try:
lines = worker.check_output(
[
'pbuilder',
'execute',
'--aptcache',
'',
'--basetgz',
'{}/output.tar.gz'.format(worker.scratch),
'--bindmounts',
'{}'.format(worker.scratch),
'--',
'{}/script'.format(worker.scratch),
test_package,
worker.scratch,
],
universal_newlines=True).strip().splitlines()
logger.info('%r', lines)
worker.check_call([
worker.command_wrapper,
'--chdir',
worker.scratch,
'--',
'pbuilder',
'build',
'--aptcache',
'',
'--basetgz',
'{}/output.tar.gz'.format(worker.scratch),
'{}.dsc'.format(test_package),
])
except Exception:
if keep:
worker.copy_to_host(
'{}/output.tar.gz'.format(worker.scratch),
out + '.new')
raise
worker.copy_to_host('{}/output.tar.gz'.format(worker.scratch),
out + '.new')
os.rename(out + '.new', out)
logger.info('Created tarball %s', tarball)
def run(args):
if args.suite is None:
if args.default_suite is not None:
args.suite = args.default_suite
else:
raise ArgumentError('--suite must be specified')
architecture = args.architecture
mirrors = args.get_mirrors()
storage = args.storage
suite = args.suite
uri = args._uri
vendor = args.vendor
worker_argv = args.worker
worker_suite = args.worker_suite
apt_key = args.apt_key
apt_key_package = args.apt_key_package
os.makedirs(storage, exist_ok=True)
for suite in (worker_suite, suite):
for ancestor in suite.hierarchy:
mirror = mirrors.lookup_suite(ancestor)
if mirror is None:
raise ArgumentError(
'No mirror configured for {}'.format(ancestor))
if uri is None:
uri = mirrors.lookup_suite(suite)
if args._merged_usr:
basename = 'minbase-merged-usr'
else:
basename = 'minbase'
minbase_tarball = '{arch}/{vendor}/{suite}/{basename}.tar.gz'.format(
arch=architecture,
basename=basename,
vendor=vendor,
suite=suite,
)
logger.info('Creating tarball %s...', minbase_tarball)
with VirtWorker(
worker_argv,
mirrors=mirrors,
storage=storage,
suite=worker_suite,
) as worker:
logger.info('Installing debootstrap')
worker.check_call([
'env',
'DEBIAN_FRONTEND=noninteractive',
'apt-get',
'-y',
'-t', worker_suite.apt_suite,
'--no-install-recommends',
'install',
'debootstrap',
'python3',
])
debootstrap_version = worker.dpkg_version('debootstrap')
if apt_key_package is not None:
worker.call([
'env',
'DEBIAN_FRONTEND=noninteractive',
'apt-get',
'-y',
'-t', worker_suite.apt_suite,
'--no-install-recommends',
'install',
apt_key_package,
])
debootstrap_args = []
if worker.call(['test', '-f', apt_key]) == 0:
logger.info('Found apt key worker:{}'.format(apt_key))
debootstrap_args.append('--keyring={}'.format(apt_key))
elif os.path.exists(apt_key):
logger.info('Found apt key host:{}, copying to worker:{}'.format(
apt_key, '{}/apt-key.gpg'.format(worker.scratch)))
worker.copy_to_guest(
apt_key, '{}/apt-key.gpg'.format(worker.scratch))
debootstrap_args.append('--keyring={}/apt-key.gpg'.format(
worker.scratch))
else:
logger.warning(
'Apt key host:{} not found; leaving it out and hoping for the '
'best'.format(apt_key))
debootstrap_args.append('--components={}'.format(
','.join(args.components)))
if debootstrap_version >= Version('1.0.86~'):
if args._merged_usr:
debootstrap_args.append('--merged-usr')
else:
# piuparts really doesn't like merged /usr
debootstrap_args.append('--no-merged-usr')
worker.check_call([
'env', 'DEBIAN_FRONTEND=noninteractive',
worker.command_wrapper,
'--',
'debootstrap',
'--arch={}'.format(architecture),
'--components={}'.format(','.join(args.components)),
'--variant=minbase',
'--verbose',
] + debootstrap_args + [
str(suite),
'{}/chroot'.format(worker.scratch),
uri,
'/usr/share/debootstrap/scripts/{}'.format(
args.debootstrap_script),
])
worker.check_call([
'chroot', '{}/chroot'.format(worker.scratch),
'apt-get', 'clean',
])
worker.check_call([
'tar', '-C', '{}/chroot'.format(worker.scratch),
'-f', '{}/output.tar.gz'.format(worker.scratch),
'-z', '-c', '.',
])
out = os.path.join(storage, minbase_tarball)
os.makedirs(os.path.dirname(out) or os.curdir, exist_ok=True)
worker.copy_to_host(
'{}/output.tar.gz'.format(worker.scratch), out + '.new')
# FIXME: smoke-test it?
os.rename(out + '.new', out)
logger.info('Created tarball %s', minbase_tarball)
def __init__(self, buildable, *, vendor):
self.buildable = buildable
self._product_prefix = None
self.arch_wildcards = set()
self.archs = []
self.binary_packages = []
self.changes_produced = {}
self.dirname = None
self.dsc = None
self.dsc_name = None
self.indep = False
self.logs = {}
self.merged_changes = OrderedDict()
self.nominal_suite = None
self.source_from_archive = False
self.source_package = None
self.sourceful_changes_name = None
self.suite = None
self.together_with = None
self.vendor = vendor
self._version = None
if os.path.exists(self.buildable):
if os.path.isdir(self.buildable):
changelog = os.path.join(self.buildable, 'debian', 'changelog')
changelog = Changelog(open(changelog))
self.source_package = changelog.get_package()
self.nominal_suite = changelog.distributions
self._version = Version(changelog.version)
control = os.path.join(self.buildable, 'debian', 'control')
if len(changelog.distributions.split()) != 1:
raise ArgumentError('Cannot build for multiple '
'distributions at once')
for paragraph in Deb822.iter_paragraphs(open(control)):
self.arch_wildcards |= set(
paragraph.get('architecture', '').split())
binary = paragraph.get('package')
if binary is not None:
self.binary_packages.append(binary)
elif self.buildable.endswith('.changes'):
self.dirname = os.path.dirname(self.buildable)
self.sourceful_changes_name = self.buildable
sourceful_changes = Changes(open(self.buildable))
if 'source' not in sourceful_changes['architecture']:
raise ArgumentError('Changes file {!r} must be '
'sourceful'.format(self.buildable))
self.nominal_suite = sourceful_changes['distribution']
for f in sourceful_changes['files']:
if f['name'].endswith('.dsc'):
if self.dsc_name is not None:
raise ArgumentError('Changes file {!r} contained '
'more than one .dsc '
'file'.format(self.buildable))
self.dsc_name = os.path.join(self.dirname, f['name'])
if self.dsc_name is None:
raise ArgumentError('Changes file {!r} did not contain a '
'.dsc file'.format(self.buildable))
self.dsc = Dsc(open(self.dsc_name))
elif self.buildable.endswith('.dsc'):
self.dirname = os.path.dirname(self.buildable)
self.dsc_name = self.buildable
self.dsc = Dsc(open(self.dsc_name))
else:
raise ArgumentError('buildable must be .changes, .dsc or '
'directory, not {!r}'.format(
self.buildable))
else:
self.source_from_archive = True
if '_' in self.buildable:
source, version = self.buildable.split('_', 1)
else:
source = self.buildable
version = None
self.source_package = source
if version is not None:
self._version = Version(version)
if self.dsc is not None:
self.source_package = self.dsc['source']
self._version = Version(self.dsc['version'])
self.arch_wildcards = set(self.dsc['architecture'].split())
self.binary_packages = [
p.strip() for p in self.dsc['binary'].split(',')
]
def __init__(self,
buildable,
*,
binary_version_suffix='',
link_builds=(),
orig_dirs=('..', ),
output_dir=None,
output_parent,
vendor):
self.buildable = buildable
self._product_prefix = None
self._source_version = None
self.arch_wildcards = set()
self.archs = []
self.autopkgtest_failures = []
self.binary_packages = []
self.binary_version_suffix = binary_version_suffix
self.changes_produced = {}
self.dirname = None
self.dsc = None
self.dsc_name = None
self.indep = False
self.indep_together_with = None
self.link_builds = link_builds
self.logs = {}
self.merged_changes = OrderedDict()
self.nominal_suite = None
self.orig_dirs = orig_dirs
self.output_dir = output_dir
self.piuparts_failures = []
self.source_from_archive = False
self.source_package = None
self.source_together_with = None
self.sourceful_changes_name = None
self.suite = None
self.vendor = vendor
if os.path.exists(self.buildable):
if os.path.isdir(self.buildable):
changelog = os.path.join(self.buildable, 'debian', 'changelog')
changelog = Changelog(open(changelog))
self.source_package = changelog.get_package()
self.nominal_suite = changelog.distributions
self._source_version = Version(changelog.version)
control = os.path.join(self.buildable, 'debian', 'control')
if len(changelog.distributions.split()) != 1:
raise ArgumentError(
'Cannot build for multiple distributions at once')
for paragraph in Deb822.iter_paragraphs(open(control)):
self.arch_wildcards |= set(
paragraph.get('architecture', '').split())
binary = paragraph.get('package')
if binary is not None:
self.binary_packages.append(binary)
elif self.buildable.endswith('.changes'):
self.dirname = os.path.dirname(self.buildable) or os.curdir
self.sourceful_changes_name = self.buildable
sourceful_changes = Changes(open(self.buildable))
if 'source' not in sourceful_changes['architecture'].split():
raise ArgumentError(
'Changes file {!r} must be sourceful'.format(
self.buildable))
self.nominal_suite = sourceful_changes['distribution']
for f in sourceful_changes['files']:
if f['name'].endswith('.dsc'):
if self.dsc_name is not None:
raise ArgumentError(
'Changes file {!r} contained more than one '
'.dsc file'.format(self.buildable))
self.dsc_name = os.path.join(self.dirname, f['name'])
if self.dsc_name is None:
raise ArgumentError(
'Changes file {!r} did not contain a .dsc file'.format(
self.buildable))
self.dsc = Dsc(open(self.dsc_name))
elif self.buildable.endswith('.dsc'):
self.dirname = os.path.dirname(self.buildable) or os.curdir
self.dsc_name = self.buildable
self.dsc = Dsc(open(self.dsc_name))
else:
raise ArgumentError(
'buildable must be .changes, .dsc or directory, not '
'{!r}'.format(self.buildable))
else:
self.source_from_archive = True
if '_' in self.buildable:
source, version = self.buildable.split('_', 1)
else:
source = self.buildable
version = None
self.source_package = source
if version is not None:
self._source_version = Version(version)
if self.dsc is not None:
self.source_package = self.dsc['source']
self._source_version = Version(self.dsc['version'])
self.arch_wildcards = set(self.dsc['architecture'].split())
self.binary_packages = [
p.strip() for p in self.dsc['binary'].split(',')
]
if self._source_version is not None:
self._binary_version = Version(
str(self._source_version) + self.binary_version_suffix)
timestamp = time.strftime('%Y%m%dt%H%M%S', time.gmtime())
if self.output_dir is None:
if self._binary_version is None:
dirname = '{}_{}'.format(self.source_package, timestamp)
else:
dirname = '{}_{}_{}'.format(self.source_package,
self._binary_version, timestamp)
self.output_dir = os.path.join(output_parent, dirname)
# For convenience, create a symbolic link for the latest build of
# each source package: hello_latest -> hello_2.10-1_20170319t102623
unversioned_symlink = os.path.join(output_parent,
self.source_package + '_latest')
with suppress(FileNotFoundError):
os.unlink(unversioned_symlink)
os.symlink(dirname, unversioned_symlink)
# If we know the version, also create a symbolic link for the
# latest build of each source/version pair:
# hello_2.10-1 -> hello_2.10-1_20170319t102623
if self._binary_version is not None:
versioned_symlink = os.path.join(
output_parent, '{}_{}'.format(self.source_package,
self._binary_version))
with suppress(FileNotFoundError):
os.unlink(versioned_symlink)
os.symlink(dirname, versioned_symlink)
# It's OK if the output directory exists but is empty.
with suppress(FileNotFoundError):
os.rmdir(self.output_dir)
# Otherwise, if someone already created this, we'll just crash out.
os.mkdir(self.output_dir)
if self.dsc is not None:
abs_file = os.path.abspath(self.dsc_name)
abs_dir, base = os.path.split(abs_file)
os.symlink(abs_file, os.path.join(self.output_dir, base))
for l in self.link_builds:
symlink = os.path.join(l, base)
with suppress(FileNotFoundError):
os.unlink(symlink)
os.symlink(abs_file, symlink)
for f in self.dsc['files']:
abs_file = os.path.join(abs_dir, f['name'])
os.symlink(abs_file, os.path.join(self.output_dir, f['name']))
for l in self.link_builds:
symlink = os.path.join(l, f['name'])
with suppress(FileNotFoundError):
os.unlink(symlink)
os.symlink(abs_file, symlink)