def run(args):
if args.suite is None:
if args.default_suite is not None:
args.suite = args.default_suite
else:
raise ArgumentError('--suite must be specified')
worker = VirtWorker(
args.piuparts_worker,
mirrors=args.get_mirrors(),
storage=args.storage,
suite=args.piuparts_worker_suite,
)
failures = _piuparts(
args._things,
architecture=args.architecture,
extra_repositories=args._extra_repository,
mirrors=args.get_mirrors(),
storage=args.storage,
suite=args.suite,
tarballs=args.get_piuparts_tarballs(
architecture=args.architecture,
suite=args.suite,
vendor=args.vendor,
),
vendor=args.vendor,
worker=worker,
)
for failure in sorted(failures):
logger.error('%s failed testing', failure)
def run(args, really=True):
if args.suite is None:
if args.default_suite is not None:
args.suite = args.default_suite
else:
raise ArgumentError('--suite must be specified')
mirrors = args.get_mirrors()
worker = VirtWorker(
args.worker,
mirrors=mirrors,
storage=args.storage,
suite=args.worker_suite,
)
if (args.lxc_worker == args.worker
and args.lxc_worker_suite == args.worker_suite):
lxc_worker = worker
else:
lxc_worker = VirtWorker(
args.lxc_worker,
mirrors=mirrors,
storage=args.storage,
suite=args.lxc_worker_suite,
)
failures = _autopkgtest(
args._things,
architecture=args.architecture,
built_binaries=args._built_binaries,
extra_repositories=args._extra_repository,
lxc_24bit_subnet=args.lxc_24bit_subnet,
lxc_worker=lxc_worker,
worker=worker,
mirrors=mirrors,
modes=args.autopkgtest,
qemu_ram_size=args.qemu_ram_size,
# use the misc worker instead of a specific schroot worker
schroot_worker=None,
storage=args.storage,
suite=args.suite,
vendor=args.vendor,
)
for failure in sorted(failures):
logger.error('%s failed testing: %s', failure, failure.failures)
def run(args):
if args.suite is None:
args.suite = args.default_suite
architecture = args.architecture
mirrors = args.get_mirrors()
security_uri = args._security_uri
storage = args.storage
suite = args.suite
uri = args._uri
vendor = args.vendor
worker_argv = args.worker
worker_suite = args.worker_suite
apt_key_package = args.apt_key_package
lxc_24bit_subnet = args.lxc_24bit_subnet
os.makedirs(storage, exist_ok=True)
rootfs_tarball = '{arch}/{vendor}/{suite}/lxc-rootfs.tar.gz'.format(
arch=architecture,
vendor=vendor,
suite=suite,
)
meta_tarball = '{arch}/{vendor}/{suite}/lxc-meta.tar.gz'.format(
arch=architecture,
vendor=vendor,
suite=suite,
)
logger.info('Creating tarballs %s, %s...', rootfs_tarball, meta_tarball)
with VirtWorker(
worker_argv,
mirrors=mirrors,
storage=storage,
suite=worker_suite,
) as worker:
logger.info('Installing debootstrap etc.')
worker.check_call([
'env',
'DEBIAN_FRONTEND=noninteractive',
'apt-get',
'-y',
'-t', worker_suite.apt_suite,
'install',
'debootstrap',
'lxc',
'python3',
])
set_up_lxc_net(worker, lxc_24bit_subnet)
# FIXME: The lxc templates only allow installing the apt keyring
# to use, and do not allow passing --keyring to debootstrap
keyring = apt_key_package
if keyring is not None:
worker.call([
'env',
'DEBIAN_FRONTEND=noninteractive',
'apt-get',
'-y',
'-t', worker_suite.apt_suite,
'--no-install-recommends',
'install',
keyring,
])
# FIXME: This is silly, but it's a limitation of the lxc templates.
# We have to provide exactly two apt URLs.
security_suite = args.get_suite(vendor, str(suite) + '-security')
if uri is None:
uri = mirrors.lookup_suite(suite)
if security_uri is None:
security_uri = mirrors.lookup_suite(security_suite)
argv = [
'env', 'DEBIAN_FRONTEND=noninteractive',
worker.command_wrapper,
'--',
'lxc-create',
'--template={}'.format(vendor),
'--name={}-{}-{}'.format(vendor, suite, architecture),
'--',
'--release={}'.format(suite),
'--arch={}'.format(architecture),
'--mirror={}'.format(uri),
'--security-mirror={}'.format(security_uri),
]
if str(vendor) == 'ubuntu':
argv.append('--variant=minbase')
worker.check_call(argv)
worker.check_call([
'tar', '-C',
'/var/lib/lxc/{}-{}-{}/rootfs'.format(vendor, suite, architecture),
'-f', '{}/rootfs.tar.gz'.format(worker.scratch),
'--exclude=./var/cache/apt/archives/*.deb',
'-z', '-c', '.',
])
worker.check_call([
'tar', '-C',
'/var/lib/lxc/{}-{}-{}'.format(vendor, suite, architecture),
'-f', '{}/meta.tar.gz'.format(worker.scratch),
'-z', '-c', 'config',
])
out = os.path.join(storage, rootfs_tarball)
os.makedirs(os.path.dirname(out) or os.curdir, exist_ok=True)
worker.copy_to_host(
'{}/rootfs.tar.gz'.format(worker.scratch), out + '.new')
# FIXME: smoke-test it?
os.rename(out + '.new', out)
out = os.path.join(storage, meta_tarball)
os.makedirs(os.path.dirname(out) or os.curdir, exist_ok=True)
worker.copy_to_host(
'{}/meta.tar.gz'.format(worker.scratch), out + '.new')
# FIXME: smoke-test it?
os.rename(out + '.new', out)
logger.info('Created tarballs %s, %s', rootfs_tarball, meta_tarball)
def run(args):
if args.suite is None:
if args.worker_suite is not None:
args.suite = args.worker_suite
else:
raise ArgumentError('--suite must be specified')
architecture = args.architecture
keep = args._keep
kernel_package = args.get_kernel_package(architecture)
mirrors = args.get_mirrors()
out = args.write_qemu_image
qemu_image_size = args.qemu_image_size
storage = args.storage
suite = args.suite
uri = args._uri
vmdebootstrap_options = args.vmdebootstrap_options
if uri is None:
uri = mirrors.lookup_suite(suite)
try:
version = subprocess.check_output(
['dpkg-query', '-W', '-f${Version}', 'vmdebootstrap'],
universal_newlines=True).rstrip('\n')
except:
# non-dpkg host, guess a recent version
version = Version('1.7')
else:
version = Version(version)
with TemporaryDirectory(prefix='vectis-bootstrap-') as scratch:
argv = [
'sudo',
os.path.join(
os.path.dirname(__file__), os.pardir,
'vectis-command-wrapper'),
'--',
]
argv.extend(
vmdebootstrap_argv(
version,
architecture=architecture,
kernel_package=kernel_package,
qemu_image_size=qemu_image_size,
suite=suite,
uri=uri,
),
)
argv.extend(vmdebootstrap_options)
argv.append(
'--customize={}'.format(os.path.join(
os.path.dirname(__file__), os.pardir, 'setup-testbed')))
argv.append('--owner={}'.format(pwd.getpwuid(os.getuid())[0]))
argv.append('--image={}/output.raw'.format(scratch))
subprocess.check_call(argv)
subprocess.check_call([
'qemu-img', 'convert', '-f', 'raw',
'-O', 'qcow2', '-c', '-p',
'{}/output.raw'.format(scratch),
'{}/output.qcow2'.format(scratch),
])
os.makedirs(os.path.dirname(out) or os.curdir, exist_ok=True)
shutil.move('{}/output.qcow2'.format(scratch), out + '.new')
try:
with VirtWorker(
['qemu', '{}.new'.format(out)],
storage=storage,
suite=suite,
mirrors=mirrors) as worker:
worker.check_call([
'env',
'DEBIAN_FRONTEND=noninteractive',
'apt-get',
'-y',
'--no-install-recommends',
'-t', suite.apt_suite,
'install',
'python3',
'sbuild',
'schroot',
])
except:
if not keep:
os.remove(out + '.new')
raise
else:
os.rename(out + '.new', out)
def run(args):
if args.suite is None:
if args.default_suite is not None:
args.suite = args.default_suite
else:
raise ArgumentError('--suite must be specified')
apt_update = args._apt_update
argv = args._argv
chdir = args._chdir
mirrors = args.get_mirrors()
input_ = args._input
output_dir = args.output_dir
output_parent = args.output_parent
qemu_image = args.qemu_image
qemu_ram_size = args.qemu_ram_size
shell_command = args._shell_command
storage = args.storage
suite = args.suite
timestamp = time.strftime('%Y%m%dt%H%M%S', time.gmtime())
if output_dir is None:
output_dir = os.path.join(output_parent,
'vectis-run_{}'.format(timestamp))
with suppress(FileNotFoundError):
os.rmdir(output_dir)
os.mkdir(output_dir)
if shell_command is None and not argv:
raise ArgumentError(
'Usage: vectis run -- PROGRAM [$1 [$2...]] or vectis run '
'-c "shell one-liner" [$0 [$1 [$2...]]]')
for suite in (suite, ):
for ancestor in suite.hierarchy:
mirror = mirrors.lookup_suite(ancestor)
if mirror is None:
raise ArgumentError(
'No mirror configured for {}'.format(ancestor))
virt = ['qemu']
if qemu_ram_size is not None:
virt.append('--ram-size={}'.format(qemu_ram_size // _1M))
virt.append(qemu_image)
with VirtWorker(
virt,
apt_update=apt_update,
mirrors=mirrors,
storage=storage,
suite=suite,
) as worker:
worker_input = worker.scratch + '/in'
temp = worker.scratch + '/tmp'
artifacts = worker.scratch + '/out'
worker.check_call(['mkdir', artifacts, worker_input, temp])
if chdir == 'in':
chdir = worker_input
elif chdir == 'out':
chdir = artifacts
elif chdir == 'tmp':
chdir = temp
elif chdir[0] != '/' and chdir != '.':
raise ArgumentError(
"Argument to --chdir must be 'in', 'out', 'tmp', "
"'.' or absolute")
wrapper = [
'sh',
'-c',
'cd "$1" && shift && exec "$@"',
'sh',
chdir,
'env',
'AUTOPKGTEST_ARTIFACTS={}'.format(artifacts),
'ADT_ARTIFACTS={}'.format(artifacts),
'VECTIS_OUT={}'.format(artifacts),
'VECTIS_TMP={}'.format(temp),
'AUTOPKGTEST_TMP={}'.format(temp),
'ADTTMP={}'.format(temp),
]
if input_ is not None:
if os.path.isdir(input_):
worker.copy_to_guest(os.path.join(input_, ''),
worker_input + '/')
else:
worker_input = worker_input + '/' + os.path.basename(input_)
worker.copy_to_guest(input_, worker_input)
wrapper.append('VECTIS_IN={}'.format(worker_input))
try:
if shell_command is not None:
worker.check_call(wrapper + ['sh', '-c', shell_command] + argv)
else:
worker.check_call(wrapper + argv)
finally:
if worker.call(['rmdir', artifacts],
stderr=subprocess.DEVNULL) == 0:
logger.info('Command produced no artifacts')
os.rmdir(output_dir)
else:
worker.copy_to_host(artifacts + '/',
os.path.join(output_dir, ''))
logger.info('Artifacts produced by command are in %s',
output_dir)
def run(args):
if args.suite is None:
if args.default_suite is not None:
args.suite = args.default_suite
else:
raise ArgumentError('--suite must be specified')
# From argv or configuration
architecture = args.architecture
components = args.components
debootstrap_script = args.debootstrap_script
keep = args._keep
mirrors = args.get_mirrors()
storage = args.storage
suite = args.suite
test_package = args._test_package
uri = args._uri
vendor = args.vendor
worker_argv = args.worker
worker_suite = args.worker_suite
# From configuration
apt_key = args.apt_key
apt_key_package = args.apt_key_package
os.makedirs(storage, exist_ok=True)
for suite in (worker_suite, suite):
for ancestor in suite.hierarchy:
mirror = mirrors.lookup_suite(ancestor)
if mirror is None:
raise ArgumentError(
'No mirror configured for {}'.format(ancestor))
if uri is None:
uri = mirrors.lookup_suite(suite)
sbuild_tarball = '{arch}/{vendor}/{suite}/sbuild.tar.gz'.format(
arch=architecture,
vendor=vendor,
suite=suite,
)
logger.info('Creating tarball %s...', sbuild_tarball)
with VirtWorker(
worker_argv,
mirrors=mirrors,
storage=storage,
suite=worker_suite,
) as worker:
logger.info('Installing debootstrap and sbuild')
worker.check_call([
'env',
'DEBIAN_FRONTEND=noninteractive',
'apt-get',
'-y',
'--no-install-recommends',
'-t',
worker_suite.apt_suite,
'install',
'debootstrap',
'python3',
'sbuild',
'schroot',
])
keyring = apt_key_package
if keyring is not None:
worker.call([
'env',
'DEBIAN_FRONTEND=noninteractive',
'apt-get',
'-y',
'-t',
worker_suite.apt_suite,
'--no-install-recommends',
'install',
keyring,
])
debootstrap_args = []
if worker.call(['test', '-f', apt_key]) == 0:
logger.info('Found apt key worker:{}'.format(apt_key))
debootstrap_args.append('--keyring={}'.format(apt_key))
elif os.path.exists(apt_key):
logger.info('Found apt key host:{}, copying to worker:{}'.format(
apt_key, '{}/apt-key.gpg'.format(worker.scratch)))
worker.copy_to_guest(apt_key,
'{}/apt-key.gpg'.format(worker.scratch))
debootstrap_args.append('--keyring={}/apt-key.gpg'.format(
worker.scratch))
else:
logger.warning(
'Apt key host:{} not found; leaving it out and hoping '
'for the best'.format(apt_key))
debootstrap_args.append('--components={}'.format(','.join(components)))
worker.check_call([
'env',
'DEBIAN_FRONTEND=noninteractive',
worker.command_wrapper,
'--',
'sbuild-createchroot',
'--arch={}'.format(architecture),
'--include=fakeroot,sudo,vim',
'--components={}'.format(','.join(components)),
'--make-sbuild-tarball={}/output.tar.gz'.format(worker.scratch),
] + debootstrap_args + [
str(suite),
'{}/chroot'.format(worker.scratch),
uri,
'/usr/share/debootstrap/scripts/{}'.format(debootstrap_script),
])
out = os.path.join(storage, sbuild_tarball)
os.makedirs(os.path.dirname(out) or os.curdir, exist_ok=True)
# Smoke-test the new tarball before being prepared to use it.
if test_package:
try:
lines = worker.check_output(
[
'schroot',
'-c',
'{}-{}-sbuild'.format(suite, architecture),
'--',
'sh',
'-c',
'apt-get update >&2 && '
'( apt-cache showsrc --only-source "$1" || '
' apt-cache showsrc "$1" ) | '
'sed -ne "s/^Version: *//p"',
'sh', # argv[0]
test_package,
],
universal_newlines=True).strip().splitlines()
version = sorted(map(Version, lines))[-1]
buildable = '{}_{}'.format(test_package, version)
worker.check_call([
worker.command_wrapper,
'--chdir',
worker.scratch,
'--',
'runuser',
'-u',
'sbuild',
'--',
'sbuild',
'--arch',
architecture,
'-c',
'{}-{}-sbuild'.format(suite, architecture),
'-d',
'whatever',
'--no-run-lintian',
buildable,
])
except Exception:
if keep:
worker.copy_to_host(
'{}/output.tar.gz'.format(worker.scratch),
out + '.new')
raise
worker.copy_to_host('{}/output.tar.gz'.format(worker.scratch),
out + '.new')
os.rename(out + '.new', out)
logger.info('Created tarball %s', sbuild_tarball)
def run(args):
if args.suite is None:
if args.default_suite is not None:
args.suite = args.default_suite
else:
raise ArgumentError('--suite must be specified')
apt_key = args.apt_key
apt_key_package = args.apt_key_package
architecture = args.architecture
components = args.components
keep = args._keep
kernel_package = args.get_kernel_package(architecture)
include = args._include
mirrors = args.get_mirrors()
out = args.write_qemu_image
qemu_image_size = args.qemu_image_size
storage = args.storage
uri = args._uri
vendor = args.vendor
suite = args.get_suite(vendor, args.suite)
vmdebootstrap_options = args.vmdebootstrap_options
vmdebootstrap_worker = args.vmdebootstrap_worker
vmdebootstrap_worker_suite = args.vmdebootstrap_worker_suite
default_dir = os.path.join(storage, architecture, str(vendor), str(suite))
if uri is None:
uri = mirrors.lookup_suite(suite)
if False:
created, out = new_ubuntu_cloud(
architecture=architecture,
default_dir=default_dir,
out=out,
qemu_image_size=qemu_image_size,
suite=suite,
uri=uri,
vendor=vendor,
)
else:
created, out = new(
apt_key=apt_key,
apt_key_package=apt_key_package,
architecture=architecture,
components=components,
default_dir=default_dir,
kernel_package=kernel_package,
include=include,
merged_usr=args._merged_usr,
mirrors=mirrors,
out=out,
qemu_image_size=qemu_image_size,
storage=storage,
suite=suite,
uri=uri,
vmdebootstrap_options=vmdebootstrap_options,
vmdebootstrap_worker=vmdebootstrap_worker,
vmdebootstrap_worker_suite=vmdebootstrap_worker_suite,
)
try:
with VirtWorker(
['qemu', created],
mirrors=mirrors,
storage=storage,
suite=suite,
) as worker:
worker.set_up_apt()
worker.check_call(['apt-get', '-y', 'update'])
worker.check_call([
'env',
'DEBIAN_FRONTEND=noninteractive',
'apt-get',
'-y',
'--no-install-recommends',
'install',
'python3',
'sbuild',
'schroot',
])
except Exception:
if keep:
if created != out + '.new':
os.rename(created, out + '.new')
else:
os.remove(created)
raise
else:
os.rename(created, out)
def new(*,
apt_key,
apt_key_package,
architecture,
components,
default_dir,
include=(),
kernel_package,
merged_usr,
mirrors,
out,
qemu_image_size,
storage,
suite,
uri,
vmdebootstrap_options,
vmdebootstrap_worker,
vmdebootstrap_worker_suite):
for suite in (vmdebootstrap_worker_suite, suite):
for ancestor in suite.hierarchy:
mirror = mirrors.lookup_suite(ancestor)
if mirror is None:
raise ArgumentError(
'No mirror configured for {}'.format(ancestor))
with VirtWorker(
vmdebootstrap_worker,
mirrors=mirrors,
storage=storage,
suite=vmdebootstrap_worker_suite,
) as worker:
worker.check_call([
'env',
'DEBIAN_FRONTEND=noninteractive',
'apt-get',
'-y',
'upgrade',
])
worker_packages = [
'autopkgtest',
'grub2-common',
'python3',
'qemu-utils',
'vmdebootstrap',
]
# Optional (x86 only, but necessary for wheezy)
optional_worker_packages = [
'extlinux',
'mbr',
]
keyring = apt_key_package
if keyring is not None:
optional_worker_packages.append(keyring)
worker.check_call([
'env',
'DEBIAN_FRONTEND=noninteractive',
'apt-get',
'-y',
'-t',
vmdebootstrap_worker_suite.apt_suite,
'--no-install-recommends',
'install',
] + worker_packages)
# Failure is ignored for these non-critical packages
for p in optional_worker_packages:
worker.call([
'env', 'DEBIAN_FRONTEND=noninteractive', 'apt-get', '-y', '-t',
vmdebootstrap_worker_suite.apt_suite,
'--no-install-recommends', 'install', p
])
version = worker.dpkg_version('vmdebootstrap')
debootstrap_version = worker.dpkg_version('debootstrap')
argv, debootstrap_args, default_name = vmdebootstrap_argv(
version,
architecture=architecture,
components=components,
debootstrap_version=debootstrap_version,
include=include,
kernel_package=kernel_package,
qemu_image_size=qemu_image_size,
suite=suite,
uri=uri,
merged_usr=merged_usr,
)
argv.extend(vmdebootstrap_options)
if worker.call(['test', '-f', apt_key]) == 0:
logger.info('Found apt key worker:{}'.format(apt_key))
debootstrap_args.append('keyring={}'.format(apt_key))
elif os.path.exists(apt_key):
logger.info('Found apt key host:{}, copying to worker:{}'.format(
apt_key, '{}/apt-key.gpg'.format(worker.scratch)))
worker.copy_to_guest(apt_key,
'{}/apt-key.gpg'.format(worker.scratch))
debootstrap_args.append('keyring={}/apt-key.gpg'.format(
worker.scratch))
else:
logger.warning('Apt key host:{} not found; leaving it out and '
'hoping for the best'.format(apt_key))
if debootstrap_args:
argv.append('--debootstrapopts={}'.format(
' '.join(debootstrap_args)))
worker.copy_to_guest(
os.path.join(os.path.dirname(__file__), '..', 'setup-testbed'),
'{}/setup-testbed'.format(worker.scratch))
worker.check_call(
['chmod', '0755', '{}/setup-testbed'.format(worker.scratch)])
worker.check_call([
'env',
'DEBIAN_FRONTEND=noninteractive',
worker.command_wrapper,
'--',
] + argv + [
'--customize={}/setup-testbed'.format(worker.scratch),
'--image={}/output.raw'.format(worker.scratch),
])
worker.check_call([
'qemu-img',
'convert',
'-f',
'raw',
'-O',
'qcow2',
'-c',
'-p',
'{}/output.raw'.format(worker.scratch),
'{}/output.qcow2'.format(worker.scratch),
])
if out is None:
out = os.path.join(default_dir, default_name)
os.makedirs(os.path.dirname(out) or os.curdir, exist_ok=True)
worker.copy_to_host('{}/output.qcow2'.format(worker.scratch),
out + '.new')
return out + '.new', out
def run(args):
if args.suite is None:
args.suite = args.default_suite
architecture = args.architecture
mirrors = args.get_mirrors()
storage = args.storage
suite = args.suite
uri = args._uri
vendor = args.vendor
worker_argv = args.lxd_worker
worker_suite = args.lxd_worker_suite
lxc_24bit_subnet = args.lxc_24bit_subnet
if uri is None:
uri = mirrors.lookup_suite(suite)
os.makedirs(storage, exist_ok=True)
tarball = '{arch}/{vendor}/{suite}/lxd-autopkgtest.tar.gz'.format(
arch=architecture,
vendor=vendor,
suite=suite,
)
logger.info('Creating tarball %s...', tarball)
with VirtWorker(
worker_argv,
mirrors=mirrors,
storage=storage,
suite=worker_suite,
) as worker:
logger.info('Installing debootstrap etc.')
worker.check_call([
'env',
'DEBIAN_FRONTEND=noninteractive',
'apt-get',
'-y',
'-t',
worker_suite.apt_suite,
'install',
'autopkgtest',
'debootstrap',
'lxd',
'lxd-client',
'python3',
])
worker.check_call([
'lxd',
'init',
'--auto',
'--debug',
'--verbose',
])
set_up_lxd_net(worker, lxc_24bit_subnet)
worker.check_call([
'env',
'MIRROR={}'.format(uri),
'RELEASE={}'.format(suite),
worker.command_wrapper,
'--',
'autopkgtest-build-lxd',
'images:{}/{}/{}'.format(vendor, suite, architecture),
])
info = worker.check_output([
'lxc',
'image',
'info',
'autopkgtest/{}/{}/{}'.format(vendor, suite, architecture),
])
for line in info.splitlines():
if line.startswith(b'Fingerprint: '):
fingerprint = line.split(b':', 1)[1].strip().decode('ascii')
break
else:
raise Exception('Cannot find image fingerprint')
worker.check_call([
'lxc',
'image',
'export',
'autopkgtest/{}/{}/{}'.format(vendor, suite, architecture),
worker.scratch,
])
out = os.path.join(storage, tarball)
os.makedirs(os.path.dirname(out) or os.curdir, exist_ok=True)
worker.copy_to_host('{}/{}.tar.gz'.format(worker.scratch, fingerprint),
out + '.new')
os.rename(out + '.new', out)
logger.info('Created tarball %s', tarball)
def run(args):
if args.suite is None:
if args.default_suite is not None:
args.suite = args.default_suite
else:
raise ArgumentError('--suite must be specified')
# From argv or configuration
architecture = args.architecture
components = args.components
keep = args._keep
mirrors = args.get_mirrors()
storage = args.storage
suite = args.suite
test_package = args._test_package
uri = args._uri
vendor = args.vendor
worker_argv = args.worker
worker_suite = args.worker_suite
# From configuration
apt_key = args.apt_key
apt_key_package = args.apt_key_package
os.makedirs(storage, exist_ok=True)
for suite in (worker_suite, suite):
for ancestor in suite.hierarchy:
mirror = mirrors.lookup_suite(ancestor)
if mirror is None:
raise ArgumentError(
'No mirror configured for {}'.format(ancestor))
if uri is None:
uri = mirrors.lookup_suite(suite)
tarball = '{arch}/{vendor}/{suite}/pbuilder.tar.gz'.format(
arch=architecture,
vendor=vendor,
suite=suite,
)
logger.info('Creating tarball %s...', tarball)
with VirtWorker(
worker_argv,
mirrors=mirrors,
storage=storage,
suite=worker_suite,
) as worker:
logger.info('Installing debootstrap and pbuilder')
worker.check_call([
'env',
'DEBIAN_FRONTEND=noninteractive',
'apt-get',
'-y',
'--no-install-recommends',
'-t',
worker_suite.apt_suite,
'install',
'debootstrap',
'python3',
'pbuilder',
])
keyring = apt_key_package
if keyring is not None:
worker.call([
'env',
'DEBIAN_FRONTEND=noninteractive',
'apt-get',
'-y',
'-t',
worker_suite.apt_suite,
'--no-install-recommends',
'install',
keyring,
])
pbuilder_args = [
'create',
'--aptcache',
'',
'--architecture',
architecture,
'--components',
' '.join(components),
'--basetgz',
'{}/output.tar.gz'.format(worker.scratch),
'--mirror',
uri,
'--distribution',
str(suite),
]
debootstrap_args = []
if worker.call(['test', '-f', apt_key]) == 0:
logger.info('Found apt key worker:{}'.format(apt_key))
pbuilder_args.append('--keyring')
pbuilder_args.append(apt_key)
debootstrap_args.append('--keyring={}'.format(apt_key))
elif os.path.exists(apt_key):
logger.info('Found apt key host:{}, copying to worker:{}'.format(
apt_key, '{}/apt-key.gpg'.format(worker.scratch)))
worker.copy_to_guest(apt_key,
'{}/apt-key.gpg'.format(worker.scratch))
pbuilder_args.append('--keyring')
pbuilder_args.append('{}/apt-key.gpg'.format(worker.scratch))
debootstrap_args.append('--keyring={}/apt-key.gpg'.format(
worker.scratch))
else:
logger.warning(
'Apt key host:{} not found; leaving it out and hoping '
'for the best'.format(apt_key))
for arg in debootstrap_args:
pbuilder_args.append('--debootstrapopts')
pbuilder_args.append(arg)
worker.check_call([
'touch',
'/root/.pbuilderrc',
])
logger.info('pbuilder %r', pbuilder_args)
worker.check_call([
'env',
'DEBIAN_FRONTEND=noninteractive',
worker.command_wrapper,
'--',
'pbuilder',
] + pbuilder_args)
out = os.path.join(storage, tarball)
os.makedirs(os.path.dirname(out) or os.curdir, exist_ok=True)
# Smoke-test the new tarball before being prepared to use it.
if test_package:
with TemporaryDirectory(prefix='vectis-pbuilder-') as tmp:
with AtomicWriter(os.path.join(tmp, 'script')) as writer:
writer.write(
textwrap.dedent('''\
#!/bin/sh
set -e
set -u
cd "$2"
perl -ne 'print $_; if (s/^deb\s/deb-src /) { print $_ }' \
< /etc/apt/sources.list \
> /etc/apt/sources.list.new
mv /etc/apt/sources.list.new /etc/apt/sources.list
apt-get update >&2
apt-get --download-only source "$1" >&2
mv *.dsc "$1.dsc"
'''))
worker.copy_to_guest(os.path.join(tmp, 'script'),
'{}/script'.format(worker.scratch))
worker.check_call([
'chmod',
'0755',
'{}/script'.format(worker.scratch),
])
try:
lines = worker.check_output(
[
'pbuilder',
'execute',
'--aptcache',
'',
'--basetgz',
'{}/output.tar.gz'.format(worker.scratch),
'--bindmounts',
'{}'.format(worker.scratch),
'--',
'{}/script'.format(worker.scratch),
test_package,
worker.scratch,
],
universal_newlines=True).strip().splitlines()
logger.info('%r', lines)
worker.check_call([
worker.command_wrapper,
'--chdir',
worker.scratch,
'--',
'pbuilder',
'build',
'--aptcache',
'',
'--basetgz',
'{}/output.tar.gz'.format(worker.scratch),
'{}.dsc'.format(test_package),
])
except Exception:
if keep:
worker.copy_to_host(
'{}/output.tar.gz'.format(worker.scratch),
out + '.new')
raise
worker.copy_to_host('{}/output.tar.gz'.format(worker.scratch),
out + '.new')
os.rename(out + '.new', out)
logger.info('Created tarball %s', tarball)
def run(args):
if args.suite is None:
if args.default_suite is not None:
args.suite = args.default_suite
else:
raise ArgumentError('--suite must be specified')
architecture = args.architecture
mirrors = args.get_mirrors()
storage = args.storage
suite = args.suite
uri = args._uri
vendor = args.vendor
worker_argv = args.worker
worker_suite = args.worker_suite
apt_key = args.apt_key
apt_key_package = args.apt_key_package
os.makedirs(storage, exist_ok=True)
for suite in (worker_suite, suite):
for ancestor in suite.hierarchy:
mirror = mirrors.lookup_suite(ancestor)
if mirror is None:
raise ArgumentError(
'No mirror configured for {}'.format(ancestor))
if uri is None:
uri = mirrors.lookup_suite(suite)
if args._merged_usr:
basename = 'minbase-merged-usr'
else:
basename = 'minbase'
minbase_tarball = '{arch}/{vendor}/{suite}/{basename}.tar.gz'.format(
arch=architecture,
basename=basename,
vendor=vendor,
suite=suite,
)
logger.info('Creating tarball %s...', minbase_tarball)
with VirtWorker(
worker_argv,
mirrors=mirrors,
storage=storage,
suite=worker_suite,
) as worker:
logger.info('Installing debootstrap')
worker.check_call([
'env',
'DEBIAN_FRONTEND=noninteractive',
'apt-get',
'-y',
'-t', worker_suite.apt_suite,
'--no-install-recommends',
'install',
'debootstrap',
'python3',
])
debootstrap_version = worker.dpkg_version('debootstrap')
if apt_key_package is not None:
worker.call([
'env',
'DEBIAN_FRONTEND=noninteractive',
'apt-get',
'-y',
'-t', worker_suite.apt_suite,
'--no-install-recommends',
'install',
apt_key_package,
])
debootstrap_args = []
if worker.call(['test', '-f', apt_key]) == 0:
logger.info('Found apt key worker:{}'.format(apt_key))
debootstrap_args.append('--keyring={}'.format(apt_key))
elif os.path.exists(apt_key):
logger.info('Found apt key host:{}, copying to worker:{}'.format(
apt_key, '{}/apt-key.gpg'.format(worker.scratch)))
worker.copy_to_guest(
apt_key, '{}/apt-key.gpg'.format(worker.scratch))
debootstrap_args.append('--keyring={}/apt-key.gpg'.format(
worker.scratch))
else:
logger.warning(
'Apt key host:{} not found; leaving it out and hoping for the '
'best'.format(apt_key))
debootstrap_args.append('--components={}'.format(
','.join(args.components)))
if debootstrap_version >= Version('1.0.86~'):
if args._merged_usr:
debootstrap_args.append('--merged-usr')
else:
# piuparts really doesn't like merged /usr
debootstrap_args.append('--no-merged-usr')
worker.check_call([
'env', 'DEBIAN_FRONTEND=noninteractive',
worker.command_wrapper,
'--',
'debootstrap',
'--arch={}'.format(architecture),
'--components={}'.format(','.join(args.components)),
'--variant=minbase',
'--verbose',
] + debootstrap_args + [
str(suite),
'{}/chroot'.format(worker.scratch),
uri,
'/usr/share/debootstrap/scripts/{}'.format(
args.debootstrap_script),
])
worker.check_call([
'chroot', '{}/chroot'.format(worker.scratch),
'apt-get', 'clean',
])
worker.check_call([
'tar', '-C', '{}/chroot'.format(worker.scratch),
'-f', '{}/output.tar.gz'.format(worker.scratch),
'-z', '-c', '.',
])
out = os.path.join(storage, minbase_tarball)
os.makedirs(os.path.dirname(out) or os.curdir, exist_ok=True)
worker.copy_to_host(
'{}/output.tar.gz'.format(worker.scratch), out + '.new')
# FIXME: smoke-test it?
os.rename(out + '.new', out)
logger.info('Created tarball %s', minbase_tarball)
def run(args):
components = args.components
link_builds = args.link_builds
orig_dirs = args.orig_dirs
output_dir = args.output_dir
output_parent = args.output_parent
mirrors = args.get_mirrors()
sbuild_options = list(args._sbuild_options)
storage = args.storage
vendor = args.vendor
deb_build_options = set()
if 'DEB_BUILD_OPTIONS' in os.environ:
for arg in os.environ['DEB_BUILD_OPTIONS'].split():
deb_build_options.add(arg)
for arg in args._add_deb_build_option:
deb_build_options.add(arg)
for arg in deb_build_options:
if arg == 'parallel' or arg.startswith('parallel='):
break
else:
deb_build_options.add('parallel={}'.format(args.parallel))
profiles = set()
if args._build_profiles is not None:
for arg in args._build_profiles.split(','):
profiles.add(arg)
elif 'DEB_BUILD_PROFILES' in os.environ:
for arg in os.environ['DEB_BUILD_PROFILES'].split():
profiles.add(arg)
for arg in args._add_build_profile:
profiles.add(arg)
db_options = []
if args._versions_since:
db_options.append('-v{}'.format(args._versions_since))
if args._include_orig_source is not None:
MAP = {
'yes': 'a',
'always': 'a',
'force': 'a',
'a': 'a',
'auto': 'i',
'maybe': 'i',
'i': 'i',
'no': 'd',
'never': 'd',
'd': 'd',
}
db_options.append('-s{}'.format(MAP[args._include_orig_source]))
ds_options = []
if args.dpkg_source_diff_ignore is ...:
ds_options.append('-i')
elif args.dpkg_source_diff_ignore is not None:
ds_options.append('-i{}'.format(args.dpkg_source_diff_ignore))
for pattern in args.dpkg_source_tar_ignore:
if pattern is ...:
ds_options.append('-I')
else:
ds_options.append('-I{}'.format(pattern))
for pattern in args.dpkg_source_extend_diff_ignore:
ds_options.append('--extend-diff-ignore={}'.format(pattern))
buildables = []
for a in (args._buildables or ['.']):
buildable = Buildable(a,
binary_version_suffix=args._append_to_version,
link_builds=link_builds,
orig_dirs=orig_dirs,
output_dir=output_dir,
output_parent=output_parent,
vendor=vendor)
buildable.select_suite(args, args.suite)
buildables.append(buildable)
for suite in (buildable.suite, args.sbuild_worker_suite):
assert isinstance(suite, Suite)
for ancestor in suite.hierarchy:
mirror = mirrors.lookup_suite(ancestor)
if mirror is None:
raise ArgumentError(
'No mirror configured for {}'.format(ancestor))
sbuild_worker = VirtWorker(
args.sbuild_worker,
mirrors=mirrors,
storage=storage,
suite=args.sbuild_worker_suite,
)
with sbuild_worker as worker:
default_architecture = worker.dpkg_architecture
_sbuild(
buildables,
archs=args._archs,
components=components,
deb_build_options=deb_build_options,
dpkg_buildpackage_options=db_options,
dpkg_source_options=ds_options,
extra_repositories=args._extra_repository,
indep=args._indep,
indep_together=args.sbuild_indep_together,
mirrors=mirrors,
profiles=profiles,
build_source=args._build_source,
sbuild_options=sbuild_options,
source_only=args._source_only,
storage=storage,
source_together=args.sbuild_source_together,
vendor=vendor,
worker=worker,
)
if (args.worker == args.sbuild_worker
and args.worker_suite == args.sbuild_worker_suite):
misc_worker = sbuild_worker
else:
misc_worker = VirtWorker(
args.worker,
mirrors=mirrors,
storage=storage,
suite=args.worker_suite,
)
if (args.piuparts_worker == args.sbuild_worker
and args.piuparts_worker_suite == args.sbuild_worker_suite):
piuparts_worker = sbuild_worker
else:
piuparts_worker = VirtWorker(
args.piuparts_worker,
mirrors=mirrors,
storage=storage,
suite=args.piuparts_worker_suite,
)
if (args.lxc_worker == args.sbuild_worker
and args.lxc_worker_suite == args.sbuild_worker_suite):
lxc_worker = sbuild_worker
else:
lxc_worker = VirtWorker(
args.lxc_worker,
mirrors=mirrors,
storage=storage,
suite=args.lxc_worker_suite,
)
_autopkgtest(
buildables,
default_architecture,
components=components,
extra_repositories=args._extra_repository,
lxc_24bit_subnet=args.lxc_24bit_subnet,
lxc_worker=lxc_worker,
mirrors=mirrors,
modes=args.autopkgtest,
qemu_ram_size=args.qemu_ram_size,
schroot_worker=sbuild_worker,
storage=storage,
vendor=vendor,
worker=misc_worker,
)
if args.piuparts_tarballs:
_piuparts(
buildables,
default_architecture,
components=components,
extra_repositories=args._extra_repository,
mirrors=mirrors,
storage=storage,
tarballs=args.piuparts_tarballs,
vendor=vendor,
worker=piuparts_worker,
)
_summarize(buildables)
_lintian(buildables)
if args._reprepro_dir:
_publish(buildables, args._reprepro_dir, args._reprepro_suite)
# We print these separately, right at the end, so that if you built more
# than one thing, the last screenful of information is the really
# important bit for testing/signing/upload
for buildable in buildables:
logger.info(
'Merged changes files from %s:\n\t%s',
buildable,
'\n\t'.join(buildable.merged_changes.values()),
)
if buildable.autopkgtest_failures:
logger.error('Autopkgtest failures for %s:', buildable)
for x in buildable.autopkgtest_failures:
logger.error('- %s', x)
if buildable.piuparts_failures:
logger.error('Piuparts failures for %s:', buildable)
for x in buildable.piuparts_failures:
logger.error('- %s', x)
for buildable in buildables:
logger.info(
'Output directory for %s: %s',
buildable,
buildable.output_dir,
)