def _fixup(parameters, src_path):
"""Fix up installed paths.
Custom-built Python sometimes creates pythonX.Ym rather than
pythonX.Y header directory (same for libpython).
"""
del src_path # Unused.
header_dir_path = _add_version(parameters, 'include/python{}.{}')
if not header_dir_path.exists():
alt_header_dir_path = ASSERT.predicate(
_add_version(parameters, 'include/python{}.{}m'),
Path.is_dir,
)
LOG.info('symlink cpython headers')
with scripts.using_sudo():
scripts.ln(alt_header_dir_path.name, header_dir_path)
libpython_path = _add_version(parameters, 'lib/libpython{}.{}.so')
if parameters['shared'] and not libpython_path.exists():
alt_libpython_path = ASSERT.predicate(
_add_version(parameters, 'lib/libpython{}.{}m.so'),
Path.is_file,
)
LOG.info('symlink cpython library')
with scripts.using_sudo():
scripts.ln(alt_libpython_path.name, libpython_path)
def austerity(parameters):
src_path = _get_src_path(parameters)
with scripts.using_sudo():
for path in src_path.iterdir():
if path.name not in ('stage', ):
scripts.rm(path, recursive=True)
scripts.mkdir(src_path / '.git')
def chown(path):
user = getpass.getuser()
with scripts.using_sudo():
scripts.chown(
user,
grp.getgrgid(pwd.getpwnam(user).pw_gid).gr_name,
path,
)
def _install():
if Path('/usr/local/include/boost').exists():
LOG.info('skip: install boost')
return
LOG.info('install boost')
with scripts.using_sudo():
scripts.run(['./b2', 'install'])
scripts.run(['ldconfig'])
def austerity(parameters):
if parameters['//bases:build-xar-image']:
LOG.info('do nothing in xar builds')
return
with scripts.using_sudo():
for path in _get_src_path(parameters).iterdir():
if path.name not in ('Makefile', 'python'):
scripts.rm(path, recursive=True)
def build(parameters):
LOG.info('install package %s version %s', package, version)
with scripts.using_sudo():
scripts.run([
parameters['//third-party/cpython:pip'],
'install',
'%s==%s' % (package, version),
])
def trim(parameters):
if parameters['//bases:build-xar-image']:
LOG.info('do nothing in xar builds')
return
LOG.info('remove unused python modules')
with scripts.using_sudo():
modules_dir_path = parameters['modules']
for module in parameters['unused-modules']:
scripts.rm(modules_dir_path / module, recursive=True)
def build(parameters):
ASSERT.is_(parameters['inside-builder-pod'], True)
ASSERT.all(parameters['roots'], _is_root_dir)
with scripts.using_sudo():
# We should run `apt-get update` even when we are not upgrading
# the full system because some packages may be removed from the
# distro repo while our local package index still has it.
scripts.apt_get_update()
scripts.mkdir(parameters['drydock'])
def _install(parameters, src_path):
del src_path # Unused.
if parameters['python'].exists():
LOG.info('skip: install cpython')
return
LOG.info('install cpython')
with scripts.using_sudo():
# (Probably a bug?) When optimizations are enabled, this will
# re-run `make run_profile_task`.
scripts.make(['install'])
if parameters['shared']:
scripts.run(['ldconfig'])
def ops_db_client_build(parameters):
src_path = ASSERT.predicate(
shipyard2.rules.pythons\
.find_package(parameters, foreman.get_relpath()) /
'scripts' /
'ops-db-client',
Path.is_file,
)
dst_path = Path('/usr/local/bin/ops-db-client')
if dst_path.exists():
LOG.info('skip: install ops-db-client')
return
LOG.info('install ops-db-client')
with scripts.using_sudo():
scripts.cp(src_path, dst_path)
def build(parameters):
src_path = parameters['//bases:drydock'] / foreman.get_relpath()
src_path /= src_path.name
if (src_path / 'capnpc-java').exists():
LOG.info('skip: build capnproto-java')
return
LOG.info('build capnproto-java')
bin_path = _get_var_path('exec_prefix') / 'bin'
header_path = _get_var_path('includedir') / 'capnp'
with scripts.using_cwd(src_path):
scripts.make()
with scripts.using_sudo():
scripts.cp('capnpc-java', bin_path)
scripts.cp('compiler/src/main/schema/capnp/java.capnp',
header_path)
def build(parameters):
src_path = parameters['//bases:drydock'] / foreman.get_relpath()
src_path /= src_path.name
if (src_path / 'c++/.libs/libcapnp.so').exists():
LOG.info('skip: build capnproto')
return
LOG.info('build capnproto')
with scripts.using_cwd(src_path / 'c++'):
scripts.run(['autoreconf', '-i'])
scripts.run(['./configure'])
# Skip `make check` for now.
scripts.make()
with scripts.using_sudo():
scripts.make(['install'])
scripts.run(['ldconfig'])
def build(parameters):
src_path = _find_project(parameters, foreman.get_relpath())
root_path = _find_root_project(src_path)
ASSERT.false(src_path.samefile(root_path))
output_path = src_path / ('build/libs/%s-all.jar' % src_path.name)
task = ':'.join(src_path.relative_to(root_path).parts)
task = ':%s:shadowJar' % task
target_dir_path = parameters[root_project + ':packages']
if (target_dir_path / output_path.name).exists():
LOG.info('skip: run task %s', task)
return
LOG.info('run task %s', task)
with scripts.using_cwd(root_path):
scripts.run(['./gradlew', task])
with scripts.using_sudo():
scripts.mkdir(target_dir_path)
scripts.cp(output_path, target_dir_path)
def base_build(parameters):
version = ASSERT.not_none(parameters['%s/version' % shipyard2.BASE])
image_paths = [
utils.get_image_path(parameters, shipyard2.BASE),
utils.get_builder_image_path(parameters, shipyard2.BASE),
]
if all(map(Path.is_file, image_paths)):
LOG.info('skip: build base: %s %s', version, image_paths)
return
ASSERT.not_any(image_paths, Path.is_file)
LOG.info('build base: %s %s', version, image_paths)
for image_path in image_paths:
scripts.mkdir(image_path.parent)
with contextlib.ExitStack() as stack:
_build_base(stack, version, image_paths[0], image_paths[1])
for image_path in image_paths:
utils.chown(image_path)
with scripts.using_sudo():
for image_path in image_paths:
ctr_scripts.ctr_import_image(image_path)
def _build(parameters, make_global_options):
# `sudo --preserve-env` does not preserve PYTHONPATH (in case you
# are curious, you may run `sudo sudo -V` to get the list of
# preserved variables).
with scripts.using_sudo(), scripts.preserving_sudo_envs(['PYTHONPATH']):
scripts.run([
parameters['//third-party/cpython:pip'],
'install',
# Use `--no-deps` (`python3 setup.py install` does not
# support this, by the way) so that we won't implicitly
# install dependencies (you must explicitly specify them).
'--no-deps',
# Because we add a few Python package to PYTHONPATH, such as
# g1.bases, we need to force their installation (otherwise
# pip would consider them already installed).
'--upgrade',
'--force-reinstall',
*_build_get_global_options(parameters, make_global_options),
'.',
])
def build(parameters):
src_path = parameters['//bases:drydock'] / foreman.get_relpath()
src_path /= src_path.name
build_dir_path = src_path / 'build'
if build_dir_path.exists():
LOG.info('skip: build nng')
return
LOG.info('build nng')
scripts.mkdir(build_dir_path)
with scripts.using_cwd(build_dir_path):
scripts.run([
'cmake',
*('-D', 'BUILD_SHARED_LIBS:BOOL=ON'),
*('-G', 'Ninja'),
'..',
])
scripts.run(['ninja'])
# Skip `ninja test` for now.
with scripts.using_sudo():
scripts.run(['ninja', 'install'])
scripts.run(['ldconfig'])
def _build_base(stack, version, base_path, base_builder_path):
# Use base output directory for intermediate data.
tempdir_path = stack.enter_context(
tempfile.TemporaryDirectory(dir=base_path.parent)
)
LOG.info('generate base and base-builder under: %s', tempdir_path)
base_builder_rootfs_path = Path(tempdir_path) / 'base-builder'
stack.enter_context(scripts.using_sudo())
stack.callback(scripts.rm, base_builder_rootfs_path, recursive=True)
ctr_scripts.ctr([
'images',
'build-base',
*('--prune-stash-path', base_builder_rootfs_path),
shipyard2.BASE,
version,
base_path,
])
ctr_scripts.ctr_build_image(
utils.get_builder_name(shipyard2.BASE),
version,
base_builder_rootfs_path,
base_builder_path,
)
def web_server_setup(parameters):
del parameters # Unused.
with scripts.using_sudo():
scripts.mkdir('/srv/web')
scripts.cp(foreman.to_path('web-server/index.html'), '/srv/web')
def haproxy_setup(parameters):
del parameters # Unused.
with scripts.using_sudo():
scripts.mkdir(HAPROXY_PATH)
def austerity(parameters):
with scripts.using_sudo():
for path in _get_src_path(parameters).iterdir():
if path.name not in ('include', 'out.gn'):
scripts.rm(path, recursive=True)
def install(parameters):
with scripts.using_sudo():
scripts.apt_get_install(parameters[parameter_packages])
def database_setup(parameters):
del parameters # Unused.
with scripts.using_sudo():
scripts.mkdir(OPS_DB_PATH)
def cleanup(parameters):
ASSERT.is_(parameters['inside-builder-pod'], True)
ASSERT.all(parameters['roots'], _is_root_dir)
with scripts.using_sudo():
scripts.apt_get_clean()