def configure(self):
return [
self.run(
ShellQuoted(
'LDFLAGS="$LDFLAGS -L"{p}"/lib -Wl,-rpath="{p}"/lib" '
'CFLAGS="$CFLAGS -I"{p}"/include" '
'CPPFLAGS="$CPPFLAGS -I"{p}"/include" '
'PY_PREFIX={p} '
'./configure --prefix={p}').format(
p=self.option('prefix'))),
]
def fbcode_builder_spec(builder):
return {
"depends_on":
[folly, wangle, fizz, sodium, zstd, mvfst, proxygen_quic],
"steps": [
# Tests for the full build with no QUIC/HTTP3
# Proxygen is the last step, so we are still in its working dir.
builder.step("Run proxygen tests",
[builder.run(ShellQuoted("make test"))])
],
}
def fbcode_builder_spec(builder):
builder.add_option("thom311/libnl:git_hash", "libnl3_2_25")
builder.add_option("openr/build:cmake_defines", {"ADD_ROOT_TESTS": "OFF"})
maybe_curl_patch = []
patch = path_join(
builder.option("projects_dir"),
"../shipit_projects/openr/build/fix-route-obj-attr-list.patch",
)
if not builder.has_option("shipit_project_dir"):
maybe_curl_patch = [
builder.run(
ShellQuoted(
"curl -O https://raw.githubusercontent.com/facebook/openr/master/"
"build/fix-route-obj-attr-list.patch"))
]
patch = "fix-route-obj-attr-list.patch"
libnl_build_commands = maybe_curl_patch + [
builder.run(ShellQuoted("git apply {p}").format(p=patch)),
builder.run(ShellQuoted("./autogen.sh")),
builder.configure(),
builder.make_and_install(),
]
return {
"depends_on": [folly, fbthrift, fbzmq, re2],
"steps": [
builder.github_project_workdir("thom311/libnl", "."),
builder.step("Build and install thom311/libnl",
libnl_build_commands),
builder.fb_github_project_workdir("openr/build", "facebook"),
builder.step(
"Build and install openr/build",
[
builder.cmake_configure("openr/build"),
# we need the pythonpath to find the thrift compiler
builder.run(
ShellQuoted(
'PYTHONPATH="$PYTHONPATH:"{p}/lib/python2.7/site-packages '
"make -j {n}").format(
p=builder.option("prefix"),
n=builder.option("make_parallelism"),
)),
builder.run(ShellQuoted("sudo make install")),
builder.run(ShellQuoted("sudo ldconfig")),
],
),
builder.step(
"Run openr tests",
[
builder.run(
ShellQuoted("CTEST_OUTPUT_ON_FAILURE=TRUE make test"))
],
),
],
}
def cmake_configure(self, name):
cmake_defines = {
'BUILD_SHARED_LIBS': 'ON',
'CMAKE_INSTALL_PREFIX': self.option('prefix'),
}
cmake_defines.update(
self.option('{0}:cmake_defines'.format(name), {})
)
return [
self.run(ShellQuoted(
'CXXFLAGS="$CXXFLAGS -fPIC -isystem "{p}"/include" '
'CFLAGS="$CFLAGS -fPIC -isystem "{p}"/include" '
'cmake {args} ..'
).format(
p=self.option('prefix'),
args=shell_join(' ', (
ShellQuoted('-D{k}={v}').format(k=k, v=v)
for k, v in cmake_defines.items()
)),
)),
]
def fbcode_builder_spec(builder):
builder.add_option('jedisct1/libsodium:git_hash', 'stable')
return {
'steps': [
builder.github_project_workdir('jedisct1/libsodium', '.'),
builder.step('Build and install jedisct1/libsodium', [
builder.run(ShellQuoted('./autogen.sh')),
builder.configure(),
builder.make_and_install(),
]),
],
}
def configure(self, name=None):
autoconf_options = {}
if name is not None:
autoconf_options.update(
self.option("{0}:autoconf_options".format(name), {}))
return [
self.run(
ShellQuoted(
'LDFLAGS="$LDFLAGS -L"{p}"/lib -Wl,-rpath="{p}"/lib" '
'CFLAGS="$CFLAGS -I"{p}"/include" '
'CPPFLAGS="$CPPFLAGS -I"{p}"/include" '
"PY_PREFIX={p} "
"./configure --prefix={p} {args}").format(
p=self.option("prefix"),
args=shell_join(
" ",
(ShellQuoted("{k}={v}").format(k=k, v=v)
for k, v in autoconf_options.items()),
),
)),
]
def fbcode_builder_spec(builder):
return {
'depends_on': [fbthrift],
'steps': [
builder.step("Install thrift python modules", [
builder.workdir(
path_join(builder.option('projects_dir'),
"fbthrift/thrift/lib/py")),
builder.run(ShellQuoted("sudo python setup.py install")),
]),
],
}
def rust_toolchain(self):
actions = []
if self.option("rust_toolchain", False):
(toolchain, is_bootstrap) = self.option("rust_toolchain")
rust_dir = path_join(self.option("prefix"), "rust")
actions = [
self.set_env("CARGO_HOME", rust_dir),
self.set_env("RUSTUP_HOME", rust_dir),
self.set_env("RUSTC_BOOTSTRAP", "1" if is_bootstrap else "0"),
self.run(
ShellQuoted(
"curl -sSf https://build.travis-ci.com/files/rustup-init.sh"
" | sh -s --"
" --default-toolchain={r} "
" --profile=minimal"
" --no-modify-path"
" -y").format(p=rust_dir, r=toolchain)),
self.set_env(
"PATH",
ShellQuoted("{p}:$PATH").format(
p=path_join(rust_dir, "bin")),
),
self.run(ShellQuoted("rustup update")),
self.run(ShellQuoted("rustc --version")),
self.run(ShellQuoted("rustup --version")),
self.run(ShellQuoted("cargo --version")),
]
return actions
def fbcode_builder_spec(builder):
return {
'depends_on': [folly, proxygen, fbthrift],
'steps': [
builder.fb_github_project_workdir('bistro/bistro'),
builder.step(
'Build bistro',
[
# Future: should this share some code with `cmake_install()`?
builder.run(
ShellQuoted(
'PYTHONPATH="$PYTHONPATH:"{p}/lib/python2.7/site-packages '
'./cmake/run-cmake.sh Debug -DCMAKE_INSTALL_PREFIX={p}'
).format(p=builder.option('prefix'))),
builder.workdir('cmake/Debug'),
builder.parallel_make(),
]),
builder.step('Run bistro tests', [
builder.run(ShellQuoted('ctest')),
]),
]
}
def fbcode_builder_spec(builder):
# This API should change rarely, so build the latest tag instead of master.
builder.add_option('no1msd/mstch:git_hash',
ShellQuoted('$(git describe --abbrev=0 --tags)'))
return {
'depends_on': [folly, fizz, fmt, sodium, rsocket, wangle, zstd],
'steps': [
# This isn't a separete spec, since only fbthrift uses mstch.
builder.github_project_workdir('no1msd/mstch', 'build'),
builder.cmake_install('no1msd/mstch'),
builder.fb_github_cmake_install('fbthrift/thrift'),
],
}
def fbcode_builder_spec(builder):
builder.add_option(
'hyperic/sigar:autoconf_options', {'CFLAGS' : '-fgnu89-inline'})
return {
'steps': [
builder.github_project_workdir('hyperic/sigar', '.'),
builder.step('Build and install sigar', [
builder.run(ShellQuoted('./autogen.sh')),
builder.configure('hyperic/sigar'),
builder.make_and_install(),
]),
],
}
def fbcode_builder_spec(builder):
# This API should change rarely, so build the latest tag instead of master.
builder.add_option(
"no1msd/mstch:git_hash", ShellQuoted("$(git describe --abbrev=0 --tags)")
)
return {
"depends_on": [zstd],
"steps": [
# This isn't a separete spec, since only fbthrift uses mstch.
builder.github_project_workdir("no1msd/mstch", "build"),
builder.cmake_install("no1msd/mstch"),
builder.fb_github_cmake_install("LogDevice/logdevice/_build"),
],
}
def github_project_workdir(self, project, path):
# Only check out a non-default branch if requested. This especially
# makes sense when building from a local repo.
git_hash = self.option(
'{0}:git_hash'.format(project),
# Any repo that has a hash in deps/github_hashes defaults to
# that, with the goal of making builds maximally consistent.
self._github_hashes.get(project, ''))
maybe_change_branch = [
self.run(ShellQuoted('git checkout {hash}').format(hash=git_hash)),
] if git_hash else []
base_dir = self.option('projects_dir')
local_repo_dir = self.option('{0}:local_repo_dir'.format(project), '')
return self.step('Check out {0}, workdir {1}'.format(project, path), [
self.workdir(base_dir),
self.run(
ShellQuoted('git clone https://github.com/{p}').format(
p=project)) if not local_repo_dir else
self.copy_local_repo(local_repo_dir, os.path.basename(project)),
self.workdir(path_join(base_dir, os.path.basename(project), path)),
] + maybe_change_branch)
def fbcode_builder_spec(builder):
# This API should change rarely, so build the latest tag instead of master.
builder.add_option('facebook/zstd:git_hash',
ShellQuoted('$(git describe --abbrev=0 --tags)'))
return {
'steps': [
builder.github_project_workdir('facebook/zstd', '.'),
builder.step('Build and install zstd', [
builder.make_and_install(make_vars={
'PREFIX': builder.option('prefix'),
})
]),
],
}
def fbcode_builder_spec(builder):
return {
'depends_on': [folly, proxygen, fbthrift],
'steps': [
builder.fb_github_project_workdir('bistro/bistro'),
builder.step(
'Build bistro',
[
# Future: should this share some code with `cmake_install()`?
builder.run(
ShellQuoted(
'PATH="$PATH:"{p}/bin '
'TEMPLATES_PATH={p}/include/thrift/templates '
'./cmake/run-cmake.sh Debug -DCMAKE_INSTALL_PREFIX={p}'
).format(p=builder.option('prefix'))),
builder.workdir('cmake/Debug'),
builder.parallel_make(),
]),
builder.step('Run bistro tests', [
builder.run(ShellQuoted('ctest --output-on-failure')),
]),
]
}
def fbcode_builder_spec(builder):
builder.add_option("jedisct1/libsodium:git_hash", "stable")
return {
"steps": [
builder.github_project_workdir("jedisct1/libsodium", "."),
builder.step(
"Build and install jedisct1/libsodium",
[
builder.run(ShellQuoted("./autogen.sh")),
builder.configure(),
builder.make_and_install(),
],
),
],
}
def setup(self):
return self.step('Setup', [self.create_python_venv()] + [
self.run(
ShellQuoted("""
case "$OSTYPE" in
darwin*)
http_proxy= https_proxy= ./tools/lfs/lfs.py \\
download homebrew.tar.gz \\
-l ./watchman/facebook/lego/.lfs-pointers
rm -rf /var/tmp/homebrew
tar xzf homebrew.tar.gz -C /var/tmp
;;
esac
"""))
])
def fbcode_builder_spec(builder):
builder.add_option('wangle/_build:cmake_defines', {
'BUILD_SHARED_LIBS': 'OFF',
'BUILD_TESTS': 'ON',
})
return {
'depends_on': [gmock, folly, fizz, sodium],
'steps': [
builder.fb_github_cmake_install('wangle/_build', '../wangle'),
builder.step('Run wangle tests', [
builder.run(
ShellQuoted('ctest --output-on-failure -j {n}').format(
n=builder.option('make_parallelism'), ))
]),
]
}
def fbcode_builder_spec(builder):
# This API should change rarely, so build the latest tag instead of master.
builder.add_option("no1msd/mstch:git_hash",
ShellQuoted("$(git describe --abbrev=0 --tags)"))
builder.add_option("PYTHON_VENV", "ON")
builder.add_option("LogDevice/logdevice/_build:cmake_defines",
{"BUILD_SUBMODULES": "OFF"})
return {
"depends_on": [zstd, fmt, folly, fizz, wangle, sodium, rsocket],
"steps": [
# This isn't a separete spec, since only fbthrift uses mstch.
builder.github_project_workdir("no1msd/mstch", "build"),
builder.cmake_install("no1msd/mstch"),
builder.fb_github_cmake_install("fbthrift/thrift"),
builder.fb_github_cmake_install("LogDevice/logdevice/_build",
github_org="facebookincubator"),
],
}
def fbcode_builder_spec(builder):
builder.add_option("folly/_build:cmake_defines", {
"BUILD_SHARED_LIBS": "OFF",
"BUILD_TESTS": "ON"
})
return {
"depends_on": [fmt, gmock],
"steps": [
builder.fb_github_cmake_install("folly/_build"),
builder.step(
"Run folly tests",
[
builder.run(
ShellQuoted("ctest --output-on-failure -j {n}").format(
n=builder.option("make_parallelism")))
],
),
],
}
def fbcode_builder_spec(builder):
builder.add_option('thom311/libnl:git_hash', 'libnl3_2_25')
builder.add_option('openr/openr/build:cmake_defines',
{'ADD_ROOT_TESTS': 'OFF'})
maybe_curl_patch = []
patch = path_join(
builder.option('projects_dir'),
'../shipit_projects/openr/build/fix-route-obj-attr-list.patch')
if not builder.has_option('shipit_project_dir'):
maybe_curl_patch = [
builder.run(
ShellQuoted(
'curl -O https://raw.githubusercontent.com/facebook/openr/master/'
'build/fix-route-obj-attr-list.patch'))
]
patch = 'fix-route-obj-attr-list.patch'
libnl_build_commands = maybe_curl_patch + [
builder.run(ShellQuoted('git apply {p}').format(p=patch)),
builder.run(ShellQuoted('./autogen.sh')),
builder.configure(),
builder.make_and_install()
]
return {
'depends_on': [folly, fbthrift, fbzmq, re2],
'steps': [
builder.github_project_workdir('thom311/libnl', '.'),
builder.step('Build and install thom311/libnl',
libnl_build_commands),
builder.fb_github_project_workdir('openr/openr/build', 'facebook'),
builder.step(
'Build and install openr/openr/build',
[
builder.cmake_configure('openr/openr/build'),
# we need the pythonpath to find the thrift compiler
builder.run(
ShellQuoted(
'PYTHONPATH="$PYTHONPATH:"{p}/lib/python2.7/site-packages '
'make -j {n}').format(
p=builder.option('prefix'),
n=builder.option('make_parallelism'))),
builder.run(ShellQuoted('make install')),
]),
builder.step('Run openr tests', [
builder.run(
ShellQuoted('CTEST_OUTPUT_ON_FAILURE=TRUE make test')),
]),
],
}
def fbcode_builder_spec(builder):
# This API should change rarely, so build the latest tag instead of master.
builder.add_option(
"facebook/zstd:git_hash",
ShellQuoted("$(git describe --abbrev=0 --tags origin/master)"),
)
return {
"steps": [
builder.github_project_workdir("facebook/zstd", "."),
builder.step(
"Build and install zstd",
[
builder.make_and_install(
make_vars={
"PREFIX": builder.option("prefix"),
})
],
),
],
}
def fbcode_builder_spec(builder):
builder.add_option(
"wangle/_build:cmake_defines",
{
"BUILD_SHARED_LIBS": "OFF",
"BUILD_TESTS": "ON",
},
)
return {
"depends_on": [gmock, fmt, folly, fizz, sodium],
"steps": [
builder.fb_github_cmake_install("wangle/_build", "../wangle"),
builder.step(
"Run wangle tests",
[
builder.run(
ShellQuoted("ctest --output-on-failure -j {n}").format(
n=builder.option("make_parallelism"), ))
],
),
],
}
def fbcode_builder_spec(builder):
# This API should change rarely, so build the latest tag instead of master.
builder.add_option('no1msd/mstch:git_hash',
ShellQuoted('$(git describe --abbrev=0 --tags)'))
builder.add_option('rsocket/rsocket-cpp/build:cmake_defines',
{'BUILD_TESTS': 'OFF'})
builder.add_option('krb5/krb5:git_hash', 'krb5-1.16.1-final')
return {
'depends_on': [folly, fizz, sodium, wangle, zstd],
'steps': [
# This isn't a separete spec, since only fbthrift uses mstch.
builder.github_project_workdir('no1msd/mstch', 'build'),
builder.cmake_install('no1msd/mstch'),
builder.github_project_workdir('krb5/krb5', 'src'),
builder.autoconf_install('krb5/krb5'),
builder.github_project_workdir('rsocket/rsocket-cpp', 'build'),
builder.step('configuration for rsocket', [
builder.cmake_configure('rsocket/rsocket-cpp/build'),
]),
builder.cmake_install('rsocket/rsocket-cpp'),
builder.fb_github_cmake_install('fbthrift/thrift'),
],
}
def setup(self):
# Please add RPM-based OSes here as appropriate.
#
# To allow exercising non-root installs -- we change users after the
# system packages are installed. TODO: For users not defined in the
# image, we should probably `useradd`.
return self.step(
'Setup',
[
ShellQuoted('FROM {img}').format(
# Docker can't deal with quotes. Oh well.
img=ShellQuoted(self.option('os_image')), ),
ShellQuoted('ENV CFLAGS=-O'),
ShellQuoted('ENV CXXFLAGS=-O'),
ShellQuoted('ENV CPPFLAGS=-O'),
# /bin/sh syntax is a pain
ShellQuoted('SHELL ["/bin/bash", "-c"]'),
] + self.install_debian_deps() + [self._change_user()])
def parallel_make(self, make_vars=None):
return self.run(
ShellQuoted("make -j {n} VERBOSE=1 {vars}").format(
n=self.option("make_parallelism"),
vars=self._make_vars(make_vars),
))
def _make_vars(self, make_vars):
return shell_join(
" ",
(ShellQuoted("{k}={v}").format(k=k, v=v)
for k, v in ({} if make_vars is None else make_vars).items()),
)
def _render_impl(self, actions):
next_workdir = None
shipit_projects = []
build_steps = []
for action in recursively_flatten_list(actions):
if isinstance(action, LegocastleFBCheckout):
next_workdir = ShellQuoted('"$(hg root)/"{d}', ).format(
d=path_join(self.option('shipit_project_dir'),
action.project_and_path), )
shipit_projects.append(
action.project_and_path.split('/', 1)[0])
elif isinstance(action, LegocastleStep):
pre_actions = [ShellQuoted('set -ex')]
if action.name != 'Setup' and \
self.option("PYTHON_VENV", "OFF") == "ON":
pre_actions.append(self.python_venv())
pre_actions.append(
ShellQuoted("""
case "$OSTYPE" in
darwin*)
BREW_PREFIX=/var/tmp/homebrew
# The apple-provided flex and bison tools are too old to successfully
# build thrift. Ensure that we resolve to the homebrew versions.
# Note that homebrew doesn't link these into its bin dir to avoid
# these newer versions taking precedence, so we need to reach into
# the cellar path. The glob is to make this script less prone to
# problems if/when the version is bumped.
BISON_BIN=$(echo $BREW_PREFIX/Cellar/bison/*/bin)
FLEX_BIN=$(echo $BREW_PREFIX/Cellar/flex/*/bin)
export CMAKE_SYSTEM_PREFIX_PATH=$BREW_PREFIX
export PKG_CONFIG_PATH=$BREW_PREFIX/opt/openssl/lib/pkgconfig
export PATH=$BISON_BIN:$FLEX_BIN:$BREW_PREFIX/bin:$PATH
export HOMEBREW_NO_AUTO_UPDATE=1
export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/
;;
esac
"""))
if next_workdir is not None:
pre_actions.append(self.workdir(next_workdir))
next_workdir = None
shell_steps = []
for a in itertools.chain(pre_actions, action.actions):
if isinstance(a, LegocastleWorkdir):
# Pass the last working directory to the next step,
# since Legocastle doesn't remember it.
next_workdir = a.dir
shell_steps.append(
ShellQuoted(
# Don't evaluate {d} twice.
'_wd={d} ; mkdir -p "$_wd" && cd "$_wd"').
format(d=a.dir))
else:
shell_steps.append(a)
build_steps.append({
'name':
action.name,
'shell':
raw_shell(shell_join('\n', shell_steps)),
'required':
True,
})
else:
raise LegocastleStepError(
'You have a top-level action {0} that is not a step()'.
format(repr(action)))
return shipit_projects, build_steps
def _render_impl(self, actions):
# This exists to transfer the current working directory to the next
# step, since Legocastle doesn't remember it. Although this is
# potentially quadratic, we have to replay ALL workdir calls from
# the previous step because these might contain variable expansions,
# making it impossible to distinguish relative directories from
# absolute. Consider this step:
# [
# builder.fb_github_project_workdir('foo/bar')
# ...
# builder.workdir('baz') # Now should be at foo/bar/baz
# ]
#
# If we just replayed the last workdir('baz'), we would not end up
# in '<github_prefix>/foo/bar/baz', but rather in `<unknown>/baz`.
next_step_workdirs = []
shipit_projects = []
build_steps = []
for action in recursively_flatten_list(actions):
if isinstance(action, LegocastleFBCheckout):
# This is guaranteed to be absolute, so drop the unnecessary
# history.
next_step_workdirs = [
ShellQuoted('"$(hg root)/"{d}', ).format(d=path_join(
self.option('shipit_project_dir'),
action.project_and_path), )
]
shipit_projects.append(
action.project_and_path.split('/', 1)[0])
elif isinstance(action, LegocastleStep):
pre_actions = [ShellQuoted('set -ex')]
if action.name != 'Setup' and \
self.option("PYTHON_VENV", "OFF") == "ON":
pre_actions.extend(self.python_venv())
pre_actions.append(
ShellQuoted("""
case "$OSTYPE" in
darwin*)
BREW_PREFIX=/var/tmp/homebrew
# The apple-provided flex and bison tools are too old to successfully
# build thrift. Ensure that we resolve to the homebrew versions.
# Note that homebrew doesn't link these into its bin dir to avoid
# these newer versions taking precedence, so we need to reach into
# the cellar path. The glob is to make this script less prone to
# problems if/when the version is bumped.
BISON_BIN=$(echo $BREW_PREFIX/Cellar/bison/*/bin)
FLEX_BIN=$(echo $BREW_PREFIX/Cellar/flex/*/bin)
export CMAKE_SYSTEM_PREFIX_PATH=$BREW_PREFIX
export PKG_CONFIG_PATH=$BREW_PREFIX/opt/openssl/lib/pkgconfig
export PATH=$BISON_BIN:$FLEX_BIN:$BREW_PREFIX/bin:$PATH
export HOMEBREW_NO_AUTO_UPDATE=1
export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/
;;
esac
"""))
pre_actions.extend(self.workdir(w) for w in next_step_workdirs)
shell_steps = []
for a in itertools.chain(pre_actions, action.actions):
if isinstance(a, LegocastleWorkdir):
next_step_workdirs.append(a.dir)
shell_steps.append(
ShellQuoted(
# Don't evaluate {d} twice.
'_wd={d} ; mkdir -p "$_wd" && cd "$_wd"').
format(d=a.dir))
else:
shell_steps.append(a)
build_steps.append({
'name':
action.name,
'shell':
raw_shell(shell_join('\n', shell_steps)),
'required':
True,
})
else:
raise LegocastleStepError(
'You have a top-level action {0} that is not a step()'.
format(repr(action)))
return shipit_projects, build_steps
def autoconf_install(self, name):
return self.step('Build and install {0}'.format(name), [
self.run(ShellQuoted('autoreconf -ivf')),
] + self.configure() + self.make_and_install())
