def rust_toolchain(self):
actions = []
if self.option("rust_toolchain", False):
(toolchain, is_bootstrap) = self.option("rust_toolchain")
rust_dir = path_join(self.option("prefix"), "rust")
actions = [
self.set_env("CARGO_HOME", rust_dir),
self.set_env("RUSTUP_HOME", rust_dir),
self.set_env("RUSTC_BOOTSTRAP", "1" if is_bootstrap else "0"),
self.run(
ShellQuoted(
"curl -sSf https://build.travis-ci.com/files/rustup-init.sh"
" | sh -s --"
" --default-toolchain={r} "
" --profile=minimal"
" --no-modify-path"
" -y").format(p=rust_dir, r=toolchain)),
self.set_env(
"PATH",
ShellQuoted("{p}:$PATH").format(
p=path_join(rust_dir, "bin")),
),
self.run(ShellQuoted("rustup update")),
self.run(ShellQuoted("rustc --version")),
self.run(ShellQuoted("rustup --version")),
self.run(ShellQuoted("cargo --version")),
]
return actions
def github_project_workdir(self, project, path):
# Only check out a non-default branch if requested. This especially
# makes sense when building from a local repo.
git_hash = self.option(
"{0}:git_hash".format(project),
# Any repo that has a hash in deps/github_hashes defaults to
# that, with the goal of making builds maximally consistent.
self._github_hashes.get(project, ""),
)
maybe_change_branch = ([
self.run(ShellQuoted("git checkout {hash}").format(hash=git_hash))
] if git_hash else [])
base_dir = self.option("projects_dir")
local_repo_dir = self.option("{0}:local_repo_dir".format(project), "")
return self.step(
"Check out {0}, workdir {1}".format(project, path),
[
self.workdir(base_dir),
self.run(
ShellQuoted("if [[ ! -d {d} ]]; then \n"
"\tgit clone https://github.com/{p}\n"
"fi").
format(p=project,
d=path_join(base_dir, os.path.basename(project))))
if not local_repo_dir else self.copy_local_repo(
local_repo_dir, os.path.basename(project)),
self.workdir(
path_join(base_dir, os.path.basename(project), path)),
] + maybe_change_branch,
)
def github_project_workdir(self, project, path):
# Only check out a non-default branch if requested. This especially
# makes sense when building from a local repo.
git_hash = self.option(
'{0}:git_hash'.format(project),
# Any repo that has a hash in deps/github_hashes defaults to
# that, with the goal of making builds maximally consistent.
self._github_hashes.get(project, ''))
maybe_change_branch = [
self.run(ShellQuoted('git checkout {hash}').format(hash=git_hash)),
] if git_hash else []
base_dir = self.option('projects_dir')
git_patch = self.option('{0}:git_patch'.format(project), '')
maybe_apply_patch = [
self.run(
ShellQuoted('git apply {patch}').format(
patch=path_join(base_dir, git_patch))),
] if git_patch else []
local_repo_dir = self.option('{0}:local_repo_dir'.format(project), '')
return self.step('Check out {0}, workdir {1}'.format(project, path), [
self.workdir(base_dir),
self.run(
ShellQuoted('git clone https://github.com/{p}').format(
p=project)) if not local_repo_dir else
self.copy_local_repo(local_repo_dir, os.path.basename(project)),
self.workdir(path_join(base_dir, os.path.basename(project), path)),
] + maybe_change_branch + maybe_apply_patch)
def fbcode_builder_spec(builder):
builder.add_option("watchman/_build:cmake_defines",
{"BUILD_SHARED_LIBS": "OFF"})
projects = builder.option("projects_dir")
return {
"depends_on": [gmock, folly, fbthrift],
"steps": [
builder.fb_github_cmake_install("watchman/_build", ".."),
builder.step(
"Run watchman tests",
[
builder.run(
ShellQuoted("ctest --output-on-failure -j {n}").format(
n=builder.option("make_parallelism"))),
builder.run(
ShellQuoted(
"cd ../ && ./runtests.py --concurrency {n} "
"--watchman-path _build/watchman --pybuild-dir {p}"
).format(
n=builder.option("make_parallelism"),
p=path_join(
projects,
"../shipit_projects/watchman/_build/python"),
)),
],
),
],
}
def fbcode_builder_spec(builder):
builder.add_option("watchman/_build:cmake_defines", {"BUILD_SHARED_LIBS": "OFF"})
projects = builder.option("projects_dir")
return {
"depends_on": [gmock, folly, fbthrift],
"steps": [
builder.fb_github_cmake_install("watchman/_build", ".."),
builder.step(
"Run watchman tests",
[
builder.run(
ShellQuoted("ctest --output-on-failure -j {n}").format(
n=builder.option("make_parallelism")
)
),
builder.run(
ShellQuoted(
"cd ../ && ./runtests.py --concurrency {n} "
"--watchman-path _build/watchman --pybuild-dir {p}"
).format(
n=builder.option("make_parallelism"),
p=path_join(
projects, "../shipit_projects/watchman/_build/python"
),
)
),
],
),
],
}
def create_python_venv(self):
action = []
if self.option("PYTHON_VENV", "OFF") == "ON":
action = self.run(
ShellQuoted("python3 -m venv {p}").format(
p=path_join(self.option('prefix'), "venv")))
return (action)
def github_project_workdir(self, project, path):
# Only check out a non-default branch if requested. This especially
# makes sense when building from a local repo.
git_hash = self.option(
'{0}:git_hash'.format(project),
# Any repo that has a hash in deps/github_hashes defaults to
# that, with the goal of making builds maximally consistent.
self._github_hashes.get(project, '')
)
maybe_change_branch = [
self.run(ShellQuoted('git checkout {hash}').format(hash=git_hash)),
] if git_hash else []
base_dir = self.option('projects_dir')
local_repo_dir = self.option('{0}:local_repo_dir'.format(project), '')
return self.step('Check out {0}, workdir {1}'.format(project, path), [
self.workdir(base_dir),
self.run(
ShellQuoted('git clone {opts} https://github.com/{p}').format(
p=project,
opts=ShellQuoted(self.option('{}:git_clone_opts'.format(project), '')))
) if not local_repo_dir else self.copy_local_repo(
local_repo_dir, os.path.basename(project)
),
self.workdir(path_join(base_dir, os.path.basename(project), path)),
] + maybe_change_branch)
def python_venv(self):
# To both avoid calling venv activate on each RUN command AND to ensure
# it is present when the resulting container is run add to PATH
actions = []
if self.option("PYTHON_VENV", "OFF") == "ON":
actions = ShellQuoted('ENV PATH={p}:$PATH').format(
p=path_join(self.option('prefix'), "venv", "bin"))
return (actions)
def python_venv(self):
# To both avoid calling venv activate on each RUN command AND to ensure
# it is present when the resulting container is run add to PATH
actions = []
if self.option("PYTHON_VENV", "OFF") == "ON":
actions = ShellQuoted('ENV PATH={p}:$PATH').format(
p=path_join(self.option('prefix'), "venv", "bin"))
return(actions)
def fbcode_builder_spec(builder):
builder.add_option("thom311/libnl:git_hash", "libnl3_2_25")
builder.add_option("openr/build:cmake_defines", {"ADD_ROOT_TESTS": "OFF"})
maybe_curl_patch = []
patch = path_join(
builder.option("projects_dir"),
"../shipit_projects/openr/build/fix-route-obj-attr-list.patch",
)
if not builder.has_option("shipit_project_dir"):
maybe_curl_patch = [
builder.run(
ShellQuoted(
"curl -O https://raw.githubusercontent.com/facebook/openr/master/"
"build/fix-route-obj-attr-list.patch"))
]
patch = "fix-route-obj-attr-list.patch"
libnl_build_commands = maybe_curl_patch + [
builder.run(ShellQuoted("git apply {p}").format(p=patch)),
builder.run(ShellQuoted("./autogen.sh")),
builder.configure(),
builder.make_and_install(),
]
return {
"depends_on": [folly, fbthrift, fbzmq, re2],
"steps": [
builder.github_project_workdir("thom311/libnl", "."),
builder.step("Build and install thom311/libnl",
libnl_build_commands),
builder.fb_github_project_workdir("openr/build", "facebook"),
builder.step(
"Build and install openr/build",
[
builder.cmake_configure("openr/build"),
# we need the pythonpath to find the thrift compiler
builder.run(
ShellQuoted(
'PYTHONPATH="$PYTHONPATH:"{p}/lib/python2.7/site-packages '
"make -j {n}").format(
p=builder.option("prefix"),
n=builder.option("make_parallelism"),
)),
builder.run(ShellQuoted("sudo make install")),
builder.run(ShellQuoted("sudo ldconfig")),
],
),
builder.step(
"Run openr tests",
[
builder.run(
ShellQuoted("CTEST_OUTPUT_ON_FAILURE=TRUE make test"))
],
),
],
}
def python_venv(self):
actions = []
if self.option("PYTHON_VENV", "OFF") == "ON":
actions.append(ShellQuoted("source {p}").format(
p=path_join(self.option('prefix'), "venv", "bin", "activate")))
actions.append(self.run(
ShellQuoted("python3 -m pip install {deps}").format(
deps=shell_join(' ', (ShellQuoted(dep) for dep in
self.python_deps())))))
return(actions)
def fbcode_builder_spec(builder):
return {
'depends_on': [fbthrift],
'steps': [
builder.step("Install thrift python modules", [
builder.workdir(
path_join(builder.option('projects_dir'),
"fbthrift/thrift/lib/py")),
builder.run(ShellQuoted("sudo python setup.py install")),
]),
],
}
def fbcode_builder_spec(builder):
builder.enable_rust_toolchain()
return {
"depends_on": [fbthrift],
"steps": [
builder.set_env(
"THRIFT", path_join(builder.option("prefix"), "bin",
"thrift1")),
builder.fb_github_cargo_build("rust-shed/",
github_org="facebookexperimental"),
],
}
def fbcode_builder_spec(builder):
builder.add_option('thom311/libnl:git_hash', 'libnl3_2_25')
builder.add_option('openr/openr/build:cmake_defines',
{'ADD_ROOT_TESTS': 'OFF'})
maybe_curl_patch = []
patch = path_join(
builder.option('projects_dir'),
'../shipit_projects/openr/build/fix-route-obj-attr-list.patch')
if not builder.has_option('shipit_project_dir'):
maybe_curl_patch = [
builder.run(
ShellQuoted(
'curl -O https://raw.githubusercontent.com/facebook/openr/master/'
'build/fix-route-obj-attr-list.patch'))
]
patch = 'fix-route-obj-attr-list.patch'
libnl_build_commands = maybe_curl_patch + [
builder.run(ShellQuoted('git apply {p}').format(p=patch)),
builder.run(ShellQuoted('./autogen.sh')),
builder.configure(),
builder.make_and_install()
]
return {
'depends_on': [folly, fbthrift, fbzmq, re2],
'steps': [
builder.github_project_workdir('thom311/libnl', '.'),
builder.step('Build and install thom311/libnl',
libnl_build_commands),
builder.fb_github_project_workdir('openr/openr/build', 'facebook'),
builder.step(
'Build and install openr/openr/build',
[
builder.cmake_configure('openr/openr/build'),
# we need the pythonpath to find the thrift compiler
builder.run(
ShellQuoted(
'PYTHONPATH="$PYTHONPATH:"{p}/lib/python2.7/site-packages '
'make -j {n}').format(
p=builder.option('prefix'),
n=builder.option('make_parallelism'))),
builder.run(ShellQuoted('make install')),
]),
builder.step('Run openr tests', [
builder.run(
ShellQuoted('CTEST_OUTPUT_ON_FAILURE=TRUE make test')),
]),
],
}
def _render_impl(self, actions):
next_workdir = None
shipit_projects = []
build_steps = []
for action in recursively_flatten_list(actions):
if isinstance(action, LegocastleFBCheckout):
next_workdir = ShellQuoted('"$(hg root)/"{d}', ).format(
d=path_join(self.option('shipit_project_dir'),
action.project_and_path), )
shipit_projects.append(
action.project_and_path.split('/', 1)[0])
elif isinstance(action, LegocastleStep):
pre_actions = [ShellQuoted('set -ex')]
if next_workdir is not None:
pre_actions.append(self.workdir(next_workdir))
next_workdir = None
shell_steps = []
for a in itertools.chain(pre_actions, action.actions):
if isinstance(a, LegocastleWorkdir):
# Pass the last working directory to the next step,
# since Legocastle doesn't remember it.
next_workdir = a.dir
shell_steps.append(
ShellQuoted(
# Don't evaluate {d} twice.
'_wd={d} ; mkdir -p "$_wd" && cd "$_wd"').
format(d=a.dir))
else:
shell_steps.append(a)
build_steps.append({
'name':
action.name,
'shell':
raw_shell(shell_join('\n', shell_steps)),
'required':
True,
})
else:
raise LegocastleStepError(
'You have a top-level action {0} that is not a step()'.
format(repr(action)))
return shipit_projects, build_steps
def github_project_workdir(self, project, path):
# Only check out a non-default branch if requested. This especially
# makes sense when building from a local repo.
git_hash = self.option('{0}:git_hash'.format(project), '')
maybe_change_branch = [
self.run(ShellQuoted('git checkout {hash}').format(hash=git_hash)),
] if git_hash else []
base_dir = self.option('projects_dir')
local_repo_dir = self.option('{0}:local_repo_dir'.format(project), '')
return self.step('Check out {0}, workdir {1}'.format(project, path), [
self.workdir(base_dir),
self.run(
ShellQuoted('git clone https://github.com/{p}').format(
p=project)) if not local_repo_dir else
self.copy_local_repo(local_repo_dir, os.path.basename(project)),
self.workdir(path_join(base_dir, os.path.basename(project), path)),
] + maybe_change_branch)
def github_project_workdir(self, project, path):
# Only check out a non-default branch if requested. This especially
# makes sense when building from a local repo.
git_hash = self.option('{0}:git_hash'.format(project), '')
maybe_change_branch = [
self.run(ShellQuoted('git checkout {hash}').format(hash=git_hash)),
] if git_hash else []
base_dir = self.option('projects_dir')
local_repo_dir = self.option('{0}:local_repo_dir'.format(project), '')
return self.step('Check out {0}, workdir {1}'.format(project, path), [
self.workdir(base_dir),
self.run(
ShellQuoted('git clone https://github.com/{p}')
.format(p=project)
) if not local_repo_dir else self.copy_local_repo(
local_repo_dir, os.path.basename(project)
),
self.workdir(path_join(base_dir, os.path.basename(project), path)),
] + maybe_change_branch)
def python_venv(self):
action = []
if self.option("PYTHON_VENV", "OFF") == "ON":
action = ShellQuoted("source {p}").format(
p=path_join(self.option('prefix'), "venv", "bin", "activate"))
return (action)
def python_venv(self):
action = []
if self.option("PYTHON_VENV", "OFF") == "ON":
action = ShellQuoted("source {p}").format(
p=path_join(self.option('prefix'), "venv", "bin", "activate"))
return(action)
def create_python_venv(self):
action = []
if self.option("PYTHON_VENV", "OFF") == "ON":
action = self.run(ShellQuoted("python3 -m venv {p}").format(
p=path_join(self.option('prefix'), "venv")))
return(action)
def _render_impl(self, actions):
next_workdir = None
shipit_projects = []
build_steps = []
for action in recursively_flatten_list(actions):
if isinstance(action, LegocastleFBCheckout):
next_workdir = ShellQuoted('"$(hg root)/"{d}', ).format(
d=path_join(self.option('shipit_project_dir'),
action.project_and_path), )
shipit_projects.append(
action.project_and_path.split('/', 1)[0])
elif isinstance(action, LegocastleStep):
pre_actions = [ShellQuoted('set -ex')]
if action.name != 'Setup' and \
self.option("PYTHON_VENV", "OFF") == "ON":
pre_actions.append(self.python_venv())
pre_actions.append(
ShellQuoted("""
case "$OSTYPE" in
darwin*)
BREW_PREFIX=/var/tmp/homebrew
# The apple-provided flex and bison tools are too old to successfully
# build thrift. Ensure that we resolve to the homebrew versions.
# Note that homebrew doesn't link these into its bin dir to avoid
# these newer versions taking precedence, so we need to reach into
# the cellar path. The glob is to make this script less prone to
# problems if/when the version is bumped.
BISON_BIN=$(echo $BREW_PREFIX/Cellar/bison/*/bin)
FLEX_BIN=$(echo $BREW_PREFIX/Cellar/flex/*/bin)
export CMAKE_SYSTEM_PREFIX_PATH=$BREW_PREFIX
export PKG_CONFIG_PATH=$BREW_PREFIX/opt/openssl/lib/pkgconfig
export PATH=$BISON_BIN:$FLEX_BIN:$BREW_PREFIX/bin:$PATH
export HOMEBREW_NO_AUTO_UPDATE=1
export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/
;;
esac
"""))
if next_workdir is not None:
pre_actions.append(self.workdir(next_workdir))
next_workdir = None
shell_steps = []
for a in itertools.chain(pre_actions, action.actions):
if isinstance(a, LegocastleWorkdir):
# Pass the last working directory to the next step,
# since Legocastle doesn't remember it.
next_workdir = a.dir
shell_steps.append(
ShellQuoted(
# Don't evaluate {d} twice.
'_wd={d} ; mkdir -p "$_wd" && cd "$_wd"').
format(d=a.dir))
else:
shell_steps.append(a)
build_steps.append({
'name':
action.name,
'shell':
raw_shell(shell_join('\n', shell_steps)),
'required':
True,
})
else:
raise LegocastleStepError(
'You have a top-level action {0} that is not a step()'.
format(repr(action)))
return shipit_projects, build_steps
def _render_impl(self, actions):
# This exists to transfer the current working directory to the next
# step, since Legocastle doesn't remember it. Although this is
# potentially quadratic, we have to replay ALL workdir calls from
# the previous step because these might contain variable expansions,
# making it impossible to distinguish relative directories from
# absolute. Consider this step:
# [
# builder.fb_github_project_workdir('foo/bar')
# ...
# builder.workdir('baz') # Now should be at foo/bar/baz
# ]
#
# If we just replayed the last workdir('baz'), we would not end up
# in '<github_prefix>/foo/bar/baz', but rather in `<unknown>/baz`.
next_step_workdirs = []
shipit_projects = []
build_steps = []
for action in recursively_flatten_list(actions):
if isinstance(action, LegocastleFBCheckout):
# This is guaranteed to be absolute, so drop the unnecessary
# history.
next_step_workdirs = [
ShellQuoted('"$(hg root)/"{d}', ).format(d=path_join(
self.option('shipit_project_dir'),
action.project_and_path), )
]
shipit_projects.append(
action.project_and_path.split('/', 1)[0])
elif isinstance(action, LegocastleStep):
pre_actions = [ShellQuoted('set -ex')]
if action.name != 'Setup' and \
self.option("PYTHON_VENV", "OFF") == "ON":
pre_actions.extend(self.python_venv())
pre_actions.append(
ShellQuoted("""
case "$OSTYPE" in
darwin*)
BREW_PREFIX=/var/tmp/homebrew
# The apple-provided flex and bison tools are too old to successfully
# build thrift. Ensure that we resolve to the homebrew versions.
# Note that homebrew doesn't link these into its bin dir to avoid
# these newer versions taking precedence, so we need to reach into
# the cellar path. The glob is to make this script less prone to
# problems if/when the version is bumped.
BISON_BIN=$(echo $BREW_PREFIX/Cellar/bison/*/bin)
FLEX_BIN=$(echo $BREW_PREFIX/Cellar/flex/*/bin)
export CMAKE_SYSTEM_PREFIX_PATH=$BREW_PREFIX
export PKG_CONFIG_PATH=$BREW_PREFIX/opt/openssl/lib/pkgconfig
export PATH=$BISON_BIN:$FLEX_BIN:$BREW_PREFIX/bin:$PATH
export HOMEBREW_NO_AUTO_UPDATE=1
export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/
;;
esac
"""))
pre_actions.extend(self.workdir(w) for w in next_step_workdirs)
shell_steps = []
for a in itertools.chain(pre_actions, action.actions):
if isinstance(a, LegocastleWorkdir):
next_step_workdirs.append(a.dir)
shell_steps.append(
ShellQuoted(
# Don't evaluate {d} twice.
'_wd={d} ; mkdir -p "$_wd" && cd "$_wd"').
format(d=a.dir))
else:
shell_steps.append(a)
build_steps.append({
'name':
action.name,
'shell':
raw_shell(shell_join('\n', shell_steps)),
'required':
True,
})
else:
raise LegocastleStepError(
'You have a top-level action {0} that is not a step()'.
format(repr(action)))
return shipit_projects, build_steps
