def run(self, shell_cmd):
return ShellQuoted('RUN {cmd}').format(cmd=shell_cmd)
def parse_args_to_fbcode_builder_opts(add_args_fn, top_level_opts, opts, help):
"""
Provides some standard arguments: --debug, --option, --shell-quoted-option
Then, calls `add_args_fn(parser)` to add application-specific arguments.
`opts` are first used as defaults for the various command-line
arguments. Then, the parsed arguments are mapped back into `opts`,
which then become the values for `FBCodeBuilder.option()`, to be used
both by the builder and by `get_steps_fn()`.
`help` is printed in response to the `--help` argument.
"""
top_level_opts = set(top_level_opts)
parser = argparse.ArgumentParser(
description=help, formatter_class=argparse.RawDescriptionHelpFormatter)
add_args_fn(parser)
parser.add_argument(
"--option",
nargs=2,
metavar=("KEY", "VALUE"),
action="append",
default=[(k, v) for k, v in opts.items()
if k not in top_level_opts and not isinstance(v, ShellQuoted)
],
help="Set project-specific options. These are assumed to be raw "
"strings, to be shell-escaped as needed. Default: %(default)s.",
)
parser.add_argument(
"--shell-quoted-option",
nargs=2,
metavar=("KEY", "VALUE"),
action="append",
default=[(k, raw_shell(v)) for k, v in opts.items()
if k not in top_level_opts and isinstance(v, ShellQuoted)],
help="Set project-specific options. These are assumed to be shell-"
"quoted, and may be used in commands as-is. Default: %(default)s.",
)
parser.add_argument("--debug", action="store_true", help="Log more")
args = parser.parse_args()
logging.basicConfig(
level=logging.DEBUG if args.debug else logging.INFO,
format="%(levelname)s: %(message)s",
)
# Map command-line args back into opts.
logging.debug("opts before command-line arguments: {0}".format(opts))
new_opts = {}
for key in top_level_opts:
val = getattr(args, key)
# Allow clients to unset a default by passing a value of None in opts
if val is not None:
new_opts[key] = val
for key, val in args.option:
new_opts[key] = val
for key, val in args.shell_quoted_option:
new_opts[key] = ShellQuoted(val)
logging.debug("opts after command-line arguments: {0}".format(new_opts))
return new_opts
def set_env(self, key, value):
return ShellQuoted("ENV {key}={val}").format(key=key, val=value)
def step(self, name, actions):
assert '\n' not in name, 'Name {0} would span > 1 line'.format(name)
b = ShellQuoted('')
return [ShellQuoted('### {0} ###'.format(name)), b] + actions + [b]
def debian_ccache_setup_steps(self):
source_ccache_tgz = self.option("ccache_tgz", "")
if not source_ccache_tgz:
logging.info("Docker ccache not enabled")
return []
dest_ccache_tgz = os.path.join(self.option("docker_context_dir"),
"ccache.tgz")
try:
try:
os.link(source_ccache_tgz, dest_ccache_tgz)
except OSError:
logging.exception(
"Hard-linking {s} to {d} failed, falling back to copy".
format(s=source_ccache_tgz, d=dest_ccache_tgz))
shutil.copyfile(source_ccache_tgz, dest_ccache_tgz)
except Exception:
logging.exception(
"Failed to copy or link {s} to {d}, aborting".format(
s=source_ccache_tgz, d=dest_ccache_tgz))
raise
return [
# Separate layer so that in development we avoid re-downloads.
self.run(ShellQuoted("apt-get install -yq ccache")),
ShellQuoted("ADD ccache.tgz /"),
ShellQuoted(
# Set CCACHE_DIR before the `ccache` invocations below.
"ENV CCACHE_DIR=/ccache "
# No clang support for now, so it's easiest to hardcode gcc.
'CC="ccache gcc" CXX="ccache g++" '
# Always log for ease of debugging. For real FB projects,
# this log is several megabytes, so dumping it to stdout
# would likely exceed the Travis log limit of 4MB.
#
# On a local machine, `docker cp` will get you the data. To
# get the data out from Travis, I would compress and dump
# uuencoded bytes to the log -- for Bistro this was about
# 600kb or 8000 lines:
#
# apt-get install sharutils
# bzip2 -9 < /tmp/ccache.log | uuencode -m ccache.log.bz2
"CCACHE_LOGFILE=/tmp/ccache.log"),
self.run(
ShellQuoted(
# Future: Skipping this part made this Docker step instant,
# saving ~1min of build time. It's unclear if it is the
# chown or the du, but probably the chown -- since a large
# part of the cost is incurred at image save time.
#
# ccache.tgz may be empty, or may have the wrong
# permissions.
"mkdir -p /ccache && time chown -R nobody /ccache && "
"time du -sh /ccache && "
# Reset stats so `docker_build_with_ccache.sh` can print
# useful values at the end of the run.
"echo === Prev run stats === && ccache -s && ccache -z && "
# Record the current time to let travis_build.sh figure out
# the number of bytes in the cache that are actually used --
# this is crucial for tuning the maximum cache size.
"date +%s > /FBCODE_BUILDER_CCACHE_START_TIME && "
# The build running as `nobody` should be able to write here
"chown nobody /tmp/ccache.log")),
]
def install_debian_deps(self):
actions = [
self.run(
ShellQuoted('apt-get update && apt-get install -yq {deps}').format(
deps=shell_join(' ', (
ShellQuoted(dep) for dep in self.debian_deps())))
),
]
gcc_version = self.option('gcc_version')
# We need some extra packages to be able to install GCC 4.9 on 14.04.
if self.option('os_image') == 'ubuntu:14.04' and gcc_version == '4.9':
actions.append(self.run(ShellQuoted(
'apt-get install -yq software-properties-common && '
'add-apt-repository ppa:ubuntu-toolchain-r/test && '
'apt-get update'
)))
# Make the selected GCC the default before building anything
actions.extend([
self.run(ShellQuoted('apt-get install -yq {c} {cpp}').format(
c=ShellQuoted('gcc-{v}').format(v=gcc_version),
cpp=ShellQuoted('g++-{v}').format(v=gcc_version),
)),
self.run(ShellQuoted(
'update-alternatives --install /usr/bin/gcc gcc {c} 40 '
'--slave /usr/bin/g++ g++ {cpp}'
).format(
c=ShellQuoted('/usr/bin/gcc-{v}').format(v=gcc_version),
cpp=ShellQuoted('/usr/bin/g++-{v}').format(v=gcc_version),
)),
self.run(ShellQuoted('update-alternatives --config gcc')),
])
# Ubuntu 14.04 comes with a CMake version that is too old for mstch.
if self.option('os_image') == 'ubuntu:14.04':
actions.append(self.run(ShellQuoted(
'apt-get install -yq software-properties-common && '
'add-apt-repository ppa:george-edison55/cmake-3.x && '
'apt-get update && '
'apt-get upgrade -yq cmake'
)))
# Debian 8.6 comes with a CMake version that is too old for folly.
if self.option('os_image') == 'debian:8.6':
actions.append(self.run(ShellQuoted(
'echo deb http://ftp.debian.org/debian jessie-backports main '
'>> /etc/apt/sources.list.d/jessie-backports.list && '
'apt-get update && '
'apt-get -yq -t jessie-backports install cmake'
)))
actions.extend(self.debian_ccache_setup_steps())
return self.step('Install packages for Debian-based OS', actions)
def workdir(self, dir):
return [
ShellQuoted("mkdir -p {d} && cd {d}").format(d=dir),
]
def parallel_make(self, make_vars=None):
return self.run(
ShellQuoted('make -j {n} {vars}').format(
n=self.option('make_parallelism'),
vars=self._make_vars(make_vars),
))
def make_lego_jobs(args, project_dirs):
'''
Compute the lego job specifications.
Returns a tuple of (shipit_projects, job_specs)
'''
install_dir = ShellQuoted( # BOX_DIR is our fbcode path.
'"$BOX_DIR/opensource/fbcode_builder/facebook_ci"')
# The following project-specific options may be overridden from the
# command-line options. If they are not specified on the command line or
# in the project configuration file, then the following defaults are used.
project_option_defaults = {
'prefix': install_dir,
'make_parallelism': 8,
'projects_dir': install_dir,
}
# Process all of the project configs
all_shipit_projects = set()
children_jobs = []
for project in project_dirs:
logging.debug('Processing %s', project)
config_path = os.path.join(project,
'facebook_fbcode_builder_config.py')
config = read_fbcode_builder_config(config_path)
for opts_key, lego_type in LEGO_OPTS_MAP.items():
config_opts = config.get(opts_key)
if not config_opts:
continue
# Construct the options for this project.
# Use everything listed in config_opts, unless the value is None.
project_opts = {
opt_name: value
for opt_name, value in config_opts.items() if value is not None
}
# Allow options specified on the command line to override the
# config's legocastle_opts data.
# For options that weren't provided in either place, use the default
# value from project_option_defaults.
for opt_name, default_value in project_option_defaults.items():
cli_value = getattr(args, opt_name)
if cli_value is not None:
project_opts[opt_name] = cli_value
elif opt_name not in config_opts:
project_opts[opt_name] = default_value
# The shipit_project_dir option cannot be overridden on a per-project
# basis. We emit this data in a single location that must be consisten
# across all of the projects we are building.
project_opts['shipit_project_dir'] = args.shipit_project_dir
builder = LegocastleFBCodeBuilder(**project_opts)
steps = build_fbcode_builder_config(config)(builder)
lego_spec = make_lego_spec(builder, type=lego_type)
shipit_projects, lego_spec['args']['steps'] = builder.render(steps)
all_shipit_projects.update(shipit_projects)
children_jobs.append(lego_spec)
return all_shipit_projects, children_jobs
def make_user_sudo(self):
return ShellQuoted('RUN echo "{user} ALL=(root) NOPASSWD:ALL" > '
'/etc/sudoers.d/{user} && chmod 0440 /etc/sudoers.d/'
'{user}').format(user=self._user())
def parallel_make(self, make_vars=None):
return self.run(
ShellQuoted("make -j {n} VERBOSE=1 {vars}").format(
n=self.option("make_parallelism"),
vars=self._make_vars(make_vars),
))
def python_venv(self):
action = []
if self.option("PYTHON_VENV", "OFF") == "ON":
action = ShellQuoted("source {p}").format(
p=path_join(self.option('prefix'), "venv", "bin", "activate"))
return(action)
def create_python_venv(self):
action = []
if self.option("PYTHON_VENV", "OFF") == "ON":
action = self.run(ShellQuoted("python3 -m venv {p}").format(
p=path_join(self.option('prefix'), "venv")))
return(action)
def install_debian_deps(self):
actions = [
self.run(
ShellQuoted('apt-get update && apt-get install -yq '
'autoconf-archive '
'bison '
'build-essential '
'cmake '
'curl '
'flex '
'git '
'gperf '
'joe '
'libboost-all-dev '
'libcap-dev '
'libdouble-conversion-dev '
'libevent-dev '
'libgflags-dev '
'libgoogle-glog-dev '
'libkrb5-dev '
'libnuma-dev '
'libsasl2-dev '
'libsnappy-dev '
'libsqlite3-dev '
'libssl-dev '
'libtool '
'netcat-openbsd '
'pkg-config '
'unzip '
'wget')),
]
gcc_version = self.option('gcc_version')
# We need some extra packages to be able to install GCC 4.9 on 14.04.
if self.option('os_image') == 'ubuntu:14.04' and gcc_version == '4.9':
actions.append(
self.run(
ShellQuoted(
'apt-get install -yq software-properties-common && '
'add-apt-repository ppa:ubuntu-toolchain-r/test && '
'apt-get update')))
# Make the selected GCC the default before building anything
actions.extend([
self.run(
ShellQuoted('apt-get install -yq {c} {cpp}').format(
c=ShellQuoted('gcc-{v}').format(v=gcc_version),
cpp=ShellQuoted('g++-{v}').format(v=gcc_version),
)),
self.run(
ShellQuoted(
'update-alternatives --install /usr/bin/gcc gcc {c} 40 '
'--slave /usr/bin/g++ g++ {cpp}').format(
c=ShellQuoted('/usr/bin/gcc-{v}').format(
v=gcc_version),
cpp=ShellQuoted('/usr/bin/g++-{v}').format(
v=gcc_version),
)),
self.run(ShellQuoted('update-alternatives --config gcc')),
])
# Ubuntu 14.04 comes with a CMake version that is too old for mstch.
if self.option('os_image') == 'ubuntu:14.04':
actions.append(
self.run(
ShellQuoted(
'apt-get install -yq software-properties-common && '
'add-apt-repository ppa:george-edison55/cmake-3.x && '
'apt-get update && '
'apt-get upgrade -yq cmake')))
actions.extend(self.debian_ccache_setup_steps())
return self.step('Install packages for Debian-based OS', actions)
def run(self, shell_cmd):
return ShellQuoted("{cmd}").format(cmd=shell_cmd)
def _make_vars(self, make_vars):
return shell_join(
' ',
(ShellQuoted('{k}={v}').format(k=k, v=v)
for k, v in ({} if make_vars is None else make_vars).items()))
def copy_local_repo(self, dir, dest_name):
return [
ShellQuoted("cp -r {dir} {dest_name}").format(dir=dir, dest_name=dest_name),
]
def autoconf_install(self, name):
return self.step('Build and install {0}'.format(name), [
self.run(ShellQuoted('autoreconf -ivf')),
] + self.configure() + self.make_and_install())
def step(self, name, actions):
assert "\n" not in name, "Name {0} would span > 1 line".format(name)
b = ShellQuoted("")
return [ShellQuoted("### {0} ###".format(name)), b] + actions + [b]
def _change_user(self):
return ShellQuoted("USER {u}").format(u=self._user())
def fbcode_builder_spec(builder):
builder.add_option("thom311/libnl:git_hash", "libnl3_2_25")
builder.add_option("openr/build:cmake_defines", {"ADD_ROOT_TESTS": "OFF"})
maybe_curl_patch = []
patch = path_join(
builder.option("projects_dir"),
"../shipit_projects/openr/build/fix-route-obj-attr-list.patch",
)
if not builder.has_option("shipit_project_dir"):
maybe_curl_patch = [
builder.run(
ShellQuoted(
"curl -O https://raw.githubusercontent.com/facebook/openr/master/"
"build/fix-route-obj-attr-list.patch"
)
)
]
patch = "fix-route-obj-attr-list.patch"
libnl_build_commands = maybe_curl_patch + [
builder.run(ShellQuoted("git apply {p}").format(p=patch)),
builder.run(ShellQuoted("./autogen.sh")),
builder.configure(),
builder.make_and_install(),
]
return {
"depends_on": [folly, fbthrift, python_fbthrift, fbzmq, python_fbzmq, re2],
"steps": [
builder.github_project_workdir("thom311/libnl", "."),
builder.step("Build and install thom311/libnl", libnl_build_commands),
builder.fb_github_project_workdir("openr/build", "facebook"),
builder.step(
"Build and install openr/build",
[
builder.cmake_configure("openr/build"),
# we need the pythonpath to find the thrift compiler
builder.run(
ShellQuoted(
'PYTHONPATH="$PYTHONPATH:"{p}/lib/python2.7/site-packages '
"make -j {n}"
).format(
p=builder.option("prefix"),
n=builder.option("make_parallelism"),
)
),
builder.run(ShellQuoted("sudo make install")),
builder.run(ShellQuoted("sudo ldconfig")),
],
),
builder.step(
"Install OpenR python modules",
[
builder.workdir(
path_join(builder.option("projects_dir"), "openr/openr/py")
),
builder.run(
ShellQuoted(
"sudo pip install cffi future pathlib 'networkx==2.2'"
)
),
builder.run(ShellQuoted("sudo python setup.py build")),
builder.run(ShellQuoted("sudo python setup.py install")),
],
),
builder.step(
"Run openr tests",
[
builder.workdir(
path_join(builder.option("projects_dir"), "openr/build")
),
builder.run(ShellQuoted("CTEST_OUTPUT_ON_FAILURE=TRUE make test")),
],
),
],
}