def configure(self): return [ self.run( ShellQuoted( 'LDFLAGS="$LDFLAGS -L"{p}"/lib -Wl,-rpath="{p}"/lib" ' 'CFLAGS="$CFLAGS -I"{p}"/include" ' 'CPPFLAGS="$CPPFLAGS -I"{p}"/include" ' 'PY_PREFIX={p} ' './configure --prefix={p}').format( p=self.option('prefix'))), ]
def fbcode_builder_spec(builder): return { "depends_on": [folly, wangle, fizz, sodium, zstd, mvfst, proxygen_quic], "steps": [ # Tests for the full build with no QUIC/HTTP3 # Proxygen is the last step, so we are still in its working dir. builder.step("Run proxygen tests", [builder.run(ShellQuoted("make test"))]) ], }
def fbcode_builder_spec(builder): builder.add_option("thom311/libnl:git_hash", "libnl3_2_25") builder.add_option("openr/build:cmake_defines", {"ADD_ROOT_TESTS": "OFF"}) maybe_curl_patch = [] patch = path_join( builder.option("projects_dir"), "../shipit_projects/openr/build/fix-route-obj-attr-list.patch", ) if not builder.has_option("shipit_project_dir"): maybe_curl_patch = [ builder.run( ShellQuoted( "curl -O https://raw.githubusercontent.com/facebook/openr/master/" "build/fix-route-obj-attr-list.patch")) ] patch = "fix-route-obj-attr-list.patch" libnl_build_commands = maybe_curl_patch + [ builder.run(ShellQuoted("git apply {p}").format(p=patch)), builder.run(ShellQuoted("./autogen.sh")), builder.configure(), builder.make_and_install(), ] return { "depends_on": [folly, fbthrift, fbzmq, re2], "steps": [ builder.github_project_workdir("thom311/libnl", "."), builder.step("Build and install thom311/libnl", libnl_build_commands), builder.fb_github_project_workdir("openr/build", "facebook"), builder.step( "Build and install openr/build", [ builder.cmake_configure("openr/build"), # we need the pythonpath to find the thrift compiler builder.run( ShellQuoted( 'PYTHONPATH="$PYTHONPATH:"{p}/lib/python2.7/site-packages ' "make -j {n}").format( p=builder.option("prefix"), n=builder.option("make_parallelism"), )), builder.run(ShellQuoted("sudo make install")), builder.run(ShellQuoted("sudo ldconfig")), ], ), builder.step( "Run openr tests", [ builder.run( ShellQuoted("CTEST_OUTPUT_ON_FAILURE=TRUE make test")) ], ), ], }
def cmake_configure(self, name): cmake_defines = { 'BUILD_SHARED_LIBS': 'ON', 'CMAKE_INSTALL_PREFIX': self.option('prefix'), } cmake_defines.update( self.option('{0}:cmake_defines'.format(name), {}) ) return [ self.run(ShellQuoted( 'CXXFLAGS="$CXXFLAGS -fPIC -isystem "{p}"/include" ' 'CFLAGS="$CFLAGS -fPIC -isystem "{p}"/include" ' 'cmake {args} ..' ).format( p=self.option('prefix'), args=shell_join(' ', ( ShellQuoted('-D{k}={v}').format(k=k, v=v) for k, v in cmake_defines.items() )), )), ]
def fbcode_builder_spec(builder): builder.add_option('jedisct1/libsodium:git_hash', 'stable') return { 'steps': [ builder.github_project_workdir('jedisct1/libsodium', '.'), builder.step('Build and install jedisct1/libsodium', [ builder.run(ShellQuoted('./autogen.sh')), builder.configure(), builder.make_and_install(), ]), ], }
def configure(self, name=None): autoconf_options = {} if name is not None: autoconf_options.update( self.option("{0}:autoconf_options".format(name), {})) return [ self.run( ShellQuoted( 'LDFLAGS="$LDFLAGS -L"{p}"/lib -Wl,-rpath="{p}"/lib" ' 'CFLAGS="$CFLAGS -I"{p}"/include" ' 'CPPFLAGS="$CPPFLAGS -I"{p}"/include" ' "PY_PREFIX={p} " "./configure --prefix={p} {args}").format( p=self.option("prefix"), args=shell_join( " ", (ShellQuoted("{k}={v}").format(k=k, v=v) for k, v in autoconf_options.items()), ), )), ]
def fbcode_builder_spec(builder): return { 'depends_on': [fbthrift], 'steps': [ builder.step("Install thrift python modules", [ builder.workdir( path_join(builder.option('projects_dir'), "fbthrift/thrift/lib/py")), builder.run(ShellQuoted("sudo python setup.py install")), ]), ], }
def rust_toolchain(self): actions = [] if self.option("rust_toolchain", False): (toolchain, is_bootstrap) = self.option("rust_toolchain") rust_dir = path_join(self.option("prefix"), "rust") actions = [ self.set_env("CARGO_HOME", rust_dir), self.set_env("RUSTUP_HOME", rust_dir), self.set_env("RUSTC_BOOTSTRAP", "1" if is_bootstrap else "0"), self.run( ShellQuoted( "curl -sSf https://build.travis-ci.com/files/rustup-init.sh" " | sh -s --" " --default-toolchain={r} " " --profile=minimal" " --no-modify-path" " -y").format(p=rust_dir, r=toolchain)), self.set_env( "PATH", ShellQuoted("{p}:$PATH").format( p=path_join(rust_dir, "bin")), ), self.run(ShellQuoted("rustup update")), self.run(ShellQuoted("rustc --version")), self.run(ShellQuoted("rustup --version")), self.run(ShellQuoted("cargo --version")), ] return actions
def fbcode_builder_spec(builder): return { 'depends_on': [folly, proxygen, fbthrift], 'steps': [ builder.fb_github_project_workdir('bistro/bistro'), builder.step( 'Build bistro', [ # Future: should this share some code with `cmake_install()`? builder.run( ShellQuoted( 'PYTHONPATH="$PYTHONPATH:"{p}/lib/python2.7/site-packages ' './cmake/run-cmake.sh Debug -DCMAKE_INSTALL_PREFIX={p}' ).format(p=builder.option('prefix'))), builder.workdir('cmake/Debug'), builder.parallel_make(), ]), builder.step('Run bistro tests', [ builder.run(ShellQuoted('ctest')), ]), ] }
def fbcode_builder_spec(builder): # This API should change rarely, so build the latest tag instead of master. builder.add_option('no1msd/mstch:git_hash', ShellQuoted('$(git describe --abbrev=0 --tags)')) return { 'depends_on': [folly, fizz, fmt, sodium, rsocket, wangle, zstd], 'steps': [ # This isn't a separete spec, since only fbthrift uses mstch. builder.github_project_workdir('no1msd/mstch', 'build'), builder.cmake_install('no1msd/mstch'), builder.fb_github_cmake_install('fbthrift/thrift'), ], }
def fbcode_builder_spec(builder): builder.add_option( 'hyperic/sigar:autoconf_options', {'CFLAGS' : '-fgnu89-inline'}) return { 'steps': [ builder.github_project_workdir('hyperic/sigar', '.'), builder.step('Build and install sigar', [ builder.run(ShellQuoted('./autogen.sh')), builder.configure('hyperic/sigar'), builder.make_and_install(), ]), ], }
def fbcode_builder_spec(builder): # This API should change rarely, so build the latest tag instead of master. builder.add_option( "no1msd/mstch:git_hash", ShellQuoted("$(git describe --abbrev=0 --tags)") ) return { "depends_on": [zstd], "steps": [ # This isn't a separete spec, since only fbthrift uses mstch. builder.github_project_workdir("no1msd/mstch", "build"), builder.cmake_install("no1msd/mstch"), builder.fb_github_cmake_install("LogDevice/logdevice/_build"), ], }
def github_project_workdir(self, project, path): # Only check out a non-default branch if requested. This especially # makes sense when building from a local repo. git_hash = self.option( '{0}:git_hash'.format(project), # Any repo that has a hash in deps/github_hashes defaults to # that, with the goal of making builds maximally consistent. self._github_hashes.get(project, '')) maybe_change_branch = [ self.run(ShellQuoted('git checkout {hash}').format(hash=git_hash)), ] if git_hash else [] base_dir = self.option('projects_dir') local_repo_dir = self.option('{0}:local_repo_dir'.format(project), '') return self.step('Check out {0}, workdir {1}'.format(project, path), [ self.workdir(base_dir), self.run( ShellQuoted('git clone https://github.com/{p}').format( p=project)) if not local_repo_dir else self.copy_local_repo(local_repo_dir, os.path.basename(project)), self.workdir(path_join(base_dir, os.path.basename(project), path)), ] + maybe_change_branch)
def fbcode_builder_spec(builder): # This API should change rarely, so build the latest tag instead of master. builder.add_option('facebook/zstd:git_hash', ShellQuoted('$(git describe --abbrev=0 --tags)')) return { 'steps': [ builder.github_project_workdir('facebook/zstd', '.'), builder.step('Build and install zstd', [ builder.make_and_install(make_vars={ 'PREFIX': builder.option('prefix'), }) ]), ], }
def fbcode_builder_spec(builder): return { 'depends_on': [folly, proxygen, fbthrift], 'steps': [ builder.fb_github_project_workdir('bistro/bistro'), builder.step( 'Build bistro', [ # Future: should this share some code with `cmake_install()`? builder.run( ShellQuoted( 'PATH="$PATH:"{p}/bin ' 'TEMPLATES_PATH={p}/include/thrift/templates ' './cmake/run-cmake.sh Debug -DCMAKE_INSTALL_PREFIX={p}' ).format(p=builder.option('prefix'))), builder.workdir('cmake/Debug'), builder.parallel_make(), ]), builder.step('Run bistro tests', [ builder.run(ShellQuoted('ctest --output-on-failure')), ]), ] }
def fbcode_builder_spec(builder): builder.add_option("jedisct1/libsodium:git_hash", "stable") return { "steps": [ builder.github_project_workdir("jedisct1/libsodium", "."), builder.step( "Build and install jedisct1/libsodium", [ builder.run(ShellQuoted("./autogen.sh")), builder.configure(), builder.make_and_install(), ], ), ], }
def setup(self): return self.step('Setup', [self.create_python_venv()] + [ self.run( ShellQuoted(""" case "$OSTYPE" in darwin*) http_proxy= https_proxy= ./tools/lfs/lfs.py \\ download homebrew.tar.gz \\ -l ./watchman/facebook/lego/.lfs-pointers rm -rf /var/tmp/homebrew tar xzf homebrew.tar.gz -C /var/tmp ;; esac """)) ])
def fbcode_builder_spec(builder): builder.add_option('wangle/_build:cmake_defines', { 'BUILD_SHARED_LIBS': 'OFF', 'BUILD_TESTS': 'ON', }) return { 'depends_on': [gmock, folly, fizz, sodium], 'steps': [ builder.fb_github_cmake_install('wangle/_build', '../wangle'), builder.step('Run wangle tests', [ builder.run( ShellQuoted('ctest --output-on-failure -j {n}').format( n=builder.option('make_parallelism'), )) ]), ] }
def fbcode_builder_spec(builder): # This API should change rarely, so build the latest tag instead of master. builder.add_option("no1msd/mstch:git_hash", ShellQuoted("$(git describe --abbrev=0 --tags)")) builder.add_option("PYTHON_VENV", "ON") builder.add_option("LogDevice/logdevice/_build:cmake_defines", {"BUILD_SUBMODULES": "OFF"}) return { "depends_on": [zstd, fmt, folly, fizz, wangle, sodium, rsocket], "steps": [ # This isn't a separete spec, since only fbthrift uses mstch. builder.github_project_workdir("no1msd/mstch", "build"), builder.cmake_install("no1msd/mstch"), builder.fb_github_cmake_install("fbthrift/thrift"), builder.fb_github_cmake_install("LogDevice/logdevice/_build", github_org="facebookincubator"), ], }
def fbcode_builder_spec(builder): builder.add_option("folly/_build:cmake_defines", { "BUILD_SHARED_LIBS": "OFF", "BUILD_TESTS": "ON" }) return { "depends_on": [fmt, gmock], "steps": [ builder.fb_github_cmake_install("folly/_build"), builder.step( "Run folly tests", [ builder.run( ShellQuoted("ctest --output-on-failure -j {n}").format( n=builder.option("make_parallelism"))) ], ), ], }
def fbcode_builder_spec(builder): builder.add_option('thom311/libnl:git_hash', 'libnl3_2_25') builder.add_option('openr/openr/build:cmake_defines', {'ADD_ROOT_TESTS': 'OFF'}) maybe_curl_patch = [] patch = path_join( builder.option('projects_dir'), '../shipit_projects/openr/build/fix-route-obj-attr-list.patch') if not builder.has_option('shipit_project_dir'): maybe_curl_patch = [ builder.run( ShellQuoted( 'curl -O https://raw.githubusercontent.com/facebook/openr/master/' 'build/fix-route-obj-attr-list.patch')) ] patch = 'fix-route-obj-attr-list.patch' libnl_build_commands = maybe_curl_patch + [ builder.run(ShellQuoted('git apply {p}').format(p=patch)), builder.run(ShellQuoted('./autogen.sh')), builder.configure(), builder.make_and_install() ] return { 'depends_on': [folly, fbthrift, fbzmq, re2], 'steps': [ builder.github_project_workdir('thom311/libnl', '.'), builder.step('Build and install thom311/libnl', libnl_build_commands), builder.fb_github_project_workdir('openr/openr/build', 'facebook'), builder.step( 'Build and install openr/openr/build', [ builder.cmake_configure('openr/openr/build'), # we need the pythonpath to find the thrift compiler builder.run( ShellQuoted( 'PYTHONPATH="$PYTHONPATH:"{p}/lib/python2.7/site-packages ' 'make -j {n}').format( p=builder.option('prefix'), n=builder.option('make_parallelism'))), builder.run(ShellQuoted('make install')), ]), builder.step('Run openr tests', [ builder.run( ShellQuoted('CTEST_OUTPUT_ON_FAILURE=TRUE make test')), ]), ], }
def fbcode_builder_spec(builder): # This API should change rarely, so build the latest tag instead of master. builder.add_option( "facebook/zstd:git_hash", ShellQuoted("$(git describe --abbrev=0 --tags origin/master)"), ) return { "steps": [ builder.github_project_workdir("facebook/zstd", "."), builder.step( "Build and install zstd", [ builder.make_and_install( make_vars={ "PREFIX": builder.option("prefix"), }) ], ), ], }
def fbcode_builder_spec(builder): builder.add_option( "wangle/_build:cmake_defines", { "BUILD_SHARED_LIBS": "OFF", "BUILD_TESTS": "ON", }, ) return { "depends_on": [gmock, fmt, folly, fizz, sodium], "steps": [ builder.fb_github_cmake_install("wangle/_build", "../wangle"), builder.step( "Run wangle tests", [ builder.run( ShellQuoted("ctest --output-on-failure -j {n}").format( n=builder.option("make_parallelism"), )) ], ), ], }
def fbcode_builder_spec(builder): # This API should change rarely, so build the latest tag instead of master. builder.add_option('no1msd/mstch:git_hash', ShellQuoted('$(git describe --abbrev=0 --tags)')) builder.add_option('rsocket/rsocket-cpp/build:cmake_defines', {'BUILD_TESTS': 'OFF'}) builder.add_option('krb5/krb5:git_hash', 'krb5-1.16.1-final') return { 'depends_on': [folly, fizz, sodium, wangle, zstd], 'steps': [ # This isn't a separete spec, since only fbthrift uses mstch. builder.github_project_workdir('no1msd/mstch', 'build'), builder.cmake_install('no1msd/mstch'), builder.github_project_workdir('krb5/krb5', 'src'), builder.autoconf_install('krb5/krb5'), builder.github_project_workdir('rsocket/rsocket-cpp', 'build'), builder.step('configuration for rsocket', [ builder.cmake_configure('rsocket/rsocket-cpp/build'), ]), builder.cmake_install('rsocket/rsocket-cpp'), builder.fb_github_cmake_install('fbthrift/thrift'), ], }
def setup(self): # Please add RPM-based OSes here as appropriate. # # To allow exercising non-root installs -- we change users after the # system packages are installed. TODO: For users not defined in the # image, we should probably `useradd`. return self.step( 'Setup', [ ShellQuoted('FROM {img}').format( # Docker can't deal with quotes. Oh well. img=ShellQuoted(self.option('os_image')), ), ShellQuoted('ENV CFLAGS=-O'), ShellQuoted('ENV CXXFLAGS=-O'), ShellQuoted('ENV CPPFLAGS=-O'), # /bin/sh syntax is a pain ShellQuoted('SHELL ["/bin/bash", "-c"]'), ] + self.install_debian_deps() + [self._change_user()])
def parallel_make(self, make_vars=None): return self.run( ShellQuoted("make -j {n} VERBOSE=1 {vars}").format( n=self.option("make_parallelism"), vars=self._make_vars(make_vars), ))
def _make_vars(self, make_vars): return shell_join( " ", (ShellQuoted("{k}={v}").format(k=k, v=v) for k, v in ({} if make_vars is None else make_vars).items()), )
def _render_impl(self, actions): next_workdir = None shipit_projects = [] build_steps = [] for action in recursively_flatten_list(actions): if isinstance(action, LegocastleFBCheckout): next_workdir = ShellQuoted('"$(hg root)/"{d}', ).format( d=path_join(self.option('shipit_project_dir'), action.project_and_path), ) shipit_projects.append( action.project_and_path.split('/', 1)[0]) elif isinstance(action, LegocastleStep): pre_actions = [ShellQuoted('set -ex')] if action.name != 'Setup' and \ self.option("PYTHON_VENV", "OFF") == "ON": pre_actions.append(self.python_venv()) pre_actions.append( ShellQuoted(""" case "$OSTYPE" in darwin*) BREW_PREFIX=/var/tmp/homebrew # The apple-provided flex and bison tools are too old to successfully # build thrift. Ensure that we resolve to the homebrew versions. # Note that homebrew doesn't link these into its bin dir to avoid # these newer versions taking precedence, so we need to reach into # the cellar path. The glob is to make this script less prone to # problems if/when the version is bumped. BISON_BIN=$(echo $BREW_PREFIX/Cellar/bison/*/bin) FLEX_BIN=$(echo $BREW_PREFIX/Cellar/flex/*/bin) export CMAKE_SYSTEM_PREFIX_PATH=$BREW_PREFIX export PKG_CONFIG_PATH=$BREW_PREFIX/opt/openssl/lib/pkgconfig export PATH=$BISON_BIN:$FLEX_BIN:$BREW_PREFIX/bin:$PATH export HOMEBREW_NO_AUTO_UPDATE=1 export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/ ;; esac """)) if next_workdir is not None: pre_actions.append(self.workdir(next_workdir)) next_workdir = None shell_steps = [] for a in itertools.chain(pre_actions, action.actions): if isinstance(a, LegocastleWorkdir): # Pass the last working directory to the next step, # since Legocastle doesn't remember it. next_workdir = a.dir shell_steps.append( ShellQuoted( # Don't evaluate {d} twice. '_wd={d} ; mkdir -p "$_wd" && cd "$_wd"'). format(d=a.dir)) else: shell_steps.append(a) build_steps.append({ 'name': action.name, 'shell': raw_shell(shell_join('\n', shell_steps)), 'required': True, }) else: raise LegocastleStepError( 'You have a top-level action {0} that is not a step()'. format(repr(action))) return shipit_projects, build_steps
def _render_impl(self, actions): # This exists to transfer the current working directory to the next # step, since Legocastle doesn't remember it. Although this is # potentially quadratic, we have to replay ALL workdir calls from # the previous step because these might contain variable expansions, # making it impossible to distinguish relative directories from # absolute. Consider this step: # [ # builder.fb_github_project_workdir('foo/bar') # ... # builder.workdir('baz') # Now should be at foo/bar/baz # ] # # If we just replayed the last workdir('baz'), we would not end up # in '<github_prefix>/foo/bar/baz', but rather in `<unknown>/baz`. next_step_workdirs = [] shipit_projects = [] build_steps = [] for action in recursively_flatten_list(actions): if isinstance(action, LegocastleFBCheckout): # This is guaranteed to be absolute, so drop the unnecessary # history. next_step_workdirs = [ ShellQuoted('"$(hg root)/"{d}', ).format(d=path_join( self.option('shipit_project_dir'), action.project_and_path), ) ] shipit_projects.append( action.project_and_path.split('/', 1)[0]) elif isinstance(action, LegocastleStep): pre_actions = [ShellQuoted('set -ex')] if action.name != 'Setup' and \ self.option("PYTHON_VENV", "OFF") == "ON": pre_actions.extend(self.python_venv()) pre_actions.append( ShellQuoted(""" case "$OSTYPE" in darwin*) BREW_PREFIX=/var/tmp/homebrew # The apple-provided flex and bison tools are too old to successfully # build thrift. Ensure that we resolve to the homebrew versions. # Note that homebrew doesn't link these into its bin dir to avoid # these newer versions taking precedence, so we need to reach into # the cellar path. The glob is to make this script less prone to # problems if/when the version is bumped. BISON_BIN=$(echo $BREW_PREFIX/Cellar/bison/*/bin) FLEX_BIN=$(echo $BREW_PREFIX/Cellar/flex/*/bin) export CMAKE_SYSTEM_PREFIX_PATH=$BREW_PREFIX export PKG_CONFIG_PATH=$BREW_PREFIX/opt/openssl/lib/pkgconfig export PATH=$BISON_BIN:$FLEX_BIN:$BREW_PREFIX/bin:$PATH export HOMEBREW_NO_AUTO_UPDATE=1 export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/ ;; esac """)) pre_actions.extend(self.workdir(w) for w in next_step_workdirs) shell_steps = [] for a in itertools.chain(pre_actions, action.actions): if isinstance(a, LegocastleWorkdir): next_step_workdirs.append(a.dir) shell_steps.append( ShellQuoted( # Don't evaluate {d} twice. '_wd={d} ; mkdir -p "$_wd" && cd "$_wd"'). format(d=a.dir)) else: shell_steps.append(a) build_steps.append({ 'name': action.name, 'shell': raw_shell(shell_join('\n', shell_steps)), 'required': True, }) else: raise LegocastleStepError( 'You have a top-level action {0} that is not a step()'. format(repr(action))) return shipit_projects, build_steps
def autoconf_install(self, name): return self.step('Build and install {0}'.format(name), [ self.run(ShellQuoted('autoreconf -ivf')), ] + self.configure() + self.make_and_install())