def rust_toolchain(self): actions = [] if self.option("rust_toolchain", False): (toolchain, is_bootstrap) = self.option("rust_toolchain") rust_dir = path_join(self.option("prefix"), "rust") actions = [ self.set_env("CARGO_HOME", rust_dir), self.set_env("RUSTUP_HOME", rust_dir), self.set_env("RUSTC_BOOTSTRAP", "1" if is_bootstrap else "0"), self.run( ShellQuoted( "curl -sSf https://build.travis-ci.com/files/rustup-init.sh" " | sh -s --" " --default-toolchain={r} " " --profile=minimal" " --no-modify-path" " -y").format(p=rust_dir, r=toolchain)), self.set_env( "PATH", ShellQuoted("{p}:$PATH").format( p=path_join(rust_dir, "bin")), ), self.run(ShellQuoted("rustup update")), self.run(ShellQuoted("rustc --version")), self.run(ShellQuoted("rustup --version")), self.run(ShellQuoted("cargo --version")), ] return actions
def github_project_workdir(self, project, path): # Only check out a non-default branch if requested. This especially # makes sense when building from a local repo. git_hash = self.option( "{0}:git_hash".format(project), # Any repo that has a hash in deps/github_hashes defaults to # that, with the goal of making builds maximally consistent. self._github_hashes.get(project, ""), ) maybe_change_branch = ([ self.run(ShellQuoted("git checkout {hash}").format(hash=git_hash)) ] if git_hash else []) base_dir = self.option("projects_dir") local_repo_dir = self.option("{0}:local_repo_dir".format(project), "") return self.step( "Check out {0}, workdir {1}".format(project, path), [ self.workdir(base_dir), self.run( ShellQuoted("if [[ ! -d {d} ]]; then \n" "\tgit clone https://github.com/{p}\n" "fi"). format(p=project, d=path_join(base_dir, os.path.basename(project)))) if not local_repo_dir else self.copy_local_repo( local_repo_dir, os.path.basename(project)), self.workdir( path_join(base_dir, os.path.basename(project), path)), ] + maybe_change_branch, )
def github_project_workdir(self, project, path): # Only check out a non-default branch if requested. This especially # makes sense when building from a local repo. git_hash = self.option( '{0}:git_hash'.format(project), # Any repo that has a hash in deps/github_hashes defaults to # that, with the goal of making builds maximally consistent. self._github_hashes.get(project, '')) maybe_change_branch = [ self.run(ShellQuoted('git checkout {hash}').format(hash=git_hash)), ] if git_hash else [] base_dir = self.option('projects_dir') git_patch = self.option('{0}:git_patch'.format(project), '') maybe_apply_patch = [ self.run( ShellQuoted('git apply {patch}').format( patch=path_join(base_dir, git_patch))), ] if git_patch else [] local_repo_dir = self.option('{0}:local_repo_dir'.format(project), '') return self.step('Check out {0}, workdir {1}'.format(project, path), [ self.workdir(base_dir), self.run( ShellQuoted('git clone https://github.com/{p}').format( p=project)) if not local_repo_dir else self.copy_local_repo(local_repo_dir, os.path.basename(project)), self.workdir(path_join(base_dir, os.path.basename(project), path)), ] + maybe_change_branch + maybe_apply_patch)
def fbcode_builder_spec(builder): builder.add_option("watchman/_build:cmake_defines", {"BUILD_SHARED_LIBS": "OFF"}) projects = builder.option("projects_dir") return { "depends_on": [gmock, folly, fbthrift], "steps": [ builder.fb_github_cmake_install("watchman/_build", ".."), builder.step( "Run watchman tests", [ builder.run( ShellQuoted("ctest --output-on-failure -j {n}").format( n=builder.option("make_parallelism"))), builder.run( ShellQuoted( "cd ../ && ./runtests.py --concurrency {n} " "--watchman-path _build/watchman --pybuild-dir {p}" ).format( n=builder.option("make_parallelism"), p=path_join( projects, "../shipit_projects/watchman/_build/python"), )), ], ), ], }
def fbcode_builder_spec(builder): builder.add_option("watchman/_build:cmake_defines", {"BUILD_SHARED_LIBS": "OFF"}) projects = builder.option("projects_dir") return { "depends_on": [gmock, folly, fbthrift], "steps": [ builder.fb_github_cmake_install("watchman/_build", ".."), builder.step( "Run watchman tests", [ builder.run( ShellQuoted("ctest --output-on-failure -j {n}").format( n=builder.option("make_parallelism") ) ), builder.run( ShellQuoted( "cd ../ && ./runtests.py --concurrency {n} " "--watchman-path _build/watchman --pybuild-dir {p}" ).format( n=builder.option("make_parallelism"), p=path_join( projects, "../shipit_projects/watchman/_build/python" ), ) ), ], ), ], }
def create_python_venv(self): action = [] if self.option("PYTHON_VENV", "OFF") == "ON": action = self.run( ShellQuoted("python3 -m venv {p}").format( p=path_join(self.option('prefix'), "venv"))) return (action)
def github_project_workdir(self, project, path): # Only check out a non-default branch if requested. This especially # makes sense when building from a local repo. git_hash = self.option( '{0}:git_hash'.format(project), # Any repo that has a hash in deps/github_hashes defaults to # that, with the goal of making builds maximally consistent. self._github_hashes.get(project, '') ) maybe_change_branch = [ self.run(ShellQuoted('git checkout {hash}').format(hash=git_hash)), ] if git_hash else [] base_dir = self.option('projects_dir') local_repo_dir = self.option('{0}:local_repo_dir'.format(project), '') return self.step('Check out {0}, workdir {1}'.format(project, path), [ self.workdir(base_dir), self.run( ShellQuoted('git clone {opts} https://github.com/{p}').format( p=project, opts=ShellQuoted(self.option('{}:git_clone_opts'.format(project), ''))) ) if not local_repo_dir else self.copy_local_repo( local_repo_dir, os.path.basename(project) ), self.workdir(path_join(base_dir, os.path.basename(project), path)), ] + maybe_change_branch)
def python_venv(self): # To both avoid calling venv activate on each RUN command AND to ensure # it is present when the resulting container is run add to PATH actions = [] if self.option("PYTHON_VENV", "OFF") == "ON": actions = ShellQuoted('ENV PATH={p}:$PATH').format( p=path_join(self.option('prefix'), "venv", "bin")) return (actions)
def python_venv(self): # To both avoid calling venv activate on each RUN command AND to ensure # it is present when the resulting container is run add to PATH actions = [] if self.option("PYTHON_VENV", "OFF") == "ON": actions = ShellQuoted('ENV PATH={p}:$PATH').format( p=path_join(self.option('prefix'), "venv", "bin")) return(actions)
def fbcode_builder_spec(builder): builder.add_option("thom311/libnl:git_hash", "libnl3_2_25") builder.add_option("openr/build:cmake_defines", {"ADD_ROOT_TESTS": "OFF"}) maybe_curl_patch = [] patch = path_join( builder.option("projects_dir"), "../shipit_projects/openr/build/fix-route-obj-attr-list.patch", ) if not builder.has_option("shipit_project_dir"): maybe_curl_patch = [ builder.run( ShellQuoted( "curl -O https://raw.githubusercontent.com/facebook/openr/master/" "build/fix-route-obj-attr-list.patch")) ] patch = "fix-route-obj-attr-list.patch" libnl_build_commands = maybe_curl_patch + [ builder.run(ShellQuoted("git apply {p}").format(p=patch)), builder.run(ShellQuoted("./autogen.sh")), builder.configure(), builder.make_and_install(), ] return { "depends_on": [folly, fbthrift, fbzmq, re2], "steps": [ builder.github_project_workdir("thom311/libnl", "."), builder.step("Build and install thom311/libnl", libnl_build_commands), builder.fb_github_project_workdir("openr/build", "facebook"), builder.step( "Build and install openr/build", [ builder.cmake_configure("openr/build"), # we need the pythonpath to find the thrift compiler builder.run( ShellQuoted( 'PYTHONPATH="$PYTHONPATH:"{p}/lib/python2.7/site-packages ' "make -j {n}").format( p=builder.option("prefix"), n=builder.option("make_parallelism"), )), builder.run(ShellQuoted("sudo make install")), builder.run(ShellQuoted("sudo ldconfig")), ], ), builder.step( "Run openr tests", [ builder.run( ShellQuoted("CTEST_OUTPUT_ON_FAILURE=TRUE make test")) ], ), ], }
def python_venv(self): actions = [] if self.option("PYTHON_VENV", "OFF") == "ON": actions.append(ShellQuoted("source {p}").format( p=path_join(self.option('prefix'), "venv", "bin", "activate"))) actions.append(self.run( ShellQuoted("python3 -m pip install {deps}").format( deps=shell_join(' ', (ShellQuoted(dep) for dep in self.python_deps()))))) return(actions)
def fbcode_builder_spec(builder): return { 'depends_on': [fbthrift], 'steps': [ builder.step("Install thrift python modules", [ builder.workdir( path_join(builder.option('projects_dir'), "fbthrift/thrift/lib/py")), builder.run(ShellQuoted("sudo python setup.py install")), ]), ], }
def fbcode_builder_spec(builder): builder.enable_rust_toolchain() return { "depends_on": [fbthrift], "steps": [ builder.set_env( "THRIFT", path_join(builder.option("prefix"), "bin", "thrift1")), builder.fb_github_cargo_build("rust-shed/", github_org="facebookexperimental"), ], }
def fbcode_builder_spec(builder): builder.add_option('thom311/libnl:git_hash', 'libnl3_2_25') builder.add_option('openr/openr/build:cmake_defines', {'ADD_ROOT_TESTS': 'OFF'}) maybe_curl_patch = [] patch = path_join( builder.option('projects_dir'), '../shipit_projects/openr/build/fix-route-obj-attr-list.patch') if not builder.has_option('shipit_project_dir'): maybe_curl_patch = [ builder.run( ShellQuoted( 'curl -O https://raw.githubusercontent.com/facebook/openr/master/' 'build/fix-route-obj-attr-list.patch')) ] patch = 'fix-route-obj-attr-list.patch' libnl_build_commands = maybe_curl_patch + [ builder.run(ShellQuoted('git apply {p}').format(p=patch)), builder.run(ShellQuoted('./autogen.sh')), builder.configure(), builder.make_and_install() ] return { 'depends_on': [folly, fbthrift, fbzmq, re2], 'steps': [ builder.github_project_workdir('thom311/libnl', '.'), builder.step('Build and install thom311/libnl', libnl_build_commands), builder.fb_github_project_workdir('openr/openr/build', 'facebook'), builder.step( 'Build and install openr/openr/build', [ builder.cmake_configure('openr/openr/build'), # we need the pythonpath to find the thrift compiler builder.run( ShellQuoted( 'PYTHONPATH="$PYTHONPATH:"{p}/lib/python2.7/site-packages ' 'make -j {n}').format( p=builder.option('prefix'), n=builder.option('make_parallelism'))), builder.run(ShellQuoted('make install')), ]), builder.step('Run openr tests', [ builder.run( ShellQuoted('CTEST_OUTPUT_ON_FAILURE=TRUE make test')), ]), ], }
def _render_impl(self, actions): next_workdir = None shipit_projects = [] build_steps = [] for action in recursively_flatten_list(actions): if isinstance(action, LegocastleFBCheckout): next_workdir = ShellQuoted('"$(hg root)/"{d}', ).format( d=path_join(self.option('shipit_project_dir'), action.project_and_path), ) shipit_projects.append( action.project_and_path.split('/', 1)[0]) elif isinstance(action, LegocastleStep): pre_actions = [ShellQuoted('set -ex')] if next_workdir is not None: pre_actions.append(self.workdir(next_workdir)) next_workdir = None shell_steps = [] for a in itertools.chain(pre_actions, action.actions): if isinstance(a, LegocastleWorkdir): # Pass the last working directory to the next step, # since Legocastle doesn't remember it. next_workdir = a.dir shell_steps.append( ShellQuoted( # Don't evaluate {d} twice. '_wd={d} ; mkdir -p "$_wd" && cd "$_wd"'). format(d=a.dir)) else: shell_steps.append(a) build_steps.append({ 'name': action.name, 'shell': raw_shell(shell_join('\n', shell_steps)), 'required': True, }) else: raise LegocastleStepError( 'You have a top-level action {0} that is not a step()'. format(repr(action))) return shipit_projects, build_steps
def github_project_workdir(self, project, path): # Only check out a non-default branch if requested. This especially # makes sense when building from a local repo. git_hash = self.option('{0}:git_hash'.format(project), '') maybe_change_branch = [ self.run(ShellQuoted('git checkout {hash}').format(hash=git_hash)), ] if git_hash else [] base_dir = self.option('projects_dir') local_repo_dir = self.option('{0}:local_repo_dir'.format(project), '') return self.step('Check out {0}, workdir {1}'.format(project, path), [ self.workdir(base_dir), self.run( ShellQuoted('git clone https://github.com/{p}').format( p=project)) if not local_repo_dir else self.copy_local_repo(local_repo_dir, os.path.basename(project)), self.workdir(path_join(base_dir, os.path.basename(project), path)), ] + maybe_change_branch)
def github_project_workdir(self, project, path): # Only check out a non-default branch if requested. This especially # makes sense when building from a local repo. git_hash = self.option('{0}:git_hash'.format(project), '') maybe_change_branch = [ self.run(ShellQuoted('git checkout {hash}').format(hash=git_hash)), ] if git_hash else [] base_dir = self.option('projects_dir') local_repo_dir = self.option('{0}:local_repo_dir'.format(project), '') return self.step('Check out {0}, workdir {1}'.format(project, path), [ self.workdir(base_dir), self.run( ShellQuoted('git clone https://github.com/{p}') .format(p=project) ) if not local_repo_dir else self.copy_local_repo( local_repo_dir, os.path.basename(project) ), self.workdir(path_join(base_dir, os.path.basename(project), path)), ] + maybe_change_branch)
def python_venv(self): action = [] if self.option("PYTHON_VENV", "OFF") == "ON": action = ShellQuoted("source {p}").format( p=path_join(self.option('prefix'), "venv", "bin", "activate")) return (action)
def python_venv(self): action = [] if self.option("PYTHON_VENV", "OFF") == "ON": action = ShellQuoted("source {p}").format( p=path_join(self.option('prefix'), "venv", "bin", "activate")) return(action)
def create_python_venv(self): action = [] if self.option("PYTHON_VENV", "OFF") == "ON": action = self.run(ShellQuoted("python3 -m venv {p}").format( p=path_join(self.option('prefix'), "venv"))) return(action)
def _render_impl(self, actions): next_workdir = None shipit_projects = [] build_steps = [] for action in recursively_flatten_list(actions): if isinstance(action, LegocastleFBCheckout): next_workdir = ShellQuoted('"$(hg root)/"{d}', ).format( d=path_join(self.option('shipit_project_dir'), action.project_and_path), ) shipit_projects.append( action.project_and_path.split('/', 1)[0]) elif isinstance(action, LegocastleStep): pre_actions = [ShellQuoted('set -ex')] if action.name != 'Setup' and \ self.option("PYTHON_VENV", "OFF") == "ON": pre_actions.append(self.python_venv()) pre_actions.append( ShellQuoted(""" case "$OSTYPE" in darwin*) BREW_PREFIX=/var/tmp/homebrew # The apple-provided flex and bison tools are too old to successfully # build thrift. Ensure that we resolve to the homebrew versions. # Note that homebrew doesn't link these into its bin dir to avoid # these newer versions taking precedence, so we need to reach into # the cellar path. The glob is to make this script less prone to # problems if/when the version is bumped. BISON_BIN=$(echo $BREW_PREFIX/Cellar/bison/*/bin) FLEX_BIN=$(echo $BREW_PREFIX/Cellar/flex/*/bin) export CMAKE_SYSTEM_PREFIX_PATH=$BREW_PREFIX export PKG_CONFIG_PATH=$BREW_PREFIX/opt/openssl/lib/pkgconfig export PATH=$BISON_BIN:$FLEX_BIN:$BREW_PREFIX/bin:$PATH export HOMEBREW_NO_AUTO_UPDATE=1 export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/ ;; esac """)) if next_workdir is not None: pre_actions.append(self.workdir(next_workdir)) next_workdir = None shell_steps = [] for a in itertools.chain(pre_actions, action.actions): if isinstance(a, LegocastleWorkdir): # Pass the last working directory to the next step, # since Legocastle doesn't remember it. next_workdir = a.dir shell_steps.append( ShellQuoted( # Don't evaluate {d} twice. '_wd={d} ; mkdir -p "$_wd" && cd "$_wd"'). format(d=a.dir)) else: shell_steps.append(a) build_steps.append({ 'name': action.name, 'shell': raw_shell(shell_join('\n', shell_steps)), 'required': True, }) else: raise LegocastleStepError( 'You have a top-level action {0} that is not a step()'. format(repr(action))) return shipit_projects, build_steps
def _render_impl(self, actions): # This exists to transfer the current working directory to the next # step, since Legocastle doesn't remember it. Although this is # potentially quadratic, we have to replay ALL workdir calls from # the previous step because these might contain variable expansions, # making it impossible to distinguish relative directories from # absolute. Consider this step: # [ # builder.fb_github_project_workdir('foo/bar') # ... # builder.workdir('baz') # Now should be at foo/bar/baz # ] # # If we just replayed the last workdir('baz'), we would not end up # in '<github_prefix>/foo/bar/baz', but rather in `<unknown>/baz`. next_step_workdirs = [] shipit_projects = [] build_steps = [] for action in recursively_flatten_list(actions): if isinstance(action, LegocastleFBCheckout): # This is guaranteed to be absolute, so drop the unnecessary # history. next_step_workdirs = [ ShellQuoted('"$(hg root)/"{d}', ).format(d=path_join( self.option('shipit_project_dir'), action.project_and_path), ) ] shipit_projects.append( action.project_and_path.split('/', 1)[0]) elif isinstance(action, LegocastleStep): pre_actions = [ShellQuoted('set -ex')] if action.name != 'Setup' and \ self.option("PYTHON_VENV", "OFF") == "ON": pre_actions.extend(self.python_venv()) pre_actions.append( ShellQuoted(""" case "$OSTYPE" in darwin*) BREW_PREFIX=/var/tmp/homebrew # The apple-provided flex and bison tools are too old to successfully # build thrift. Ensure that we resolve to the homebrew versions. # Note that homebrew doesn't link these into its bin dir to avoid # these newer versions taking precedence, so we need to reach into # the cellar path. The glob is to make this script less prone to # problems if/when the version is bumped. BISON_BIN=$(echo $BREW_PREFIX/Cellar/bison/*/bin) FLEX_BIN=$(echo $BREW_PREFIX/Cellar/flex/*/bin) export CMAKE_SYSTEM_PREFIX_PATH=$BREW_PREFIX export PKG_CONFIG_PATH=$BREW_PREFIX/opt/openssl/lib/pkgconfig export PATH=$BISON_BIN:$FLEX_BIN:$BREW_PREFIX/bin:$PATH export HOMEBREW_NO_AUTO_UPDATE=1 export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/ ;; esac """)) pre_actions.extend(self.workdir(w) for w in next_step_workdirs) shell_steps = [] for a in itertools.chain(pre_actions, action.actions): if isinstance(a, LegocastleWorkdir): next_step_workdirs.append(a.dir) shell_steps.append( ShellQuoted( # Don't evaluate {d} twice. '_wd={d} ; mkdir -p "$_wd" && cd "$_wd"'). format(d=a.dir)) else: shell_steps.append(a) build_steps.append({ 'name': action.name, 'shell': raw_shell(shell_join('\n', shell_steps)), 'required': True, }) else: raise LegocastleStepError( 'You have a top-level action {0} that is not a step()'. format(repr(action))) return shipit_projects, build_steps