Ejemplo n.º 1
0
 def rust_toolchain(self):
     actions = []
     if self.option("rust_toolchain", False):
         (toolchain, is_bootstrap) = self.option("rust_toolchain")
         rust_dir = path_join(self.option("prefix"), "rust")
         actions = [
             self.set_env("CARGO_HOME", rust_dir),
             self.set_env("RUSTUP_HOME", rust_dir),
             self.set_env("RUSTC_BOOTSTRAP", "1" if is_bootstrap else "0"),
             self.run(
                 ShellQuoted(
                     "curl -sSf https://build.travis-ci.com/files/rustup-init.sh"
                     " | sh -s --"
                     "   --default-toolchain={r} "
                     "   --profile=minimal"
                     "   --no-modify-path"
                     "   -y").format(p=rust_dir, r=toolchain)),
             self.set_env(
                 "PATH",
                 ShellQuoted("{p}:$PATH").format(
                     p=path_join(rust_dir, "bin")),
             ),
             self.run(ShellQuoted("rustup update")),
             self.run(ShellQuoted("rustc --version")),
             self.run(ShellQuoted("rustup --version")),
             self.run(ShellQuoted("cargo --version")),
         ]
     return actions
Ejemplo n.º 2
0
    def github_project_workdir(self, project, path):
        # Only check out a non-default branch if requested. This especially
        # makes sense when building from a local repo.
        git_hash = self.option(
            "{0}:git_hash".format(project),
            # Any repo that has a hash in deps/github_hashes defaults to
            # that, with the goal of making builds maximally consistent.
            self._github_hashes.get(project, ""),
        )
        maybe_change_branch = ([
            self.run(ShellQuoted("git checkout {hash}").format(hash=git_hash))
        ] if git_hash else [])

        base_dir = self.option("projects_dir")

        local_repo_dir = self.option("{0}:local_repo_dir".format(project), "")
        return self.step(
            "Check out {0}, workdir {1}".format(project, path),
            [
                self.workdir(base_dir),
                self.run(
                    ShellQuoted("if [[ ! -d {d} ]]; then \n"
                                "\tgit clone https://github.com/{p}\n"
                                "fi").
                    format(p=project,
                           d=path_join(base_dir, os.path.basename(project))))
                if not local_repo_dir else self.copy_local_repo(
                    local_repo_dir, os.path.basename(project)),
                self.workdir(
                    path_join(base_dir, os.path.basename(project), path)),
            ] + maybe_change_branch,
        )
Ejemplo n.º 3
0
    def github_project_workdir(self, project, path):
        # Only check out a non-default branch if requested. This especially
        # makes sense when building from a local repo.
        git_hash = self.option(
            '{0}:git_hash'.format(project),
            # Any repo that has a hash in deps/github_hashes defaults to
            # that, with the goal of making builds maximally consistent.
            self._github_hashes.get(project, ''))
        maybe_change_branch = [
            self.run(ShellQuoted('git checkout {hash}').format(hash=git_hash)),
        ] if git_hash else []

        base_dir = self.option('projects_dir')
        git_patch = self.option('{0}:git_patch'.format(project), '')
        maybe_apply_patch = [
            self.run(
                ShellQuoted('git apply {patch}').format(
                    patch=path_join(base_dir, git_patch))),
        ] if git_patch else []

        local_repo_dir = self.option('{0}:local_repo_dir'.format(project), '')
        return self.step('Check out {0}, workdir {1}'.format(project, path), [
            self.workdir(base_dir),
            self.run(
                ShellQuoted('git clone https://github.com/{p}').format(
                    p=project)) if not local_repo_dir else
            self.copy_local_repo(local_repo_dir, os.path.basename(project)),
            self.workdir(path_join(base_dir, os.path.basename(project), path)),
        ] + maybe_change_branch + maybe_apply_patch)
def fbcode_builder_spec(builder):
    builder.add_option("watchman/_build:cmake_defines",
                       {"BUILD_SHARED_LIBS": "OFF"})

    projects = builder.option("projects_dir")

    return {
        "depends_on": [gmock, folly, fbthrift],
        "steps": [
            builder.fb_github_cmake_install("watchman/_build", ".."),
            builder.step(
                "Run watchman tests",
                [
                    builder.run(
                        ShellQuoted("ctest --output-on-failure -j {n}").format(
                            n=builder.option("make_parallelism"))),
                    builder.run(
                        ShellQuoted(
                            "cd ../ && ./runtests.py --concurrency {n} "
                            "--watchman-path _build/watchman --pybuild-dir {p}"
                        ).format(
                            n=builder.option("make_parallelism"),
                            p=path_join(
                                projects,
                                "../shipit_projects/watchman/_build/python"),
                        )),
                ],
            ),
        ],
    }
Ejemplo n.º 5
0
def fbcode_builder_spec(builder):
    builder.add_option("watchman/_build:cmake_defines", {"BUILD_SHARED_LIBS": "OFF"})

    projects = builder.option("projects_dir")

    return {
        "depends_on": [gmock, folly, fbthrift],
        "steps": [
            builder.fb_github_cmake_install("watchman/_build", ".."),
            builder.step(
                "Run watchman tests",
                [
                    builder.run(
                        ShellQuoted("ctest --output-on-failure -j {n}").format(
                            n=builder.option("make_parallelism")
                        )
                    ),
                    builder.run(
                        ShellQuoted(
                            "cd ../ && ./runtests.py --concurrency {n} "
                            "--watchman-path _build/watchman --pybuild-dir {p}"
                        ).format(
                            n=builder.option("make_parallelism"),
                            p=path_join(
                                projects, "../shipit_projects/watchman/_build/python"
                            ),
                        )
                    ),
                ],
            ),
        ],
    }
Ejemplo n.º 6
0
 def create_python_venv(self):
     action = []
     if self.option("PYTHON_VENV", "OFF") == "ON":
         action = self.run(
             ShellQuoted("python3 -m venv {p}").format(
                 p=path_join(self.option('prefix'), "venv")))
     return (action)
Ejemplo n.º 7
0
    def github_project_workdir(self, project, path):
        # Only check out a non-default branch if requested. This especially
        # makes sense when building from a local repo.
        git_hash = self.option(
            '{0}:git_hash'.format(project),
            # Any repo that has a hash in deps/github_hashes defaults to
            # that, with the goal of making builds maximally consistent.
            self._github_hashes.get(project, '')
        )
        maybe_change_branch = [
            self.run(ShellQuoted('git checkout {hash}').format(hash=git_hash)),
        ] if git_hash else []

        base_dir = self.option('projects_dir')

        local_repo_dir = self.option('{0}:local_repo_dir'.format(project), '')
        return self.step('Check out {0}, workdir {1}'.format(project, path), [
            self.workdir(base_dir),
            self.run(
                ShellQuoted('git clone {opts} https://github.com/{p}').format(
                    p=project,
                    opts=ShellQuoted(self.option('{}:git_clone_opts'.format(project), '')))
            ) if not local_repo_dir else self.copy_local_repo(
                local_repo_dir, os.path.basename(project)
            ),
            self.workdir(path_join(base_dir, os.path.basename(project), path)),
        ] + maybe_change_branch)
Ejemplo n.º 8
0
 def python_venv(self):
     # To both avoid calling venv activate on each RUN command AND to ensure
     # it is present when the resulting container is run add to PATH
     actions = []
     if self.option("PYTHON_VENV", "OFF") == "ON":
         actions = ShellQuoted('ENV PATH={p}:$PATH').format(
             p=path_join(self.option('prefix'), "venv", "bin"))
     return (actions)
 def python_venv(self):
     # To both avoid calling venv activate on each RUN command AND to ensure
     # it is present when the resulting container is run add to PATH
     actions = []
     if self.option("PYTHON_VENV", "OFF") == "ON":
         actions = ShellQuoted('ENV PATH={p}:$PATH').format(
             p=path_join(self.option('prefix'), "venv", "bin"))
     return(actions)
def fbcode_builder_spec(builder):
    builder.add_option("thom311/libnl:git_hash", "libnl3_2_25")
    builder.add_option("openr/build:cmake_defines", {"ADD_ROOT_TESTS": "OFF"})
    maybe_curl_patch = []
    patch = path_join(
        builder.option("projects_dir"),
        "../shipit_projects/openr/build/fix-route-obj-attr-list.patch",
    )

    if not builder.has_option("shipit_project_dir"):
        maybe_curl_patch = [
            builder.run(
                ShellQuoted(
                    "curl -O https://raw.githubusercontent.com/facebook/openr/master/"
                    "build/fix-route-obj-attr-list.patch"))
        ]
        patch = "fix-route-obj-attr-list.patch"
    libnl_build_commands = maybe_curl_patch + [
        builder.run(ShellQuoted("git apply {p}").format(p=patch)),
        builder.run(ShellQuoted("./autogen.sh")),
        builder.configure(),
        builder.make_and_install(),
    ]

    return {
        "depends_on": [folly, fbthrift, fbzmq, re2],
        "steps": [
            builder.github_project_workdir("thom311/libnl", "."),
            builder.step("Build and install thom311/libnl",
                         libnl_build_commands),
            builder.fb_github_project_workdir("openr/build", "facebook"),
            builder.step(
                "Build and install openr/build",
                [
                    builder.cmake_configure("openr/build"),
                    # we need the pythonpath to find the thrift compiler
                    builder.run(
                        ShellQuoted(
                            'PYTHONPATH="$PYTHONPATH:"{p}/lib/python2.7/site-packages '
                            "make -j {n}").format(
                                p=builder.option("prefix"),
                                n=builder.option("make_parallelism"),
                            )),
                    builder.run(ShellQuoted("sudo make install")),
                    builder.run(ShellQuoted("sudo ldconfig")),
                ],
            ),
            builder.step(
                "Run openr tests",
                [
                    builder.run(
                        ShellQuoted("CTEST_OUTPUT_ON_FAILURE=TRUE make test"))
                ],
            ),
        ],
    }
Ejemplo n.º 11
0
    def python_venv(self):
        actions = []
        if self.option("PYTHON_VENV", "OFF") == "ON":
            actions.append(ShellQuoted("source {p}").format(
                p=path_join(self.option('prefix'), "venv", "bin", "activate")))

            actions.append(self.run(
                ShellQuoted("python3 -m pip install {deps}").format(
                    deps=shell_join(' ', (ShellQuoted(dep) for dep in
                        self.python_deps())))))
        return(actions)
Ejemplo n.º 12
0
def fbcode_builder_spec(builder):
    return {
        'depends_on': [fbthrift],
        'steps': [
            builder.step("Install thrift python modules", [
                builder.workdir(
                    path_join(builder.option('projects_dir'),
                              "fbthrift/thrift/lib/py")),
                builder.run(ShellQuoted("sudo python setup.py install")),
            ]),
        ],
    }
Ejemplo n.º 13
0
def fbcode_builder_spec(builder):
    builder.enable_rust_toolchain()
    return {
        "depends_on": [fbthrift],
        "steps": [
            builder.set_env(
                "THRIFT", path_join(builder.option("prefix"), "bin",
                                    "thrift1")),
            builder.fb_github_cargo_build("rust-shed/",
                                          github_org="facebookexperimental"),
        ],
    }
Ejemplo n.º 14
0
def fbcode_builder_spec(builder):
    builder.add_option('thom311/libnl:git_hash', 'libnl3_2_25')
    builder.add_option('openr/openr/build:cmake_defines',
                       {'ADD_ROOT_TESTS': 'OFF'})
    maybe_curl_patch = []
    patch = path_join(
        builder.option('projects_dir'),
        '../shipit_projects/openr/build/fix-route-obj-attr-list.patch')

    if not builder.has_option('shipit_project_dir'):
        maybe_curl_patch = [
            builder.run(
                ShellQuoted(
                    'curl -O https://raw.githubusercontent.com/facebook/openr/master/'
                    'build/fix-route-obj-attr-list.patch'))
        ]
        patch = 'fix-route-obj-attr-list.patch'
    libnl_build_commands = maybe_curl_patch + [
        builder.run(ShellQuoted('git apply {p}').format(p=patch)),
        builder.run(ShellQuoted('./autogen.sh')),
        builder.configure(),
        builder.make_and_install()
    ]

    return {
        'depends_on': [folly, fbthrift, fbzmq, re2],
        'steps': [
            builder.github_project_workdir('thom311/libnl', '.'),
            builder.step('Build and install thom311/libnl',
                         libnl_build_commands),
            builder.fb_github_project_workdir('openr/openr/build', 'facebook'),
            builder.step(
                'Build and install openr/openr/build',
                [
                    builder.cmake_configure('openr/openr/build'),
                    # we need the pythonpath to find the thrift compiler
                    builder.run(
                        ShellQuoted(
                            'PYTHONPATH="$PYTHONPATH:"{p}/lib/python2.7/site-packages '
                            'make -j {n}').format(
                                p=builder.option('prefix'),
                                n=builder.option('make_parallelism'))),
                    builder.run(ShellQuoted('make install')),
                ]),
            builder.step('Run openr tests', [
                builder.run(
                    ShellQuoted('CTEST_OUTPUT_ON_FAILURE=TRUE make test')),
            ]),
        ],
    }
    def _render_impl(self, actions):
        next_workdir = None
        shipit_projects = []
        build_steps = []
        for action in recursively_flatten_list(actions):
            if isinstance(action, LegocastleFBCheckout):
                next_workdir = ShellQuoted('"$(hg root)/"{d}', ).format(
                    d=path_join(self.option('shipit_project_dir'),
                                action.project_and_path), )
                shipit_projects.append(
                    action.project_and_path.split('/', 1)[0])
            elif isinstance(action, LegocastleStep):
                pre_actions = [ShellQuoted('set -ex')]
                if next_workdir is not None:
                    pre_actions.append(self.workdir(next_workdir))

                next_workdir = None
                shell_steps = []
                for a in itertools.chain(pre_actions, action.actions):
                    if isinstance(a, LegocastleWorkdir):
                        # Pass the last working directory to the next step,
                        # since Legocastle doesn't remember it.
                        next_workdir = a.dir
                        shell_steps.append(
                            ShellQuoted(
                                # Don't evaluate {d} twice.
                                '_wd={d} ; mkdir -p "$_wd" && cd "$_wd"').
                            format(d=a.dir))
                    else:
                        shell_steps.append(a)

                build_steps.append({
                    'name':
                    action.name,
                    'shell':
                    raw_shell(shell_join('\n', shell_steps)),
                    'required':
                    True,
                })
            else:
                raise LegocastleStepError(
                    'You have a top-level action {0} that is not a step()'.
                    format(repr(action)))

        return shipit_projects, build_steps
Ejemplo n.º 16
0
    def github_project_workdir(self, project, path):
        # Only check out a non-default branch if requested. This especially
        # makes sense when building from a local repo.
        git_hash = self.option('{0}:git_hash'.format(project), '')
        maybe_change_branch = [
            self.run(ShellQuoted('git checkout {hash}').format(hash=git_hash)),
        ] if git_hash else []

        base_dir = self.option('projects_dir')
        local_repo_dir = self.option('{0}:local_repo_dir'.format(project), '')
        return self.step('Check out {0}, workdir {1}'.format(project, path), [
            self.workdir(base_dir),
            self.run(
                ShellQuoted('git clone https://github.com/{p}').format(
                    p=project)) if not local_repo_dir else
            self.copy_local_repo(local_repo_dir, os.path.basename(project)),
            self.workdir(path_join(base_dir, os.path.basename(project), path)),
        ] + maybe_change_branch)
Ejemplo n.º 17
0
    def github_project_workdir(self, project, path):
        # Only check out a non-default branch if requested. This especially
        # makes sense when building from a local repo.
        git_hash = self.option('{0}:git_hash'.format(project), '')
        maybe_change_branch = [
            self.run(ShellQuoted('git checkout {hash}').format(hash=git_hash)),
        ] if git_hash else []

        base_dir = self.option('projects_dir')
        local_repo_dir = self.option('{0}:local_repo_dir'.format(project), '')
        return self.step('Check out {0}, workdir {1}'.format(project, path), [
            self.workdir(base_dir),
            self.run(
                ShellQuoted('git clone https://github.com/{p}')
                    .format(p=project)
            ) if not local_repo_dir else self.copy_local_repo(
                local_repo_dir, os.path.basename(project)
            ),
            self.workdir(path_join(base_dir, os.path.basename(project), path)),
        ] + maybe_change_branch)
Ejemplo n.º 18
0
 def python_venv(self):
     action = []
     if self.option("PYTHON_VENV", "OFF") == "ON":
         action = ShellQuoted("source {p}").format(
             p=path_join(self.option('prefix'), "venv", "bin", "activate"))
     return (action)
Ejemplo n.º 19
0
 def python_venv(self):
     action = []
     if self.option("PYTHON_VENV", "OFF") == "ON":
         action = ShellQuoted("source {p}").format(
             p=path_join(self.option('prefix'), "venv", "bin", "activate"))
     return(action)
Ejemplo n.º 20
0
 def create_python_venv(self):
     action = []
     if self.option("PYTHON_VENV", "OFF") == "ON":
         action = self.run(ShellQuoted("python3 -m venv {p}").format(
             p=path_join(self.option('prefix'), "venv")))
     return(action)
    def _render_impl(self, actions):
        next_workdir = None
        shipit_projects = []
        build_steps = []
        for action in recursively_flatten_list(actions):
            if isinstance(action, LegocastleFBCheckout):
                next_workdir = ShellQuoted('"$(hg root)/"{d}', ).format(
                    d=path_join(self.option('shipit_project_dir'),
                                action.project_and_path), )
                shipit_projects.append(
                    action.project_and_path.split('/', 1)[0])
            elif isinstance(action, LegocastleStep):
                pre_actions = [ShellQuoted('set -ex')]
                if action.name != 'Setup' and \
                        self.option("PYTHON_VENV", "OFF") == "ON":
                    pre_actions.append(self.python_venv())
                pre_actions.append(
                    ShellQuoted("""
case "$OSTYPE" in
  darwin*)
    BREW_PREFIX=/var/tmp/homebrew

    # The apple-provided flex and bison tools are too old to successfully
    # build thrift.  Ensure that we resolve to the homebrew versions.
    # Note that homebrew doesn't link these into its bin dir to avoid
    # these newer versions taking precedence, so we need to reach into
    # the cellar path.  The glob is to make this script less prone to
    # problems if/when the version is bumped.
    BISON_BIN=$(echo $BREW_PREFIX/Cellar/bison/*/bin)
    FLEX_BIN=$(echo $BREW_PREFIX/Cellar/flex/*/bin)

    export CMAKE_SYSTEM_PREFIX_PATH=$BREW_PREFIX
    export PKG_CONFIG_PATH=$BREW_PREFIX/opt/openssl/lib/pkgconfig
    export PATH=$BISON_BIN:$FLEX_BIN:$BREW_PREFIX/bin:$PATH
    export HOMEBREW_NO_AUTO_UPDATE=1
    export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/
  ;;
esac
"""))
                if next_workdir is not None:
                    pre_actions.append(self.workdir(next_workdir))

                next_workdir = None
                shell_steps = []
                for a in itertools.chain(pre_actions, action.actions):
                    if isinstance(a, LegocastleWorkdir):
                        # Pass the last working directory to the next step,
                        # since Legocastle doesn't remember it.
                        next_workdir = a.dir
                        shell_steps.append(
                            ShellQuoted(
                                # Don't evaluate {d} twice.
                                '_wd={d} ; mkdir -p "$_wd" && cd "$_wd"').
                            format(d=a.dir))
                    else:
                        shell_steps.append(a)

                build_steps.append({
                    'name':
                    action.name,
                    'shell':
                    raw_shell(shell_join('\n', shell_steps)),
                    'required':
                    True,
                })
            else:
                raise LegocastleStepError(
                    'You have a top-level action {0} that is not a step()'.
                    format(repr(action)))

        return shipit_projects, build_steps
Ejemplo n.º 22
0
    def _render_impl(self, actions):
        # This exists to transfer the current working directory to the next
        # step, since Legocastle doesn't remember it.  Although this is
        # potentially quadratic, we have to replay ALL workdir calls from
        # the previous step because these might contain variable expansions,
        # making it impossible to distinguish relative directories from
        # absolute. Consider this step:
        #     [
        #         builder.fb_github_project_workdir('foo/bar')
        #         ...
        #         builder.workdir('baz')  # Now should be at foo/bar/baz
        #     ]
        #
        # If we just replayed the last workdir('baz'), we would not end up
        # in '<github_prefix>/foo/bar/baz', but rather in `<unknown>/baz`.
        next_step_workdirs = []
        shipit_projects = []
        build_steps = []
        for action in recursively_flatten_list(actions):
            if isinstance(action, LegocastleFBCheckout):
                # This is guaranteed to be absolute, so drop the unnecessary
                # history.
                next_step_workdirs = [
                    ShellQuoted('"$(hg root)/"{d}', ).format(d=path_join(
                        self.option('shipit_project_dir'),
                        action.project_and_path), )
                ]
                shipit_projects.append(
                    action.project_and_path.split('/', 1)[0])
            elif isinstance(action, LegocastleStep):
                pre_actions = [ShellQuoted('set -ex')]
                if action.name != 'Setup' and \
                        self.option("PYTHON_VENV", "OFF") == "ON":
                    pre_actions.extend(self.python_venv())
                pre_actions.append(
                    ShellQuoted("""
case "$OSTYPE" in
  darwin*)
    BREW_PREFIX=/var/tmp/homebrew

    # The apple-provided flex and bison tools are too old to successfully
    # build thrift.  Ensure that we resolve to the homebrew versions.
    # Note that homebrew doesn't link these into its bin dir to avoid
    # these newer versions taking precedence, so we need to reach into
    # the cellar path.  The glob is to make this script less prone to
    # problems if/when the version is bumped.
    BISON_BIN=$(echo $BREW_PREFIX/Cellar/bison/*/bin)
    FLEX_BIN=$(echo $BREW_PREFIX/Cellar/flex/*/bin)

    export CMAKE_SYSTEM_PREFIX_PATH=$BREW_PREFIX
    export PKG_CONFIG_PATH=$BREW_PREFIX/opt/openssl/lib/pkgconfig
    export PATH=$BISON_BIN:$FLEX_BIN:$BREW_PREFIX/bin:$PATH
    export HOMEBREW_NO_AUTO_UPDATE=1
    export SDKROOT=/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/
  ;;
esac
"""))
                pre_actions.extend(self.workdir(w) for w in next_step_workdirs)

                shell_steps = []
                for a in itertools.chain(pre_actions, action.actions):
                    if isinstance(a, LegocastleWorkdir):
                        next_step_workdirs.append(a.dir)
                        shell_steps.append(
                            ShellQuoted(
                                # Don't evaluate {d} twice.
                                '_wd={d} ; mkdir -p "$_wd" && cd "$_wd"').
                            format(d=a.dir))
                    else:
                        shell_steps.append(a)

                build_steps.append({
                    'name':
                    action.name,
                    'shell':
                    raw_shell(shell_join('\n', shell_steps)),
                    'required':
                    True,
                })
            else:
                raise LegocastleStepError(
                    'You have a top-level action {0} that is not a step()'.
                    format(repr(action)))

        return shipit_projects, build_steps