def run(self, args): opts = setup_build_options(args) loader = ManifestLoader(opts) manifest = loader.load_manifest(args.project) if args.recursive: projects = loader.manifests_in_dependency_order() else: projects = [manifest] for m in projects: fetcher = loader.create_fetcher(m) fetcher.update()
def run(self, args): opts = setup_build_options(args) loader = ManifestLoader(opts) loader.ctx_gen.set_value_for_project(args.project, "test", "on") manifest = loader.load_manifest(args.project) if args.recursive: manifests = loader.manifests_in_dependency_order() else: manifests = [manifest] for m in manifests: fetcher = loader.create_fetcher(m) print(fetcher.get_src_dir())
def run(self, args): opts = setup_build_options(args) loader = ManifestLoader(opts) loader.ctx_gen.set_value_for_project(args.project, "test", "on") manifest = loader.load_manifest(args.project) projects = loader.manifests_in_dependency_order() if args.recursive: manifests = projects else: manifests = [manifest] for m in manifests: inst_dir = loader.get_project_install_dir(m) print(inst_dir)
def run(self, args): opts = setup_build_options(args) ctx_gen = opts.get_context_generator(facebook_internal=args.facebook_internal) if args.enable_tests: ctx_gen.set_value_for_project(args.project, "test", "on") loader = ManifestLoader(opts, ctx_gen) manifest = loader.load_manifest(args.project) projects = loader.manifests_in_dependency_order() # Accumulate the install directories so that the build steps # can find their dep installation install_dirs = [] for m in projects: inst_dir = loader.get_project_install_dir(m) install_dirs.append(inst_dir) if m == manifest: dep_munger = create_dyn_dep_munger(opts, install_dirs) dep_munger.process_deps(args.destdir, args.final_install_prefix)
def write_job_for_platform(self, platform, args): build_opts = setup_build_options(args, platform) ctx_gen = build_opts.get_context_generator() loader = ManifestLoader(build_opts, ctx_gen) manifest = loader.load_manifest(args.project) manifest_ctx = loader.ctx_gen.get_context(manifest.name) # Some projects don't do anything "useful" as a leaf project, only # as a dep for a leaf project. Check for those here; we don't want # to waste the effort scheduling them on CI. # We do this by looking at the builder type in the manifest file # rather than creating a builder and checking its type because we # don't know enough to create the full builder instance here. if manifest.get("build", "builder", ctx=manifest_ctx) == "nop": return None # We want to be sure that we're running things with python 3 # but python versioning is honestly a bit of a frustrating mess. # `python` may be version 2 or version 3 depending on the system. # python3 may not be a thing at all! # Assume an optimistic default py3 = "python3" if build_opts.is_linux(): job_name = "linux" runs_on = "ubuntu-18.04" elif build_opts.is_windows(): # We're targeting the windows-2016 image because it has # Visual Studio 2017 installed, and at the time of writing, # the version of boost in the manifests (1.69) is not # buildable with Visual Studio 2019 job_name = "windows" runs_on = "windows-2016" # The windows runners are python 3 by default; python2.exe # is available if needed. py3 = "python" else: job_name = "mac" runs_on = "macOS-latest" os.makedirs(args.output_dir, exist_ok=True) output_file = os.path.join(args.output_dir, f"getdeps_{job_name}.yml") with open(output_file, "w") as out: # Deliberate line break here because the @ and the generated # symbols are meaningful to our internal tooling when they # appear in a single token out.write("# This file was @") out.write("generated by getdeps.py\n") out.write(f""" name: {job_name} on: push: branches: - master pull_request: branches: - master jobs: """) getdeps = f"{py3} build/fbcode_builder/getdeps.py --allow-system-packages" out.write(" build:\n") out.write(" runs-on: %s\n" % runs_on) out.write(" steps:\n") out.write(" - uses: actions/checkout@v1\n") if build_opts.is_windows(): # cmake relies on BOOST_ROOT but GH deliberately don't set it in order # to avoid versioning issues: # https://github.com/actions/virtual-environments/issues/319 # Instead, set the version we think we need; this is effectively # coupled with the boost manifest # This is the unusual syntax for setting an env var for the rest of # the steps in a workflow: # https://help.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable out.write(" - name: Export boost environment\n") out.write( ' run: "echo ::set-env name=BOOST_ROOT::%BOOST_ROOT_1_69_0%"\n' ) out.write(" shell: cmd\n") # The git installation may not like long filenames, so tell it # that we want it to use them! out.write(" - name: Fix Git config\n") out.write( " run: git config --system core.longpaths true\n") else: out.write(" - name: Install system deps\n") out.write( f" run: sudo {getdeps} install-system-deps --recursive {manifest.name}\n" ) projects = loader.manifests_in_dependency_order() for m in projects: if m != manifest: out.write(" - name: Fetch %s\n" % m.name) out.write( f" run: {getdeps} fetch --no-tests {m.name}\n") for m in projects: if m != manifest: out.write(" - name: Build %s\n" % m.name) out.write( f" run: {getdeps} build --no-tests {m.name}\n") out.write(" - name: Build %s\n" % manifest.name) project_prefix = "" if not build_opts.is_windows(): project_prefix = (" --project-install-prefix %s:/usr/local" % manifest.name) out.write( f" run: {getdeps} build --src-dir=. {manifest.name} {project_prefix}\n" ) out.write(" - name: Copy artifacts\n") if build_opts.is_linux(): # Strip debug info from the binaries, but only on linux. # While the `strip` utility is also available on macOS, # attempting to strip there results in an error. # The `strip` utility is not available on Windows. strip = " --strip" else: strip = "" out.write( f" run: {getdeps} fixup-dyn-deps{strip} " f"--src-dir=. {manifest.name} _artifacts/{job_name} {project_prefix} " f"--final-install-prefix /usr/local\n") out.write(" - uses: actions/upload-artifact@master\n") out.write(" with:\n") out.write(" name: %s\n" % manifest.name) out.write(" path: _artifacts\n") out.write(" - name: Test %s\n" % manifest.name) out.write( f" run: {getdeps} test --src-dir=. {manifest.name} {project_prefix}\n" )
def write_job_for_platform(self, platform, args): # noqa: C901 build_opts = setup_build_options(args, platform) ctx_gen = build_opts.get_context_generator() loader = ManifestLoader(build_opts, ctx_gen) manifest = loader.load_manifest(args.project) manifest_ctx = loader.ctx_gen.get_context(manifest.name) run_on = self.get_run_on(args) # Some projects don't do anything "useful" as a leaf project, only # as a dep for a leaf project. Check for those here; we don't want # to waste the effort scheduling them on CI. # We do this by looking at the builder type in the manifest file # rather than creating a builder and checking its type because we # don't know enough to create the full builder instance here. if manifest.get("build", "builder", ctx=manifest_ctx) == "nop": return None # We want to be sure that we're running things with python 3 # but python versioning is honestly a bit of a frustrating mess. # `python` may be version 2 or version 3 depending on the system. # python3 may not be a thing at all! # Assume an optimistic default py3 = "python3" if build_opts.is_linux(): artifacts = "linux" runs_on = f"ubuntu-{args.ubuntu_version}" elif build_opts.is_windows(): artifacts = "windows" runs_on = "windows-2019" # The windows runners are python 3 by default; python2.exe # is available if needed. py3 = "python" else: artifacts = "mac" runs_on = "macOS-latest" os.makedirs(args.output_dir, exist_ok=True) job_file_prefix = "getdeps_" if args.job_file_prefix: job_file_prefix = args.job_file_prefix output_file = os.path.join(args.output_dir, f"{job_file_prefix}{artifacts}.yml") if args.job_name_prefix: job_name = args.job_name_prefix + artifacts.capitalize() else: job_name = artifacts with open(output_file, "w") as out: # Deliberate line break here because the @ and the generated # symbols are meaningful to our internal tooling when they # appear in a single token out.write("# This file was @") out.write("generated by getdeps.py\n") out.write(f""" name: {job_name} on:{run_on} jobs: """) getdepscmd = f"{py3} build/fbcode_builder/getdeps.py" out.write(" build:\n") out.write(" runs-on: %s\n" % runs_on) out.write(" steps:\n") out.write(" - uses: actions/checkout@v2\n") if build_opts.is_windows(): # cmake relies on BOOST_ROOT but GH deliberately don't set it in order # to avoid versioning issues: # https://github.com/actions/virtual-environments/issues/319 # Instead, set the version we think we need; this is effectively # coupled with the boost manifest # This is the unusual syntax for setting an env var for the rest of # the steps in a workflow: # https://github.blog/changelog/2020-10-01-github-actions-deprecating-set-env-and-add-path-commands/ out.write(" - name: Export boost environment\n") out.write( ' run: "echo BOOST_ROOT=%BOOST_ROOT_1_78_0% >> %GITHUB_ENV%"\n' ) out.write(" shell: cmd\n") # The git installation may not like long filenames, so tell it # that we want it to use them! out.write(" - name: Fix Git config\n") out.write( " run: git config --system core.longpaths true\n") allow_sys_arg = "" if (build_opts.allow_system_packages and build_opts.host_type.get_package_manager()): sudo_arg = "sudo " allow_sys_arg = " --allow-system-packages" if build_opts.host_type.get_package_manager() == "deb": out.write(" - name: Update system package info\n") out.write(f" run: {sudo_arg}apt-get update\n") out.write(" - name: Install system deps\n") if build_opts.is_darwin(): # brew is installed as regular user sudo_arg = "" out.write( f" run: {sudo_arg}python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive {manifest.name}\n" ) projects = loader.manifests_in_dependency_order() main_repo_url = manifest.get_repo_url(manifest_ctx) has_same_repo_dep = False for m in projects: if m != manifest: if m.name == "rust": out.write(" - name: Install Rust Stable\n") out.write(" uses: actions-rs/toolchain@v1\n") out.write(" with:\n") out.write(" toolchain: stable\n") out.write(" default: true\n") out.write(" profile: minimal\n") else: ctx = loader.ctx_gen.get_context(m.name) if m.get_repo_url(ctx) != main_repo_url: out.write(" - name: Fetch %s\n" % m.name) out.write( f" run: {getdepscmd}{allow_sys_arg} fetch --no-tests {m.name}\n" ) for m in projects: if m != manifest: if m.name == "rust": continue else: src_dir_arg = "" ctx = loader.ctx_gen.get_context(m.name) if main_repo_url and m.get_repo_url( ctx) == main_repo_url: # Its in the same repo, so src-dir is also . src_dir_arg = "--src-dir=. " has_same_repo_dep = True out.write(" - name: Build %s\n" % m.name) out.write( f" run: {getdepscmd}{allow_sys_arg} build {src_dir_arg}--no-tests {m.name}\n" ) out.write(" - name: Build %s\n" % manifest.name) project_prefix = "" if not build_opts.is_windows(): project_prefix = (" --project-install-prefix %s:/usr/local" % manifest.name) # If we have dep from same repo, we already built it and don't want to rebuild it again no_deps_arg = "" if has_same_repo_dep: no_deps_arg = "--no-deps " out.write( f" run: {getdepscmd}{allow_sys_arg} build {no_deps_arg}--src-dir=. {manifest.name} {project_prefix}\n" ) out.write(" - name: Copy artifacts\n") if build_opts.is_linux(): # Strip debug info from the binaries, but only on linux. # While the `strip` utility is also available on macOS, # attempting to strip there results in an error. # The `strip` utility is not available on Windows. strip = " --strip" else: strip = "" out.write( f" run: {getdepscmd}{allow_sys_arg} fixup-dyn-deps{strip} " f"--src-dir=. {manifest.name} _artifacts/{artifacts} {project_prefix} " f"--final-install-prefix /usr/local\n") out.write(" - uses: actions/upload-artifact@v2\n") out.write(" with:\n") out.write(" name: %s\n" % manifest.name) out.write(" path: _artifacts\n") if manifest.get("github.actions", "run_tests", ctx=manifest_ctx) != "off": out.write(" - name: Test %s\n" % manifest.name) out.write( f" run: {getdepscmd}{allow_sys_arg} test --src-dir=. {manifest.name} {project_prefix}\n" )
def write_job_for_platform(self, out, args, build_opts): ctx_gen = build_opts.get_context_generator() loader = ManifestLoader(build_opts, ctx_gen) manifest = loader.load_manifest(args.project) manifest_ctx = loader.ctx_gen.get_context(manifest.name) # Some projects don't do anything "useful" as a leaf project, only # as a dep for a leaf project. Check for those here; we don't want # to waste the effort scheduling them on CI. # We do this by looking at the builder type in the manifest file # rather than creating a builder and checking its type because we # don't know enough to create the full builder instance here. if manifest.get("build", "builder", ctx=manifest_ctx) == "nop": return None if build_opts.is_linux(): job_name = "linux" runs_on = "ubuntu-18.04" elif build_opts.is_windows(): # We're targeting the windows-2016 image because it has # Visual Studio 2017 installed, and at the time of writing, # the version of boost in the manifests (1.69) is not # buildable with Visual Studio 2019 job_name = "windows" runs_on = "windows-2016" else: job_name = "mac" runs_on = "macOS-latest" out.write(" %s:\n" % job_name) out.write(" runs-on: %s\n" % runs_on) out.write(" steps:\n") out.write(" - uses: actions/checkout@v1\n") projects = loader.manifests_in_dependency_order() for m in projects: if m != manifest: out.write(" - name: Fetch %s\n" % m.name) out.write( " run: python build/fbcode_builder/getdeps.py fetch " "--no-tests %s\n" % m.name) for m in projects: if m != manifest: out.write(" - name: Build %s\n" % m.name) out.write( " run: python build/fbcode_builder/getdeps.py build " "--no-tests %s\n" % m.name) out.write(" - name: Build %s\n" % manifest.name) out.write( " run: python build/fbcode_builder/getdeps.py build --src-dir=. %s\n" % manifest.name) out.write(" - name: Copy artifacts\n") out.write( " run: python build/fbcode_builder/getdeps.py fixup-dyn-deps " "--src-dir=. %s _artifacts/%s\n" % (manifest.name, job_name)) out.write(" - uses: actions/upload-artifact@master\n") out.write(" with:\n") out.write(" name: %s\n" % manifest.name) out.write(" path: _artifacts\n") out.write(" - name: Test %s\n" % manifest.name) out.write( " run: python build/fbcode_builder/getdeps.py test --src-dir=. %s\n" % manifest.name)
def write_job_for_platform(self, out, args, build_opts): ctx_gen = build_opts.get_context_generator() loader = ManifestLoader(build_opts, ctx_gen) manifest = loader.load_manifest(args.project) manifest_ctx = loader.ctx_gen.get_context(manifest.name) # Some projects don't do anything "useful" as a leaf project, only # as a dep for a leaf project. Check for those here; we don't want # to waste the effort scheduling them on CI. # We do this by looking at the builder type in the manifest file # rather than creating a builder and checking its type because we # don't know enough to create the full builder instance here. if manifest.get("build", "builder", ctx=manifest_ctx) == "nop": return None # We want to be sure that we're running things with python 3 # but python versioning is honestly a bit of a frustrating mess. # `python` may be version 2 or version 3 depending on the system. # python3 may not be a thing at all! # Assume an optimistic default py3 = "python3" if build_opts.is_linux(): job_name = "linux" runs_on = "ubuntu-18.04" elif build_opts.is_windows(): # We're targeting the windows-2016 image because it has # Visual Studio 2017 installed, and at the time of writing, # the version of boost in the manifests (1.69) is not # buildable with Visual Studio 2019 job_name = "windows" runs_on = "windows-2016" # The windows runners are python 3 by default; python2.exe # is available if needed. py3 = "python" else: job_name = "mac" runs_on = "macOS-latest" getdeps = f"{py3} build/fbcode_builder/getdeps.py --allow-system-packages" out.write(" %s:\n" % job_name) out.write(" runs-on: %s\n" % runs_on) out.write(" steps:\n") out.write(" - uses: actions/checkout@v1\n") if build_opts.is_windows(): # The git installation may not like long filenames, so tell it # that we want it to use them! out.write(" - name: Fix Git config\n") out.write(" run: git config --system core.longpaths true\n") else: out.write(" - name: Install system deps\n") out.write( f" run: sudo {getdeps} install-system-deps --recursive {manifest.name}\n" ) projects = loader.manifests_in_dependency_order() for m in projects: if m != manifest: out.write(" - name: Fetch %s\n" % m.name) out.write(f" run: {getdeps} fetch --no-tests {m.name}\n") for m in projects: if m != manifest: out.write(" - name: Build %s\n" % m.name) out.write(f" run: {getdeps} build --no-tests {m.name}\n") out.write(" - name: Build %s\n" % manifest.name) project_prefix = "" if not build_opts.is_windows(): project_prefix = " --project-install-prefix %s:/usr/local" % manifest.name out.write( f" run: {getdeps} build --src-dir=. {manifest.name} {project_prefix}\n" ) out.write(" - name: Copy artifacts\n") out.write( f" run: {getdeps} fixup-dyn-deps " f"--src-dir=. {manifest.name} _artifacts/{job_name} --final-install-prefix /usr/local\n" ) out.write(" - uses: actions/upload-artifact@master\n") out.write(" with:\n") out.write(" name: %s\n" % manifest.name) out.write(" path: _artifacts\n") out.write(" - name: Test %s\n" % manifest.name) out.write( f" run: {getdeps} test --src-dir=. {manifest.name} {project_prefix}\n" )