def main() -> None:
"""Main program.
Exits zero on successful program termination, non-zero otherwise.
"""
configure_reproducible_wheels()
parser = argparse.ArgumentParser(
description="Resolve and fetch artifacts transitively from PyPI")
parser.add_argument(
"--requirements",
action="store",
required=True,
help="Path to requirements.txt from where to install dependencies",
)
arguments.parse_common_args(parser)
args = parser.parse_args()
deserialized_args = dict(vars(args))
arguments.deserialize_structured_args(deserialized_args)
# Pip is run with the working directory changed to the folder containing the requirements.txt file, to allow for
# relative requirements to be correctly resolved. The --wheel-dir is therefore required to be repointed back to the
# current calling working directory (the repo root in .../external/name), where the wheel files should be written to
pip_args = ([sys.executable, "-m", "pip"] +
(["--isolated"] if args.isolated else []) +
["wheel", "-r", args.requirements] +
["--wheel-dir", os.getcwd()] +
deserialized_args["extra_pip_args"])
env = os.environ.copy()
env.update(deserialized_args["environment"])
# Assumes any errors are logged by pip so do nothing. This command will fail if pip fails
subprocess.run(pip_args,
check=True,
env=env,
cwd=str(pathlib.Path(args.requirements).parent.resolve()))
extras = requirements.parse_extras(args.requirements)
repo_label = "@%s" % args.repo
targets = [
'"%s%s"' % (
repo_label,
bazel.extract_wheel(whl, extras,
deserialized_args["pip_data_exclude"],
args.enable_implicit_namespace_pkgs),
) for whl in glob.glob("*.whl")
]
with open("requirements.bzl", "w") as requirement_file:
requirement_file.write(
bazel.generate_requirements_file_contents(repo_label, targets))
def main() -> None:
parser = argparse.ArgumentParser(
description=
"Build and/or fetch a single wheel based on the requirement passed in")
parser.add_argument(
"--requirement",
action="store",
required=True,
help="A single PEP508 requirement specifier string.",
)
arguments.parse_common_args(parser)
args = parser.parse_args()
deserialized_args = dict(vars(args))
arguments.deserialize_structured_args(deserialized_args)
configure_reproducible_wheels()
pip_args = ([sys.executable, "-m", "pip"] +
(["--isolated"] if args.isolated else []) +
["wheel", "--no-deps"] + deserialized_args["extra_pip_args"])
requirement_file = NamedTemporaryFile(mode='wb', delete=False)
try:
requirement_file.write(args.requirement.encode("utf-8"))
requirement_file.flush()
# Close the file so pip is allowed to read it when running on Windows.
# For more information, see: https://bugs.python.org/issue14243
requirement_file.close()
# Requirement specific args like --hash can only be passed in a requirements file,
# so write our single requirement into a temp file in case it has any of those flags.
pip_args.extend(["-r", requirement_file.name])
env = os.environ.copy()
env.update(deserialized_args["environment"])
# Assumes any errors are logged by pip so do nothing. This command will fail if pip fails
subprocess.run(pip_args, check=True, env=env)
finally:
try:
os.unlink(requirement_file.name)
except OSError as e:
if e.errno != errno.ENOENT:
raise
name, extras_for_pkg = requirements._parse_requirement_for_extra(
args.requirement)
extras = {name: extras_for_pkg} if extras_for_pkg and name else dict()
whl = next(iter(glob.glob("*.whl")))
bazel.extract_wheel(whl,
extras,
deserialized_args["pip_data_exclude"],
args.enable_implicit_namespace_pkgs,
incremental=True,
incremental_repo_prefix=bazel.whl_library_repo_prefix(
args.repo))
def test_deserialize_structured_args(self) -> None:
serialized_args = {
"pip_data_exclude": json.dumps({"arg": ["**.foo"]}),
"environment": json.dumps({"arg": {"PIP_DO_SOMETHING": "True"}}),
}
args = arguments.deserialize_structured_args(serialized_args)
self.assertEqual(args["pip_data_exclude"], ["**.foo"])
self.assertEqual(args["environment"], {"PIP_DO_SOMETHING": "True"})
self.assertEqual(args["extra_pip_args"], [])
def test_arguments(self) -> None:
parser = argparse.ArgumentParser()
parser = arguments.parse_common_args(parser)
repo_name = "foo"
index_url = "--index_url=pypi.org/simple"
extra_pip_args = [index_url]
args_dict = vars(parser.parse_args(
args=["--repo", repo_name, f"--extra_pip_args={json.dumps({'arg': extra_pip_args})}"]))
args_dict = arguments.deserialize_structured_args(args_dict)
self.assertIn("repo", args_dict)
self.assertIn("extra_pip_args", args_dict)
self.assertEqual(args_dict["pip_data_exclude"], [])
self.assertEqual(args_dict["enable_implicit_namespace_pkgs"], False)
self.assertEqual(args_dict["repo"], repo_name)
self.assertEqual(args_dict["extra_pip_args"], extra_pip_args)
def generate_parsed_requirements_contents(all_args: argparse.Namespace) -> str:
"""
Parse each requirement from the requirements_lock file, and prepare arguments for each
repository rule, which will represent the individual requirements.
Generates a requirements.bzl file containing a macro (install_deps()) which instantiates
a repository rule for each requirment in the lock file.
"""
args = dict(vars(all_args))
args = arguments.deserialize_structured_args(args)
args.setdefault("python_interpreter", sys.executable)
# Pop this off because it wont be used as a config argument to the whl_library rule.
requirements_lock = args.pop("requirements_lock")
repo_prefix = bazel.whl_library_repo_prefix(args["repo"])
install_req_and_lines = parse_install_requirements(requirements_lock, args["extra_pip_args"])
repo_names_and_reqs = repo_names_and_requirements(install_req_and_lines, repo_prefix)
all_requirements = ", ".join(
[bazel.sanitised_repo_library_label(ir.name, repo_prefix=repo_prefix) for ir, _ in install_req_and_lines]
)
all_whl_requirements = ", ".join(
[bazel.sanitised_repo_file_label(ir.name, repo_prefix=repo_prefix) for ir, _ in install_req_and_lines]
)
return textwrap.dedent("""\
load("@rules_python//python/pip_install:pip_repository.bzl", "whl_library")
all_requirements = [{all_requirements}]
all_whl_requirements = [{all_whl_requirements}]
_packages = {repo_names_and_reqs}
_config = {args}
def _clean_name(name):
return name.replace("-", "_").replace(".", "_").lower()
def requirement(name):
return "@{repo_prefix}" + _clean_name(name) + "//:{py_library_label}"
def whl_requirement(name):
return "@{repo_prefix}" + _clean_name(name) + "//:{wheel_file_label}"
def data_requirement(name):
return "@{repo_prefix}" + _clean_name(name) + "//:{data_label}"
def dist_info_requirement(name):
return "@{repo_prefix}" + _clean_name(name) + "//:{dist_info_label}"
def entry_point(pkg, script = None):
if not script:
script = pkg
return "@{repo_prefix}" + _clean_name(pkg) + "//:{entry_point_prefix}_" + script
def install_deps():
for name, requirement in _packages:
whl_library(
name = name,
requirement = requirement,
**_config,
)
""".format(
all_requirements=all_requirements,
all_whl_requirements=all_whl_requirements,
repo_names_and_reqs=repo_names_and_reqs,
args=args,
repo_prefix=repo_prefix,
py_library_label=bazel.PY_LIBRARY_LABEL,
wheel_file_label=bazel.WHEEL_FILE_LABEL,
data_label=bazel.DATA_LABEL,
dist_info_label=bazel.DIST_INFO_LABEL,
entry_point_prefix=bazel.WHEEL_ENTRY_POINT_PREFIX,
)
)