def main():
args = parser.parse_args()
# https://github.com/pypa/pip/blob/9.0.1/pip/__init__.py#L209
if pip_main(["wheel", "-w", args.directory, "-r", args.input]):
sys.exit(1)
# Enumerate the .whl files we downloaded.
def list_whls():
dir = args.directory + '/'
for root, unused_dirnames, filenames in os.walk(dir):
for fname in filenames:
if fname.endswith('.whl'):
yield os.path.join(root, fname)
def whl_library(wheel):
# Indentation here matters. whl_library must be within the scope
# of the function below. We also avoid reimporting an existing WHL.
return """
if "{repo_name}" not in native.existing_rules():
whl_library(
name = "{repo_name}",
whl = "@{name}//:{path}",
requirements = "@{name}//:requirements.bzl",
)""".format(name=args.name,
repo_name=wheel.repository_name(),
path=wheel.basename())
whls = [Wheel(path) for path in list_whls()]
with open(args.output, 'w') as f:
f.write("""\
# Install pip requirements.
#
# Generated from {input}
load("@io_bazel_rules_python//python:whl.bzl", "whl_library")
def pip_install():
{whl_libraries}
_packages = {{
{mappings}
}}
all_packages = _packages.values()
def package(name):
name = name.replace("-", "_")
return _packages[name]
""".format(input=args.input,
whl_libraries='\n'.join(map(whl_library, whls)),
mappings=','.join([
'"%s": "@%s//:pkg"' %
(wheel.distribution(), wheel.repository_name())
for wheel in whls
])))
def main():
args = _parse_args()
# https://github.com/pypa/pip/blob/9.0.1/pip/__init__.py#L209
if _pip_main(["wheel", "-w", args.directory, "-r", args.input]):
sys.exit(1)
# Enumerate the .whl files we downloaded.
def list_whl_files():
dir_ = args.directory + '/'
for root, unused_dirnames, filenames in os.walk(dir_):
for fname in filenames:
if fname.endswith('.whl'):
yield os.path.join(root, fname)
wheels = [Wheel(path) for path in list_whl_files()]
bzl_file_content = _make_bzl_file_content(
wheels=wheels,
reqs_repo_name=args.name,
input_requirements_file_path=args.input)
with open(args.output, 'w') as file_obj:
file_obj.write(bzl_file_content)
def main():
args = parser.parse_args()
# https://github.com/pypa/pip/blob/9.0.1/pip/__init__.py#L209
if pip_main(["wheel", "-w", args.directory, "-r", args.input]):
sys.exit(1)
# Enumerate the .whl files we downloaded.
def list_whls():
dir = args.directory + '/'
for root, unused_dirnames, filenames in os.walk(dir):
for fname in filenames:
if fname.endswith('.whl'):
yield os.path.join(root, fname)
whls = [Wheel(path) for path in list_whls()]
possible_extras = determine_possible_extras(whls)
def whl_library(wheel):
# Indentation here matters. whl_library must be within the scope
# of the function below. We also avoid reimporting an existing WHL.
return """
if "{repo_name}" not in native.existing_rules():
whl_library(
name = "{repo_name}",
python_interpreter = "{python_interpreter}",
whl = "@{name}//:{path}",
requirements = "@{name}//:requirements.bzl",
extras = [{extras}]
)""".format(name=args.name,
repo_name=wheel.repository_name(),
python_interpreter=args.python_interpreter,
path=wheel.basename(),
extras=','.join([
'"%s"' % extra for extra in possible_extras.get(wheel, [])
]))
whl_targets = ','.join([
','.join([
'"%s": "@%s//:pkg"' %
(whl.distribution().lower(), whl.repository_name())
] + [
# For every extra that is possible from this requirements.txt
'"%s[%s]": "@%s//:%s"' %
(whl.distribution().lower(), extra.lower(), whl.repository_name(),
extra) for extra in possible_extras.get(whl, [])
]) for whl in whls
])
with open(args.output, 'w') as f:
f.write("""\
# Install pip requirements.
#
# Generated from {input}
load("@io_bazel_rules_python//python:whl.bzl", "whl_library")
def pip_install():
{whl_libraries}
_requirements = {{
{mappings}
}}
all_requirements = _requirements.values()
def requirement(name):
name_key = name.replace("-", "_").lower()
if name_key not in _requirements:
fail("Could not find pip-provided dependency: '%s'" % name)
return _requirements[name_key]
""".format(input=args.input,
whl_libraries='\n'.join(map(whl_library, whls)) if whls else "pass",
mappings=whl_targets))
def list_whls(dir):
for root, _, filenames in os.walk(dir + "/"):
for fname in filenames:
if fname.endswith('.whl'):
yield Wheel(os.path.join(root, fname))
def genbuild(args):
whl = Wheel(args.whl)
extra_deps = args.add_dependency or []
drop_deps = {d: None for d in args.drop_dependency or []}
external_deps = [d for d in itertools.chain(whl.dependencies(), extra_deps) if d not in drop_deps]
contents = []
add_build_content = args.add_build_content or []
for name in add_build_content:
with open(name) as f:
contents.append(f.read() + '\n')
contents = '\n'.join(contents)
parser = whl.entrypoints()
entrypoints_build = ''
if parser:
if parser.has_section('console_scripts'):
for name, location in parser.items('console_scripts'):
# Assumes it doesn't depend on extras. TODO(conrado): fix
entrypoint_file = 'entrypoint_%s.py' % name
with open(os.path.join(args.directory, entrypoint_file), 'w') as f:
f.write("""from %s import %s as main; main()""" % tuple(location.split(":")))
attrs = []
attrs += [("name", '"entrypoint_%s"' % name)]
attrs += [("srcs", '["%s"]' % entrypoint_file)]
attrs += [("main", '"%s"' % entrypoint_file)]
if args.python_version:
attrs += [("python_version", '"%s"' % args.python_version)]
attrs += [("deps", '[":pkg"]')]
entrypoints_build += """
py_binary(
{attrs}
)
""".format(attrs=",\n ".join(['{} = {}'.format(k, v) for k, v in attrs]))
attrs = []
if args.patches:
attrs += [("patches", '["%s"]' % '", "'.join(args.patches))]
if args.patch_tool:
attrs += [("patch_tool", '"%s"' % args.patch_tool)]
if args.patch_args:
attrs += [("patch_args", '["%s"]' % '", "'.join(args.patch_args))]
if args.patch_cmds:
attrs += [("patch_cmds", '["%s"]' % '", "'.join(args.patch_cmds))]
attrs += [("distribution", '"%s"' % whl.distribution().lower())]
attrs += [("version", '"%s"' % whl.version())]
attrs += [("wheel_size", "%s" % os.path.getsize(args.whl))]
if args.python_version:
attrs += [("python_version", '"%s"' % args.python_version)]
with open(os.path.join(args.directory, 'BUILD'), 'w') as f:
f.write("""
package(default_visibility = ["//visibility:public"])
load("@io_bazel_rules_python//python:python.bzl", "extract_wheel")
load("@{repository}//:requirements.bzl", "requirement")
filegroup(
name = "wheel",
data = ["{wheel}"],
)
extract_wheel(
name = "extracted",
wheel = "{wheel}",
{attrs}
)
py_library(
name = "pkg",
imports = ["extracted"],
deps = [
":extracted",{dependencies}
],
)
{extras}
{entrypoints_build}
{contents}""".format(
wheel = whl.basename(),
repository=args.repository,
dependencies=''.join([
('\n "%s",' % d) if d[0] == "@" else ('\n requirement("%s"),' % d)
for d in sorted(external_deps)
]),
attrs=",\n ".join(['{} = {}'.format(k, v) for k, v in attrs]),
extras='\n\n'.join([
"""py_library(
name = "{extra}",
deps = [
":pkg",{deps}
],
)""".format(extra=extra,
deps=','.join([
'requirement("%s")' % dep
for dep in sorted(whl.dependencies(extra))
]))
for extra in args.extras or []
]),
entrypoints_build=entrypoints_build,
contents=contents))
def extract(args): whl = Wheel(args.whl) whl.expand(args.directory)
def build_wheel(distribution,
version,
directory,
cache_key=None,
build_dir=None,
build_env=None,
build_deps=None,
sha256=None,
pip_args=None,
resolving=False):
env = {}
home = None
if build_dir:
home = build_dir.rstrip("/") + ".home"
if os.path.isdir(home):
shutil.rmtree(home)
os.makedirs(home)
else:
home = tempfile.mkdtemp()
cmd = ["wheel"]
cmd += ["-w", directory]
# Allowing "pip wheel" to download setup_requires packages with easy_install would
# poke a hole to our wheel version locking scheme, making wheel builds non-deterministic.
# Disable easy_install as instructed here:
# https://pip.pypa.io/en/stable/reference/pip_install/#controlling-setup-requires
# We set HOME to the current directory so pip will look at this file; see:
# https://docs.python.org/2/install/index.html#distutils-configuration-files
env["HOME"] = home
with open(os.path.join(home, ".pydistutils.cfg"), "w") as f:
f.write("[easy_install]\nallow_hosts = ''\n")
for d in build_deps or []:
Wheel(d).expand(home)
# Process .pth files of the extracted build deps.
with open(os.path.join(home, "sitecustomize.py"), "w") as f:
f.write("import site; import os; site.addsitedir(os.path.dirname(__file__))")
# Set PYTHONPATH so that all extracted buildtime dependencies are available.
env["PYTHONPATH"] = ":".join(os.environ.get("PYTHONPATH", "").split(":") + [home])
env["CFLAGS"] = " ".join([
"-D__DATE__=\"redacted\"",
"-D__TIMESTAMP__=\"redacted\"",
"-D__TIME__=\"redacted\"",
"-Wno-builtin-macro-redefine",
])
# Set any other custom env variables the user wants to add to the wheel build.
env.update(dict([x.split("=", 1) for x in build_env or []]))
# For determinism, canonicalize distribution name to lowercase here, since, lo and
# behold, the wheel contents may be different depending on the case passed to
# "pip wheel" command...
cmd += ["%s==%s" % (distribution.lower(), version)]
cmd += ["--no-cache-dir"]
cmd += ["--no-deps"]
# Build the wheel in a deterministic path so that any debug symbols have stable
# paths and the resulting wheel has a higher chance of being deterministic.
if build_dir:
if os.path.isdir(build_dir):
shutil.rmtree(build_dir)
cmd += ["--build", build_dir]
cmd += pip_args or []
locally_built = False
with CaptureOutput() as output:
if pip_main(cmd, env):
print("pip command failed: " + str(cmd))
sys.exit(1)
if re.search(r"Running setup\.py bdist_wheel", output.stdout.getvalue().decode()):
locally_built = True
wheels = wheels_from_dir(directory)
assert len(wheels) == 1
wheel = wheels[0]
if locally_built:
# The wheel was built locally. For determinism, we need to strip timestamps
# from the zip-file.
strip_wheel(wheel)
computed_sha256 = digest(wheel.path())
if sha256 and computed_sha256 != sha256:
if resolving:
if locally_built:
# If we built the wheel locally and the sha256 had changed from the previous one,
# build the wheel again to make sure we get the same digest again.
os.rename(wheel.path(), wheel.path() + ".0")
if pip_main(cmd, env):
sys.exit(1)
strip_wheel(wheel)
second_sha256 = digest(wheel.path())
if computed_sha256 != second_sha256:
os.rename(wheel.path(), wheel.path() + ".1")
print("Wheel build not deterministic:")
print(" %s.0: %s" % (wheel.path(), computed_sha256))
print(" %s.1: %s" % (wheel.path(), second_sha256))
sys.exit(1)
os.remove(wheel.path() + ".0")
else:
# If the user supplied an expected sha256, the built wheel should match it.
print("\033[0;33mWARNING:\033[0m Built wheel %s digest %s does not match expected digest %s." % (wheel.path(), computed_sha256, sha256))
shutil.rmtree(home)
return computed_sha256
def extract(args):
# Extract the files into the current directory
whl = Wheel(args.whl)
whl.expand(args.directory)
extra_deps = args.add_dependency or []
drop_deps = {d: None for d in args.drop_dependency or []}
imports = ['.']
external_deps = [
d for d in itertools.chain(whl.dependencies(), extra_deps)
if d not in drop_deps
]
contents = []
add_build_content = args.add_build_content or []
for name in add_build_content:
with open(name) as f:
contents.append(f.read() + '\n')
contents = '\n'.join(contents)
parser = whl.entrypoints()
entrypoints_build = ''
if parser:
if parser.has_section('console_scripts'):
for name, location in parser.items('console_scripts'):
# Assumes it doesn't depend on extras. TODO(conrado): fix
entrypoint_file = 'entrypoint_%s.py' % name
with open(os.path.join(args.directory, entrypoint_file),
'w') as f:
f.write("""from %s import %s as main; main()""" %
tuple(location.split(":")))
entrypoints_build += """
py_binary(
name = "{entrypoint_name}",
srcs = ["{entrypoint_file}"],
main = "{entrypoint_file}",
deps = [":pkg"],
)""".format(entrypoint_name='entrypoint_%s' % name,
entrypoint_file=entrypoint_file)
with open(os.path.join(args.directory, 'BUILD'), 'w') as f:
f.write("""
package(default_visibility = ["//visibility:public"])
load("@{repository}//:requirements.bzl", "requirement")
filegroup(
name = "wheel",
data = ["{wheel}"],
)
py_library(
name = "pkg",
srcs = glob(["**/*.py"]),
data = glob(["**/*"], exclude=["**/*.py", "**/* *", "*.whl", "BUILD", "WORKSPACE"]),
# This makes this directory a top-level in the python import
# search path for anything that depends on this.
imports = [{imports}],
deps = [
{dependencies}
],
)
{extras}
{entrypoints_build}
{contents}""".format(wheel=whl.basename(),
repository=args.repository,
dependencies=''.join([
('\n "%s",' % d) if d[0] == "@" else
('\n requirement("%s"),' % d)
for d in sorted(external_deps)
]),
imports=','.join(['"%s"' % i for i in sorted(imports)]),
extras='\n\n'.join([
"""py_library(
name = "{extra}",
deps = [
":pkg",{deps}
],
)""".format(extra=extra,
deps=','.join([
'requirement("%s")' % dep
for dep in sorted(whl.dependencies(extra))
])) for extra in args.extras or []
]),
entrypoints_build=entrypoints_build,
contents=contents))