def _pip_compile(*args):
"""
Performs pip-compile (from piptools) with a twist.
We force editable requirements to use GIT repository (parameter obtain=True) so we have setuptools_scm working on
them (axado.runner uses setuptools_scm).
"""
from contextlib import contextmanager
@contextmanager
def replaced_argv(args):
import sys
argv = sys.argv
sys.argv = [''] + list(args)
yield
sys.argv = argv
from pip.req.req_install import InstallRequirement
try:
InstallRequirement.update_editable_
except AttributeError:
InstallRequirement.update_editable_ = InstallRequirement.update_editable
InstallRequirement.update_editable = lambda s, _o: InstallRequirement.update_editable_(
s, True)
with replaced_argv(args):
from piptools.scripts.compile import cli
try:
cli()
except SystemExit as e:
return e.code
def compile_file(filename, pip_args):
output_filename = str(filename.with_suffix(".txt").name)
with chdir(filename.parent):
try:
compile.cli(pip_args + [
"-o",
output_filename,
"--verbose",
"--generate-hashes",
"--",
str(filename.name),
])
return 0
except SystemExit as e:
if e.code != 0:
print(e)
print("Could not compile {}".format(filename))
else:
# forces all end lines to be '\n'
with open(output_filename, 'rb') as f:
output_content = f.read()
fixed_content = b''.join(
line.rstrip(b'\r\n') + b'\n'
for line in output_content.splitlines(True))
with open(output_filename, 'wb') as f:
f.write(fixed_content)
return e.code
def generate_requirements_file(import_mapping: dict, dest_file: str,
code_dir_abs_path: str, mapping_dict: dict):
if not os.path.exists(dest_file):
with open(dest_file, 'w'):
pass
sys.path.insert(0, code_dir_abs_path)
import_list = flatten_nested_lst(list(import_mapping.values()))
# normalize api.utils -> api and then remove duplicates
import_set = set([imp.split(".")[0] for imp in import_list])
local_import_set, faulty_import_set = get_local_faulty_imports(
import_mapping, code_dir_abs_path)
# Filter 1: Remove local and error-generating imports
non_local_import_set = import_set - local_import_set - faulty_import_set
vers = ".".join(python_version().split(".")[:-1])
builtin_pkg_set = get_builtin_libs(vers)
# Filter 2: Remove builtin packages
third_party_imports = list(
non_local_import_set.difference(builtin_pkg_set))
# Filter 3: Apply mapping according to json file
pip_package_list = apply_mapping(third_party_imports, mapping_dict)
with open(dest_file, 'w') as f:
for pkg in pip_package_list:
f.write(f"{pkg}\n")
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
# Need to change the args before running pip-compile
sys.argv = ["", dest_file]
print("Generating dependencies from top-level imports...")
cli()
def run(args, env):
import piptools.scripts.compile as pip_compile # type: ignore
try:
return pip_compile.cli(args) # type: ignore
except SystemExit:
pass
def main() -> None:
"""Regenerates the 'requirements.txt' file using the 'requirements.in'
file to produce a deterministic list of all the dependencies that should be
in the 'third_party/python_libs' folder.
"""
# This code is copied from the pip-compile script. We cannot use the
# pip-compile script because we installed pip-tools to our own local
# oppia_tools directory.
#
# In a normal installation, piptools adds the pip-compile script to the
# user's bin directory and the pip-tools libraries to one of the default
# python system path directories. When the pip-compile script executes, it
# can import 'scripts.compile' correctly from the default sys.path. However,
# since we are installing piptools to a local directory, the pip-compile
# script will not be able to find the pip tools python packages. Therefore,
# we need to write our own script and manually add our local pip-tools
# directory to the python system path in order to import the required
# libraries correctly.
compile.cli() # pylint: disable=no-value-for-parameter
from piptools.scripts import compile
if __name__ == "__main__": # pragma: no branch
compile.cli()
#!/usr/bin/env python
"""Wrapper for the pip-tools compile entry point"""
from piptools.scripts import compile as pip_compile
if __name__ == '__main__':
pip_compile.cli()
# $(rootpath) in the workspace root gives ./requirements.in
if update_target_pkg == ".":
update_target_pkg = ""
update_command = os.getenv("CUSTOM_COMPILE_COMMAND") or "bazel run //%s:%s" % (
update_target_pkg, update_target_name)
os.environ["CUSTOM_COMPILE_COMMAND"] = update_command
sys.argv.append("--generate-hashes")
sys.argv.append("--output-file")
sys.argv.append(requirements_txt if UPDATE else requirements_out)
sys.argv.append(requirements_in)
if UPDATE:
print("Updating " + requirements_txt)
cli()
else:
# cli will exit(0) on success
try:
print("Checking " + requirements_txt)
cli()
print("cli() should exit", file=sys.stderr)
sys.exit(1)
except SystemExit:
golden = open(requirements_txt).readlines()
out = open(requirements_out).readlines()
if golden != out:
import difflib
print(''.join(difflib.unified_diff(golden, out)), file=sys.stderr)
print(