def extract_license_from_sdist(
destination: Path,
sdist: Path,
license_directories: Dict[str, str],
license_fallback_urls: Dict[str, str],
) -> None:
def extract_from_source_tarfile(sdist: Path) -> bool:
ext = sdist.suffixes[-1][1:]
with tarfile.open(sdist, mode="r:{}".format(ext)) as tar:
return find_and_extract_license(destination, tar, tar.getmembers(),
license_directories)
def extract_from_source_zipfile(sdist: Path) -> bool:
with zipfile.ZipFile(sdist) as zip:
return find_and_extract_license(destination, zip, zip.infolist(),
license_directories)
if sdist.suffixes[-2:-1] == [".tar"]:
found = extract_from_source_tarfile(sdist)
elif sdist.suffixes[-1] == ".zip":
found = extract_from_source_zipfile(sdist)
else:
raise NotImplementedError("new sdist type!")
if found:
return
UI.log("License not found in {}".format(sdist.name))
get_license_fallback(destination, sdist.name, license_directories,
license_fallback_urls)
def update(verbose: bool, location: Path, package: Optional[str]) -> None:
UI.verbose = verbose
location = Path(location)
try:
with UI.task("Load configuration"):
config = load_configuration(location)
with UI.task("Updating requirements"):
update_requirements(config, package)
except VendoringError as e:
UI.show_error(e)
def update_requirements(config: Configuration, package: Optional[str]) -> None:
requirements = config.base_directory / config.requirements
packages = parse_pinned_packages(requirements)
for pkg in packages:
if package is None or pkg.name == package:
pkg.version = determine_latest_release(pkg.name)
UI.log(f"Rewriting {requirements}")
with requirements.open("w", encoding="utf-8") as f:
f.writelines(f"{p}{linesep}" for p in packages)
def determine_latest_release(name: str) -> str:
UI.log(f"Determining latest version for {name}...")
try:
r = requests.get(f"https://pypi.org/pypi/{name}/json")
retval = str(r.json()["info"]["version"])
except Exception as e:
raise VendoringError(
f"Could not determine latest version for {name}: {e!r}")
UI.log(f"Got {retval}")
return retval
def detect_vendored_libs(destination: Path, files_to_skip: List[str]) -> List[str]:
retval = []
for item in destination.iterdir():
if item.is_dir():
retval.append(item.name)
elif item.name.endswith(".pyi"): # generated stubs
continue
elif item.name not in files_to_skip:
if not item.name.endswith(".py"):
UI.warn(f"Got unexpected non-Python file: {item}")
continue
retval.append(item.name[:-3])
return retval
def sync(verbose: bool, location: Optional[str]) -> None:
UI.verbose = verbose
if location is None:
project_path = Path()
else:
project_path = Path(location)
print(f"Working in {project_path}")
try:
with UI.task("Load configuration"):
config = load_configuration(project_path)
with UI.task("Clean existing libraries"):
cleanup_existing_vendored(config)
with UI.task("Add vendored libraries"):
libraries = vendor_libraries(config)
with UI.task("Fetch licenses"):
fetch_licenses(config)
with UI.task("Generate static-typing stubs"):
generate_stubs(config, libraries)
except VendoringError as e:
UI.show_error(e)
def run(command: List[str], *, working_directory: Optional[Path]) -> None:
UI.log("Running {}".format(" ".join(map(shlex.quote, command))))
p = subprocess.Popen(
command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
encoding="utf-8",
cwd=working_directory,
)
while True:
retcode = p.poll()
line = p.stdout.readline().rstrip()
if line:
with UI.indent():
UI.log(line)
if retcode is not None:
break
def extract_license_member(
destination: Path,
tar: SDistArchive,
member: SDistMember,
name: str,
license_directories: Dict[str, str],
) -> None:
mpath = Path(name) # relative path inside the sdist
dirname = list(mpath.parents)[-2].name # -1 is .
libname = libname_from_dir(dirname)
dest = license_destination(destination, libname, mpath.name,
license_directories)
UI.log("Extracting {} into {}".format(name, dest.relative_to(destination)))
try:
fileobj = tar.extractfile(member) # type: ignore
dest.write_bytes(fileobj.read()) # type: ignore
except AttributeError: # zipfile
dest.write_bytes(tar.read(member)) # type: ignore
def find_and_extract_license(
destination: Path,
tar: SDistArchive,
members: Iterable[SDistMember],
license_directories: Dict[str, str],
) -> bool:
found = False
for member in members:
try:
license_directories,
name = member.name # type: ignore
except AttributeError: # zipfile
name = member.filename # type: ignore
if "LICENSE" in name or "COPYING" in name:
if "/test" in name:
# some testing licenses in html5lib and distlib
UI.log("Ignoring {}".format(name))
continue
found = True
extract_license_member(destination, tar, member, name,
license_directories)
return found
def run(command: List[str], *, working_directory: Optional[Path]) -> None:
cmd = " ".join(map(shlex.quote, command))
UI.log(f"Running {cmd}")
p = subprocess.Popen(
command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
encoding="utf-8",
cwd=working_directory,
)
while True:
retcode = p.poll()
line = p.stdout.readline().rstrip()
if line:
with UI.indent():
UI.log(line)
if retcode is not None:
break
if retcode:
raise VendoringError(
f"Command exited with non-zero exit code: {retcode}")
def load_configuration(directory: Path) -> Configuration:
# Read the contents of the file.
file = directory / "pyproject.toml"
UI.log(f"Will attempt to load {file}.")
try:
file_contents = file.read_text(encoding="utf8")
except IOError as read_error:
raise ConfigurationError(
"Could not read pyproject.toml.") from read_error
else:
UI.log("Read configuration file.")
try:
parsed_contents = parse_toml(file_contents)
except TomlDecodeError as toml_error:
raise ConfigurationError(
"Could not parse pyproject.toml.") from toml_error
else:
UI.log("Parsed configuration file.")
if ("tool" not in parsed_contents
or not isinstance(parsed_contents["tool"], dict)
or "vendoring" not in parsed_contents["tool"]
or not isinstance(parsed_contents["tool"]["vendoring"], dict)):
raise ConfigurationError(
"Can not load `tool.vendoring` from pyproject.toml")
tool_config = parsed_contents["tool"]["vendoring"]
try:
retval = Configuration.load_from_dict(tool_config, location=directory)
except ConfigurationError as e:
raise ConfigurationError(
"Could not load values from [tool.vendoring] in pyproject.toml.\n"
f" REASON: {e}")
else:
UI.log("Validated configuration.")
return retval
def sync(verbose: bool, location: Path) -> None:
UI.verbose = verbose
location = Path(location)
try:
with UI.task("Load configuration"):
config = load_configuration(location)
with UI.task("Clean existing libraries"):
cleanup_existing_vendored(config)
with UI.task("Add vendored libraries"):
libraries = vendor_libraries(config)
with UI.task("Fetch licenses"):
fetch_licenses(config)
with UI.task("Generate static-typing stubs"):
generate_stubs(config, libraries)
except VendoringError as e:
UI.show_error(e)
def download_url(url: str, dest: Path) -> None:
UI.log("Downloading {}".format(url))
r = requests.get(url, allow_redirects=True)
r.raise_for_status()
dest.write_bytes(r.content)
