def get_wheel_list_to_install(self):
os.chdir(self.curdir)
in_bin = os.path.relpath(self.in_bin, start=self.curdir)
our_whl_path = os.path.join(in_bin, "ourwheel")
our_wheels = []
our_wheels_set = set()
if os.path.exists(our_whl_path):
our_wheels = [
os.path.join(our_whl_path, whl)
for whl in os.listdir(our_whl_path) if whl.endswith('.whl')
]
our_wheels_set = set(
[parse_wheel_filename(whl).project for whl in our_wheels])
ext_whl_path = os.path.join(self.in_bin, "extwheel")
ext_wheels = []
ext_src = []
if os.path.exists(ext_whl_path):
ext_wheels = [
os.path.join(in_bin, "extwheel", whl)
for whl in os.listdir(os.path.join(self.in_bin, "extwheel"))
if whl.endswith('.whl')
and parse_wheel_filename(whl).project not in our_wheels_set
]
ext_src = [
os.path.join(in_bin, "extwheel", whl)
for whl in os.listdir(os.path.join(self.in_bin, "extwheel"))
if whl.endswith('tar.gz') or whl.endswith('tar.bz2')
]
return ext_wheels + our_wheels + ext_src
def test_parse_wheel_filename_path() -> None:
parsed = parse_wheel_filename('dist/foo-1.0-py3-none-any.whl')
assert parsed == ParsedWheelFilename(
project='foo',
version='1.0',
build=None,
python_tags=['py3'],
abi_tags=['none'],
platform_tags=['any'],
)
assert str(parsed) == 'foo-1.0-py3-none-any.whl'
def test_parse_wheel_filename_path() -> None:
parsed = parse_wheel_filename("dist/foo-1.0-py3-none-any.whl")
assert parsed == ParsedWheelFilename(
project="foo",
version="1.0",
build=None,
python_tags=["py3"],
abi_tags=["none"],
platform_tags=["any"],
)
assert str(parsed) == "foo-1.0-py3-none-any.whl"
def is_compatible(package):
"""
Check whether the given python package is a wheel compatible with the
current platform and python interpreter.
Compatibility is based on https://www.python.org/dev/peps/pep-0425/
"""
try:
w = parse_wheel_filename(package)
for systag in tags.sys_tags():
for tag in w.tag_triples():
if systag in tags.parse_tag(tag):
return True
except InvalidFilenameError:
return False
def from_wheel(cls, path: Union[str, os.PathLike]) -> 'WheelContents':
""" Construct a `WheelContents` from the wheel at the given path """
whlname = parse_wheel_filename(path)
with open(path, 'rb') as fp, ZipFile(fp) as zf:
dist_info_dir, data_dir = find_wheel_dirs(
zf.namelist(),
whlname.project,
whlname.version,
)
if data_dir is None:
data_dir = f'{whlname.project}-{whlname.version}.data'
wc = cls(dist_info_dir=dist_info_dir, data_dir=data_dir)
try:
wheel_info = zf.getinfo(f'{dist_info_dir}/WHEEL')
except KeyError:
raise WheelValidationError('No WHEEL file in wheel')
with zf.open(wheel_info) as wf:
for line in TextIOWrapper(wf, 'utf-8'):
m = ROOT_IS_PURELIB_RGX.fullmatch(line)
if m:
rip = m.group(1)
if rip.lower() == 'true':
wc.root_is_purelib = True
elif rip.lower() == 'false':
wc.root_is_purelib = False
else:
raise WheelValidationError(
f'Invalid Root-Is-Purelib value in WHEEL'
f' file: {rip!r}')
break
else:
raise WheelValidationError(
'Root-Is-Purelib header not found in WHEEL file')
try:
record_info = zf.getinfo(f'{dist_info_dir}/RECORD')
except KeyError:
raise WheelValidationError('No RECORD file in wheel')
with zf.open(record_info) as rf:
wc.add_record_file(TextIOWrapper(rf, 'utf-8', newline=''))
wc.validate_tree()
return wc
def test_bad_path() -> None:
with pytest.raises(InvalidFilenameError) as excinfo:
parse_wheel_filename(os.path.join('dist', 'foo-0.1.whl'))
assert excinfo.value.filename == 'foo-0.1.whl'
assert str(excinfo.value) == "Invalid wheel filename: 'foo-0.1.whl'"
def test_bad_filename(filename: str) -> None:
with pytest.raises(InvalidFilenameError) as excinfo:
parse_wheel_filename(filename)
assert excinfo.value.filename == filename
assert str(excinfo.value) == f'Invalid wheel filename: {filename!r}'
def test_parse_wheel_filename(filename: str,
expected: ParsedWheelFilename) -> None:
parsed = parse_wheel_filename(filename)
assert parsed == expected
assert str(parsed) == filename
def __init__(self, path):
self.path = Path(path)
self.parsed_filename = parse_wheel_filename(self.path)
self.fp = None
self.zipfile = None
self._dist_info = None
def update_data_from_pypi():
for package in tqdm(packages, unit="packages"):
resp = http.request("GET", f"https://pypi.org/pypi/{package}/json")
if resp.status != 200:
continue
try:
resp = json.loads(resp.data.decode("utf-8"))
except Exception:
continue
try:
version = Version(resp["info"]["version"])
except InvalidVersion: # The latest release has an invalid version, skip
continue
latest_version = max(to_versions(resp["releases"].keys()))
# Favor pre-releases over non-pre-releases
if version < latest_version:
version = latest_version
new_resp = http.request(
"GET",
f"https://pypi.org/pypi/{package}/{latest_version}/json")
if new_resp.status == 200:
resp = json.loads(new_resp.data.decode("utf-8"))
# Get the exact string for the version that we found
for strv in resp["releases"]:
try:
if Version(strv) == version:
str_version = strv
break
except InvalidVersion:
continue
else:
raise ValueError("???")
# Check to see if we already have this version or not
with closing(db.cursor()) as cur:
cur.execute(
"SELECT name FROM packages WHERE name = ? AND version = ?;",
(package, str_version),
)
if cur.fetchone():
continue
# If we don't have 'requires_dist' information install
# locally and investigate the installed package
if False and resp["info"]["requires_dist"] is None:
new_resp = get_metadata_by_install(package, resp)
if new_resp is not None:
resp = new_resp
requires_python = resp["info"]["requires_python"] or ""
urequires_dist = [
normalize_requires_dist(x)
for x in resp["info"]["requires_dist"] or []
]
urequires_dist = sorted(urequires_dist, key=requires_dist_sort_key)
requires_dist = {"specifiers": [], "dists": []}
requires_extras = {}
yanked = []
releases = resp["releases"][str_version]
uploaded_at = None if not releases else min(x["upload_time"]
for x in releases)
wheel_filenames = [
x["filename"] for x in releases if x["filename"].endswith(".whl")
]
has_binary_wheel = False
for filename in wheel_filenames:
try:
whl = parse_wheel_filename(filename)
except InvalidFilenameError:
continue
python_tags, abi_tags, platform_tags = (
whl.python_tags,
whl.abi_tags,
whl.platform_tags,
)
for wheel_data in itertools.product(python_tags, abi_tags,
platform_tags):
py, abi, plat = wheel_data
db.execute(
"""
INSERT INTO wheels (
name, version, filename, python, abi, platform
) VALUES (?, ?, ?, ?, ?, ?);
""",
(package, str_version, filename, py, abi, plat),
)
if abi_tags == ["none"] and platform_tags == ["any"]:
continue
has_binary_wheel = True
db.execute(
"""
INSERT OR IGNORE INTO packages (
name, version, requires_python, has_binary_wheel, uploaded_at
) VALUES (?, ?, ?, ?, ?);
""",
(package, str_version, requires_python, has_binary_wheel,
uploaded_at),
)
db.commit()
for req in urequires_dist:
extras = get_extras(req)
req_no_specifiers = dist_from_requires_dist(req)
specifier = specifier_from_requires_dist(req).replace(
req_no_specifiers + " ", "", 1)
if extras:
for extra in extras:
db.execute(
"""
INSERT OR IGNORE INTO deps (
name,
version,
dep_name,
dep_specifier,
extra
) VALUES (?, ?, ?, ?, ?);
""",
(package, str_version, req_no_specifiers, specifier,
extra),
)
else:
db.execute(
"""
INSERT OR IGNORE INTO deps (
name,
version,
dep_name,
dep_specifier
) VALUES (?, ?, ?, ?);
""",
(package, str_version, req_no_specifiers, specifier),
)
requires_dist["dists"] = sorted(set(requires_dist["dists"]))
for extra, extra_info in list(requires_extras.items()):
requires_extras[extra]["dists"] = sorted(set(extra_info["dists"]))
for relv, downloads in resp["releases"].items():
for download in downloads:
if download["yanked"]:
yanked.append(relv)
break
yanked = sorted_versions(set(yanked))
if yanked:
db.execute(
"UPDATE packages SET yanked=1 WHERE name=? AND version=?;",
(package, str_version),
)
db.commit()
def wheel_sort_key(filename):
"""
Returns a sort key for the given wheel filename that will be used to select
the "preferred" or "default" wheel to display for a given project &
version.
General rules:
- It is assumed that only wheels for the same version of the same project
are ever compared, and so those parts of the filename are ignored.
- Prefer more general wheels (e.g., pure Python) to more specific (e.g.,
platform specific).
- Prefer compability with more versions to fewer.
- "any" is the most preferred platform.
- "none" is the most preferred ABI.
- Prefer compability with higher versions to lower.
- Unrecognized values are ignored if possible, otherwise sorted at the
bottom.
Specific, arbitrary preferences:
- Sort by Python tag first, then platform tag, then ABI tag, then
"pyver-abi-platform" string (as a tiebreaker), then build tag.
- Filenames that can't be parsed sort the lowest and sort relative to each
other based on filename.
- Python implementations: py (generic) > cp (CPython) > pp (PyPy) > jy
(Jython) > ip (IronPython) > everything else
- Platforms: any > manylinux > Linux > Windows > Mac OS X > everything else
"""
try:
whlname = parse_wheel_filename(filename)
except ValueError:
return (0, filename)
if whlname.build is not None:
n = re.fullmatch(r"(?P<buildno>\d+)(?P<buildstr>[^-]*)", whlname.build)
if not n:
return (0, filename)
build_rank = (int(n.group("buildno")), n.group("buildstr"))
else:
build_rank = (-1, "")
pyver_rank = []
for py in whlname.python_tags:
n = re.fullmatch(r"(\w+?)(\d[\d_]*)", py)
if not n:
return (0, filename)
pyver_rank.append((
PYTHON_PREFERENCES[n.group(1)],
VersionNoDot(n.group(2)),
))
pyver_rank.sort(reverse=True)
### TODO: distlib expects wheels to have only one ABI tag in their filename
### while wheel_inspect does not. If the latter turns out to be the
### correct approach, adjust this code to handle multiple tags.
abi = whlname.abi_tags[0]
### TODO: Should abi3 be given some rank?
if abi == "none":
abi_rank = (1, )
else:
n = re.fullmatch(r"(\wp)(\d+)(\w*)", abi)
if n:
py_imp, py_ver, flags = n.groups()
abi_rank = (0, PYTHON_PREFERENCES[py_imp], VersionNoDot(py_ver),
flags)
else:
abi_rank = (0, -1, -1, "")
platform_rank = []
for plat in whlname.platform_tags:
for rank, rgx in enumerate([
r"macosx_10_(?P<version>\d+)_(?P<arch>\w+)",
"macosx",
"win32",
"win64",
"win_amd64",
r"linux_(?P<arch>\w+)",
r"manylinux(?P<version>\d+)_(?P<arch>\w+)",
"any",
]):
n = re.fullmatch(rgx, plat)
if n:
d = n.groupdict()
version = d.get("version")
version = int(version) if version is not None else -1
arch = ARCH_PREFERENCES[d.get("arch")]
platform_rank.append((rank, version, arch))
break
else:
### TODO: Don't discard
pass
platform_rank.sort(reverse=True)
tiebreaker = "{}-{}-{}".format(
".".join(whlname.python_tags),
".".join(whlname.abi_tags),
".".join(whlname.platform_tags),
)
return (1, pyver_rank, platform_rank, abi_rank, tiebreaker, build_rank)
def generate_install(self):
os.chdir(self.curdir)
os.chdir(self.output_dir)
lines = []
scmd = '%s\\vs_BuildTools.exe --layout %s/%s --lang en-US %s ' % (n(
self.bin_dir), n(
self.bin_dir), self.msvc.version, self.msvc.get_add_line())
lines.append(scmd)
scmd = '%s\\%s\\vs_BuildTools.exe -p %s ' % (n(
self.bin_dir), self.msvc.version, self.msvc.get_add_line())
lines.append(scmd)
# scmd = R'%s\wdk\wdksetup.exe /q ' % (self.bin_dir.replace('/', '\\') )
# lines.append(scmd)
scmd = fR'{self.bin_dirw}\wdk10\wdksetup.exe /q '
lines.append(scmd)
scmd = fR'{self.bin_dirw}\wdk8.1\wdksetup.exe /q '
lines.append(scmd)
scmd = fR'{self.bin_dirw}\msvc2013\vs_ultimate.exe /NoRestart /Passive '
lines.append(scmd)
scmd = fR'{self.bin_dirw}\wsdk8\sdksetup.exe /q '
lines.append(scmd)
self.lines2bat("10-install-msvc", lines)
# do not append installs
lines = []
scmd = '%s\\%s /passive TargetDir=%s ' % (n(
self.bin_dir), self.python.artifact_name(), self.pythondir)
lines.append(scmd)
scmd = "%s\\python -m pip install wheel cython clcache " % (
self.pythondir)
lines.append(scmd)
for ut_ in self.utilities + [self.tess, self.imagick]:
install_dir = self.pythondir
try:
install_dir = os.path.join(self.buildroot, ut_.version)
except:
pass
try:
install_dir = ut_.path
except:
pass
# if 'magick' in ut_.url:
# wtf = 1
lines += ut_.install_me_lines(self.bin_dir, install_dir)
our_wheels = [
n(os.path.join(self.ourwheel_dir, whl))
for whl in os.listdir(self.ourwheel_dir) if whl.endswith('.whl')
]
our_wheels_set = set([
parse_wheel_filename(whl.replace('\\', '/')).project
for whl in our_wheels
])
ext_wheels = [
n(os.path.join(self.extwheel_dir, whl))
for whl in os.listdir(self.extwheel_dir) if whl.endswith('.whl')
and parse_wheel_filename(whl).project not in our_wheels_set
]
ext_src = [
n(os.path.join(self.extwheel_dir, whl))
for whl in os.listdir(self.extwheel_dir)
if whl.endswith('tar.gz') or whl.endswith('tar.bz2')
]
for wheel in ext_wheels + our_wheels:
scmd = '%s\\python -m pip install --upgrade %s ' % (
self.pythondir, wheel) # --force-reinstall
lines.append(scmd)
self.lines2bat("11-install", lines)
pass