def test_redirected_project_page():
responses.add(
method=responses.GET,
url='https://nil.test/simple/project/',
status=301,
headers={'Location': 'https://test.nil/simple/project/'},
)
responses.add(
method=responses.GET,
url='https://test.nil/simple/project/',
body='<a href="../files/project-0.1.0.tar.gz">project-0.1.0.tar.gz</a>',
content_type='text/html',
)
simple = PyPISimple('https://nil.test/simple/')
assert simple.get_project_files('project') == [
DistributionPackage(
filename='project-0.1.0.tar.gz',
project='project',
version='0.1.0',
package_type='sdist',
url="https://test.nil/simple/files/project-0.1.0.tar.gz",
requires_python=None,
has_sig=False,
yanked=None,
),
]
def test_latin2_declarations(content_type, body_decl):
# This test is deliberately weird in order to make sure the code is
# actually paying attention to the encoding declarations and not just
# assuming UTF-8 because the input happens to be valid UTF-8.
responses.add(
method=responses.GET,
url='https://test.nil/simple/project/',
body=body_decl +
b'<a href="../files/project-0.1.0-p\xC3\xBF42-none-any.whl">project-0.1.0-p\xC3\xBF42-none-any.whl</a>',
content_type=content_type,
)
simple = PyPISimple('https://test.nil/simple/')
assert simple.get_project_files('project') == [
DistributionPackage(
filename=u'project-0.1.0-p\u0102\u017C42-none-any.whl',
project='project',
version='0.1.0',
package_type='wheel',
url=
u"https://test.nil/simple/files/project-0.1.0-p\u0102\u017C42-none-any.whl",
requires_python=None,
has_sig=False,
yanked=None,
),
]
class SimpleRepo(PythonDistributionRepo):
""" PEP-503 (aka pypi/simple) compliant repository """
_username: t.Optional[str]
_password: t.Optional[str]
url: str
client: PyPISimple
def __init__(self, url: str, auth: t.Tuple[str, str] = None) -> None:
self.url = url
self._username, self._password = auth or (None, None)
self.client = PyPISimple(endpoint=url)
def project_files(self, project: t.Any) -> t.Any:
return self.client.get_project_files(project)
def download(self, project: t.Any, file: t.Any, dest: t.Any) -> t.Any:
dists = [x for x in self.project_files(project) if x.filename == file]
if not dists:
raise NotAvailable(f"{file} is not available in project{project}")
dist = dists[0]
full_fname = os.path.join(dest, dist.filename)
download_file(dist.url, full_fname)
return full_fname
def upload(self, project: t.Any, file: t.Any) -> t.Any:
settings = twine.settings.Settings(
repository_url=self.url, username=self._username, password=self._password
)
# TODO: use twine Repository object instead of command?
twine_upload(settings, [file])
def _get_latest_versions(*, package_name, endpoint):
pypi = PyPISimple(endpoint=endpoint)
packages = pypi.get_project_files(package_name)
try:
latest_version = packages[-1].version
except IndexError:
# Package not found
latest_version = None
return (package_name, latest_version)
def get_hashes(dist_path: str,
simple_url: str) -> Tuple[str, Union[str, None]]:
"""Get local and remote hashes for a distribution
Returns a pair (local_hash, remote_hash).
The algorithm for the hashes returned is based on the information provided
by remote's Simple API. Based on PEP 503, the API can return any type of a
hash, as long as it is supported by the builtin hashlib module.
If the Simple API doesn't return any hash, raises UnsupportedAPIError.
If the package couldn't be found, an md5 has of local package is returned
along with a None, i.e. ('<md5>', None).
"""
client = PyPISimple(simple_url)
package = PackageFile.from_filename(dist_path, comment=None)
name = package.safe_name
filename = os.path.basename(dist_path)
_, local_version, local_package_type = parse_filename(filename,
project_hint=name)
remote_dists = client.get_project_files(name)
for remote_dist in remote_dists:
if (versions_equal(local_version, remote_dist.version)
and remote_dist.package_type == local_package_type):
break
else:
return hash_file(dist_path, hashlib.md5), None
try:
algo, remote_hash = url_hash_fragment(remote_dist.url)
except Exception as exc:
raise UnsupportedAPIError("API doesn't support hashes.") from exc
if algo not in hashlib.algorithms_guaranteed:
raise UnsupportedAPIError(f"API returned an unsupported hash: {algo}.")
hashlib_cls = getattr(hashlib, algo)
local_hash = hash_file(dist_path, hashlib_cls)
return local_hash, remote_hash
def test_project_hint_received():
"""
Test that the argument to ``get_project_files()`` is used to disambiguate
filenames
"""
with open(join(dirname(__file__), os.pardir, 'data',
'aws-adfs-ebsco.html')) as fp:
responses.add(
method=responses.GET,
url='https://test.nil/simple/aws-adfs-ebsco/',
body=fp.read(),
content_type='text/html',
)
simple = PyPISimple('https://test.nil/simple/')
with pytest.warns(DeprecationWarning):
files = simple.get_project_files('aws-adfs-ebsco')
assert files == [
DistributionPackage(
filename='aws-adfs-ebsco-0.3.6-2.tar.gz',
project='aws-adfs-ebsco',
version='0.3.6-2',
package_type='sdist',
url=
"https://files.pythonhosted.org/packages/13/b7/a69bdbf294db5ba0973ee45a2b2ce7045030cd922e1c0ca052d102c45b95/aws-adfs-ebsco-0.3.6-2.tar.gz#sha256=6eadd17408e1f26a313bc75afaa3011333bc2915461c446720bafd7608987e1e",
requires_python=None,
has_sig=None,
yanked=None,
),
DistributionPackage(
filename='aws-adfs-ebsco-0.3.7-1.tar.gz',
project='aws-adfs-ebsco',
version='0.3.7-1',
package_type='sdist',
url=
"https://files.pythonhosted.org/packages/86/8a/46c2a99113cfbb7d6c089b2128ca9e4faaea1f6a1d4e17577fd9a3396bee/aws-adfs-ebsco-0.3.7-1.tar.gz#sha256=7992abc36d0061896a3f06f055e053ffde9f3fcf483340adfa675c65ebfb3f8d",
requires_python=None,
has_sig=None,
yanked=None,
),
]
def test_utf8_declarations(content_type, body_decl):
responses.add(
method=responses.GET,
url='https://test.nil/simple/project/',
body=body_decl +
b'<a href="../files/project-0.1.0-p\xC3\xBF42-none-any.whl">project-0.1.0-p\xC3\xBF42-none-any.whl</a>',
content_type=content_type,
)
simple = PyPISimple('https://test.nil/simple/')
assert simple.get_project_files('project') == [
DistributionPackage(
filename=u'project-0.1.0-p\xFF42-none-any.whl',
project='project',
version='0.1.0',
package_type='wheel',
url=
u"https://test.nil/simple/files/project-0.1.0-p\xFF42-none-any.whl",
requires_python=None,
has_sig=False,
yanked=None,
),
]
class PyPiClient:
def __init__(self, cache_path: "str"):
self.client = PyPISimple()
self.session = requests.Session()
self.cache_path = cache_path
@functools.lru_cache(maxsize=2)
def list_projects(self):
try:
return self.load_cached_mirror()
except:
return list(self.client.get_projects())
def new_projects(self):
cached_projects = set(self.load_cached_mirror())
found_projects = set(self.list_projects())
return set.difference(cached_projects, found_projects)
def download_mirror(self):
projects = self.list_projects()
with open(self.cache_path, 'w', encoding='utf-8-sig') as f:
json.dump(projects, fp=f)
def load_cached_mirror(self):
with open(self.cache_path, 'r', encoding='utf-8-sig') as f:
doc = json.load(fp=f)
return doc
@functools.lru_cache(maxsize=20)
def extract_latest_project_file(self, project):
project_files = self.client.get_project_files(project)
max_version = packaging.version.parse('-1')
max_file = None
for project_file in project_files:
if project_file.version is None:
version = packaging.version.parse('0.0.0')
else:
version = packaging.version.parse(project_file.version)
if project_file.package_type in ('sdist', 'wheel') and (
version > max_version or (version == max_version)
and project_file.package_type == 'wheel'):
max_file, max_version = project_file, version
if not max_file:
raise MissingFilesError('Missing files?')
return max_version, max_file
@functools.lru_cache(maxsize=20)
def extract_setup_py(self, project):
version, project_file = self.extract_latest_project_file(project)
response = self.session.get(project_file.url, stream=True, timeout=10)
fp = io.BytesIO(response.content)
if project_file.filename.endswith(
'.zip') or project_file.filename.endswith('.whl'):
zf = zipfile.ZipFile(fp)
if project_file.filename.endswith('.zip'):
nameinfos = list(
[ni for ni in zf.namelist() if ni.endswith('/setup.py')])
if nameinfos:
return ('setup.py', version, zf.read(nameinfos[0]))
raise MissingSetupError()
else:
nameinfos = list(
[ni for ni in zf.namelist() if ni.endswith('/METADATA')])
if nameinfos:
return ('metadata', version, zf.read(nameinfos[0]))
raise MissingManifestError()
else:
tf = tarfile.open(fileobj=fp)
nameinfos = list(ni for ni in tf.getnames()
if ni.endswith('/setup.py'))
if nameinfos:
data = tf.extractfile(nameinfos[0])
return ('setup.py', version, data.read())
raise MissingSetupError()
def test_session(content_type):
session_dir = join(dirname(__file__), 'data', 'session01')
with open(join(session_dir, 'simple.html')) as fp:
responses.add(
method=responses.GET,
url='https://test.nil/simple/',
body=fp.read(),
content_type=content_type,
)
responses.add(
method=responses.GET,
url='https://test.nil/simple/',
body='This URL should only be requested once.',
status=500,
)
with open(join(session_dir, 'in-place.html')) as fp:
responses.add(
method=responses.GET,
url='https://test.nil/simple/in-place/',
body=fp.read(),
content_type=content_type,
)
responses.add(
method=responses.GET,
url='https://test.nil/simple/nonexistent/',
body='Does not exist',
status=404,
)
simple = PyPISimple('https://test.nil/simple/')
assert list(simple.get_projects()) == ['in_place', 'foo', 'BAR']
assert simple.get_project_url(
'IN.PLACE') == 'https://test.nil/simple/in-place/'
assert simple.get_project_files('IN.PLACE') == [
DistributionPackage(
filename='in_place-0.1.1-py2.py3-none-any.whl',
project='in_place',
version='0.1.1',
package_type='wheel',
url=
"https://files.pythonhosted.org/packages/34/81/2baaaa588ee1a6faa6354b7c9bc365f1b3da867707cd136dfedff7c06608/in_place-0.1.1-py2.py3-none-any.whl#sha256=e0732b6bdc2f1bfc4e1b96c1de2fbbd053bb2a9534547474a0485baa339bfa97",
requires_python=None,
has_sig=False,
yanked=None,
),
DistributionPackage(
filename='in_place-0.1.1.tar.gz',
project='in_place',
version='0.1.1',
package_type='sdist',
url=
"https://files.pythonhosted.org/packages/b9/ba/f1c67fb32c37ba4263326ae4ac6fd00b128025c9289b2fb31a60a0a22f90/in_place-0.1.1.tar.gz#sha256=ffa729fd0b818ac750aa31bafc886f266380e1c8557ba38f70f422d2f6a77e23",
requires_python=None,
has_sig=False,
yanked=None,
),
DistributionPackage(
filename='in_place-0.2.0-py2.py3-none-any.whl',
project='in_place',
version='0.2.0',
package_type='wheel',
url=
"https://files.pythonhosted.org/packages/9f/46/9f5679f3b2068e10b33c16a628a78b2b36531a9df08671bd0104f11d8461/in_place-0.2.0-py2.py3-none-any.whl#sha256=e1ad42a41dfde02092b411b1634a4be228e28c27553499a81ef04b377b28857c",
requires_python=None,
has_sig=False,
yanked=None,
),
DistributionPackage(
filename='in_place-0.2.0.tar.gz',
project='in_place',
version='0.2.0',
package_type='sdist',
url=
"https://files.pythonhosted.org/packages/f0/51/c30f1fad2b857f7b5d5ff76ec01f1f80dd0f2ab6b6afcde7b2aed54faa7e/in_place-0.2.0.tar.gz#sha256=ff783dca5d06f85b8d084871abd11a170d732423edb48c53ccb68c55fcbbeb76",
requires_python=None,
has_sig=False,
yanked=None,
),
DistributionPackage(
filename='in_place-0.3.0-py2.py3-none-any.whl',
project='in_place',
version='0.3.0',
package_type='wheel',
url=
"https://files.pythonhosted.org/packages/6f/84/ced31e646df335f8cd1b7884e3740b8c012314a28504542ef5631cdc1449/in_place-0.3.0-py2.py3-none-any.whl#sha256=af5ce9bd309f85a6bbe4119acbc0a67cda68f0ae616f0a76a947addc62791fda",
requires_python=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, <4",
has_sig=False,
yanked=None,
),
DistributionPackage(
filename='in_place-0.3.0.tar.gz',
project='in_place',
version='0.3.0',
package_type='sdist',
url=
"https://files.pythonhosted.org/packages/b6/cd/1dc736d5248420b15dd1546c2938aec7e6dab134e698e0768f54f1757af7/in_place-0.3.0.tar.gz#sha256=4758db1457c8addcd5f5b15ef870eab66b238e46e7d784bff99ab1b2126660ea",
requires_python=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, <4",
has_sig=False,
yanked=None,
),
]
assert simple.get_project_files('nonexistent') == []
