def test_redirected_project_page(): responses.add( method=responses.GET, url='https://nil.test/simple/project/', status=301, headers={'Location': 'https://test.nil/simple/project/'}, ) responses.add( method=responses.GET, url='https://test.nil/simple/project/', body='<a href="../files/project-0.1.0.tar.gz">project-0.1.0.tar.gz</a>', content_type='text/html', ) simple = PyPISimple('https://nil.test/simple/') assert simple.get_project_files('project') == [ DistributionPackage( filename='project-0.1.0.tar.gz', project='project', version='0.1.0', package_type='sdist', url="https://test.nil/simple/files/project-0.1.0.tar.gz", requires_python=None, has_sig=False, yanked=None, ), ]
def test_latin2_declarations(content_type, body_decl): # This test is deliberately weird in order to make sure the code is # actually paying attention to the encoding declarations and not just # assuming UTF-8 because the input happens to be valid UTF-8. responses.add( method=responses.GET, url='https://test.nil/simple/project/', body=body_decl + b'<a href="../files/project-0.1.0-p\xC3\xBF42-none-any.whl">project-0.1.0-p\xC3\xBF42-none-any.whl</a>', content_type=content_type, ) simple = PyPISimple('https://test.nil/simple/') assert simple.get_project_files('project') == [ DistributionPackage( filename=u'project-0.1.0-p\u0102\u017C42-none-any.whl', project='project', version='0.1.0', package_type='wheel', url= u"https://test.nil/simple/files/project-0.1.0-p\u0102\u017C42-none-any.whl", requires_python=None, has_sig=False, yanked=None, ), ]
class SimpleRepo(PythonDistributionRepo): """ PEP-503 (aka pypi/simple) compliant repository """ _username: t.Optional[str] _password: t.Optional[str] url: str client: PyPISimple def __init__(self, url: str, auth: t.Tuple[str, str] = None) -> None: self.url = url self._username, self._password = auth or (None, None) self.client = PyPISimple(endpoint=url) def project_files(self, project: t.Any) -> t.Any: return self.client.get_project_files(project) def download(self, project: t.Any, file: t.Any, dest: t.Any) -> t.Any: dists = [x for x in self.project_files(project) if x.filename == file] if not dists: raise NotAvailable(f"{file} is not available in project{project}") dist = dists[0] full_fname = os.path.join(dest, dist.filename) download_file(dist.url, full_fname) return full_fname def upload(self, project: t.Any, file: t.Any) -> t.Any: settings = twine.settings.Settings( repository_url=self.url, username=self._username, password=self._password ) # TODO: use twine Repository object instead of command? twine_upload(settings, [file])
def _get_latest_versions(*, package_name, endpoint): pypi = PyPISimple(endpoint=endpoint) packages = pypi.get_project_files(package_name) try: latest_version = packages[-1].version except IndexError: # Package not found latest_version = None return (package_name, latest_version)
def get_hashes(dist_path: str, simple_url: str) -> Tuple[str, Union[str, None]]: """Get local and remote hashes for a distribution Returns a pair (local_hash, remote_hash). The algorithm for the hashes returned is based on the information provided by remote's Simple API. Based on PEP 503, the API can return any type of a hash, as long as it is supported by the builtin hashlib module. If the Simple API doesn't return any hash, raises UnsupportedAPIError. If the package couldn't be found, an md5 has of local package is returned along with a None, i.e. ('<md5>', None). """ client = PyPISimple(simple_url) package = PackageFile.from_filename(dist_path, comment=None) name = package.safe_name filename = os.path.basename(dist_path) _, local_version, local_package_type = parse_filename(filename, project_hint=name) remote_dists = client.get_project_files(name) for remote_dist in remote_dists: if (versions_equal(local_version, remote_dist.version) and remote_dist.package_type == local_package_type): break else: return hash_file(dist_path, hashlib.md5), None try: algo, remote_hash = url_hash_fragment(remote_dist.url) except Exception as exc: raise UnsupportedAPIError("API doesn't support hashes.") from exc if algo not in hashlib.algorithms_guaranteed: raise UnsupportedAPIError(f"API returned an unsupported hash: {algo}.") hashlib_cls = getattr(hashlib, algo) local_hash = hash_file(dist_path, hashlib_cls) return local_hash, remote_hash
def test_project_hint_received(): """ Test that the argument to ``get_project_files()`` is used to disambiguate filenames """ with open(join(dirname(__file__), os.pardir, 'data', 'aws-adfs-ebsco.html')) as fp: responses.add( method=responses.GET, url='https://test.nil/simple/aws-adfs-ebsco/', body=fp.read(), content_type='text/html', ) simple = PyPISimple('https://test.nil/simple/') with pytest.warns(DeprecationWarning): files = simple.get_project_files('aws-adfs-ebsco') assert files == [ DistributionPackage( filename='aws-adfs-ebsco-0.3.6-2.tar.gz', project='aws-adfs-ebsco', version='0.3.6-2', package_type='sdist', url= "https://files.pythonhosted.org/packages/13/b7/a69bdbf294db5ba0973ee45a2b2ce7045030cd922e1c0ca052d102c45b95/aws-adfs-ebsco-0.3.6-2.tar.gz#sha256=6eadd17408e1f26a313bc75afaa3011333bc2915461c446720bafd7608987e1e", requires_python=None, has_sig=None, yanked=None, ), DistributionPackage( filename='aws-adfs-ebsco-0.3.7-1.tar.gz', project='aws-adfs-ebsco', version='0.3.7-1', package_type='sdist', url= "https://files.pythonhosted.org/packages/86/8a/46c2a99113cfbb7d6c089b2128ca9e4faaea1f6a1d4e17577fd9a3396bee/aws-adfs-ebsco-0.3.7-1.tar.gz#sha256=7992abc36d0061896a3f06f055e053ffde9f3fcf483340adfa675c65ebfb3f8d", requires_python=None, has_sig=None, yanked=None, ), ]
def test_utf8_declarations(content_type, body_decl): responses.add( method=responses.GET, url='https://test.nil/simple/project/', body=body_decl + b'<a href="../files/project-0.1.0-p\xC3\xBF42-none-any.whl">project-0.1.0-p\xC3\xBF42-none-any.whl</a>', content_type=content_type, ) simple = PyPISimple('https://test.nil/simple/') assert simple.get_project_files('project') == [ DistributionPackage( filename=u'project-0.1.0-p\xFF42-none-any.whl', project='project', version='0.1.0', package_type='wheel', url= u"https://test.nil/simple/files/project-0.1.0-p\xFF42-none-any.whl", requires_python=None, has_sig=False, yanked=None, ), ]
class PyPiClient: def __init__(self, cache_path: "str"): self.client = PyPISimple() self.session = requests.Session() self.cache_path = cache_path @functools.lru_cache(maxsize=2) def list_projects(self): try: return self.load_cached_mirror() except: return list(self.client.get_projects()) def new_projects(self): cached_projects = set(self.load_cached_mirror()) found_projects = set(self.list_projects()) return set.difference(cached_projects, found_projects) def download_mirror(self): projects = self.list_projects() with open(self.cache_path, 'w', encoding='utf-8-sig') as f: json.dump(projects, fp=f) def load_cached_mirror(self): with open(self.cache_path, 'r', encoding='utf-8-sig') as f: doc = json.load(fp=f) return doc @functools.lru_cache(maxsize=20) def extract_latest_project_file(self, project): project_files = self.client.get_project_files(project) max_version = packaging.version.parse('-1') max_file = None for project_file in project_files: if project_file.version is None: version = packaging.version.parse('0.0.0') else: version = packaging.version.parse(project_file.version) if project_file.package_type in ('sdist', 'wheel') and ( version > max_version or (version == max_version) and project_file.package_type == 'wheel'): max_file, max_version = project_file, version if not max_file: raise MissingFilesError('Missing files?') return max_version, max_file @functools.lru_cache(maxsize=20) def extract_setup_py(self, project): version, project_file = self.extract_latest_project_file(project) response = self.session.get(project_file.url, stream=True, timeout=10) fp = io.BytesIO(response.content) if project_file.filename.endswith( '.zip') or project_file.filename.endswith('.whl'): zf = zipfile.ZipFile(fp) if project_file.filename.endswith('.zip'): nameinfos = list( [ni for ni in zf.namelist() if ni.endswith('/setup.py')]) if nameinfos: return ('setup.py', version, zf.read(nameinfos[0])) raise MissingSetupError() else: nameinfos = list( [ni for ni in zf.namelist() if ni.endswith('/METADATA')]) if nameinfos: return ('metadata', version, zf.read(nameinfos[0])) raise MissingManifestError() else: tf = tarfile.open(fileobj=fp) nameinfos = list(ni for ni in tf.getnames() if ni.endswith('/setup.py')) if nameinfos: data = tf.extractfile(nameinfos[0]) return ('setup.py', version, data.read()) raise MissingSetupError()
def test_session(content_type): session_dir = join(dirname(__file__), 'data', 'session01') with open(join(session_dir, 'simple.html')) as fp: responses.add( method=responses.GET, url='https://test.nil/simple/', body=fp.read(), content_type=content_type, ) responses.add( method=responses.GET, url='https://test.nil/simple/', body='This URL should only be requested once.', status=500, ) with open(join(session_dir, 'in-place.html')) as fp: responses.add( method=responses.GET, url='https://test.nil/simple/in-place/', body=fp.read(), content_type=content_type, ) responses.add( method=responses.GET, url='https://test.nil/simple/nonexistent/', body='Does not exist', status=404, ) simple = PyPISimple('https://test.nil/simple/') assert list(simple.get_projects()) == ['in_place', 'foo', 'BAR'] assert simple.get_project_url( 'IN.PLACE') == 'https://test.nil/simple/in-place/' assert simple.get_project_files('IN.PLACE') == [ DistributionPackage( filename='in_place-0.1.1-py2.py3-none-any.whl', project='in_place', version='0.1.1', package_type='wheel', url= "https://files.pythonhosted.org/packages/34/81/2baaaa588ee1a6faa6354b7c9bc365f1b3da867707cd136dfedff7c06608/in_place-0.1.1-py2.py3-none-any.whl#sha256=e0732b6bdc2f1bfc4e1b96c1de2fbbd053bb2a9534547474a0485baa339bfa97", requires_python=None, has_sig=False, yanked=None, ), DistributionPackage( filename='in_place-0.1.1.tar.gz', project='in_place', version='0.1.1', package_type='sdist', url= "https://files.pythonhosted.org/packages/b9/ba/f1c67fb32c37ba4263326ae4ac6fd00b128025c9289b2fb31a60a0a22f90/in_place-0.1.1.tar.gz#sha256=ffa729fd0b818ac750aa31bafc886f266380e1c8557ba38f70f422d2f6a77e23", requires_python=None, has_sig=False, yanked=None, ), DistributionPackage( filename='in_place-0.2.0-py2.py3-none-any.whl', project='in_place', version='0.2.0', package_type='wheel', url= "https://files.pythonhosted.org/packages/9f/46/9f5679f3b2068e10b33c16a628a78b2b36531a9df08671bd0104f11d8461/in_place-0.2.0-py2.py3-none-any.whl#sha256=e1ad42a41dfde02092b411b1634a4be228e28c27553499a81ef04b377b28857c", requires_python=None, has_sig=False, yanked=None, ), DistributionPackage( filename='in_place-0.2.0.tar.gz', project='in_place', version='0.2.0', package_type='sdist', url= "https://files.pythonhosted.org/packages/f0/51/c30f1fad2b857f7b5d5ff76ec01f1f80dd0f2ab6b6afcde7b2aed54faa7e/in_place-0.2.0.tar.gz#sha256=ff783dca5d06f85b8d084871abd11a170d732423edb48c53ccb68c55fcbbeb76", requires_python=None, has_sig=False, yanked=None, ), DistributionPackage( filename='in_place-0.3.0-py2.py3-none-any.whl', project='in_place', version='0.3.0', package_type='wheel', url= "https://files.pythonhosted.org/packages/6f/84/ced31e646df335f8cd1b7884e3740b8c012314a28504542ef5631cdc1449/in_place-0.3.0-py2.py3-none-any.whl#sha256=af5ce9bd309f85a6bbe4119acbc0a67cda68f0ae616f0a76a947addc62791fda", requires_python=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, <4", has_sig=False, yanked=None, ), DistributionPackage( filename='in_place-0.3.0.tar.gz', project='in_place', version='0.3.0', package_type='sdist', url= "https://files.pythonhosted.org/packages/b6/cd/1dc736d5248420b15dd1546c2938aec7e6dab134e698e0768f54f1757af7/in_place-0.3.0.tar.gz#sha256=4758db1457c8addcd5f5b15ef870eab66b238e46e7d784bff99ab1b2126660ea", requires_python=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, <4", has_sig=False, yanked=None, ), ] assert simple.get_project_files('nonexistent') == []