def test_build_repository_with_specified_root(self, mock): from papaye.proxy import PyPiProxy from papaye.models import Package, ReleaseFile, Root mock.return_value = self.pypi_response info_dict = json.loads(self.pypi_response.content.decode('utf-8')) root = Root() proxy = PyPiProxy(self.request, 'pyramid') result = proxy.build_repository(with_metadata=True, root=root) self.assertIsInstance(result, Package) for release in result.releases.values(): self.assertIn(release.__name__, info_dict['releases'].keys()) self.assertEqual(len(result.releases.keys()), len(info_dict['releases'].keys())) self.assertEqual(len(result['1.5'].release_files.keys()), 1) self.assertIsInstance(result['1.5'].release_files.values()[0], ReleaseFile) self.assertTrue( getattr(result['1.5'].release_files.values()[0], 'pypi_url', None)) self.assertEqual( result['1.5'].release_files.values()[0].pypi_url, "https://pypi.python.org/packages/source/p/pyramid/pyramid-1.5.tar.gz" ) self.assertIsNotNone(result['1.5'].metadata) assert result.__parent__ is root
def test_smart_merge(self, mock): from papaye.proxy import PyPiProxy from papaye.factories import repository_root_factory from papaye.models import Package, Release, ReleaseFile mock.return_value = self.pypi_response root = repository_root_factory(self.request) # Existing releases root['pyramid'] = Package(name='pyramid') root['pyramid']['1.4'] = Release(name='1.4', version='1.4', metadata={}) root['pyramid']['1.4']['pyramid-1.4.tar.gz'] = ReleaseFile( filename='pyramid-1.4.tar.gz', content=b'', md5_digest='12345' ) package = Package(name='pyramid') package['1.5'] = Release(name='1.5', version='1.5', metadata={}) package['1.5']['pyramid-1.5.tar.gz'] = ReleaseFile( filename='pyramid-1.5.tar.gz', content=b'', md5_digest='12345' ) proxy = PyPiProxy(self.request, 'pyramid') result = proxy.smart_merge(root, package) self.assertEqual([key for key in result.releases.keys()], ['1.4', '1.5'])
def not_found(request): settings = request.registry.settings proxy = settings.get('papaye.proxy', False) proxy = True if proxy and proxy == 'true' else False if not proxy: return HTTPNotFound() try: proxy = PyPiProxy(request, request.matchdict['traverse'][0]) package = proxy.build_repository() if not package: return HTTPNotFound() if len(request.matchdict['traverse']) == 1: view = ListReleaseFileView(package, request) elif len(request.matchdict['traverse']) == 2: context = package[request.matchdict['traverse'][1]] view = ListReleaseFileByReleaseView(context, request) elif len(request.matchdict['traverse']) == 3: release_file = package[request.matchdict['traverse'][1]][ request.matchdict['traverse'][2]] package_name, release_name, _ = request.matchdict['traverse'] download_release_from_pypi.delay(package_name, release_name) return HTTPTemporaryRedirect(location=release_file.pypi_url) except KeyError: return HTTPNotFound() return view()
def download_release_from_pypi(config, package_name, release_name): try: repository_is_updated = False conn = get_connection(config) proxy = PyPiProxy(conn, package_name) root = repository_root_factory(conn) package = proxy.build_repository(release_name=release_name, with_metadata=True) if not package: logger.error('Package {} not found on PYPI'.format(package_name)) for release_file in package[release_name].release_files.values(): logger.info('Download file "{}"'.format(release_file.filename)) release_file.set_content( requests.get(release_file.pypi_url).content) with release_file.content.open() as content: binary_content = content.read() if hashlib.md5( binary_content).hexdigest() != release_file.md5_digest: continue release_file.size = len(binary_content) repository_is_updated = True package = proxy.smart_merge(root, package) if repository_is_updated: root[package.name] = package for release in package: release.__parent__ = package for release_file in release: release_file.__parent__ = release transaction.commit() except: transaction.abort()
def not_found(request, stop=None): if not proxy_activated(request.registry.settings) or stop is not None: return HTTPNotFound() proxy = PyPiProxy() package_name = request.matchdict['traverse'][0] if package_name in request.root: local_package = request.root[package_name] else: local_package = Package(package_name) local_package.fake = True merged_repository = proxy.merged_repository(local_package) if merged_repository: package = merged_repository[package_name] traversed = len(request.matchdict['traverse']) if traversed == 1: view = ListReleaseFileView(package, request) view.stop = True elif traversed == 2: context = package[request.matchdict['traverse'][1]] view = ListReleaseFileByReleaseView(context, request) elif traversed == 3: release_file = package[request.matchdict['traverse'][1]][request.matchdict['traverse'][2]] filename = request.matchdict['traverse'][2] package_name, release_name, _ = request.matchdict['traverse'] download_release_from_pypi.delay(request.registry._zodb_databases[''], package_name, release_name, filename) return HTTPTemporaryRedirect(location=release_file.pypi_url) return view() return HTTPNotFound()
def test_smart_merge_with_existing_release(self, mock): from papaye.proxy import PyPiProxy from papaye.factories import repository_root_factory from papaye.models import Package, Release, ReleaseFile mock.return_value = self.pypi_response root = repository_root_factory(self.request) # Existing releases package = Package(name='pyramid') package['1.5'] = Release(name='1.5', version='1.5', metadata={}) package['1.5']['pyramid-1.5.tar.gz'] = ReleaseFile( filename='pyramid-1.5.tar.gz', content=b'', md5_digest='12345') package = Package(name='pyramid') package['1.5'] = Release(name='1.5', version='1.5', metadata={}) package['1.5']['pyramid-1.5.tar.gz'] = ReleaseFile( filename='pyramid-1.5.tar.gz', content=b'', md5_digest='12345') proxy = PyPiProxy(self.request, 'pyramid') result = proxy.smart_merge(root, package) self.assertEqual([key for key in result.releases.keys()], [ '1.5', ]) self.assertEqual(result['1.5']['pyramid-1.5.tar.gz'].md5_digest, '12345')
def test_get_remote_informations_connection_error(self, mock): from papaye.proxy import PyPiProxy mock.side_effect = ConnectionError url = "http://pypi.python.org/pypi/pyramid/json" proxy = PyPiProxy(self.request, 'pyramid') result = proxy.get_remote_informations(url) self.assertIsNone(result)
def test_get_remote_informations_404(self, mock): from papaye.proxy import PyPiProxy mock.return_value = FakeGRequestResponse(404, b'') url = "http://pypi.python.org/pypi/pyramid/json" proxy = PyPiProxy(self.request, 'pyramid') result = proxy.get_remote_informations(url) self.assertIsNone(result)
def test_get_remote_package_name(self, mock): from papaye.proxy import PyPiProxy mock.side_effect = ConnectionError url = "http://pypi.python.org/pypi/pyramid/json" proxy = PyPiProxy() result = proxy.get_remote_package_name(url) self.assertIsNone(result)
def test_build_repository_404_error(self, mock): from papaye.proxy import PyPiProxy mock.return_value = FakeGRequestResponse(404, b'') proxy = PyPiProxy(self.request, 'pyramid') result = proxy.build_repository() self.assertIsNone(result)
def test_build_remote_repository_with_unknown_package(self, mock): from papaye.proxy import PyPiProxy mock.return_value = FakeGRequestResponse(404, b'') proxy = PyPiProxy() result = proxy.build_remote_repository('pyramid', metadata=True) assert result is None
def test_get_remote_informations(self, mock): from papaye.proxy import PyPiProxy mock.return_value = self.pypi_response url = "http://pypi.python.org/pypi/pyramid/json" proxy = PyPiProxy(self.request, 'pyramid') result = proxy.get_remote_informations(url) self.assertIsInstance(result, dict) self.assertEqual(result['info']['name'], 'pyramid')
def test_build_remote_repository_with_release_name(self, mock): from papaye.models import Root from papaye.proxy import PyPiProxy mock_proxy_response(mock) proxy = PyPiProxy() result = proxy.build_remote_repository('pyramid', release_name='1.5') assert isinstance(result, Root) assert 'pyramid' in list(result.keys()) assert len(list(result['pyramid'])) == 1
def test_build_remote_repository(self, mock): from papaye.models import Root from papaye.proxy import PyPiProxy mock_proxy_response(mock) proxy = PyPiProxy() result = proxy.build_remote_repository('pyramid') assert isinstance(result, Root) assert 'pyramid' in [e.__name__ for e in result] assert len(list(result['pyramid'])) == 81 assert not hasattr(list(result['pyramid'])[0], 'metadata')
def test_build_remote_repository_with_metadata(self, mock): from papaye.models import Root from papaye.proxy import PyPiProxy mock_proxy_response(mock) proxy = PyPiProxy() result = proxy.build_remote_repository('pyramid', metadata=True) assert isinstance(result, Root) assert 'pyramid' in list(result.keys()) assert len(list(result['pyramid'])) == 81 assert hasattr(list(result['pyramid'])[0], 'metadata') assert len(list(result['pyramid'])[0].metadata.keys()) == 13
def test_get_remote_informations(self, mock): # Given from papaye.proxy import PyPiProxy mock_proxy_response(mock) url = "http://pypi.python.org/pypi/pyramid/json" proxy = PyPiProxy() # When result = proxy.get_remote_informations(url) # Then self.assertIsInstance(result, dict) self.assertEqual(result['info']['name'], 'pyramid')
def __call__(self): package = self.context proxy = PyPiProxy() stop = hasattr(self, 'stop') and self.stop repository = package.__parent__ if stop else proxy.merged_repository(package) rfiles = [rfile for rel in repository[package.__name__] for rfile in rel] context = {'objects': ((self.request.resource_url( rfile, route_name='simple' )[:-1] + "#md5={}".format(rfile.md5_digest), rfile) for rfile in rfiles)} transaction.abort() if len(rfiles): return context elif stop: return HTTPNotFound() else: return not_found(self.request)
def test_merged_repository_without_repository_package(self, mock): from papaye.models import Package, Release, ReleaseFile, Root from papaye.proxy import PyPiProxy mock.return_value = FakeGRequestResponse(404, b'') package1 = Package(name='pyramid') package1['100.5'] = Release(version='100.5', metadata={}) package1['100.5']['pyramid-100.5.tar.gz'] = ReleaseFile( filename='pyramid-100.5.tar.gz', content=b'a existing content', md5_digest='12345' ) proxy = PyPiProxy() result = proxy.merged_repository(package1, metadata=True) assert isinstance(result, Root) assert [pack.__name__ for pack in result] == ["pyramid", ] assert len(list(result['pyramid'])) == 1
def test_merged_repository_with_release_name(self, mock): from papaye.models import Package, Release, ReleaseFile, Root from papaye.proxy import PyPiProxy mock_proxy_response(mock) package1 = Package(name='pyramid') package1['100.5'] = Release(version='100.5', metadata={}) package1['100.5']['pyramid-100.5.tar.gz'] = ReleaseFile( filename='pyramid-100.5.tar.gz', content=b'a existing content', md5_digest='12345' ) proxy = PyPiProxy() result = proxy.merged_repository(package1, release_name='1.5') assert isinstance(result, Root) assert [pack.__name__ for pack in result] == ["pyramid", ] assert len(list(result['pyramid'])) == 2 assert '1.5' in result['pyramid'].releases.keys() assert '100.5' in [rel for rel in result['pyramid'].releases]
def test_build_repository(self, mock): from papaye.proxy import PyPiProxy from papaye.models import Package, ReleaseFile mock.return_value = self.pypi_response info_dict = json.loads(self.pypi_response.content.decode('utf-8')) proxy = PyPiProxy(self.request, 'pyramid') result = proxy.build_repository() self.assertIsInstance(result, Package) for release in result.releases.values(): self.assertIn(release.__name__, info_dict['releases'].keys()) self.assertEqual(len(result.releases.keys()), len(info_dict['releases'].keys())) self.assertEqual(len(result['1.5'].release_files.keys()), 1) self.assertIsInstance(result['1.5'].release_files.values()[0], ReleaseFile) self.assertTrue(getattr(result['1.5'].release_files.values()[0], 'pypi_url', None)) self.assertEqual(result['1.5'].release_files.values()[0].pypi_url, "https://pypi.python.org/packages/source/p/pyramid/pyramid-1.5.tar.gz") self.assertIsNotNone(result['1.5'].metadata) self.assertIsInstance(result['1.5'].metadata, dict)
def download_release_from_pypi(config, package_name, release_name): conn = get_connection(config) proxy = PyPiProxy(conn, package_name) package = proxy.build_repository(release_name=release_name) if not package: logger.error('Package {} not found on PYPI'.format(package_name)) root = repository_root_factory(conn) for release_file in package[release_name].release_files.values(): logger.info('Download file "{}"'.format(release_file.filename)) release_file.set_content(requests.get(release_file.pypi_url).content) with release_file.content.open() as content: binary_content = content.read() if hashlib.md5(binary_content).hexdigest() != release_file.md5_digest: raise IOError('md5 check error') release_file.size = len(binary_content) root[package.name] = package try: transaction.commit() except: transaction.abort()
def not_found(request): settings = request.registry.settings proxy = settings.get('papaye.proxy', False) proxy = True if proxy and proxy == 'true' else False if not proxy: return HTTPNotFound() try: proxy = PyPiProxy(request, request.matchdict['traverse'][0]) package = proxy.build_repository() if not package: return HTTPNotFound() if len(request.matchdict['traverse']) == 1: view = ListReleaseFileView(package, request) elif len(request.matchdict['traverse']) == 2: context = package[request.matchdict['traverse'][1]] view = ListReleaseFileByReleaseView(context, request) elif len(request.matchdict['traverse']) == 3: release_file = package[request.matchdict['traverse'][1]][request.matchdict['traverse'][2]] package_name, release_name, _ = request.matchdict['traverse'] download_release_from_pypi.delay(package_name, release_name) return HTTPTemporaryRedirect(location=release_file.pypi_url) except KeyError: return HTTPNotFound() return view()
class ListReleaseFileView(BaseView): _repository = None def __init__(self, context, request): super().__init__(context, request) self.proxy_instance = PyPiProxy() self.stop = hasattr(self, 'stop') and self.stop @property def repository(self): if self._repository is None: if self.stop: self._repository = self.context.root else: self._repository = self.proxy_instance.merged_repository( self.context ) return self._repository def __call__(self): package = self.context root = package.root if self.repository is not root: self.repository.name = root.name rfiles = [ rfile for rel in self.repository[package.name] for rfile in rel ] context = { 'objects': (self.format_release_file(rfile) for rfile in rfiles) } transaction.abort() if len(rfiles): return context elif self.stop: return HTTPNotFound() else: return not_found(self.request) def format_release_file(self, release_file): return self.request.resource_url( release_file, route_name='simple' )[:-1] + "#md5={}".format(release_file.md5_digest), release_file
def __init__(self, context, request): super().__init__(context, request) self.proxy_instance = PyPiProxy() self.stop = hasattr(self, 'stop') and self.stop