def test_build_repository_with_specified_root(self, mock):
from papaye.proxy import PyPiProxy
from papaye.models import Package, ReleaseFile, Root
mock.return_value = self.pypi_response
info_dict = json.loads(self.pypi_response.content.decode('utf-8'))
root = Root()
proxy = PyPiProxy(self.request, 'pyramid')
result = proxy.build_repository(with_metadata=True, root=root)
self.assertIsInstance(result, Package)
for release in result.releases.values():
self.assertIn(release.__name__, info_dict['releases'].keys())
self.assertEqual(len(result.releases.keys()),
len(info_dict['releases'].keys()))
self.assertEqual(len(result['1.5'].release_files.keys()), 1)
self.assertIsInstance(result['1.5'].release_files.values()[0],
ReleaseFile)
self.assertTrue(
getattr(result['1.5'].release_files.values()[0], 'pypi_url', None))
self.assertEqual(
result['1.5'].release_files.values()[0].pypi_url,
"https://pypi.python.org/packages/source/p/pyramid/pyramid-1.5.tar.gz"
)
self.assertIsNotNone(result['1.5'].metadata)
assert result.__parent__ is root
def test_smart_merge(self, mock):
from papaye.proxy import PyPiProxy
from papaye.factories import repository_root_factory
from papaye.models import Package, Release, ReleaseFile
mock.return_value = self.pypi_response
root = repository_root_factory(self.request)
# Existing releases
root['pyramid'] = Package(name='pyramid')
root['pyramid']['1.4'] = Release(name='1.4', version='1.4', metadata={})
root['pyramid']['1.4']['pyramid-1.4.tar.gz'] = ReleaseFile(
filename='pyramid-1.4.tar.gz',
content=b'',
md5_digest='12345'
)
package = Package(name='pyramid')
package['1.5'] = Release(name='1.5', version='1.5', metadata={})
package['1.5']['pyramid-1.5.tar.gz'] = ReleaseFile(
filename='pyramid-1.5.tar.gz',
content=b'',
md5_digest='12345'
)
proxy = PyPiProxy(self.request, 'pyramid')
result = proxy.smart_merge(root, package)
self.assertEqual([key for key in result.releases.keys()], ['1.4', '1.5'])
def not_found(request):
settings = request.registry.settings
proxy = settings.get('papaye.proxy', False)
proxy = True if proxy and proxy == 'true' else False
if not proxy:
return HTTPNotFound()
try:
proxy = PyPiProxy(request, request.matchdict['traverse'][0])
package = proxy.build_repository()
if not package:
return HTTPNotFound()
if len(request.matchdict['traverse']) == 1:
view = ListReleaseFileView(package, request)
elif len(request.matchdict['traverse']) == 2:
context = package[request.matchdict['traverse'][1]]
view = ListReleaseFileByReleaseView(context, request)
elif len(request.matchdict['traverse']) == 3:
release_file = package[request.matchdict['traverse'][1]][
request.matchdict['traverse'][2]]
package_name, release_name, _ = request.matchdict['traverse']
download_release_from_pypi.delay(package_name, release_name)
return HTTPTemporaryRedirect(location=release_file.pypi_url)
except KeyError:
return HTTPNotFound()
return view()
def download_release_from_pypi(config, package_name, release_name):
try:
repository_is_updated = False
conn = get_connection(config)
proxy = PyPiProxy(conn, package_name)
root = repository_root_factory(conn)
package = proxy.build_repository(release_name=release_name,
with_metadata=True)
if not package:
logger.error('Package {} not found on PYPI'.format(package_name))
for release_file in package[release_name].release_files.values():
logger.info('Download file "{}"'.format(release_file.filename))
release_file.set_content(
requests.get(release_file.pypi_url).content)
with release_file.content.open() as content:
binary_content = content.read()
if hashlib.md5(
binary_content).hexdigest() != release_file.md5_digest:
continue
release_file.size = len(binary_content)
repository_is_updated = True
package = proxy.smart_merge(root, package)
if repository_is_updated:
root[package.name] = package
for release in package:
release.__parent__ = package
for release_file in release:
release_file.__parent__ = release
transaction.commit()
except:
transaction.abort()
def not_found(request, stop=None):
if not proxy_activated(request.registry.settings) or stop is not None:
return HTTPNotFound()
proxy = PyPiProxy()
package_name = request.matchdict['traverse'][0]
if package_name in request.root:
local_package = request.root[package_name]
else:
local_package = Package(package_name)
local_package.fake = True
merged_repository = proxy.merged_repository(local_package)
if merged_repository:
package = merged_repository[package_name]
traversed = len(request.matchdict['traverse'])
if traversed == 1:
view = ListReleaseFileView(package, request)
view.stop = True
elif traversed == 2:
context = package[request.matchdict['traverse'][1]]
view = ListReleaseFileByReleaseView(context, request)
elif traversed == 3:
release_file = package[request.matchdict['traverse'][1]][request.matchdict['traverse'][2]]
filename = request.matchdict['traverse'][2]
package_name, release_name, _ = request.matchdict['traverse']
download_release_from_pypi.delay(request.registry._zodb_databases[''], package_name, release_name, filename)
return HTTPTemporaryRedirect(location=release_file.pypi_url)
return view()
return HTTPNotFound()
def test_smart_merge_with_existing_release(self, mock):
from papaye.proxy import PyPiProxy
from papaye.factories import repository_root_factory
from papaye.models import Package, Release, ReleaseFile
mock.return_value = self.pypi_response
root = repository_root_factory(self.request)
# Existing releases
package = Package(name='pyramid')
package['1.5'] = Release(name='1.5', version='1.5', metadata={})
package['1.5']['pyramid-1.5.tar.gz'] = ReleaseFile(
filename='pyramid-1.5.tar.gz', content=b'', md5_digest='12345')
package = Package(name='pyramid')
package['1.5'] = Release(name='1.5', version='1.5', metadata={})
package['1.5']['pyramid-1.5.tar.gz'] = ReleaseFile(
filename='pyramid-1.5.tar.gz', content=b'', md5_digest='12345')
proxy = PyPiProxy(self.request, 'pyramid')
result = proxy.smart_merge(root, package)
self.assertEqual([key for key in result.releases.keys()], [
'1.5',
])
self.assertEqual(result['1.5']['pyramid-1.5.tar.gz'].md5_digest,
'12345')
def test_get_remote_informations_connection_error(self, mock):
from papaye.proxy import PyPiProxy
mock.side_effect = ConnectionError
url = "http://pypi.python.org/pypi/pyramid/json"
proxy = PyPiProxy(self.request, 'pyramid')
result = proxy.get_remote_informations(url)
self.assertIsNone(result)
def test_get_remote_informations_404(self, mock):
from papaye.proxy import PyPiProxy
mock.return_value = FakeGRequestResponse(404, b'')
url = "http://pypi.python.org/pypi/pyramid/json"
proxy = PyPiProxy(self.request, 'pyramid')
result = proxy.get_remote_informations(url)
self.assertIsNone(result)
def test_get_remote_informations_connection_error(self, mock):
from papaye.proxy import PyPiProxy
mock.side_effect = ConnectionError
url = "http://pypi.python.org/pypi/pyramid/json"
proxy = PyPiProxy(self.request, 'pyramid')
result = proxy.get_remote_informations(url)
self.assertIsNone(result)
def test_get_remote_package_name(self, mock):
from papaye.proxy import PyPiProxy
mock.side_effect = ConnectionError
url = "http://pypi.python.org/pypi/pyramid/json"
proxy = PyPiProxy()
result = proxy.get_remote_package_name(url)
self.assertIsNone(result)
def test_build_repository_404_error(self, mock):
from papaye.proxy import PyPiProxy
mock.return_value = FakeGRequestResponse(404, b'')
proxy = PyPiProxy(self.request, 'pyramid')
result = proxy.build_repository()
self.assertIsNone(result)
def test_get_remote_informations_404(self, mock):
from papaye.proxy import PyPiProxy
mock.return_value = FakeGRequestResponse(404, b'')
url = "http://pypi.python.org/pypi/pyramid/json"
proxy = PyPiProxy(self.request, 'pyramid')
result = proxy.get_remote_informations(url)
self.assertIsNone(result)
def test_build_repository_404_error(self, mock):
from papaye.proxy import PyPiProxy
mock.return_value = FakeGRequestResponse(404, b'')
proxy = PyPiProxy(self.request, 'pyramid')
result = proxy.build_repository()
self.assertIsNone(result)
def test_build_remote_repository_with_unknown_package(self, mock):
from papaye.proxy import PyPiProxy
mock.return_value = FakeGRequestResponse(404, b'')
proxy = PyPiProxy()
result = proxy.build_remote_repository('pyramid', metadata=True)
assert result is None
def test_get_remote_informations(self, mock):
from papaye.proxy import PyPiProxy
mock.return_value = self.pypi_response
url = "http://pypi.python.org/pypi/pyramid/json"
proxy = PyPiProxy(self.request, 'pyramid')
result = proxy.get_remote_informations(url)
self.assertIsInstance(result, dict)
self.assertEqual(result['info']['name'], 'pyramid')
def test_get_remote_informations(self, mock):
from papaye.proxy import PyPiProxy
mock.return_value = self.pypi_response
url = "http://pypi.python.org/pypi/pyramid/json"
proxy = PyPiProxy(self.request, 'pyramid')
result = proxy.get_remote_informations(url)
self.assertIsInstance(result, dict)
self.assertEqual(result['info']['name'], 'pyramid')
def test_build_remote_repository_with_release_name(self, mock):
from papaye.models import Root
from papaye.proxy import PyPiProxy
mock_proxy_response(mock)
proxy = PyPiProxy()
result = proxy.build_remote_repository('pyramid', release_name='1.5')
assert isinstance(result, Root)
assert 'pyramid' in list(result.keys())
assert len(list(result['pyramid'])) == 1
def test_build_remote_repository(self, mock):
from papaye.models import Root
from papaye.proxy import PyPiProxy
mock_proxy_response(mock)
proxy = PyPiProxy()
result = proxy.build_remote_repository('pyramid')
assert isinstance(result, Root)
assert 'pyramid' in [e.__name__ for e in result]
assert len(list(result['pyramid'])) == 81
assert not hasattr(list(result['pyramid'])[0], 'metadata')
def test_build_remote_repository_with_metadata(self, mock):
from papaye.models import Root
from papaye.proxy import PyPiProxy
mock_proxy_response(mock)
proxy = PyPiProxy()
result = proxy.build_remote_repository('pyramid', metadata=True)
assert isinstance(result, Root)
assert 'pyramid' in list(result.keys())
assert len(list(result['pyramid'])) == 81
assert hasattr(list(result['pyramid'])[0], 'metadata')
assert len(list(result['pyramid'])[0].metadata.keys()) == 13
def test_get_remote_informations(self, mock):
# Given
from papaye.proxy import PyPiProxy
mock_proxy_response(mock)
url = "http://pypi.python.org/pypi/pyramid/json"
proxy = PyPiProxy()
# When
result = proxy.get_remote_informations(url)
# Then
self.assertIsInstance(result, dict)
self.assertEqual(result['info']['name'], 'pyramid')
def __call__(self):
package = self.context
proxy = PyPiProxy()
stop = hasattr(self, 'stop') and self.stop
repository = package.__parent__ if stop else proxy.merged_repository(package)
rfiles = [rfile for rel in repository[package.__name__] for rfile in rel]
context = {'objects': ((self.request.resource_url(
rfile,
route_name='simple'
)[:-1] + "#md5={}".format(rfile.md5_digest), rfile) for rfile in rfiles)}
transaction.abort()
if len(rfiles):
return context
elif stop:
return HTTPNotFound()
else:
return not_found(self.request)
def test_merged_repository_without_repository_package(self, mock):
from papaye.models import Package, Release, ReleaseFile, Root
from papaye.proxy import PyPiProxy
mock.return_value = FakeGRequestResponse(404, b'')
package1 = Package(name='pyramid')
package1['100.5'] = Release(version='100.5', metadata={})
package1['100.5']['pyramid-100.5.tar.gz'] = ReleaseFile(
filename='pyramid-100.5.tar.gz',
content=b'a existing content',
md5_digest='12345'
)
proxy = PyPiProxy()
result = proxy.merged_repository(package1, metadata=True)
assert isinstance(result, Root)
assert [pack.__name__ for pack in result] == ["pyramid", ]
assert len(list(result['pyramid'])) == 1
def test_merged_repository_with_release_name(self, mock):
from papaye.models import Package, Release, ReleaseFile, Root
from papaye.proxy import PyPiProxy
mock_proxy_response(mock)
package1 = Package(name='pyramid')
package1['100.5'] = Release(version='100.5', metadata={})
package1['100.5']['pyramid-100.5.tar.gz'] = ReleaseFile(
filename='pyramid-100.5.tar.gz',
content=b'a existing content',
md5_digest='12345'
)
proxy = PyPiProxy()
result = proxy.merged_repository(package1, release_name='1.5')
assert isinstance(result, Root)
assert [pack.__name__ for pack in result] == ["pyramid", ]
assert len(list(result['pyramid'])) == 2
assert '1.5' in result['pyramid'].releases.keys()
assert '100.5' in [rel for rel in result['pyramid'].releases]
def test_build_repository(self, mock):
from papaye.proxy import PyPiProxy
from papaye.models import Package, ReleaseFile
mock.return_value = self.pypi_response
info_dict = json.loads(self.pypi_response.content.decode('utf-8'))
proxy = PyPiProxy(self.request, 'pyramid')
result = proxy.build_repository()
self.assertIsInstance(result, Package)
for release in result.releases.values():
self.assertIn(release.__name__, info_dict['releases'].keys())
self.assertEqual(len(result.releases.keys()), len(info_dict['releases'].keys()))
self.assertEqual(len(result['1.5'].release_files.keys()), 1)
self.assertIsInstance(result['1.5'].release_files.values()[0], ReleaseFile)
self.assertTrue(getattr(result['1.5'].release_files.values()[0], 'pypi_url', None))
self.assertEqual(result['1.5'].release_files.values()[0].pypi_url,
"https://pypi.python.org/packages/source/p/pyramid/pyramid-1.5.tar.gz")
self.assertIsNotNone(result['1.5'].metadata)
self.assertIsInstance(result['1.5'].metadata, dict)
def download_release_from_pypi(config, package_name, release_name):
conn = get_connection(config)
proxy = PyPiProxy(conn, package_name)
package = proxy.build_repository(release_name=release_name)
if not package:
logger.error('Package {} not found on PYPI'.format(package_name))
root = repository_root_factory(conn)
for release_file in package[release_name].release_files.values():
logger.info('Download file "{}"'.format(release_file.filename))
release_file.set_content(requests.get(release_file.pypi_url).content)
with release_file.content.open() as content:
binary_content = content.read()
if hashlib.md5(binary_content).hexdigest() != release_file.md5_digest:
raise IOError('md5 check error')
release_file.size = len(binary_content)
root[package.name] = package
try:
transaction.commit()
except:
transaction.abort()
def not_found(request):
settings = request.registry.settings
proxy = settings.get('papaye.proxy', False)
proxy = True if proxy and proxy == 'true' else False
if not proxy:
return HTTPNotFound()
try:
proxy = PyPiProxy(request, request.matchdict['traverse'][0])
package = proxy.build_repository()
if not package:
return HTTPNotFound()
if len(request.matchdict['traverse']) == 1:
view = ListReleaseFileView(package, request)
elif len(request.matchdict['traverse']) == 2:
context = package[request.matchdict['traverse'][1]]
view = ListReleaseFileByReleaseView(context, request)
elif len(request.matchdict['traverse']) == 3:
release_file = package[request.matchdict['traverse'][1]][request.matchdict['traverse'][2]]
package_name, release_name, _ = request.matchdict['traverse']
download_release_from_pypi.delay(package_name, release_name)
return HTTPTemporaryRedirect(location=release_file.pypi_url)
except KeyError:
return HTTPNotFound()
return view()
class ListReleaseFileView(BaseView):
_repository = None
def __init__(self, context, request):
super().__init__(context, request)
self.proxy_instance = PyPiProxy()
self.stop = hasattr(self, 'stop') and self.stop
@property
def repository(self):
if self._repository is None:
if self.stop:
self._repository = self.context.root
else:
self._repository = self.proxy_instance.merged_repository(
self.context
)
return self._repository
def __call__(self):
package = self.context
root = package.root
if self.repository is not root:
self.repository.name = root.name
rfiles = [
rfile for rel in self.repository[package.name]
for rfile in rel
]
context = {
'objects': (self.format_release_file(rfile) for rfile in rfiles)
}
transaction.abort()
if len(rfiles):
return context
elif self.stop:
return HTTPNotFound()
else:
return not_found(self.request)
def format_release_file(self, release_file):
return self.request.resource_url(
release_file,
route_name='simple'
)[:-1] + "#md5={}".format(release_file.md5_digest), release_file
def __init__(self, context, request):
super().__init__(context, request)
self.proxy_instance = PyPiProxy()
self.stop = hasattr(self, 'stop') and self.stop
