Ejemplo n.º 1
0
    def test_build_repository_with_specified_root(self, mock):
        from papaye.proxy import PyPiProxy
        from papaye.models import Package, ReleaseFile, Root
        mock.return_value = self.pypi_response
        info_dict = json.loads(self.pypi_response.content.decode('utf-8'))
        root = Root()

        proxy = PyPiProxy(self.request, 'pyramid')
        result = proxy.build_repository(with_metadata=True, root=root)

        self.assertIsInstance(result, Package)
        for release in result.releases.values():
            self.assertIn(release.__name__, info_dict['releases'].keys())
        self.assertEqual(len(result.releases.keys()),
                         len(info_dict['releases'].keys()))
        self.assertEqual(len(result['1.5'].release_files.keys()), 1)
        self.assertIsInstance(result['1.5'].release_files.values()[0],
                              ReleaseFile)
        self.assertTrue(
            getattr(result['1.5'].release_files.values()[0], 'pypi_url', None))
        self.assertEqual(
            result['1.5'].release_files.values()[0].pypi_url,
            "https://pypi.python.org/packages/source/p/pyramid/pyramid-1.5.tar.gz"
        )
        self.assertIsNotNone(result['1.5'].metadata)
        assert result.__parent__ is root
Ejemplo n.º 2
0
    def test_smart_merge(self, mock):
        from papaye.proxy import PyPiProxy
        from papaye.factories import repository_root_factory
        from papaye.models import Package, Release, ReleaseFile
        mock.return_value = self.pypi_response
        root = repository_root_factory(self.request)

        # Existing releases
        root['pyramid'] = Package(name='pyramid')
        root['pyramid']['1.4'] = Release(name='1.4', version='1.4', metadata={})
        root['pyramid']['1.4']['pyramid-1.4.tar.gz'] = ReleaseFile(
            filename='pyramid-1.4.tar.gz',
            content=b'',
            md5_digest='12345'
        )

        package = Package(name='pyramid')
        package['1.5'] = Release(name='1.5', version='1.5', metadata={})
        package['1.5']['pyramid-1.5.tar.gz'] = ReleaseFile(
            filename='pyramid-1.5.tar.gz',
            content=b'',
            md5_digest='12345'
        )

        proxy = PyPiProxy(self.request, 'pyramid')
        result = proxy.smart_merge(root, package)

        self.assertEqual([key for key in result.releases.keys()], ['1.4', '1.5'])
Ejemplo n.º 3
0
def not_found(request):
    settings = request.registry.settings
    proxy = settings.get('papaye.proxy', False)
    proxy = True if proxy and proxy == 'true' else False
    if not proxy:
        return HTTPNotFound()
    try:
        proxy = PyPiProxy(request, request.matchdict['traverse'][0])
        package = proxy.build_repository()
        if not package:
            return HTTPNotFound()
        if len(request.matchdict['traverse']) == 1:
            view = ListReleaseFileView(package, request)
        elif len(request.matchdict['traverse']) == 2:
            context = package[request.matchdict['traverse'][1]]
            view = ListReleaseFileByReleaseView(context, request)
        elif len(request.matchdict['traverse']) == 3:
            release_file = package[request.matchdict['traverse'][1]][
                request.matchdict['traverse'][2]]
            package_name, release_name, _ = request.matchdict['traverse']
            download_release_from_pypi.delay(package_name, release_name)
            return HTTPTemporaryRedirect(location=release_file.pypi_url)
    except KeyError:
        return HTTPNotFound()
    return view()
Ejemplo n.º 4
0
def download_release_from_pypi(config, package_name, release_name):
    try:
        repository_is_updated = False
        conn = get_connection(config)
        proxy = PyPiProxy(conn, package_name)
        root = repository_root_factory(conn)
        package = proxy.build_repository(release_name=release_name,
                                         with_metadata=True)
        if not package:
            logger.error('Package {} not found on PYPI'.format(package_name))
        for release_file in package[release_name].release_files.values():
            logger.info('Download file "{}"'.format(release_file.filename))
            release_file.set_content(
                requests.get(release_file.pypi_url).content)
            with release_file.content.open() as content:
                binary_content = content.read()
                if hashlib.md5(
                        binary_content).hexdigest() != release_file.md5_digest:
                    continue
            release_file.size = len(binary_content)
            repository_is_updated = True

        package = proxy.smart_merge(root, package)
        if repository_is_updated:
            root[package.name] = package
            for release in package:
                release.__parent__ = package
                for release_file in release:
                    release_file.__parent__ = release

        transaction.commit()
    except:
        transaction.abort()
Ejemplo n.º 5
0
def not_found(request, stop=None):
    if not proxy_activated(request.registry.settings) or stop is not None:
        return HTTPNotFound()
    proxy = PyPiProxy()
    package_name = request.matchdict['traverse'][0]
    if package_name in request.root:
        local_package = request.root[package_name]
    else:
        local_package = Package(package_name)
        local_package.fake = True

    merged_repository = proxy.merged_repository(local_package)

    if merged_repository:
        package = merged_repository[package_name]
        traversed = len(request.matchdict['traverse'])
        if traversed == 1:
            view = ListReleaseFileView(package, request)
            view.stop = True
        elif traversed == 2:
            context = package[request.matchdict['traverse'][1]]
            view = ListReleaseFileByReleaseView(context, request)
        elif traversed == 3:
            release_file = package[request.matchdict['traverse'][1]][request.matchdict['traverse'][2]]
            filename = request.matchdict['traverse'][2]
            package_name, release_name, _ = request.matchdict['traverse']
            download_release_from_pypi.delay(request.registry._zodb_databases[''], package_name, release_name, filename)
            return HTTPTemporaryRedirect(location=release_file.pypi_url)
        return view()
    return HTTPNotFound()
Ejemplo n.º 6
0
    def test_smart_merge_with_existing_release(self, mock):
        from papaye.proxy import PyPiProxy
        from papaye.factories import repository_root_factory
        from papaye.models import Package, Release, ReleaseFile
        mock.return_value = self.pypi_response
        root = repository_root_factory(self.request)

        # Existing releases
        package = Package(name='pyramid')
        package['1.5'] = Release(name='1.5', version='1.5', metadata={})
        package['1.5']['pyramid-1.5.tar.gz'] = ReleaseFile(
            filename='pyramid-1.5.tar.gz', content=b'', md5_digest='12345')

        package = Package(name='pyramid')
        package['1.5'] = Release(name='1.5', version='1.5', metadata={})
        package['1.5']['pyramid-1.5.tar.gz'] = ReleaseFile(
            filename='pyramid-1.5.tar.gz', content=b'', md5_digest='12345')

        proxy = PyPiProxy(self.request, 'pyramid')
        result = proxy.smart_merge(root, package)

        self.assertEqual([key for key in result.releases.keys()], [
            '1.5',
        ])
        self.assertEqual(result['1.5']['pyramid-1.5.tar.gz'].md5_digest,
                         '12345')
Ejemplo n.º 7
0
    def test_get_remote_informations_connection_error(self, mock):
        from papaye.proxy import PyPiProxy
        mock.side_effect = ConnectionError
        url = "http://pypi.python.org/pypi/pyramid/json"

        proxy = PyPiProxy(self.request, 'pyramid')
        result = proxy.get_remote_informations(url)
        self.assertIsNone(result)
Ejemplo n.º 8
0
    def test_get_remote_informations_404(self, mock):
        from papaye.proxy import PyPiProxy
        mock.return_value = FakeGRequestResponse(404, b'')
        url = "http://pypi.python.org/pypi/pyramid/json"

        proxy = PyPiProxy(self.request, 'pyramid')
        result = proxy.get_remote_informations(url)
        self.assertIsNone(result)
Ejemplo n.º 9
0
    def test_get_remote_informations_connection_error(self, mock):
        from papaye.proxy import PyPiProxy
        mock.side_effect = ConnectionError
        url = "http://pypi.python.org/pypi/pyramid/json"

        proxy = PyPiProxy(self.request, 'pyramid')
        result = proxy.get_remote_informations(url)
        self.assertIsNone(result)
Ejemplo n.º 10
0
    def test_get_remote_package_name(self, mock):
        from papaye.proxy import PyPiProxy
        mock.side_effect = ConnectionError
        url = "http://pypi.python.org/pypi/pyramid/json"

        proxy = PyPiProxy()
        result = proxy.get_remote_package_name(url)
        self.assertIsNone(result)
Ejemplo n.º 11
0
    def test_build_repository_404_error(self, mock):
        from papaye.proxy import PyPiProxy
        mock.return_value = FakeGRequestResponse(404, b'')

        proxy = PyPiProxy(self.request, 'pyramid')
        result = proxy.build_repository()

        self.assertIsNone(result)
Ejemplo n.º 12
0
    def test_get_remote_informations_404(self, mock):
        from papaye.proxy import PyPiProxy
        mock.return_value = FakeGRequestResponse(404, b'')
        url = "http://pypi.python.org/pypi/pyramid/json"

        proxy = PyPiProxy(self.request, 'pyramid')
        result = proxy.get_remote_informations(url)
        self.assertIsNone(result)
Ejemplo n.º 13
0
    def test_build_repository_404_error(self, mock):
        from papaye.proxy import PyPiProxy
        mock.return_value = FakeGRequestResponse(404, b'')

        proxy = PyPiProxy(self.request, 'pyramid')
        result = proxy.build_repository()

        self.assertIsNone(result)
Ejemplo n.º 14
0
    def test_build_remote_repository_with_unknown_package(self, mock):
        from papaye.proxy import PyPiProxy
        mock.return_value = FakeGRequestResponse(404, b'')
        proxy = PyPiProxy()

        result = proxy.build_remote_repository('pyramid', metadata=True)

        assert result is None
Ejemplo n.º 15
0
    def test_get_remote_informations(self, mock):
        from papaye.proxy import PyPiProxy
        mock.return_value = self.pypi_response
        url = "http://pypi.python.org/pypi/pyramid/json"

        proxy = PyPiProxy(self.request, 'pyramid')
        result = proxy.get_remote_informations(url)
        self.assertIsInstance(result, dict)
        self.assertEqual(result['info']['name'], 'pyramid')
Ejemplo n.º 16
0
    def test_get_remote_informations(self, mock):
        from papaye.proxy import PyPiProxy
        mock.return_value = self.pypi_response
        url = "http://pypi.python.org/pypi/pyramid/json"

        proxy = PyPiProxy(self.request, 'pyramid')
        result = proxy.get_remote_informations(url)
        self.assertIsInstance(result, dict)
        self.assertEqual(result['info']['name'], 'pyramid')
Ejemplo n.º 17
0
    def test_build_remote_repository_with_release_name(self, mock):
        from papaye.models import Root
        from papaye.proxy import PyPiProxy
        mock_proxy_response(mock)
        proxy = PyPiProxy()

        result = proxy.build_remote_repository('pyramid', release_name='1.5')

        assert isinstance(result, Root)
        assert 'pyramid' in list(result.keys())
        assert len(list(result['pyramid'])) == 1
Ejemplo n.º 18
0
    def test_build_remote_repository(self, mock):
        from papaye.models import Root
        from papaye.proxy import PyPiProxy
        mock_proxy_response(mock)
        proxy = PyPiProxy()

        result = proxy.build_remote_repository('pyramid')

        assert isinstance(result, Root)
        assert 'pyramid' in [e.__name__ for e in result]
        assert len(list(result['pyramid'])) == 81
        assert not hasattr(list(result['pyramid'])[0], 'metadata')
Ejemplo n.º 19
0
    def test_build_remote_repository_with_metadata(self, mock):
        from papaye.models import Root
        from papaye.proxy import PyPiProxy
        mock_proxy_response(mock)
        proxy = PyPiProxy()

        result = proxy.build_remote_repository('pyramid', metadata=True)

        assert isinstance(result, Root)
        assert 'pyramid' in list(result.keys())
        assert len(list(result['pyramid'])) == 81
        assert hasattr(list(result['pyramid'])[0], 'metadata')
        assert len(list(result['pyramid'])[0].metadata.keys()) == 13
Ejemplo n.º 20
0
    def test_get_remote_informations(self, mock):
        # Given
        from papaye.proxy import PyPiProxy
        mock_proxy_response(mock)
        url = "http://pypi.python.org/pypi/pyramid/json"
        proxy = PyPiProxy()

        # When
        result = proxy.get_remote_informations(url)

        # Then
        self.assertIsInstance(result, dict)
        self.assertEqual(result['info']['name'], 'pyramid')
Ejemplo n.º 21
0
 def __call__(self):
     package = self.context
     proxy = PyPiProxy()
     stop = hasattr(self, 'stop') and self.stop
     repository = package.__parent__ if stop else proxy.merged_repository(package)
     rfiles = [rfile for rel in repository[package.__name__] for rfile in rel]
     context = {'objects': ((self.request.resource_url(
         rfile,
         route_name='simple'
     )[:-1] + "#md5={}".format(rfile.md5_digest), rfile) for rfile in rfiles)}
     transaction.abort()
     if len(rfiles):
         return context
     elif stop:
         return HTTPNotFound()
     else:
         return not_found(self.request)
Ejemplo n.º 22
0
    def test_merged_repository_without_repository_package(self, mock):
        from papaye.models import Package, Release, ReleaseFile, Root
        from papaye.proxy import PyPiProxy
        mock.return_value = FakeGRequestResponse(404, b'')
        package1 = Package(name='pyramid')
        package1['100.5'] = Release(version='100.5', metadata={})
        package1['100.5']['pyramid-100.5.tar.gz'] = ReleaseFile(
            filename='pyramid-100.5.tar.gz',
            content=b'a existing content',
            md5_digest='12345'
        )
        proxy = PyPiProxy()
        result = proxy.merged_repository(package1, metadata=True)

        assert isinstance(result, Root)
        assert [pack.__name__ for pack in result] == ["pyramid", ]
        assert len(list(result['pyramid'])) == 1
Ejemplo n.º 23
0
    def test_merged_repository_with_release_name(self, mock):
        from papaye.models import Package, Release, ReleaseFile, Root
        from papaye.proxy import PyPiProxy
        mock_proxy_response(mock)
        package1 = Package(name='pyramid')
        package1['100.5'] = Release(version='100.5', metadata={})
        package1['100.5']['pyramid-100.5.tar.gz'] = ReleaseFile(
            filename='pyramid-100.5.tar.gz',
            content=b'a existing content',
            md5_digest='12345'
        )
        proxy = PyPiProxy()
        result = proxy.merged_repository(package1, release_name='1.5')

        assert isinstance(result, Root)
        assert [pack.__name__ for pack in result] == ["pyramid", ]
        assert len(list(result['pyramid'])) == 2
        assert '1.5' in result['pyramid'].releases.keys()
        assert '100.5' in [rel for rel in result['pyramid'].releases]
Ejemplo n.º 24
0
    def test_build_repository(self, mock):
        from papaye.proxy import PyPiProxy
        from papaye.models import Package, ReleaseFile
        mock.return_value = self.pypi_response
        info_dict = json.loads(self.pypi_response.content.decode('utf-8'))

        proxy = PyPiProxy(self.request, 'pyramid')
        result = proxy.build_repository()

        self.assertIsInstance(result, Package)
        for release in result.releases.values():
            self.assertIn(release.__name__, info_dict['releases'].keys())
        self.assertEqual(len(result.releases.keys()), len(info_dict['releases'].keys()))
        self.assertEqual(len(result['1.5'].release_files.keys()), 1)
        self.assertIsInstance(result['1.5'].release_files.values()[0], ReleaseFile)
        self.assertTrue(getattr(result['1.5'].release_files.values()[0], 'pypi_url', None))
        self.assertEqual(result['1.5'].release_files.values()[0].pypi_url,
                         "https://pypi.python.org/packages/source/p/pyramid/pyramid-1.5.tar.gz")
        self.assertIsNotNone(result['1.5'].metadata)
        self.assertIsInstance(result['1.5'].metadata, dict)
Ejemplo n.º 25
0
def download_release_from_pypi(config, package_name, release_name):
    conn = get_connection(config)
    proxy = PyPiProxy(conn, package_name)
    package = proxy.build_repository(release_name=release_name)
    if not package:
        logger.error('Package {} not found on PYPI'.format(package_name))
    root = repository_root_factory(conn)
    for release_file in package[release_name].release_files.values():
        logger.info('Download file "{}"'.format(release_file.filename))
        release_file.set_content(requests.get(release_file.pypi_url).content)
        with release_file.content.open() as content:
            binary_content = content.read()
            if hashlib.md5(binary_content).hexdigest() != release_file.md5_digest:
                raise IOError('md5 check error')
        release_file.size = len(binary_content)
    root[package.name] = package
    try:
        transaction.commit()
    except:
        transaction.abort()
Ejemplo n.º 26
0
def not_found(request):
    settings = request.registry.settings
    proxy = settings.get('papaye.proxy', False)
    proxy = True if proxy and proxy == 'true' else False
    if not proxy:
        return HTTPNotFound()
    try:
        proxy = PyPiProxy(request, request.matchdict['traverse'][0])
        package = proxy.build_repository()
        if not package:
            return HTTPNotFound()
        if len(request.matchdict['traverse']) == 1:
            view = ListReleaseFileView(package, request)
        elif len(request.matchdict['traverse']) == 2:
            context = package[request.matchdict['traverse'][1]]
            view = ListReleaseFileByReleaseView(context, request)
        elif len(request.matchdict['traverse']) == 3:
            release_file = package[request.matchdict['traverse'][1]][request.matchdict['traverse'][2]]
            package_name, release_name, _ = request.matchdict['traverse']
            download_release_from_pypi.delay(package_name, release_name)
            return HTTPTemporaryRedirect(location=release_file.pypi_url)
    except KeyError:
        return HTTPNotFound()
    return view()
Ejemplo n.º 27
0
class ListReleaseFileView(BaseView):
    _repository = None

    def __init__(self, context, request):
        super().__init__(context, request)
        self.proxy_instance = PyPiProxy()
        self.stop = hasattr(self, 'stop') and self.stop

    @property
    def repository(self):
        if self._repository is None:
            if self.stop:
                self._repository = self.context.root
            else:
                self._repository = self.proxy_instance.merged_repository(
                    self.context
                )
        return self._repository

    def __call__(self):
        package = self.context
        root = package.root
        if self.repository is not root:
            self.repository.name = root.name
        rfiles = [
            rfile for rel in self.repository[package.name]
            for rfile in rel
        ]
        context = {
            'objects': (self.format_release_file(rfile) for rfile in rfiles)
        }
        transaction.abort()
        if len(rfiles):
            return context
        elif self.stop:
            return HTTPNotFound()
        else:
            return not_found(self.request)

    def format_release_file(self, release_file):
        return self.request.resource_url(
            release_file,
            route_name='simple'
        )[:-1] + "#md5={}".format(release_file.md5_digest), release_file
Ejemplo n.º 28
0
 def __init__(self, context, request):
     super().__init__(context, request)
     self.proxy_instance = PyPiProxy()
     self.stop = hasattr(self, 'stop') and self.stop