def download_release_from_pypi(db_or_root, package_name, release_name, filename): db = isinstance(db_or_root, DB) if db: conn = db_or_root.open() root = repository_root_factory(conn) else: root = db_or_root metadata = get_release_metadata(package_name, release_name) for attempt in transaction.attempts(): with attempt: if root[package_name] is None: root[package_name] = Package(package_name) release_ready_exists = release_name in root[package_name] if not release_ready_exists: root[package_name][release_name] = Release( release_name, release_name, metadata['info'], deserialize_metadata=True ) if filename not in root[package_name][release_name]: release = [release for release in metadata['releases'][release_name] if release['filename'] == filename][0] url = release['url'] md5_digest = release['md5_digest'] content = download_file(filename, url, md5_digest) if not content: transaction.abort() return None root[package_name][release_name][filename] = ReleaseFile(filename, content, md5_digest) if db: conn.close()
def test_smart_merge_with_other_release(self): from papaye.factories import repository_root_factory from papaye.models import Package, Release, ReleaseFile from papaye.proxy import smart_merge package1 = Package(name='pyramid') package1['1.5'] = Release(name='1.5', version='1.5', metadata={}) package1['1.5']['pyramid-1.5.tar.gz'] = ReleaseFile( filename='pyramid-1.5.tar.gz', content=b'', md5_digest='12345' ) package2 = Package(name='pyramid') package2['1.6'] = Release(name='1.6', version='1.6', metadata={}) package2['1.6']['pyramid-1.6.tar.gz'] = ReleaseFile( filename='pyramid-1.6.tar.gz', content=b'', md5_digest='12345' ) result = smart_merge(package1, package2) assert isinstance(result, Package) assert len(list(result.__parent__)) == 1 assert len(list(result)) == 2 assert repository_root_factory(self.request) is not result.__parent__ result = smart_merge(package1, package2) assert isinstance(result, Package) assert len(list(result)) == 2 assert repository_root_factory(self.request) is not result.__parent__
def by_releasefilename(cls, package, release, releasefile, request): if hasattr(request, 'root') and request.root is not None: root = request.root else: root = repository_root_factory(request) if package in [pkg.__name__ for pkg in root] and root[package].get(release): return root[package][release].get(releasefile, None)
def test_download_release_from_pypi_with_existing_package(self, get_connection_mock, request_mock): from papaye.factories import repository_root_factory from papaye.models import Package, Release, ReleaseFile from papaye.tasks.download import download_release_from_pypi request = get_current_request() get_connection_mock.return_value = self.conn json_response = open(get_resource('pyramid.json'), 'rb') release_file_content = open(get_resource('pyramid-1.5.tar.gz'), 'rb') request_mock.side_effect = [ FakeGRequestResponse(200, b'', 'http://pypi.python.org/simple/pyramid/'), FakeGRequestResponse(200, json_response.read()), FakeGRequestResponse(200, release_file_content.read()), ] root = repository_root_factory(self.conn) package = Package('pyramid') release = Release('1.0', '1.0', metadata={}) release_file = ReleaseFile('pyramid-1.0.tar.gz', b'') root['pyramid'] = package root['pyramid']['1.0'] = release root['pyramid']['1.0']['pyramid-1.0.tar.gz']= release_file download_release_from_pypi(request.registry.settings, 'pyramid', '1.5') self.assertEqual(request_mock.call_count, 3) assert request_mock.call_count == 3 assert len(list(root['pyramid'])) == 2
def test_list_release_with_two_release(self): from papaye.views.simple import ListReleaseFileByReleaseView from papaye.models import Package, Release, ReleaseFile from papaye.factories import repository_root_factory # Initial data root = repository_root_factory(self.request) package = Package('my_package') package['1.0'] = Release('1.0', '1.0', metadata={}) package['1.0']['foo.tar.gz'] = ReleaseFile('foo.tar.gz', b'') package['2.0'] = Release('2.0', '2.0', metadata={}) package['2.0']['foo2.tar.gz'] = ReleaseFile('foo2.tar.gz', b'') root['my_package'] = package view1 = ListReleaseFileByReleaseView(package['1.0'], self.request) view2 = ListReleaseFileByReleaseView(package['2.0'], self.request) result1 = view1() result2 = view2() self.assertIsInstance(result1, dict) self.assertIn('objects', result1) self.assertEqual( list(result1['objects']), [('http://example.com/simple/my_package/1.0/foo.tar.gz/', package['1.0']['foo.tar.gz'])]) self.assertIsInstance(result2, dict) self.assertIn('objects', result2) self.assertEqual( list(result2['objects']), [('http://example.com/simple/my_package/2.0/foo2.tar.gz/', package['2.0']['foo2.tar.gz'])])
def test_download_release_from_pypi(self, get_connection_mock, request_mock): from papaye.tasks.download import download_release_from_pypi from papaye.factories import repository_root_factory request = get_current_request() get_connection_mock.return_value = self.conn json_response = open(get_resource('pyramid.json'), 'rb') release_file_content = open(get_resource('pyramid-1.5.tar.gz'), 'rb') request_mock.side_effect = [ FakeGRequestResponse(200, b'', 'http://pypi.python.org/simple/pyramid/'), FakeGRequestResponse(200, json_response.read()), FakeGRequestResponse(200, release_file_content.read()), ] download_release_from_pypi(request.registry.settings, 'pyramid', '1.5') self.assertEqual(request_mock.call_count, 3) root = repository_root_factory(self.conn) self.assertIn('pyramid', root) self.assertIn('1.5', root['pyramid'].releases) self.assertIn('pyramid-1.5.tar.gz', root['pyramid']['1.5'].release_files) release_file = root['pyramid']['1.5']['pyramid-1.5.tar.gz'] self.assertEqual(release_file.md5_digest, "8747658dcbab709a9c491e43d3b0d58b") self.assertEqual(release_file.filename, "pyramid-1.5.tar.gz") self.assertEqual(release_file.size, 2413504) self.assertEqual(list(root['pyramid'].releases.keys()), ['1.5', ])
def test_list_release_with_two_release(self): from papaye.views.simple import ListReleaseFileByReleaseView from papaye.models import Package, Release, ReleaseFile from papaye.factories import repository_root_factory # Initial data root = repository_root_factory(self.request) package = Package("my_package") package["1.0"] = Release("1.0", "1.0", metadata={}) package["1.0"]["foo.tar.gz"] = ReleaseFile("foo.tar.gz", b"") package["2.0"] = Release("2.0", "2.0", metadata={}) package["2.0"]["foo2.tar.gz"] = ReleaseFile("foo2.tar.gz", b"") root["my_package"] = package view1 = ListReleaseFileByReleaseView(package["1.0"], self.request) view2 = ListReleaseFileByReleaseView(package["2.0"], self.request) result1 = view1() result2 = view2() self.assertIsInstance(result1, dict) self.assertIn("objects", result1) self.assertEqual( list(result1["objects"]), [("http://example.com/simple/my_package/1.0/foo.tar.gz/", package["1.0"]["foo.tar.gz"])], ) self.assertIsInstance(result2, dict) self.assertIn("objects", result2) self.assertEqual( list(result2["objects"]), [("http://example.com/simple/my_package/2.0/foo2.tar.gz/", package["2.0"]["foo2.tar.gz"])], )
def download_release_from_pypi(config, package_name, release_name): try: repository_is_updated = False conn = get_connection(config) proxy = PyPiProxy(conn, package_name) root = repository_root_factory(conn) package = proxy.build_repository(release_name=release_name, with_metadata=True) if not package: logger.error('Package {} not found on PYPI'.format(package_name)) for release_file in package[release_name].release_files.values(): logger.info('Download file "{}"'.format(release_file.filename)) release_file.set_content( requests.get(release_file.pypi_url).content) with release_file.content.open() as content: binary_content = content.read() if hashlib.md5( binary_content).hexdigest() != release_file.md5_digest: continue release_file.size = len(binary_content) repository_is_updated = True package = proxy.smart_merge(root, package) if repository_is_updated: root[package.name] = package for release in package: release.__parent__ = package for release_file in release: release_file.__parent__ = release transaction.commit() except: transaction.abort()
def test_smart_merge(self, mock): from papaye.proxy import PyPiProxy from papaye.factories import repository_root_factory from papaye.models import Package, Release, ReleaseFile mock.return_value = self.pypi_response root = repository_root_factory(self.request) # Existing releases root['pyramid'] = Package(name='pyramid') root['pyramid']['1.4'] = Release(name='1.4', version='1.4', metadata={}) root['pyramid']['1.4']['pyramid-1.4.tar.gz'] = ReleaseFile( filename='pyramid-1.4.tar.gz', content=b'', md5_digest='12345' ) package = Package(name='pyramid') package['1.5'] = Release(name='1.5', version='1.5', metadata={}) package['1.5']['pyramid-1.5.tar.gz'] = ReleaseFile( filename='pyramid-1.5.tar.gz', content=b'', md5_digest='12345' ) proxy = PyPiProxy(self.request, 'pyramid') result = proxy.smart_merge(root, package) self.assertEqual([key for key in result.releases.keys()], ['1.4', '1.5'])
def test_smart_merge_with_existing_release(self, mock): from papaye.proxy import PyPiProxy from papaye.factories import repository_root_factory from papaye.models import Package, Release, ReleaseFile mock.return_value = self.pypi_response root = repository_root_factory(self.request) # Existing releases package = Package(name='pyramid') package['1.5'] = Release(name='1.5', version='1.5', metadata={}) package['1.5']['pyramid-1.5.tar.gz'] = ReleaseFile( filename='pyramid-1.5.tar.gz', content=b'', md5_digest='12345') package = Package(name='pyramid') package['1.5'] = Release(name='1.5', version='1.5', metadata={}) package['1.5']['pyramid-1.5.tar.gz'] = ReleaseFile( filename='pyramid-1.5.tar.gz', content=b'', md5_digest='12345') proxy = PyPiProxy(self.request, 'pyramid') result = proxy.smart_merge(root, package) self.assertEqual([key for key in result.releases.keys()], [ '1.5', ]) self.assertEqual(result['1.5']['pyramid-1.5.tar.gz'].md5_digest, '12345')
def test_download_release_from_pypi(self, get_connection_mock, request_mock): from papaye.tasks.download import download_release_from_pypi from papaye.factories import repository_root_factory request = get_current_request() get_connection_mock.return_value = self.conn json_response = open(get_resource('pyramid.json'), 'rb') release_file_content = open(get_resource('pyramid-1.5.tar.gz'), 'rb') request_mock.side_effect = [ FakeGRequestResponse(200, b'', 'http://pypi.python.org/simple/pyramid/'), FakeGRequestResponse(200, json_response.read()), FakeGRequestResponse(200, release_file_content.read()), ] download_release_from_pypi(request.registry.settings, 'pyramid', '1.5') self.assertEqual(request_mock.call_count, 3) root = repository_root_factory(self.conn) self.assertIn('pyramid', root) self.assertIn('1.5', root['pyramid'].releases) self.assertIn('pyramid-1.5.tar.gz', root['pyramid']['1.5'].release_files) release_file = root['pyramid']['1.5']['pyramid-1.5.tar.gz'] self.assertEqual(release_file.md5_digest, "8747658dcbab709a9c491e43d3b0d58b") self.assertEqual(release_file.filename, "pyramid-1.5.tar.gz") self.assertEqual(release_file.size, 2413504) self.assertEqual(list(root['pyramid'].releases.keys()), ['1.5', ]) assert root['pyramid']['1.5'].metadata is not None assert root['pyramid'].__parent__ is root assert root['pyramid']['1.5'].__parent__ is root['pyramid'] assert root['pyramid']['1.5']['pyramid-1.5.tar.gz'].__parent__ is root['pyramid']['1.5']
def test_by_name(self): from papaye.models import Package from papaye.factories import repository_root_factory root = repository_root_factory(self.request) root['package1'] = Package(name='package1') result = Package.by_name('package1', self.request) self.assertEqual(result, root['package1'])
def test_by_name(self): from papaye.models import Package from papaye.factories import repository_root_factory root = repository_root_factory(self.request) root['package1'] = Package(name='package1') result = Package.by_name('package1', self.request) self.assertEqual(result, root['package1'])
def test_by_packagename(self): from papaye.models import Release, Package from papaye.factories import repository_root_factory root = repository_root_factory(self.request) root['package1'] = Package(name='package1') root['package1']['1.0'] = Release('1.0', '1.0', metadata={}) result = Release.by_packagename('package1', self.request) self.assertEqual(result, [root['package1']['1.0'], ])
def check_database_config(config): from papaye.models import get_manager manager = get_manager(config) if manager.get_db_version() < manager.get_sw_version(): raise ConfigurationError('Your database need to be updated! Run ' '"papaye_evolve path_to_your_config_file.ini" command first') conn = config.registry._zodb_databases[''].open() if user_root_factory(conn) is None or repository_root_factory(conn) is None: raise ConfigurationError('Database does not exist! Run "papaye_init ' 'path_to_your_config_file.ini command first') return True
def check_database_config(config): from papaye.models import get_manager manager = get_manager(config) if manager.get_db_version() < manager.get_sw_version(): raise ConfigurationError('Your database need to be updated! Run ' '"papaye_evolve path_to_your_config_file.ini" command first') conn = config.registry._zodb_databases[''].open() if user_root_factory(conn) is None or repository_root_factory(conn) is None: raise ConfigurationError('Database does not exist! Run "papaye_init ' 'path_to_your_config_file.ini command first') return True
def test_by_packagename(self): from papaye.models import Release, Package from papaye.factories import repository_root_factory root = repository_root_factory(self.request) root['package1'] = Package(name='package1') root['package1']['1.0'] = Release('1.0', '1.0', metadata={}) result = Release.by_packagename('package1', self.request) self.assertEqual(result, [ root['package1']['1.0'], ])
def test_download_release_from_pypi_with_bad_md5(self, request_mock): from papaye.factories import repository_root_factory from papaye.tasks.download import download_release_from_pypi json_request_response = open(get_resource('pyramid1.4.json'), 'rb') release_file_content = io.BytesIO(b'corrupted_file') request_mock.side_effect = [ FakeGRequestResponse(200, json_request_response.read()), FakeGRequestResponse(200, release_file_content.read()), ] root = repository_root_factory(self.conn) download_release_from_pypi(root, 'pyramid', '1.5', 'pyramid-1.5.tar.gz') self.assertEqual(request_mock.call_count, 2) assert root._p_status == 'unsaved'
def test_download_release_from_pypi_with_bad_md5(self, get_connection_mock, request_mock): from papaye.tasks.download import download_release_from_pypi from papaye.factories import repository_root_factory request = get_current_request() get_connection_mock.return_value = self.conn json_response = open(get_resource('pyramid.json'), 'rb') release_file_content = io.BytesIO(b'corrupted_file') request_mock.side_effect = [ FakeGRequestResponse(200, b'', 'http://pypi.python.org/simple/pyramid/'), FakeGRequestResponse(200, json_response.read()), FakeGRequestResponse(200, release_file_content.read()), ] self.assertRaises(IOError, download_release_from_pypi, request.registry.settings, 'pyramid', '1.5') self.assertEqual(request_mock.call_count, 3) root = repository_root_factory(self.conn) self.assertNotIn('pyramid', root)
def download_release_from_pypi(config, package_name, release_name): conn = get_connection(config) proxy = PyPiProxy(conn, package_name) package = proxy.build_repository(release_name=release_name) if not package: logger.error('Package {} not found on PYPI'.format(package_name)) root = repository_root_factory(conn) for release_file in package[release_name].release_files.values(): logger.info('Download file "{}"'.format(release_file.filename)) release_file.set_content(requests.get(release_file.pypi_url).content) with release_file.content.open() as content: binary_content = content.read() if hashlib.md5(binary_content).hexdigest() != release_file.md5_digest: raise IOError('md5 check error') release_file.size = len(binary_content) root[package.name] = package try: transaction.commit() except: transaction.abort()
def test_download_release_from_pypi_with_existing_package(self, request_mock): from papaye.factories import repository_root_factory from papaye.models import Package, Release, ReleaseFile from papaye.tasks.download import download_release_from_pypi json_request_response = open(get_resource('pyramid1.4.json'), 'rb') release_file_content = open(get_resource('pyramid-1.5.tar.gz'), 'rb') request_mock.side_effect = [ FakeGRequestResponse(200, json_request_response.read()), FakeGRequestResponse(200, release_file_content.read()), ] root = repository_root_factory(self.conn) package = Package('pyramid') release = Release('1.0', '1.0', metadata={}) release_file = ReleaseFile('pyramid-1.0.tar.gz', b'') root['pyramid'] = package root['pyramid']['1.0'] = release root['pyramid']['1.0']['pyramid-1.0.tar.gz'] = release_file download_release_from_pypi(root, 'pyramid', '1.5', 'pyramid-1.5.tar.gz') assert request_mock.call_count == 2 assert len(list(root['pyramid'])) == 2
def by_packagename(cls, package, request): root = repository_root_factory(request) if package not in root: return None return list(root[package].releases.values())
def by_name(cls, name, request): if hasattr(request, 'root') and request.root is not None: root = request.root else: root = repository_root_factory(request) return root[name] if name in [package.__name__ for package in root] else None
def by_packagename(cls, package_name, request): root = repository_root_factory(request) if package_name not in [pkg.__name__ for pkg in root]: return None return list(root[package_name].releases.values())
def setup(env): env['user_root'] = user_root_factory(env['request']) env['repo_root'] = repository_root_factory(env['request']) env['set_db_version'] = functools.partial(set_database_version, env['request'])
def by_name(cls, name, request): root = repository_root_factory(request) return root[name] if name in root else None
def setup(env): env['user_root'] = user_root_factory(env['request']) env['repo_root'] = repository_root_factory(env['request']) env['default_root'] = default_root_factory(env['request']) env['set_db_version'] = functools.partial(set_database_version, env['request'])
def by_packagename(cls, package, request): root = repository_root_factory(request) if package not in root: return None return list(root[package].releases.values())
def by_name(cls, name, request): root = repository_root_factory(request) return root[name] if name in root else None