def download_release_from_pypi(db_or_root, package_name, release_name, filename):
db = isinstance(db_or_root, DB)
if db:
conn = db_or_root.open()
root = repository_root_factory(conn)
else:
root = db_or_root
metadata = get_release_metadata(package_name, release_name)
for attempt in transaction.attempts():
with attempt:
if root[package_name] is None:
root[package_name] = Package(package_name)
release_ready_exists = release_name in root[package_name]
if not release_ready_exists:
root[package_name][release_name] = Release(
release_name,
release_name,
metadata['info'],
deserialize_metadata=True
)
if filename not in root[package_name][release_name]:
release = [release for release in metadata['releases'][release_name]
if release['filename'] == filename][0]
url = release['url']
md5_digest = release['md5_digest']
content = download_file(filename, url, md5_digest)
if not content:
transaction.abort()
return None
root[package_name][release_name][filename] = ReleaseFile(filename, content, md5_digest)
if db:
conn.close()
def test_smart_merge_with_other_release(self):
from papaye.factories import repository_root_factory
from papaye.models import Package, Release, ReleaseFile
from papaye.proxy import smart_merge
package1 = Package(name='pyramid')
package1['1.5'] = Release(name='1.5', version='1.5', metadata={})
package1['1.5']['pyramid-1.5.tar.gz'] = ReleaseFile(
filename='pyramid-1.5.tar.gz',
content=b'',
md5_digest='12345'
)
package2 = Package(name='pyramid')
package2['1.6'] = Release(name='1.6', version='1.6', metadata={})
package2['1.6']['pyramid-1.6.tar.gz'] = ReleaseFile(
filename='pyramid-1.6.tar.gz',
content=b'',
md5_digest='12345'
)
result = smart_merge(package1, package2)
assert isinstance(result, Package)
assert len(list(result.__parent__)) == 1
assert len(list(result)) == 2
assert repository_root_factory(self.request) is not result.__parent__
result = smart_merge(package1, package2)
assert isinstance(result, Package)
assert len(list(result)) == 2
assert repository_root_factory(self.request) is not result.__parent__
def by_releasefilename(cls, package, release, releasefile, request):
if hasattr(request, 'root') and request.root is not None:
root = request.root
else:
root = repository_root_factory(request)
if package in [pkg.__name__ for pkg in root] and root[package].get(release):
return root[package][release].get(releasefile, None)
def test_download_release_from_pypi_with_existing_package(self, get_connection_mock, request_mock):
from papaye.factories import repository_root_factory
from papaye.models import Package, Release, ReleaseFile
from papaye.tasks.download import download_release_from_pypi
request = get_current_request()
get_connection_mock.return_value = self.conn
json_response = open(get_resource('pyramid.json'), 'rb')
release_file_content = open(get_resource('pyramid-1.5.tar.gz'), 'rb')
request_mock.side_effect = [
FakeGRequestResponse(200, b'', 'http://pypi.python.org/simple/pyramid/'),
FakeGRequestResponse(200, json_response.read()),
FakeGRequestResponse(200, release_file_content.read()),
]
root = repository_root_factory(self.conn)
package = Package('pyramid')
release = Release('1.0', '1.0', metadata={})
release_file = ReleaseFile('pyramid-1.0.tar.gz', b'')
root['pyramid'] = package
root['pyramid']['1.0'] = release
root['pyramid']['1.0']['pyramid-1.0.tar.gz']= release_file
download_release_from_pypi(request.registry.settings, 'pyramid', '1.5')
self.assertEqual(request_mock.call_count, 3)
assert request_mock.call_count == 3
assert len(list(root['pyramid'])) == 2
def test_list_release_with_two_release(self):
from papaye.views.simple import ListReleaseFileByReleaseView
from papaye.models import Package, Release, ReleaseFile
from papaye.factories import repository_root_factory
# Initial data
root = repository_root_factory(self.request)
package = Package('my_package')
package['1.0'] = Release('1.0', '1.0', metadata={})
package['1.0']['foo.tar.gz'] = ReleaseFile('foo.tar.gz', b'')
package['2.0'] = Release('2.0', '2.0', metadata={})
package['2.0']['foo2.tar.gz'] = ReleaseFile('foo2.tar.gz', b'')
root['my_package'] = package
view1 = ListReleaseFileByReleaseView(package['1.0'], self.request)
view2 = ListReleaseFileByReleaseView(package['2.0'], self.request)
result1 = view1()
result2 = view2()
self.assertIsInstance(result1, dict)
self.assertIn('objects', result1)
self.assertEqual(
list(result1['objects']),
[('http://example.com/simple/my_package/1.0/foo.tar.gz/',
package['1.0']['foo.tar.gz'])])
self.assertIsInstance(result2, dict)
self.assertIn('objects', result2)
self.assertEqual(
list(result2['objects']),
[('http://example.com/simple/my_package/2.0/foo2.tar.gz/',
package['2.0']['foo2.tar.gz'])])
def test_download_release_from_pypi(self, get_connection_mock, request_mock):
from papaye.tasks.download import download_release_from_pypi
from papaye.factories import repository_root_factory
request = get_current_request()
get_connection_mock.return_value = self.conn
json_response = open(get_resource('pyramid.json'), 'rb')
release_file_content = open(get_resource('pyramid-1.5.tar.gz'), 'rb')
request_mock.side_effect = [
FakeGRequestResponse(200, b'', 'http://pypi.python.org/simple/pyramid/'),
FakeGRequestResponse(200, json_response.read()),
FakeGRequestResponse(200, release_file_content.read()),
]
download_release_from_pypi(request.registry.settings, 'pyramid', '1.5')
self.assertEqual(request_mock.call_count, 3)
root = repository_root_factory(self.conn)
self.assertIn('pyramid', root)
self.assertIn('1.5', root['pyramid'].releases)
self.assertIn('pyramid-1.5.tar.gz', root['pyramid']['1.5'].release_files)
release_file = root['pyramid']['1.5']['pyramid-1.5.tar.gz']
self.assertEqual(release_file.md5_digest, "8747658dcbab709a9c491e43d3b0d58b")
self.assertEqual(release_file.filename, "pyramid-1.5.tar.gz")
self.assertEqual(release_file.size, 2413504)
self.assertEqual(list(root['pyramid'].releases.keys()), ['1.5', ])
def test_list_release_with_two_release(self):
from papaye.views.simple import ListReleaseFileByReleaseView
from papaye.models import Package, Release, ReleaseFile
from papaye.factories import repository_root_factory
# Initial data
root = repository_root_factory(self.request)
package = Package("my_package")
package["1.0"] = Release("1.0", "1.0", metadata={})
package["1.0"]["foo.tar.gz"] = ReleaseFile("foo.tar.gz", b"")
package["2.0"] = Release("2.0", "2.0", metadata={})
package["2.0"]["foo2.tar.gz"] = ReleaseFile("foo2.tar.gz", b"")
root["my_package"] = package
view1 = ListReleaseFileByReleaseView(package["1.0"], self.request)
view2 = ListReleaseFileByReleaseView(package["2.0"], self.request)
result1 = view1()
result2 = view2()
self.assertIsInstance(result1, dict)
self.assertIn("objects", result1)
self.assertEqual(
list(result1["objects"]),
[("http://example.com/simple/my_package/1.0/foo.tar.gz/", package["1.0"]["foo.tar.gz"])],
)
self.assertIsInstance(result2, dict)
self.assertIn("objects", result2)
self.assertEqual(
list(result2["objects"]),
[("http://example.com/simple/my_package/2.0/foo2.tar.gz/", package["2.0"]["foo2.tar.gz"])],
)
def download_release_from_pypi(config, package_name, release_name):
try:
repository_is_updated = False
conn = get_connection(config)
proxy = PyPiProxy(conn, package_name)
root = repository_root_factory(conn)
package = proxy.build_repository(release_name=release_name,
with_metadata=True)
if not package:
logger.error('Package {} not found on PYPI'.format(package_name))
for release_file in package[release_name].release_files.values():
logger.info('Download file "{}"'.format(release_file.filename))
release_file.set_content(
requests.get(release_file.pypi_url).content)
with release_file.content.open() as content:
binary_content = content.read()
if hashlib.md5(
binary_content).hexdigest() != release_file.md5_digest:
continue
release_file.size = len(binary_content)
repository_is_updated = True
package = proxy.smart_merge(root, package)
if repository_is_updated:
root[package.name] = package
for release in package:
release.__parent__ = package
for release_file in release:
release_file.__parent__ = release
transaction.commit()
except:
transaction.abort()
def test_smart_merge(self, mock):
from papaye.proxy import PyPiProxy
from papaye.factories import repository_root_factory
from papaye.models import Package, Release, ReleaseFile
mock.return_value = self.pypi_response
root = repository_root_factory(self.request)
# Existing releases
root['pyramid'] = Package(name='pyramid')
root['pyramid']['1.4'] = Release(name='1.4', version='1.4', metadata={})
root['pyramid']['1.4']['pyramid-1.4.tar.gz'] = ReleaseFile(
filename='pyramid-1.4.tar.gz',
content=b'',
md5_digest='12345'
)
package = Package(name='pyramid')
package['1.5'] = Release(name='1.5', version='1.5', metadata={})
package['1.5']['pyramid-1.5.tar.gz'] = ReleaseFile(
filename='pyramid-1.5.tar.gz',
content=b'',
md5_digest='12345'
)
proxy = PyPiProxy(self.request, 'pyramid')
result = proxy.smart_merge(root, package)
self.assertEqual([key for key in result.releases.keys()], ['1.4', '1.5'])
def test_smart_merge_with_existing_release(self, mock):
from papaye.proxy import PyPiProxy
from papaye.factories import repository_root_factory
from papaye.models import Package, Release, ReleaseFile
mock.return_value = self.pypi_response
root = repository_root_factory(self.request)
# Existing releases
package = Package(name='pyramid')
package['1.5'] = Release(name='1.5', version='1.5', metadata={})
package['1.5']['pyramid-1.5.tar.gz'] = ReleaseFile(
filename='pyramid-1.5.tar.gz', content=b'', md5_digest='12345')
package = Package(name='pyramid')
package['1.5'] = Release(name='1.5', version='1.5', metadata={})
package['1.5']['pyramid-1.5.tar.gz'] = ReleaseFile(
filename='pyramid-1.5.tar.gz', content=b'', md5_digest='12345')
proxy = PyPiProxy(self.request, 'pyramid')
result = proxy.smart_merge(root, package)
self.assertEqual([key for key in result.releases.keys()], [
'1.5',
])
self.assertEqual(result['1.5']['pyramid-1.5.tar.gz'].md5_digest,
'12345')
def test_download_release_from_pypi(self, get_connection_mock, request_mock):
from papaye.tasks.download import download_release_from_pypi
from papaye.factories import repository_root_factory
request = get_current_request()
get_connection_mock.return_value = self.conn
json_response = open(get_resource('pyramid.json'), 'rb')
release_file_content = open(get_resource('pyramid-1.5.tar.gz'), 'rb')
request_mock.side_effect = [
FakeGRequestResponse(200, b'', 'http://pypi.python.org/simple/pyramid/'),
FakeGRequestResponse(200, json_response.read()),
FakeGRequestResponse(200, release_file_content.read()),
]
download_release_from_pypi(request.registry.settings, 'pyramid', '1.5')
self.assertEqual(request_mock.call_count, 3)
root = repository_root_factory(self.conn)
self.assertIn('pyramid', root)
self.assertIn('1.5', root['pyramid'].releases)
self.assertIn('pyramid-1.5.tar.gz', root['pyramid']['1.5'].release_files)
release_file = root['pyramid']['1.5']['pyramid-1.5.tar.gz']
self.assertEqual(release_file.md5_digest, "8747658dcbab709a9c491e43d3b0d58b")
self.assertEqual(release_file.filename, "pyramid-1.5.tar.gz")
self.assertEqual(release_file.size, 2413504)
self.assertEqual(list(root['pyramid'].releases.keys()), ['1.5', ])
assert root['pyramid']['1.5'].metadata is not None
assert root['pyramid'].__parent__ is root
assert root['pyramid']['1.5'].__parent__ is root['pyramid']
assert root['pyramid']['1.5']['pyramid-1.5.tar.gz'].__parent__ is root['pyramid']['1.5']
def test_by_name(self):
from papaye.models import Package
from papaye.factories import repository_root_factory
root = repository_root_factory(self.request)
root['package1'] = Package(name='package1')
result = Package.by_name('package1', self.request)
self.assertEqual(result, root['package1'])
def test_by_name(self):
from papaye.models import Package
from papaye.factories import repository_root_factory
root = repository_root_factory(self.request)
root['package1'] = Package(name='package1')
result = Package.by_name('package1', self.request)
self.assertEqual(result, root['package1'])
def test_by_packagename(self):
from papaye.models import Release, Package
from papaye.factories import repository_root_factory
root = repository_root_factory(self.request)
root['package1'] = Package(name='package1')
root['package1']['1.0'] = Release('1.0', '1.0', metadata={})
result = Release.by_packagename('package1', self.request)
self.assertEqual(result, [root['package1']['1.0'], ])
def check_database_config(config):
from papaye.models import get_manager
manager = get_manager(config)
if manager.get_db_version() < manager.get_sw_version():
raise ConfigurationError('Your database need to be updated! Run '
'"papaye_evolve path_to_your_config_file.ini" command first')
conn = config.registry._zodb_databases[''].open()
if user_root_factory(conn) is None or repository_root_factory(conn) is None:
raise ConfigurationError('Database does not exist! Run "papaye_init '
'path_to_your_config_file.ini command first')
return True
def check_database_config(config):
from papaye.models import get_manager
manager = get_manager(config)
if manager.get_db_version() < manager.get_sw_version():
raise ConfigurationError('Your database need to be updated! Run '
'"papaye_evolve path_to_your_config_file.ini" command first')
conn = config.registry._zodb_databases[''].open()
if user_root_factory(conn) is None or repository_root_factory(conn) is None:
raise ConfigurationError('Database does not exist! Run "papaye_init '
'path_to_your_config_file.ini command first')
return True
def test_by_packagename(self):
from papaye.models import Release, Package
from papaye.factories import repository_root_factory
root = repository_root_factory(self.request)
root['package1'] = Package(name='package1')
root['package1']['1.0'] = Release('1.0', '1.0', metadata={})
result = Release.by_packagename('package1', self.request)
self.assertEqual(result, [
root['package1']['1.0'],
])
def test_download_release_from_pypi_with_bad_md5(self, request_mock):
from papaye.factories import repository_root_factory
from papaye.tasks.download import download_release_from_pypi
json_request_response = open(get_resource('pyramid1.4.json'), 'rb')
release_file_content = io.BytesIO(b'corrupted_file')
request_mock.side_effect = [
FakeGRequestResponse(200, json_request_response.read()),
FakeGRequestResponse(200, release_file_content.read()),
]
root = repository_root_factory(self.conn)
download_release_from_pypi(root, 'pyramid', '1.5', 'pyramid-1.5.tar.gz')
self.assertEqual(request_mock.call_count, 2)
assert root._p_status == 'unsaved'
def test_download_release_from_pypi_with_bad_md5(self, get_connection_mock, request_mock):
from papaye.tasks.download import download_release_from_pypi
from papaye.factories import repository_root_factory
request = get_current_request()
get_connection_mock.return_value = self.conn
json_response = open(get_resource('pyramid.json'), 'rb')
release_file_content = io.BytesIO(b'corrupted_file')
request_mock.side_effect = [
FakeGRequestResponse(200, b'', 'http://pypi.python.org/simple/pyramid/'),
FakeGRequestResponse(200, json_response.read()),
FakeGRequestResponse(200, release_file_content.read()),
]
self.assertRaises(IOError, download_release_from_pypi, request.registry.settings, 'pyramid', '1.5')
self.assertEqual(request_mock.call_count, 3)
root = repository_root_factory(self.conn)
self.assertNotIn('pyramid', root)
def download_release_from_pypi(config, package_name, release_name):
conn = get_connection(config)
proxy = PyPiProxy(conn, package_name)
package = proxy.build_repository(release_name=release_name)
if not package:
logger.error('Package {} not found on PYPI'.format(package_name))
root = repository_root_factory(conn)
for release_file in package[release_name].release_files.values():
logger.info('Download file "{}"'.format(release_file.filename))
release_file.set_content(requests.get(release_file.pypi_url).content)
with release_file.content.open() as content:
binary_content = content.read()
if hashlib.md5(binary_content).hexdigest() != release_file.md5_digest:
raise IOError('md5 check error')
release_file.size = len(binary_content)
root[package.name] = package
try:
transaction.commit()
except:
transaction.abort()
def test_download_release_from_pypi_with_existing_package(self, request_mock):
from papaye.factories import repository_root_factory
from papaye.models import Package, Release, ReleaseFile
from papaye.tasks.download import download_release_from_pypi
json_request_response = open(get_resource('pyramid1.4.json'), 'rb')
release_file_content = open(get_resource('pyramid-1.5.tar.gz'), 'rb')
request_mock.side_effect = [
FakeGRequestResponse(200, json_request_response.read()),
FakeGRequestResponse(200, release_file_content.read()),
]
root = repository_root_factory(self.conn)
package = Package('pyramid')
release = Release('1.0', '1.0', metadata={})
release_file = ReleaseFile('pyramid-1.0.tar.gz', b'')
root['pyramid'] = package
root['pyramid']['1.0'] = release
root['pyramid']['1.0']['pyramid-1.0.tar.gz'] = release_file
download_release_from_pypi(root, 'pyramid', '1.5', 'pyramid-1.5.tar.gz')
assert request_mock.call_count == 2
assert len(list(root['pyramid'])) == 2
def by_packagename(cls, package, request):
root = repository_root_factory(request)
if package not in root:
return None
return list(root[package].releases.values())
def by_name(cls, name, request):
if hasattr(request, 'root') and request.root is not None:
root = request.root
else:
root = repository_root_factory(request)
return root[name] if name in [package.__name__ for package in root] else None
def by_packagename(cls, package_name, request):
root = repository_root_factory(request)
if package_name not in [pkg.__name__ for pkg in root]:
return None
return list(root[package_name].releases.values())
def setup(env):
env['user_root'] = user_root_factory(env['request'])
env['repo_root'] = repository_root_factory(env['request'])
env['set_db_version'] = functools.partial(set_database_version,
env['request'])
def by_name(cls, name, request):
root = repository_root_factory(request)
return root[name] if name in root else None
def setup(env):
env['user_root'] = user_root_factory(env['request'])
env['repo_root'] = repository_root_factory(env['request'])
env['default_root'] = default_root_factory(env['request'])
env['set_db_version'] = functools.partial(set_database_version, env['request'])
def by_packagename(cls, package, request):
root = repository_root_factory(request)
if package not in root:
return None
return list(root[package].releases.values())
def by_name(cls, name, request):
root = repository_root_factory(request)
return root[name] if name in root else None
