def _cache_url(self, filepath):
response = urlopen_original(self.url)
fp = open(filepath, 'wb')
# when it uses file:// scheme, code is None and there is no msg attr
# but it has been successfully opened
status = b('%s %s' % (getattr(response, 'code', 200) or 200, getattr(response, 'msg', 'OK')))
headers = [b('%s: %s' % (key, value)) for key, value in list(response.headers.items())]
body = response.read()
fp.write(b('\n').join([status] + headers + [b(''), body]))
fp.close()
def test_unpack_http_url_bad_downloaded_checksum(mock_get_response, mock_unpack_file):
"""
If already-downloaded file has bad checksum, re-download.
"""
base_url = 'http://www.example.com/somepackage.tgz'
contents = b('downloaded')
download_hash = hashlib.new('sha1', contents)
link = Link(base_url + '#sha1=' + download_hash.hexdigest())
response = mock_get_response.return_value = MockResponse(contents)
response.info = lambda: {'content-type': 'application/x-tar'}
response.geturl = lambda: base_url
download_dir = mkdtemp()
try:
downloaded_file = os.path.join(download_dir, 'somepackage.tgz')
_write_file(downloaded_file, 'some contents')
unpack_http_url(link, 'location', download_cache=None, download_dir=download_dir)
# despite existence of downloaded file with bad hash, downloaded again
mock_get_response.assert_called_once_with(base_url, link)
# cached file is replaced with newly downloaded file
with open(downloaded_file) as fh:
assert fh.read() == 'downloaded'
finally:
rmtree(download_dir)
def test_unpack_http_url_bad_cache_checksum(mock_get_response,
mock_unpack_file):
"""
If cached download has bad checksum, re-download.
"""
base_url = 'http://www.example.com/somepackage.tgz'
contents = b('downloaded')
download_hash = hashlib.new('sha1', contents)
link = Link(base_url + '#sha1=' + download_hash.hexdigest())
response = mock_get_response.return_value = MockResponse(contents)
response.info = lambda: {'content-type': 'application/x-tar'}
response.geturl = lambda: base_url
cache_dir = mkdtemp()
try:
cache_file = os.path.join(cache_dir, urllib.quote(base_url, ''))
cache_ct_file = cache_file + '.content-type'
_write_file(cache_file, 'some contents')
_write_file(cache_ct_file, 'application/x-tar')
unpack_http_url(link, 'location', download_cache=cache_dir)
# despite existence of cached file with bad hash, downloaded again
mock_get_response.assert_called_once_with(base_url, link)
# cached file is replaced with newly downloaded file
with open(cache_file) as fh:
assert fh.read() == 'downloaded'
finally:
rmtree(cache_dir)
def test_searching_through_Search_class():
"""
Verify if ``pip.vcs.Search`` uses tests xmlrpclib.Transport class
"""
original_xmlrpclib_transport = pip.download.xmlrpclib_transport
pip.download.xmlrpclib_transport = fake_transport = Mock()
query = 'mylittlequerythatdoesnotexists'
dumped_xmlrpc_request = b(
xmlrpclib.dumps(({
'name': query,
'summary': query
}, 'or'), 'search'))
expected = [{
'_pypi_ordering': 100,
'name': 'foo',
'summary': 'foo summary',
'version': '1.0'
}]
fake_transport.request.return_value = (expected, )
pypi_searcher = SearchCommand(create_main_parser())
result = pypi_searcher.search(query, 'http://pypi.python.org/pypi')
try:
assert expected == result, result
fake_transport.request.assert_called_with('pypi.python.org',
'/pypi',
dumped_xmlrpc_request,
verbose=VERBOSE_FALSE)
finally:
pip.download.xmlrpclib_transport = original_xmlrpclib_transport
def _set_all_fields(self, folder):
filename = os.path.join(folder, urllib.quote(self.url, ''))
if not os.path.exists(filename):
self._cache_url(filename)
fp = open(filename, 'rb')
try:
line = fp.readline().strip()
self.code, self.msg = line.split(None, 1)
except ValueError:
raise ValueError('Bad field line: %r' % line)
self.code = int(self.code)
self.msg = u(self.msg)
for line in fp:
if line == b('\n'):
break
key, value = line.split(b(': '), 1)
self.headers[u(key)] = u(value.strip())
for line in fp:
self._body += line
fp.close()
def remove(self):
logger.info('Removing pth entries from %s:' % self.file)
fh = open(self.file, 'rb')
# windows uses '\r\n' with py3k, but uses '\n' with py2.x
lines = fh.readlines()
self._saved_lines = lines
fh.close()
if any(b('\r\n') in line for line in lines):
endline = '\r\n'
else:
endline = '\n'
for entry in self.entries:
try:
logger.info('Removing entry: %s' % entry)
lines.remove(b(entry + endline))
except ValueError:
pass
fh = open(self.file, 'wb')
fh.writelines(lines)
fh.close()
def remove(self):
logger.info("Removing pth entries from %s:" % self.file)
fh = open(self.file, "rb")
# windows uses '\r\n' with py3k, but uses '\n' with py2.x
lines = fh.readlines()
self._saved_lines = lines
fh.close()
if any(b("\r\n") in line for line in lines):
endline = "\r\n"
else:
endline = "\n"
for entry in self.entries:
try:
logger.info("Removing entry: %s" % entry)
lines.remove(b(entry + endline))
except ValueError:
pass
fh = open(self.file, "wb")
fh.writelines(lines)
fh.close()
def __init__(self, url, folder):
self.headers = emailmessage.Message()
self.code = 500
self.msg = 'Internal Server Error'
# url can be a simple string, or a urllib2.Request object
if isinstance(url, string_types):
self.url = url
else:
self.url = url.get_full_url()
for key, value in url.headers.items():
self.headers[key] = value
self._body = b('')
self._set_all_fields(folder)
def test_searching_through_Search_class():
"""
Verify if ``pip.vcs.Search`` uses tests xmlrpclib.Transport class
"""
pip.download.xmlrpclib_transport = fake_transport = Mock()
query = 'mylittlequerythatdoesnotexists'
dumped_xmlrpc_request = b(xmlrpclib.dumps(({'name': query, 'summary': query}, 'or'), 'search'))
expected = [{'_pypi_ordering': 100, 'name': 'foo', 'summary': 'foo summary', 'version': '1.0'}]
fake_transport.request.return_value = (expected,)
pypi_searcher = SearchCommand()
result = pypi_searcher.search(query, 'http://pypi.python.org/pypi')
assert expected == result, result
fake_transport.request.assert_called_with('pypi.python.org', '/pypi', dumped_xmlrpc_request, verbose=VERBOSE_FALSE)
def test_unpack_http_url_bad_cache_checksum(mock_unpack_file):
"""
If cached download has bad checksum, re-download.
"""
base_url = 'http://www.example.com/somepackage.tgz'
contents = b('downloaded')
download_hash = hashlib.new('sha1', contents)
link = Link(base_url + '#sha1=' + download_hash.hexdigest())
session = Mock()
session.get = Mock()
response = session.get.return_value = MockResponse(contents)
response.headers = {'content-type': 'application/x-tar'}
response.url = base_url
cache_dir = mkdtemp()
try:
cache_file = os.path.join(cache_dir, urllib.quote(base_url, ''))
cache_ct_file = cache_file + '.content-type'
_write_file(cache_file, 'some contents')
_write_file(cache_ct_file, 'application/x-tar')
unpack_http_url(
link,
'location',
download_cache=cache_dir,
session=session,
)
# despite existence of cached file with bad hash, downloaded again
session.get.assert_called_once_with(
"http://www.example.com/somepackage.tgz",
stream=True,
)
# cached file is replaced with newly downloaded file
with open(cache_file) as fh:
assert fh.read() == 'downloaded'
finally:
rmtree(cache_dir)
def test_unpack_http_url_bad_downloaded_checksum(mock_unpack_file):
"""
If already-downloaded file has bad checksum, re-download.
"""
base_url = 'http://www.example.com/somepackage.tgz'
contents = b('downloaded')
download_hash = hashlib.new('sha1', contents)
link = Link(base_url + '#sha1=' + download_hash.hexdigest())
session = Mock()
session.get = Mock()
response = session.get.return_value = MockResponse(contents)
response.headers = {'content-type': 'application/x-tar'}
response.url = base_url
download_dir = mkdtemp()
try:
downloaded_file = os.path.join(download_dir, 'somepackage.tgz')
_write_file(downloaded_file, 'some contents')
unpack_http_url(
link,
'location',
download_cache=None,
download_dir=download_dir,
session=session,
)
# despite existence of downloaded file with bad hash, downloaded again
session.get.assert_called_once_with(
'http://www.example.com/somepackage.tgz',
stream=True,
)
# cached file is replaced with newly downloaded file
with open(downloaded_file) as fh:
assert fh.read() == 'downloaded'
finally:
rmtree(download_dir)
def __init__(self, url, folder):
self.headers = emailmessage.Message()
# patch due to setuptools>=0.7 header processing
# easy_install fails w/o this on windows/py2
# https://github.com/pypa/pip/issues/946#issuecomment-20860320
if sys.version_info < (3,):
def getheaders(key):
return self.headers.get_all(key)
self.headers.getheaders = getheaders
self.code = 500
self.msg = 'Internal Server Error'
# url can be a simple string, or a urllib2.Request object
if isinstance(url, string_types):
self.url = url
else:
self.url = url.get_full_url()
for key, value in url.headers.items():
self.headers[key] = value
self._body = b('')
self._set_all_fields(folder)
