def test_urlopen_rate_limit(self, log, sleep, urlopen):
limit_exceeded_headers = {
'X-RateLimit-Limit': '10',
'X-RateLimit-Remaining': '0',
'X-RateLimit-Reset': '1382693522',
}
response_limit_exceeded = StringIO('{}')
response_limit_exceeded.info = lambda: limit_exceeded_headers
response_ok = StringIO('{}')
response_ok.info = lambda: {}
urlopen.side_effect = [response_limit_exceeded, response_ok]
e = github.GitHubProjectExtractor('test_project')
e.get_page('fake')
self.assertEqual(sleep.call_count, 1)
self.assertEqual(urlopen.call_count, 2)
log.warn.assert_called_once_with(
'Rate limit exceeded (10 requests/hour). '
'Sleeping until 2013-10-25 09:32:02 UTC'
)
sleep.reset_mock(); urlopen.reset_mock(); log.warn.reset_mock()
response_ok = StringIO('{}')
response_ok.info = lambda: {}
urlopen.side_effect = [response_ok]
e.get_page('fake 2')
self.assertEqual(sleep.call_count, 0)
self.assertEqual(urlopen.call_count, 1)
self.assertEqual(log.warn.call_count, 0)
def test_urlopen_rate_limit(self, log, sleep, urlopen):
limit_exceeded_headers = {
'X-RateLimit-Limit': '10',
'X-RateLimit-Remaining': '0',
'X-RateLimit-Reset': '1382693522',
}
response_limit_exceeded = StringIO('{}')
response_limit_exceeded.info = lambda: limit_exceeded_headers
response_ok = StringIO('{}')
response_ok.info = lambda: {}
urlopen.side_effect = [response_limit_exceeded, response_ok]
e = github.GitHubProjectExtractor('test_project')
e.get_page('fake')
self.assertEqual(sleep.call_count, 1)
self.assertEqual(urlopen.call_count, 2)
log.warn.assert_called_once_with(
'Rate limit exceeded (10 requests/hour). '
'Sleeping until 2013-10-25 09:32:02 UTC'
)
sleep.reset_mock()
urlopen.reset_mock()
log.warn.reset_mock()
response_ok = StringIO('{}')
response_ok.info = lambda: {}
urlopen.side_effect = [response_ok]
e.get_page('fake 2')
self.assertEqual(sleep.call_count, 0)
self.assertEqual(urlopen.call_count, 1)
self.assertEqual(log.warn.call_count, 0)
def response_mock(encoding, data):
"""Make a fake response with specific encoding and content."""
resp = StringIO(data)
info = mock.Mock()
info.get_param = mock.Mock(return_value=encoding)
resp.info = mock.Mock(return_value=info)
return resp
def mocked_urlopen(self, url):
headers = {}
if url.endswith('/test_project'):
response = StringIO(json.dumps(self.PROJECT_INFO))
elif url.endswith('/issues?state=closed'):
response = StringIO(json.dumps(self.CLOSED_ISSUES_LIST))
elif url.endswith('/issues?state=opened'):
response = StringIO(json.dumps(self.OPENED_ISSUES_LIST))
headers = {'Link': '</issues?state=opened&page=2>; rel="next"'}
elif url.endswith('/issues?state=opened&page=2'):
response = StringIO(json.dumps(self.OPENED_ISSUES_LIST_PAGE2))
elif url.endswith('/comments'):
response = StringIO(json.dumps(self.ISSUE_COMMENTS))
headers = {'Link': '</comments?page=2>; rel="next"'}
elif url.endswith('/comments?page=2'):
response = StringIO(json.dumps(self.ISSUE_COMMENTS_PAGE2))
elif url.endswith('/events'):
response = StringIO(json.dumps(self.ISSUE_EVENTS))
headers = {'Link': '</events?page=2>; rel="next"'}
elif url.endswith('/events?page=2'):
response = StringIO(json.dumps(self.ISSUE_EVENTS_PAGE2))
response.info = lambda: headers
return response
def mocked_urlopen(self, url):
headers = {}
if url.endswith('/test_project'):
response = StringIO(json.dumps(self.PROJECT_INFO))
elif url.endswith('/issues?state=closed'):
response = StringIO(json.dumps(self.CLOSED_ISSUES_LIST))
elif url.endswith('/issues?state=open'):
response = StringIO(json.dumps(self.OPENED_ISSUES_LIST))
headers = {'Link': '</issues?state=open&page=2>; rel="next"'}
elif url.endswith('/issues?state=open&page=2'):
response = StringIO(json.dumps(self.OPENED_ISSUES_LIST_PAGE2))
elif url.endswith('/comments'):
response = StringIO(json.dumps(self.ISSUE_COMMENTS))
headers = {'Link': '</comments?page=2>; rel="next"'}
elif url.endswith('/comments?page=2'):
response = StringIO(json.dumps(self.ISSUE_COMMENTS_PAGE2))
elif url.endswith('/events'):
response = StringIO(json.dumps(self.ISSUE_EVENTS))
headers = {'Link': '</events?page=2>; rel="next"'}
elif url.endswith('/events?page=2'):
response = StringIO(json.dumps(self.ISSUE_EVENTS_PAGE2))
response.info = lambda: headers
return response
def post(self, url, data):
def info():
return {'Content-Length': len(self.www[url])}
remote = StringIO(self.www[url])
remote.info = info
return remote
def urlopen_side_effect(*a, **kw):
mock_resp = StringIO('{}')
mock_resp.info = lambda: {}
urlopen.side_effect = [mock_resp]
raise urllib2.HTTPError(
'url', 403, 'msg', limit_exceeded_headers, StringIO('{}'))
def fake_file(self, string='{}', url='http://bogus', info=None):
sio = StringIO(string)
sio.info = lambda: FakeFileInfo(info)
sio.url = url
return sio
def urlopen_side_effect(*a, **kw):
mock_resp = StringIO('{}')
mock_resp.info = lambda: {}
urlopen.side_effect = [mock_resp]
raise urllib2.HTTPError(
'url', 403, 'msg', limit_exceeded_headers, StringIO('{}'))
