def test_get_url_content(request): """should fetch uncompressed URL content""" url = 'https://www.bible.com/bible/59/psa.23' web.get_url_content(url) request.assert_called_once_with(url, headers={ 'User-Agent': 'YouVersion Suggest', 'Accept-Encoding': 'gzip, deflate' })
def get_chapter_html(ref): chapter_uid = get_ref_chapter_uid(ref) url = core.get_ref_url(ref_uid=chapter_uid) entry_key = '{}.html'.format(chapter_uid) chapter_html = cache.get_cache_entry_content(entry_key) if not chapter_html: chapter_html = web.get_url_content(url) cache.add_cache_entry(entry_key, chapter_html) return chapter_html
def get_search_html(query_str, user_prefs): url = 'https://www.bible.com/search/bible?q={}&version_id={}'.format( urllib.parse.quote_plus(query_str), user_prefs['version']) entry_key = '{}/{}.html'.format(user_prefs['version'], query_str) search_html = cache.get_cache_entry_content(entry_key) if not search_html: search_html = web.get_url_content(url) cache.add_cache_entry(entry_key, search_html) return search_html
def get_search_html(query_str): version = core.get_user_prefs()['version'] url = 'https://www.bible.com/search/bible?q={}&version_id={}'.format( urllib.quote_plus(query_str.encode('utf-8')), version) entry_key = '{}/{}.html'.format(version, query_str) search_html = cache.get_cache_entry_content(entry_key) if not search_html: search_html = web.get_url_content(url) cache.add_cache_entry(entry_key, search_html) return search_html
def test_get_url_content_compressed(request): """should automatically decompress compressed URL content""" url = 'https://www.bible.com/bible/59/psa.23' gzip_buf = BytesIO() with GzipFile(fileobj=gzip_buf, mode='wb') as gzip_file: gzip_file.write(html_content.encode('utf-8')) gzipped_content = gzip_buf.getvalue() response_mock = NonCallableMock( read=Mock(return_value=gzipped_content), info=Mock(return_value=NonCallableMock(get=Mock(return_value='gzip')))) with patch('urllib.request.urlopen', return_value=response_mock): url_content = web.get_url_content(url) nose.assert_equal(url_content, html_content)
def test_get_url_content_compressed(request): """should automatically decompress compressed URL content""" url = 'https://www.bible.com/bible/59/psa.23' gzip_buf = StringIO() with GzipFile(fileobj=gzip_buf, mode='wb') as gzip_file: gzip_file.write(html_content) gzipped_content = gzip_buf.getvalue() response_mock = NonCallableMock( read=Mock(return_value=gzipped_content), info=Mock(return_value=NonCallableMock( get=Mock(return_value='gzip')))) with patch('urllib2.urlopen', return_value=response_mock): url_content = web.get_url_content(url).encode('utf-8') nose.assert_equal(url_content, html_content)
def test_get_url_content_timeout(request, urlopen): """should timeout URL content request after 3 seconds""" web.get_url_content('https://www.bible.com/bible/59/psa.23') urlopen.assert_called_once_with(request.return_value, timeout=3)
def test_get_url_content_timeout(request, urlopen): """should timeout URL content request after 3 seconds""" web.get_url_content('https://www.bible.com/bible/59/psa.23') urlopen.assert_called_once_with(request.return_value, timeout=5)