def test_get_web_page_with_user_agent_in_headers(self):
"""
Test that adding a user agent in the header when calling get_web_page() adds that user agent to the request
"""
with patch('openlp.core.lib.webpagereader.urllib.request.Request') as MockRequest, \
patch('openlp.core.lib.webpagereader.urllib.request.urlopen') as mock_urlopen, \
patch('openlp.core.lib.webpagereader._get_user_agent') as mock_get_user_agent:
# GIVEN: Mocked out objects, a fake URL and a fake header
mocked_request_object = MagicMock()
MockRequest.return_value = mocked_request_object
mocked_page_object = MagicMock()
mock_urlopen.return_value = mocked_page_object
fake_url = 'this://is.a.fake/url'
user_agent_header = ('User-Agent', 'OpenLP/2.2.0')
# WHEN: The get_web_page() method is called
returned_page = get_web_page(fake_url, header=user_agent_header)
# THEN: The correct methods are called with the correct arguments and a web page is returned
MockRequest.assert_called_with(fake_url)
mocked_request_object.add_header.assert_called_with(
user_agent_header[0], user_agent_header[1])
self.assertEqual(1, mocked_request_object.add_header.call_count,
'There should only be 1 call to add_header')
self.assertEqual(0, mock_get_user_agent.call_count,
'_get_user_agent should not have been called')
mock_urlopen.assert_called_with(mocked_request_object, timeout=30)
mocked_page_object.geturl.assert_called_with()
self.assertEqual(mocked_page_object, returned_page,
'The returned page should be the mock object')
def test_get_web_page_update_openlp(self):
"""
Test that passing "update_openlp" as true to get_web_page calls Registry().get('app').process_events()
"""
with patch('openlp.core.lib.webpagereader.urllib.request.Request') as MockRequest, \
patch('openlp.core.lib.webpagereader.urllib.request.urlopen') as mock_urlopen, \
patch('openlp.core.lib.webpagereader._get_user_agent') as mock_get_user_agent, \
patch('openlp.core.lib.webpagereader.Registry') as MockRegistry:
# GIVEN: Mocked out objects, a fake URL
mocked_request_object = MagicMock()
MockRequest.return_value = mocked_request_object
mocked_page_object = MagicMock()
mock_urlopen.return_value = mocked_page_object
mock_get_user_agent.return_value = 'user_agent'
mocked_registry_object = MagicMock()
mocked_application_object = MagicMock()
mocked_registry_object.get.return_value = mocked_application_object
MockRegistry.return_value = mocked_registry_object
fake_url = 'this://is.a.fake/url'
# WHEN: The get_web_page() method is called
returned_page = get_web_page(fake_url, update_openlp=True)
# THEN: The correct methods are called with the correct arguments and a web page is returned
MockRequest.assert_called_with(fake_url)
mocked_request_object.add_header.assert_called_with(
'User-Agent', 'user_agent')
self.assertEqual(1, mocked_request_object.add_header.call_count,
'There should only be 1 call to add_header')
mock_urlopen.assert_called_with(mocked_request_object, timeout=30)
mocked_page_object.geturl.assert_called_with()
mocked_registry_object.get.assert_called_with('application')
mocked_application_object.process_events.assert_called_with()
self.assertEqual(mocked_page_object, returned_page,
'The returned page should be the mock object')
def test_get_web_page(self):
"""
Test that the get_web_page method works correctly
"""
with patch('openlp.core.lib.webpagereader.urllib.request.Request') as MockRequest, \
patch('openlp.core.lib.webpagereader.urllib.request.urlopen') as mock_urlopen, \
patch('openlp.core.lib.webpagereader._get_user_agent') as mock_get_user_agent, \
patch('openlp.core.common.Registry') as MockRegistry:
# GIVEN: Mocked out objects and a fake URL
mocked_request_object = MagicMock()
MockRequest.return_value = mocked_request_object
mocked_page_object = MagicMock()
mock_urlopen.return_value = mocked_page_object
mock_get_user_agent.return_value = 'user_agent'
fake_url = 'this://is.a.fake/url'
# WHEN: The get_web_page() method is called
returned_page = get_web_page(fake_url)
# THEN: The correct methods are called with the correct arguments and a web page is returned
MockRequest.assert_called_with(fake_url)
mocked_request_object.add_header.assert_called_with(
'User-Agent', 'user_agent')
self.assertEqual(1, mocked_request_object.add_header.call_count,
'There should only be 1 call to add_header')
mock_get_user_agent.assert_called_with()
mock_urlopen.assert_called_with(mocked_request_object, timeout=30)
mocked_page_object.geturl.assert_called_with()
self.assertEqual(
0, MockRegistry.call_count,
'The Registry() object should have never been called')
self.assertEqual(mocked_page_object, returned_page,
'The returned page should be the mock object')
def test_get_web_page_with_header(self):
"""
Test that adding a header to the call to get_web_page() adds the header to the request
"""
with patch('openlp.core.lib.webpagereader.urllib.request.Request') as MockRequest, \
patch('openlp.core.lib.webpagereader.urllib.request.urlopen') as mock_urlopen, \
patch('openlp.core.lib.webpagereader._get_user_agent') as mock_get_user_agent:
# GIVEN: Mocked out objects, a fake URL and a fake header
mocked_request_object = MagicMock()
MockRequest.return_value = mocked_request_object
mocked_page_object = MagicMock()
mock_urlopen.return_value = mocked_page_object
mock_get_user_agent.return_value = 'user_agent'
fake_url = 'this://is.a.fake/url'
fake_header = ('Fake-Header', 'fake value')
# WHEN: The get_web_page() method is called
returned_page = get_web_page(fake_url, header=fake_header)
# THEN: The correct methods are called with the correct arguments and a web page is returned
MockRequest.assert_called_with(fake_url)
mocked_request_object.add_header.assert_called_with(
fake_header[0], fake_header[1])
self.assertEqual(2, mocked_request_object.add_header.call_count,
'There should only be 2 calls to add_header')
mock_get_user_agent.assert_called_with()
mock_urlopen.assert_called_with(mocked_request_object, timeout=30)
mocked_page_object.geturl.assert_called_with()
self.assertEqual(mocked_page_object, returned_page,
'The returned page should be the mock object')
def test_webpage_connection_retry(self):
"""
Test get_web_page will attempt CONNECTION_RETRIES+1 connections - bug 1409031
"""
# GIVEN: Initial settings and mocks
with patch.object(urllib.request, 'urlopen') as mocked_urlopen:
mocked_urlopen.side_effect = ConnectionError
# WHEN: A webpage is requested
try:
get_web_page(url='http://localhost')
except:
pass
# THEN: urlopen should have been called CONNECTION_RETRIES + 1 count
self.assertEquals(
mocked_urlopen.call_count, CONNECTION_RETRIES + 1,
'get_web_page() should have tried {} times'.format(
CONNECTION_RETRIES))
def test_get_web_page_no_url(self):
"""
Test that sending a URL of None to the get_web_page method returns None
"""
# GIVEN: A None url
test_url = None
# WHEN: We try to get the test URL
result = get_web_page(test_url)
# THEN: None should be returned
self.assertIsNone(result,
'The return value of get_web_page should be None')
def get_books_from_http(self, version):
"""
Load a list of all books a Bible contains from BibleGateway website.
:param version: The version of the Bible like NIV for New International Version
"""
log.debug('BGExtract.get_books_from_http("{version}")'.format(
version=version))
url_params = urllib.parse.urlencode({
'action':
'getVersionInfo',
'vid':
'{version}'.format(version=version)
})
reference_url = 'http://biblegateway.com/versions/?{url}#books'.format(
url=url_params)
page = get_web_page(reference_url)
if not page:
send_error_message('download')
return None
page_source = page.read()
try:
page_source = str(page_source, 'utf8')
except UnicodeDecodeError:
page_source = str(page_source, 'cp1251')
try:
soup = BeautifulSoup(page_source, 'lxml')
except Exception:
log.error('BeautifulSoup could not parse the Bible page.')
send_error_message('parse')
return None
if not soup:
send_error_message('parse')
return None
self.application.process_events()
content = soup.find('table', 'infotable')
if content:
content = content.find_all('tr')
if not content:
log.error('No books found in the Biblegateway response.')
send_error_message('parse')
return None
books = []
for book in content:
book = book.find('td')
if book:
books.append(book.contents[1])
return books
def get_soup_for_bible_ref(reference_url,
header=None,
pre_parse_regex=None,
pre_parse_substitute=None):
"""
Gets a webpage and returns a parsed and optionally cleaned soup or None.
:param reference_url: The URL to obtain the soup from.
:param header: An optional HTTP header to pass to the bible web server.
:param pre_parse_regex: A regular expression to run on the webpage. Allows manipulation of the webpage before
passing to BeautifulSoup for parsing.
:param pre_parse_substitute: The text to replace any matches to the regular expression with.
"""
if not reference_url:
return None
try:
page = get_web_page(reference_url, header, True)
except:
page = None
if not page:
send_error_message('download')
return None
page_source = page.read()
if pre_parse_regex and pre_parse_substitute is not None:
page_source = re.sub(pre_parse_regex, pre_parse_substitute,
page_source.decode())
soup = None
try:
soup = BeautifulSoup(page_source, 'lxml')
CLEANER_REGEX.sub('', str(soup))
except Exception:
log.exception('BeautifulSoup could not parse the bible page.')
if not soup:
send_error_message('parse')
return None
Registry().get('application').process_events()
return soup
def _download_index(self):
"""
Download the configuration file and kick off the theme screenshot download threads
"""
# check to see if we have web access
self.web_access = False
self.config = ConfigParser()
user_agent = 'OpenLP/' + Registry().get('application').applicationVersion()
self.application.process_events()
try:
web_config = get_web_page('{host}{name}'.format(host=self.web, name='download.cfg'),
header=('User-Agent', user_agent))
except (urllib.error.URLError, ConnectionError) as err:
msg = QtWidgets.QMessageBox()
title = translate('OpenLP.FirstTimeWizard', 'Network Error')
msg.setText('{title} {error}'.format(title=title,
error=err.code if hasattr(err, 'code') else ''))
msg.setInformativeText(translate('OpenLP.FirstTimeWizard',
'There was a network error attempting to '
'connect to retrieve initial configuration information'))
msg.setStandardButtons(msg.Ok)
ans = msg.exec()
web_config = False
if web_config:
files = web_config.read()
try:
self.config.read_string(files.decode())
self.web = self.config.get('general', 'base url')
self.songs_url = self.web + self.config.get('songs', 'directory') + '/'
self.bibles_url = self.web + self.config.get('bibles', 'directory') + '/'
self.themes_url = self.web + self.config.get('themes', 'directory') + '/'
self.web_access = True
except (NoSectionError, NoOptionError, MissingSectionHeaderError):
log.debug('A problem occured while parsing the downloaded config file')
trace_error_handler(log)
self.update_screen_list_combo()
self.application.process_events()
# TODO: Tested at home
self.downloading = translate('OpenLP.FirstTimeWizard', 'Downloading {name}...')
if self.has_run_wizard:
self.songs_check_box.setChecked(self.plugin_manager.get_plugin_by_name('songs').is_active())
self.bible_check_box.setChecked(self.plugin_manager.get_plugin_by_name('bibles').is_active())
self.presentation_check_box.setChecked(self.plugin_manager.get_plugin_by_name('presentations').is_active())
self.image_check_box.setChecked(self.plugin_manager.get_plugin_by_name('images').is_active())
self.media_check_box.setChecked(self.plugin_manager.get_plugin_by_name('media').is_active())
self.remote_check_box.setChecked(self.plugin_manager.get_plugin_by_name('remotes').is_active())
self.custom_check_box.setChecked(self.plugin_manager.get_plugin_by_name('custom').is_active())
self.song_usage_check_box.setChecked(self.plugin_manager.get_plugin_by_name('songusage').is_active())
self.alert_check_box.setChecked(self.plugin_manager.get_plugin_by_name('alerts').is_active())
self.application.set_normal_cursor()
# Sort out internet access for downloads
if self.web_access:
songs = self.config.get('songs', 'languages')
songs = songs.split(',')
for song in songs:
self.application.process_events()
title = self.config.get('songs_{song}'.format(song=song), 'title')
filename = self.config.get('songs_{song}'.format(song=song), 'filename')
sha256 = self.config.get('songs_{song}'.format(song=song), 'sha256', fallback='')
item = QtWidgets.QListWidgetItem(title, self.songs_list_widget)
item.setData(QtCore.Qt.UserRole, (filename, sha256))
item.setCheckState(QtCore.Qt.Unchecked)
item.setFlags(item.flags() | QtCore.Qt.ItemIsUserCheckable)
bible_languages = self.config.get('bibles', 'languages')
bible_languages = bible_languages.split(',')
for lang in bible_languages:
self.application.process_events()
language = self.config.get('bibles_{lang}'.format(lang=lang), 'title')
lang_item = QtWidgets.QTreeWidgetItem(self.bibles_tree_widget, [language])
bibles = self.config.get('bibles_{lang}'.format(lang=lang), 'translations')
bibles = bibles.split(',')
for bible in bibles:
self.application.process_events()
title = self.config.get('bible_{bible}'.format(bible=bible), 'title')
filename = self.config.get('bible_{bible}'.format(bible=bible), 'filename')
sha256 = self.config.get('bible_{bible}'.format(bible=bible), 'sha256', fallback='')
item = QtWidgets.QTreeWidgetItem(lang_item, [title])
item.setData(0, QtCore.Qt.UserRole, (filename, sha256))
item.setCheckState(0, QtCore.Qt.Unchecked)
item.setFlags(item.flags() | QtCore.Qt.ItemIsUserCheckable)
self.bibles_tree_widget.expandAll()
self.application.process_events()
# Download the theme screenshots
themes = self.config.get('themes', 'files').split(',')
for theme in themes:
title = self.config.get('theme_{theme}'.format(theme=theme), 'title')
filename = self.config.get('theme_{theme}'.format(theme=theme), 'filename')
sha256 = self.config.get('theme_{theme}'.format(theme=theme), 'sha256', fallback='')
screenshot = self.config.get('theme_{theme}'.format(theme=theme), 'screenshot')
worker = ThemeScreenshotWorker(self.themes_url, title, filename, sha256, screenshot)
self.theme_screenshot_workers.append(worker)
worker.screenshot_downloaded.connect(self.on_screenshot_downloaded)
thread = QtCore.QThread(self)
self.theme_screenshot_threads.append(thread)
thread.started.connect(worker.run)
worker.finished.connect(thread.quit)
worker.moveToThread(thread)
thread.start()
self.application.process_events()