def test_get_web_page_with_header(self):
"""
Test that adding a header to the call to get_web_page() adds the header to the request
"""
with patch('openlp.core.common.httputils.urllib.request.Request') as MockRequest, \
patch('openlp.core.common.httputils.urllib.request.urlopen') as mock_urlopen, \
patch('openlp.core.common.httputils.get_user_agent') as mock_get_user_agent:
# GIVEN: Mocked out objects, a fake URL and a fake header
mocked_request_object = MagicMock()
MockRequest.return_value = mocked_request_object
mocked_page_object = MagicMock()
mock_urlopen.return_value = mocked_page_object
mock_get_user_agent.return_value = 'user_agent'
fake_url = 'this://is.a.fake/url'
fake_header = ('Fake-Header', 'fake value')
# WHEN: The get_web_page() method is called
returned_page = get_web_page(fake_url, header=fake_header)
# THEN: The correct methods are called with the correct arguments and a web page is returned
MockRequest.assert_called_with(fake_url)
mocked_request_object.add_header.assert_called_with(fake_header[0], fake_header[1])
self.assertEqual(2, mocked_request_object.add_header.call_count,
'There should only be 2 calls to add_header')
mock_get_user_agent.assert_called_with()
mock_urlopen.assert_called_with(mocked_request_object, timeout=30)
mocked_page_object.geturl.assert_called_with()
self.assertEqual(mocked_page_object, returned_page, 'The returned page should be the mock object')
def _download_index(self):
"""
Download the configuration file and kick off the theme screenshot download threads
"""
# Check if the index file has already been downloaded
if self.is_index_downloaded:
return
# check to see if we have web access
self.has_web_access = False
self.config = ''
web_config = None
user_agent = 'OpenLP/' + QtWidgets.QApplication.applicationVersion()
self.application.process_events()
try:
web_config = get_web_page('{host}{name}'.format(host=self.web, name='download_3.0.json'),
headers={'User-Agent': user_agent})
except ConnectionError:
QtWidgets.QMessageBox.critical(self, translate('OpenLP.FirstTimeWizard', 'Network Error'),
translate('OpenLP.FirstTimeWizard', 'There was a network error attempting '
'to connect to retrieve initial configuration information'),
QtWidgets.QMessageBox.Ok)
if web_config and self._parse_config(web_config):
self.has_web_access = True
self.application.process_events()
self.downloading = translate('OpenLP.FirstTimeWizard', 'Downloading {name}...')
self.application.set_normal_cursor()
self.is_index_downloaded = True
def test_get_web_page_update_openlp(self):
"""
Test that passing "update_openlp" as true to get_web_page calls Registry().get('app').process_events()
"""
with patch('openlp.core.common.httputils.urllib.request.Request') as MockRequest, \
patch('openlp.core.common.httputils.urllib.request.urlopen') as mock_urlopen, \
patch('openlp.core.common.httputils.get_user_agent') as mock_get_user_agent, \
patch('openlp.core.common.httputils.Registry') as MockRegistry:
# GIVEN: Mocked out objects, a fake URL
mocked_request_object = MagicMock()
MockRequest.return_value = mocked_request_object
mocked_page_object = MagicMock()
mock_urlopen.return_value = mocked_page_object
mock_get_user_agent.return_value = 'user_agent'
mocked_registry_object = MagicMock()
mocked_application_object = MagicMock()
mocked_registry_object.get.return_value = mocked_application_object
MockRegistry.return_value = mocked_registry_object
fake_url = 'this://is.a.fake/url'
# WHEN: The get_web_page() method is called
returned_page = get_web_page(fake_url, update_openlp=True)
# THEN: The correct methods are called with the correct arguments and a web page is returned
MockRequest.assert_called_with(fake_url)
mocked_request_object.add_header.assert_called_with('User-Agent', 'user_agent')
self.assertEqual(1, mocked_request_object.add_header.call_count,
'There should only be 1 call to add_header')
mock_urlopen.assert_called_with(mocked_request_object, timeout=30)
mocked_page_object.geturl.assert_called_with()
mocked_registry_object.get.assert_called_with('application')
mocked_application_object.process_events.assert_called_with()
self.assertEqual(mocked_page_object, returned_page, 'The returned page should be the mock object')
def test_get_web_page(self):
"""
Test that the get_web_page method works correctly
"""
with patch('openlp.core.common.httputils.urllib.request.Request') as MockRequest, \
patch('openlp.core.common.httputils.urllib.request.urlopen') as mock_urlopen, \
patch('openlp.core.common.httputils.get_user_agent') as mock_get_user_agent, \
patch('openlp.core.common.Registry') as MockRegistry:
# GIVEN: Mocked out objects and a fake URL
mocked_request_object = MagicMock()
MockRequest.return_value = mocked_request_object
mocked_page_object = MagicMock()
mock_urlopen.return_value = mocked_page_object
mock_get_user_agent.return_value = 'user_agent'
fake_url = 'this://is.a.fake/url'
# WHEN: The get_web_page() method is called
returned_page = get_web_page(fake_url)
# THEN: The correct methods are called with the correct arguments and a web page is returned
MockRequest.assert_called_with(fake_url)
mocked_request_object.add_header.assert_called_with('User-Agent', 'user_agent')
self.assertEqual(1, mocked_request_object.add_header.call_count,
'There should only be 1 call to add_header')
mock_get_user_agent.assert_called_with()
mock_urlopen.assert_called_with(mocked_request_object, timeout=30)
mocked_page_object.geturl.assert_called_with()
self.assertEqual(0, MockRegistry.call_count, 'The Registry() object should have never been called')
self.assertEqual(mocked_page_object, returned_page, 'The returned page should be the mock object')
def get_soup_for_bible_ref(reference_url, header=None, pre_parse_regex=None, pre_parse_substitute=None):
"""
Gets a webpage and returns a parsed and optionally cleaned soup or None.
:param reference_url: The URL to obtain the soup from.
:param header: An optional HTTP header to pass to the bible web server.
:param pre_parse_regex: A regular expression to run on the webpage. Allows manipulation of the webpage before
passing to BeautifulSoup for parsing.
:param pre_parse_substitute: The text to replace any matches to the regular expression with.
"""
if not reference_url:
return None
try:
page = get_web_page(reference_url, header, True)
except:
page = None
if not page:
send_error_message('download')
return None
page_source = page.read()
if pre_parse_regex and pre_parse_substitute is not None:
page_source = re.sub(pre_parse_regex, pre_parse_substitute, page_source.decode())
soup = None
try:
soup = BeautifulSoup(page_source, 'lxml')
CLEANER_REGEX.sub('', str(soup))
except Exception:
log.exception('BeautifulSoup could not parse the bible page.')
if not soup:
send_error_message('parse')
return None
Registry().get('application').process_events()
return soup
def test_get_web_page_update_openlp(self, MockRegistry,
mocked_get_user_agent,
mocked_requests):
"""
Test that passing "update_openlp" as true to get_web_page calls Registry().get('app').process_events()
"""
# GIVEN: Mocked out objects, a fake URL
mocked_requests.get.return_value = MagicMock(text='text')
mocked_get_user_agent.return_value = 'user_agent'
mocked_registry_object = MagicMock()
mocked_application_object = MagicMock()
mocked_registry_object.get.return_value = mocked_application_object
MockRegistry.return_value = mocked_registry_object
fake_url = 'this://is.a.fake/url'
# WHEN: The get_web_page() method is called
returned_page = get_web_page(fake_url, update_openlp=True)
# THEN: The correct methods are called with the correct arguments and a web page is returned
mocked_requests.get.assert_called_once_with(
fake_url,
headers={'User-Agent': 'user_agent'},
proxies=None,
timeout=30.0)
mocked_get_user_agent.assert_called_once_with()
mocked_registry_object.get.assert_called_with('application')
mocked_application_object.process_events.assert_called_with()
assert returned_page == 'text', 'The returned page should be the mock object'
def test_get_web_page_with_user_agent_in_headers(self):
"""
Test that adding a user agent in the header when calling get_web_page() adds that user agent to the request
"""
with patch('openlp.core.common.httputils.urllib.request.Request') as MockRequest, \
patch('openlp.core.common.httputils.urllib.request.urlopen') as mock_urlopen, \
patch('openlp.core.common.httputils.get_user_agent') as mock_get_user_agent:
# GIVEN: Mocked out objects, a fake URL and a fake header
mocked_request_object = MagicMock()
MockRequest.return_value = mocked_request_object
mocked_page_object = MagicMock()
mock_urlopen.return_value = mocked_page_object
fake_url = 'this://is.a.fake/url'
user_agent_header = ('User-Agent', 'OpenLP/2.2.0')
# WHEN: The get_web_page() method is called
returned_page = get_web_page(fake_url, header=user_agent_header)
# THEN: The correct methods are called with the correct arguments and a web page is returned
MockRequest.assert_called_with(fake_url)
mocked_request_object.add_header.assert_called_with(user_agent_header[0], user_agent_header[1])
self.assertEqual(1, mocked_request_object.add_header.call_count,
'There should only be 1 call to add_header')
self.assertEqual(0, mock_get_user_agent.call_count, 'get_user_agent should not have been called')
mock_urlopen.assert_called_with(mocked_request_object, timeout=30)
mocked_page_object.geturl.assert_called_with()
self.assertEqual(mocked_page_object, returned_page, 'The returned page should be the mock object')
def test_webpage_connection_retry(self, mocked_requests):
"""
Test get_web_page will attempt CONNECTION_RETRIES+1 connections - bug 1409031
"""
# GIVEN: Initial settings and mocks
mocked_requests.get.side_effect = OSError('Unable to connect')
# WHEN: A webpage is requested
try:
get_web_page('http://localhost')
except Exception as e:
assert isinstance(e, ConnectionError)
# THEN: urlopen should have been called CONNECTION_RETRIES + 1 count
assert mocked_requests.get.call_count == CONNECTION_RETRIES, \
'get should have been called {} times, but was only called {} times'.format(
CONNECTION_RETRIES, mocked_requests.get.call_count)
def test_webpage_connection_retry(self):
"""
Test get_web_page will attempt CONNECTION_RETRIES+1 connections - bug 1409031
"""
# GIVEN: Initial settings and mocks
with patch.object(urllib.request, 'urlopen') as mocked_urlopen:
mocked_urlopen.side_effect = ConnectionError
# WHEN: A webpage is requested
try:
get_web_page(url='http://localhost')
except:
pass
# THEN: urlopen should have been called CONNECTION_RETRIES + 1 count
self.assertEquals(mocked_urlopen.call_count, CONNECTION_RETRIES + 1,
'get_web_page() should have tried {} times'.format(CONNECTION_RETRIES))
def test_get_web_page_no_url(self):
"""
Test that sending a URL of None to the get_web_page method returns None
"""
# GIVEN: A None url
test_url = None
# WHEN: We try to get the test URL
result = get_web_page(test_url)
# THEN: None should be returned
self.assertIsNone(result, 'The return value of get_web_page should be None')
def test_get_web_page_no_url(self):
"""
Test that sending a URL of None to the get_web_page method returns None
"""
# GIVEN: A None url
test_url = None
# WHEN: We try to get the test URL
result = get_web_page(test_url)
# THEN: None should be returned
assert result is None, 'The return value of get_web_page should be None'
def download_sha256():
"""
Download the config file to extract the sha256 and version number
"""
user_agent = 'OpenLP/' + QtWidgets.QApplication.applicationVersion()
try:
web_config = get_web_page('https://get.openlp.org/webclient/download.cfg', headers={'User-Agent': user_agent})
except ConnectionError:
return False
if not web_config:
return None
file_bits = web_config.split()
return file_bits[0], file_bits[2]
def is_stage_active(self):
"""
Is stage active - call it and see but only once
:return: if stage is active or not
"""
if self.stage_cache is None:
try:
page = get_web_page("http://localhost:4316/stage")
except:
page = None
if page:
self.stage_cache = True
else:
self.stage_cache = False
return self.stage_cache
def is_chords_active(self):
"""
Is chords active - call it and see but only once
:return: if live is active or not
"""
if self.chords_cache is None:
try:
page = get_web_page("http://localhost:4316/chords")
except:
page = None
if page:
self.chords_cache = True
else:
self.chords_cache = False
return self.chords_cache
def is_live_active(self):
"""
Is main active - call it and see but only once
:return: if live is active or not
"""
if self.live_cache is None:
try:
page = get_web_page("http://localhost:4316/main")
except Exception:
page = None
if page:
self.live_cache = True
else:
self.live_cache = False
return self.live_cache
def test_get_web_page_with_user_agent_in_headers(self, mocked_get_user_agent, mocked_requests):
"""
Test that adding a user agent in the header when calling get_web_page() adds that user agent to the request
"""
# GIVEN: Mocked out objects, a fake URL and a fake header
mocked_requests.get.return_value = MagicMock(text='text')
fake_url = 'this://is.a.fake/url'
user_agent_headers = {'User-Agent': 'OpenLP/2.2.0'}
# WHEN: The get_web_page() method is called
returned_page = get_web_page(fake_url, headers=user_agent_headers)
# THEN: The correct methods are called with the correct arguments and a web page is returned
mocked_requests.get.assert_called_once_with(fake_url, headers=user_agent_headers,
proxies=None, timeout=30.0)
assert mocked_get_user_agent.call_count == 0, 'get_user_agent() should not have been called'
assert returned_page == 'text', 'The returned page should be "test"'
def test_get_web_page(self, MockRegistry, mocked_get_user_agent, mocked_requests):
"""
Test that the get_web_page method works correctly
"""
# GIVEN: Mocked out objects and a fake URL
mocked_requests.get.return_value = MagicMock(text='text')
mocked_get_user_agent.return_value = 'user_agent'
fake_url = 'this://is.a.fake/url'
# WHEN: The get_web_page() method is called
returned_page = get_web_page(fake_url)
# THEN: The correct methods are called with the correct arguments and a web page is returned
mocked_requests.get.assert_called_once_with(fake_url, headers={'User-Agent': 'user_agent'},
proxies=None, timeout=30.0)
mocked_get_user_agent.assert_called_once_with()
assert MockRegistry.call_count == 0, 'The Registry() object should have never been called'
assert returned_page == 'text', 'The returned page should be the mock object'
def get_books_from_http(self, version):
"""
Load a list of all books a Bible contains from BibleGateway website.
:param version: The version of the Bible like NIV for New International Version
"""
log.debug('BGExtract.get_books_from_http("{version}")'.format(
version=version))
url_params = urllib.parse.urlencode({
'action':
'getVersionInfo',
'vid':
'{version}'.format(version=version)
})
reference_url = 'http://www.biblegateway.com/versions/?{url}#books'.format(
url=url_params)
page_source = get_web_page(reference_url)
if not page_source:
send_error_message('download')
return None
try:
soup = BeautifulSoup(page_source, 'lxml')
except Exception:
log.error('BeautifulSoup could not parse the Bible page.')
send_error_message('parse')
return None
if not soup:
send_error_message('parse')
return None
self.application.process_events()
content = soup.find('table', 'infotable')
if content:
content = content.find_all('tr')
if not content:
log.error('No books found in the Biblegateway response.')
send_error_message('parse')
return None
books = []
for book in content:
book = book.find('td')
if book:
books.append(book.contents[1])
return books
def test_get_web_page_with_header(self, mocked_get_user_agent, mocked_requests):
"""
Test that adding a header to the call to get_web_page() adds the header to the request
"""
# GIVEN: Mocked out objects, a fake URL and a fake header
mocked_requests.get.return_value = MagicMock(text='text')
mocked_get_user_agent.return_value = 'user_agent'
fake_url = 'this://is.a.fake/url'
fake_headers = {'Fake-Header': 'fake value'}
# WHEN: The get_web_page() method is called
returned_page = get_web_page(fake_url, headers=fake_headers)
# THEN: The correct methods are called with the correct arguments and a web page is returned
expected_headers = dict(fake_headers)
expected_headers.update({'User-Agent': 'user_agent'})
mocked_requests.get.assert_called_once_with(fake_url, headers=expected_headers,
proxies=None, timeout=30.0)
mocked_get_user_agent.assert_called_with()
assert returned_page == 'text', 'The returned page should be the mock object'
def get_books_from_http(self, version):
"""
Load a list of all books a Bible contains from BibleGateway website.
:param version: The version of the Bible like NIV for New International Version
"""
log.debug('BGExtract.get_books_from_http("{version}")'.format(version=version))
url_params = urllib.parse.urlencode({'action': 'getVersionInfo', 'vid': '{version}'.format(version=version)})
reference_url = 'http://biblegateway.com/versions/?{url}#books'.format(url=url_params)
page = get_web_page(reference_url)
if not page:
send_error_message('download')
return None
page_source = page.read()
try:
page_source = str(page_source, 'utf8')
except UnicodeDecodeError:
page_source = str(page_source, 'cp1251')
try:
soup = BeautifulSoup(page_source, 'lxml')
except Exception:
log.error('BeautifulSoup could not parse the Bible page.')
send_error_message('parse')
return None
if not soup:
send_error_message('parse')
return None
self.application.process_events()
content = soup.find('table', 'infotable')
if content:
content = content.find_all('tr')
if not content:
log.error('No books found in the Biblegateway response.')
send_error_message('parse')
return None
books = []
for book in content:
book = book.find('td')
if book:
books.append(book.contents[1])
return books
def get_soup_for_bible_ref(reference_url,
headers=None,
pre_parse_regex=None,
pre_parse_substitute=None):
"""
Gets a webpage and returns a parsed and optionally cleaned soup or None.
:param reference_url: The URL to obtain the soup from.
:param header: An optional HTTP header to pass to the bible web server.
:param pre_parse_regex: A regular expression to run on the webpage. Allows manipulation of the webpage before
passing to BeautifulSoup for parsing.
:param pre_parse_substitute: The text to replace any matches to the regular expression with.
"""
if not reference_url:
return None
try:
page_source = get_web_page(reference_url, headers, update_openlp=True)
except Exception as e:
log.exception(
'Unable to download Bible %s, unknown exception occurred',
reference_url)
page_source = None
if not page_source:
send_error_message('download')
return None
if pre_parse_regex and pre_parse_substitute is not None:
page_source = re.sub(pre_parse_regex, pre_parse_substitute,
page_source)
soup = None
try:
soup = BeautifulSoup(page_source, 'lxml')
CLEANER_REGEX.sub('', str(soup))
except Exception:
log.exception('BeautifulSoup could not parse the bible page.')
if not soup:
send_error_message('parse')
return None
Registry().get('application').process_events()
return soup
def _download_index(self):
"""
Download the configuration file and kick off the theme screenshot download threads
"""
# check to see if we have web access
self.web_access = False
self.config = ConfigParser()
user_agent = 'OpenLP/' + Registry().get(
'application').applicationVersion()
self.application.process_events()
try:
web_config = get_web_page('{host}{name}'.format(
host=self.web, name='download.cfg'),
headers={'User-Agent': user_agent})
except ConnectionError:
QtWidgets.QMessageBox.critical(
self, translate('OpenLP.FirstTimeWizard', 'Network Error'),
translate(
'OpenLP.FirstTimeWizard',
'There was a network error attempting '
'to connect to retrieve initial configuration information'
), QtWidgets.QMessageBox.Ok)
web_config = False
if web_config:
try:
self.config.read_string(web_config)
self.web = self.config.get('general', 'base url')
self.songs_url = self.web + self.config.get(
'songs', 'directory') + '/'
self.bibles_url = self.web + self.config.get(
'bibles', 'directory') + '/'
self.themes_url = self.web + self.config.get(
'themes', 'directory') + '/'
self.web_access = True
except (NoSectionError, NoOptionError, MissingSectionHeaderError):
log.debug(
'A problem occurred while parsing the downloaded config file'
)
trace_error_handler(log)
self.update_screen_list_combo()
self.application.process_events()
self.downloading = translate('OpenLP.FirstTimeWizard',
'Downloading {name}...')
if self.has_run_wizard:
self.songs_check_box.setChecked(
self.plugin_manager.get_plugin_by_name('songs').is_active())
self.bible_check_box.setChecked(
self.plugin_manager.get_plugin_by_name('bibles').is_active())
self.presentation_check_box.setChecked(
self.plugin_manager.get_plugin_by_name(
'presentations').is_active())
self.image_check_box.setChecked(
self.plugin_manager.get_plugin_by_name('images').is_active())
self.media_check_box.setChecked(
self.plugin_manager.get_plugin_by_name('media').is_active())
self.custom_check_box.setChecked(
self.plugin_manager.get_plugin_by_name('custom').is_active())
self.song_usage_check_box.setChecked(
self.plugin_manager.get_plugin_by_name(
'songusage').is_active())
self.alert_check_box.setChecked(
self.plugin_manager.get_plugin_by_name('alerts').is_active())
self.application.set_normal_cursor()
# Sort out internet access for downloads
if self.web_access:
songs = self.config.get('songs', 'languages')
songs = songs.split(',')
for song in songs:
self.application.process_events()
title = self.config.get('songs_{song}'.format(song=song),
'title')
filename = self.config.get('songs_{song}'.format(song=song),
'filename')
sha256 = self.config.get('songs_{song}'.format(song=song),
'sha256',
fallback='')
item = QtWidgets.QListWidgetItem(title, self.songs_list_widget)
item.setData(QtCore.Qt.UserRole, (filename, sha256))
item.setCheckState(QtCore.Qt.Unchecked)
item.setFlags(item.flags() | QtCore.Qt.ItemIsUserCheckable)
bible_languages = self.config.get('bibles', 'languages')
bible_languages = bible_languages.split(',')
for lang in bible_languages:
self.application.process_events()
language = self.config.get('bibles_{lang}'.format(lang=lang),
'title')
lang_item = QtWidgets.QTreeWidgetItem(self.bibles_tree_widget,
[language])
bibles = self.config.get('bibles_{lang}'.format(lang=lang),
'translations')
bibles = bibles.split(',')
for bible in bibles:
self.application.process_events()
title = self.config.get(
'bible_{bible}'.format(bible=bible), 'title')
filename = self.config.get(
'bible_{bible}'.format(bible=bible), 'filename')
sha256 = self.config.get(
'bible_{bible}'.format(bible=bible),
'sha256',
fallback='')
item = QtWidgets.QTreeWidgetItem(lang_item, [title])
item.setData(0, QtCore.Qt.UserRole, (filename, sha256))
item.setCheckState(0, QtCore.Qt.Unchecked)
item.setFlags(item.flags() | QtCore.Qt.ItemIsUserCheckable)
self.bibles_tree_widget.expandAll()
self.application.process_events()
# Download the theme screenshots
themes = self.config.get('themes', 'files').split(',')
for theme in themes:
title = self.config.get('theme_{theme}'.format(theme=theme),
'title')
filename = self.config.get('theme_{theme}'.format(theme=theme),
'filename')
sha256 = self.config.get('theme_{theme}'.format(theme=theme),
'sha256',
fallback='')
screenshot = self.config.get(
'theme_{theme}'.format(theme=theme), 'screenshot')
worker = ThemeScreenshotWorker(self.themes_url, title,
filename, sha256, screenshot)
worker.screenshot_downloaded.connect(
self.on_screenshot_downloaded)
thread_name = 'theme_screenshot_{title}'.format(title=title)
run_thread(worker, thread_name)
self.theme_screenshot_threads.append(thread_name)
self.application.process_events()