def test_get_web_page_with_header(self): """ Test that adding a header to the call to get_web_page() adds the header to the request """ with patch('openlp.core.common.httputils.urllib.request.Request') as MockRequest, \ patch('openlp.core.common.httputils.urllib.request.urlopen') as mock_urlopen, \ patch('openlp.core.common.httputils.get_user_agent') as mock_get_user_agent: # GIVEN: Mocked out objects, a fake URL and a fake header mocked_request_object = MagicMock() MockRequest.return_value = mocked_request_object mocked_page_object = MagicMock() mock_urlopen.return_value = mocked_page_object mock_get_user_agent.return_value = 'user_agent' fake_url = 'this://is.a.fake/url' fake_header = ('Fake-Header', 'fake value') # WHEN: The get_web_page() method is called returned_page = get_web_page(fake_url, header=fake_header) # THEN: The correct methods are called with the correct arguments and a web page is returned MockRequest.assert_called_with(fake_url) mocked_request_object.add_header.assert_called_with(fake_header[0], fake_header[1]) self.assertEqual(2, mocked_request_object.add_header.call_count, 'There should only be 2 calls to add_header') mock_get_user_agent.assert_called_with() mock_urlopen.assert_called_with(mocked_request_object, timeout=30) mocked_page_object.geturl.assert_called_with() self.assertEqual(mocked_page_object, returned_page, 'The returned page should be the mock object')
def _download_index(self): """ Download the configuration file and kick off the theme screenshot download threads """ # Check if the index file has already been downloaded if self.is_index_downloaded: return # check to see if we have web access self.has_web_access = False self.config = '' web_config = None user_agent = 'OpenLP/' + QtWidgets.QApplication.applicationVersion() self.application.process_events() try: web_config = get_web_page('{host}{name}'.format(host=self.web, name='download_3.0.json'), headers={'User-Agent': user_agent}) except ConnectionError: QtWidgets.QMessageBox.critical(self, translate('OpenLP.FirstTimeWizard', 'Network Error'), translate('OpenLP.FirstTimeWizard', 'There was a network error attempting ' 'to connect to retrieve initial configuration information'), QtWidgets.QMessageBox.Ok) if web_config and self._parse_config(web_config): self.has_web_access = True self.application.process_events() self.downloading = translate('OpenLP.FirstTimeWizard', 'Downloading {name}...') self.application.set_normal_cursor() self.is_index_downloaded = True
def test_get_web_page_update_openlp(self): """ Test that passing "update_openlp" as true to get_web_page calls Registry().get('app').process_events() """ with patch('openlp.core.common.httputils.urllib.request.Request') as MockRequest, \ patch('openlp.core.common.httputils.urllib.request.urlopen') as mock_urlopen, \ patch('openlp.core.common.httputils.get_user_agent') as mock_get_user_agent, \ patch('openlp.core.common.httputils.Registry') as MockRegistry: # GIVEN: Mocked out objects, a fake URL mocked_request_object = MagicMock() MockRequest.return_value = mocked_request_object mocked_page_object = MagicMock() mock_urlopen.return_value = mocked_page_object mock_get_user_agent.return_value = 'user_agent' mocked_registry_object = MagicMock() mocked_application_object = MagicMock() mocked_registry_object.get.return_value = mocked_application_object MockRegistry.return_value = mocked_registry_object fake_url = 'this://is.a.fake/url' # WHEN: The get_web_page() method is called returned_page = get_web_page(fake_url, update_openlp=True) # THEN: The correct methods are called with the correct arguments and a web page is returned MockRequest.assert_called_with(fake_url) mocked_request_object.add_header.assert_called_with('User-Agent', 'user_agent') self.assertEqual(1, mocked_request_object.add_header.call_count, 'There should only be 1 call to add_header') mock_urlopen.assert_called_with(mocked_request_object, timeout=30) mocked_page_object.geturl.assert_called_with() mocked_registry_object.get.assert_called_with('application') mocked_application_object.process_events.assert_called_with() self.assertEqual(mocked_page_object, returned_page, 'The returned page should be the mock object')
def test_get_web_page(self): """ Test that the get_web_page method works correctly """ with patch('openlp.core.common.httputils.urllib.request.Request') as MockRequest, \ patch('openlp.core.common.httputils.urllib.request.urlopen') as mock_urlopen, \ patch('openlp.core.common.httputils.get_user_agent') as mock_get_user_agent, \ patch('openlp.core.common.Registry') as MockRegistry: # GIVEN: Mocked out objects and a fake URL mocked_request_object = MagicMock() MockRequest.return_value = mocked_request_object mocked_page_object = MagicMock() mock_urlopen.return_value = mocked_page_object mock_get_user_agent.return_value = 'user_agent' fake_url = 'this://is.a.fake/url' # WHEN: The get_web_page() method is called returned_page = get_web_page(fake_url) # THEN: The correct methods are called with the correct arguments and a web page is returned MockRequest.assert_called_with(fake_url) mocked_request_object.add_header.assert_called_with('User-Agent', 'user_agent') self.assertEqual(1, mocked_request_object.add_header.call_count, 'There should only be 1 call to add_header') mock_get_user_agent.assert_called_with() mock_urlopen.assert_called_with(mocked_request_object, timeout=30) mocked_page_object.geturl.assert_called_with() self.assertEqual(0, MockRegistry.call_count, 'The Registry() object should have never been called') self.assertEqual(mocked_page_object, returned_page, 'The returned page should be the mock object')
def get_soup_for_bible_ref(reference_url, header=None, pre_parse_regex=None, pre_parse_substitute=None): """ Gets a webpage and returns a parsed and optionally cleaned soup or None. :param reference_url: The URL to obtain the soup from. :param header: An optional HTTP header to pass to the bible web server. :param pre_parse_regex: A regular expression to run on the webpage. Allows manipulation of the webpage before passing to BeautifulSoup for parsing. :param pre_parse_substitute: The text to replace any matches to the regular expression with. """ if not reference_url: return None try: page = get_web_page(reference_url, header, True) except: page = None if not page: send_error_message('download') return None page_source = page.read() if pre_parse_regex and pre_parse_substitute is not None: page_source = re.sub(pre_parse_regex, pre_parse_substitute, page_source.decode()) soup = None try: soup = BeautifulSoup(page_source, 'lxml') CLEANER_REGEX.sub('', str(soup)) except Exception: log.exception('BeautifulSoup could not parse the bible page.') if not soup: send_error_message('parse') return None Registry().get('application').process_events() return soup
def test_get_web_page_update_openlp(self, MockRegistry, mocked_get_user_agent, mocked_requests): """ Test that passing "update_openlp" as true to get_web_page calls Registry().get('app').process_events() """ # GIVEN: Mocked out objects, a fake URL mocked_requests.get.return_value = MagicMock(text='text') mocked_get_user_agent.return_value = 'user_agent' mocked_registry_object = MagicMock() mocked_application_object = MagicMock() mocked_registry_object.get.return_value = mocked_application_object MockRegistry.return_value = mocked_registry_object fake_url = 'this://is.a.fake/url' # WHEN: The get_web_page() method is called returned_page = get_web_page(fake_url, update_openlp=True) # THEN: The correct methods are called with the correct arguments and a web page is returned mocked_requests.get.assert_called_once_with( fake_url, headers={'User-Agent': 'user_agent'}, proxies=None, timeout=30.0) mocked_get_user_agent.assert_called_once_with() mocked_registry_object.get.assert_called_with('application') mocked_application_object.process_events.assert_called_with() assert returned_page == 'text', 'The returned page should be the mock object'
def test_get_web_page_with_user_agent_in_headers(self): """ Test that adding a user agent in the header when calling get_web_page() adds that user agent to the request """ with patch('openlp.core.common.httputils.urllib.request.Request') as MockRequest, \ patch('openlp.core.common.httputils.urllib.request.urlopen') as mock_urlopen, \ patch('openlp.core.common.httputils.get_user_agent') as mock_get_user_agent: # GIVEN: Mocked out objects, a fake URL and a fake header mocked_request_object = MagicMock() MockRequest.return_value = mocked_request_object mocked_page_object = MagicMock() mock_urlopen.return_value = mocked_page_object fake_url = 'this://is.a.fake/url' user_agent_header = ('User-Agent', 'OpenLP/2.2.0') # WHEN: The get_web_page() method is called returned_page = get_web_page(fake_url, header=user_agent_header) # THEN: The correct methods are called with the correct arguments and a web page is returned MockRequest.assert_called_with(fake_url) mocked_request_object.add_header.assert_called_with(user_agent_header[0], user_agent_header[1]) self.assertEqual(1, mocked_request_object.add_header.call_count, 'There should only be 1 call to add_header') self.assertEqual(0, mock_get_user_agent.call_count, 'get_user_agent should not have been called') mock_urlopen.assert_called_with(mocked_request_object, timeout=30) mocked_page_object.geturl.assert_called_with() self.assertEqual(mocked_page_object, returned_page, 'The returned page should be the mock object')
def test_webpage_connection_retry(self, mocked_requests): """ Test get_web_page will attempt CONNECTION_RETRIES+1 connections - bug 1409031 """ # GIVEN: Initial settings and mocks mocked_requests.get.side_effect = OSError('Unable to connect') # WHEN: A webpage is requested try: get_web_page('http://localhost') except Exception as e: assert isinstance(e, ConnectionError) # THEN: urlopen should have been called CONNECTION_RETRIES + 1 count assert mocked_requests.get.call_count == CONNECTION_RETRIES, \ 'get should have been called {} times, but was only called {} times'.format( CONNECTION_RETRIES, mocked_requests.get.call_count)
def test_webpage_connection_retry(self): """ Test get_web_page will attempt CONNECTION_RETRIES+1 connections - bug 1409031 """ # GIVEN: Initial settings and mocks with patch.object(urllib.request, 'urlopen') as mocked_urlopen: mocked_urlopen.side_effect = ConnectionError # WHEN: A webpage is requested try: get_web_page(url='http://localhost') except: pass # THEN: urlopen should have been called CONNECTION_RETRIES + 1 count self.assertEquals(mocked_urlopen.call_count, CONNECTION_RETRIES + 1, 'get_web_page() should have tried {} times'.format(CONNECTION_RETRIES))
def test_get_web_page_no_url(self): """ Test that sending a URL of None to the get_web_page method returns None """ # GIVEN: A None url test_url = None # WHEN: We try to get the test URL result = get_web_page(test_url) # THEN: None should be returned self.assertIsNone(result, 'The return value of get_web_page should be None')
def test_get_web_page_no_url(self): """ Test that sending a URL of None to the get_web_page method returns None """ # GIVEN: A None url test_url = None # WHEN: We try to get the test URL result = get_web_page(test_url) # THEN: None should be returned assert result is None, 'The return value of get_web_page should be None'
def download_sha256(): """ Download the config file to extract the sha256 and version number """ user_agent = 'OpenLP/' + QtWidgets.QApplication.applicationVersion() try: web_config = get_web_page('https://get.openlp.org/webclient/download.cfg', headers={'User-Agent': user_agent}) except ConnectionError: return False if not web_config: return None file_bits = web_config.split() return file_bits[0], file_bits[2]
def is_stage_active(self): """ Is stage active - call it and see but only once :return: if stage is active or not """ if self.stage_cache is None: try: page = get_web_page("http://localhost:4316/stage") except: page = None if page: self.stage_cache = True else: self.stage_cache = False return self.stage_cache
def is_chords_active(self): """ Is chords active - call it and see but only once :return: if live is active or not """ if self.chords_cache is None: try: page = get_web_page("http://localhost:4316/chords") except: page = None if page: self.chords_cache = True else: self.chords_cache = False return self.chords_cache
def is_live_active(self): """ Is main active - call it and see but only once :return: if live is active or not """ if self.live_cache is None: try: page = get_web_page("http://localhost:4316/main") except Exception: page = None if page: self.live_cache = True else: self.live_cache = False return self.live_cache
def test_get_web_page_with_user_agent_in_headers(self, mocked_get_user_agent, mocked_requests): """ Test that adding a user agent in the header when calling get_web_page() adds that user agent to the request """ # GIVEN: Mocked out objects, a fake URL and a fake header mocked_requests.get.return_value = MagicMock(text='text') fake_url = 'this://is.a.fake/url' user_agent_headers = {'User-Agent': 'OpenLP/2.2.0'} # WHEN: The get_web_page() method is called returned_page = get_web_page(fake_url, headers=user_agent_headers) # THEN: The correct methods are called with the correct arguments and a web page is returned mocked_requests.get.assert_called_once_with(fake_url, headers=user_agent_headers, proxies=None, timeout=30.0) assert mocked_get_user_agent.call_count == 0, 'get_user_agent() should not have been called' assert returned_page == 'text', 'The returned page should be "test"'
def test_get_web_page(self, MockRegistry, mocked_get_user_agent, mocked_requests): """ Test that the get_web_page method works correctly """ # GIVEN: Mocked out objects and a fake URL mocked_requests.get.return_value = MagicMock(text='text') mocked_get_user_agent.return_value = 'user_agent' fake_url = 'this://is.a.fake/url' # WHEN: The get_web_page() method is called returned_page = get_web_page(fake_url) # THEN: The correct methods are called with the correct arguments and a web page is returned mocked_requests.get.assert_called_once_with(fake_url, headers={'User-Agent': 'user_agent'}, proxies=None, timeout=30.0) mocked_get_user_agent.assert_called_once_with() assert MockRegistry.call_count == 0, 'The Registry() object should have never been called' assert returned_page == 'text', 'The returned page should be the mock object'
def get_books_from_http(self, version): """ Load a list of all books a Bible contains from BibleGateway website. :param version: The version of the Bible like NIV for New International Version """ log.debug('BGExtract.get_books_from_http("{version}")'.format( version=version)) url_params = urllib.parse.urlencode({ 'action': 'getVersionInfo', 'vid': '{version}'.format(version=version) }) reference_url = 'http://www.biblegateway.com/versions/?{url}#books'.format( url=url_params) page_source = get_web_page(reference_url) if not page_source: send_error_message('download') return None try: soup = BeautifulSoup(page_source, 'lxml') except Exception: log.error('BeautifulSoup could not parse the Bible page.') send_error_message('parse') return None if not soup: send_error_message('parse') return None self.application.process_events() content = soup.find('table', 'infotable') if content: content = content.find_all('tr') if not content: log.error('No books found in the Biblegateway response.') send_error_message('parse') return None books = [] for book in content: book = book.find('td') if book: books.append(book.contents[1]) return books
def test_get_web_page_with_header(self, mocked_get_user_agent, mocked_requests): """ Test that adding a header to the call to get_web_page() adds the header to the request """ # GIVEN: Mocked out objects, a fake URL and a fake header mocked_requests.get.return_value = MagicMock(text='text') mocked_get_user_agent.return_value = 'user_agent' fake_url = 'this://is.a.fake/url' fake_headers = {'Fake-Header': 'fake value'} # WHEN: The get_web_page() method is called returned_page = get_web_page(fake_url, headers=fake_headers) # THEN: The correct methods are called with the correct arguments and a web page is returned expected_headers = dict(fake_headers) expected_headers.update({'User-Agent': 'user_agent'}) mocked_requests.get.assert_called_once_with(fake_url, headers=expected_headers, proxies=None, timeout=30.0) mocked_get_user_agent.assert_called_with() assert returned_page == 'text', 'The returned page should be the mock object'
def get_books_from_http(self, version): """ Load a list of all books a Bible contains from BibleGateway website. :param version: The version of the Bible like NIV for New International Version """ log.debug('BGExtract.get_books_from_http("{version}")'.format(version=version)) url_params = urllib.parse.urlencode({'action': 'getVersionInfo', 'vid': '{version}'.format(version=version)}) reference_url = 'http://biblegateway.com/versions/?{url}#books'.format(url=url_params) page = get_web_page(reference_url) if not page: send_error_message('download') return None page_source = page.read() try: page_source = str(page_source, 'utf8') except UnicodeDecodeError: page_source = str(page_source, 'cp1251') try: soup = BeautifulSoup(page_source, 'lxml') except Exception: log.error('BeautifulSoup could not parse the Bible page.') send_error_message('parse') return None if not soup: send_error_message('parse') return None self.application.process_events() content = soup.find('table', 'infotable') if content: content = content.find_all('tr') if not content: log.error('No books found in the Biblegateway response.') send_error_message('parse') return None books = [] for book in content: book = book.find('td') if book: books.append(book.contents[1]) return books
def get_soup_for_bible_ref(reference_url, headers=None, pre_parse_regex=None, pre_parse_substitute=None): """ Gets a webpage and returns a parsed and optionally cleaned soup or None. :param reference_url: The URL to obtain the soup from. :param header: An optional HTTP header to pass to the bible web server. :param pre_parse_regex: A regular expression to run on the webpage. Allows manipulation of the webpage before passing to BeautifulSoup for parsing. :param pre_parse_substitute: The text to replace any matches to the regular expression with. """ if not reference_url: return None try: page_source = get_web_page(reference_url, headers, update_openlp=True) except Exception as e: log.exception( 'Unable to download Bible %s, unknown exception occurred', reference_url) page_source = None if not page_source: send_error_message('download') return None if pre_parse_regex and pre_parse_substitute is not None: page_source = re.sub(pre_parse_regex, pre_parse_substitute, page_source) soup = None try: soup = BeautifulSoup(page_source, 'lxml') CLEANER_REGEX.sub('', str(soup)) except Exception: log.exception('BeautifulSoup could not parse the bible page.') if not soup: send_error_message('parse') return None Registry().get('application').process_events() return soup
def _download_index(self): """ Download the configuration file and kick off the theme screenshot download threads """ # check to see if we have web access self.web_access = False self.config = ConfigParser() user_agent = 'OpenLP/' + Registry().get( 'application').applicationVersion() self.application.process_events() try: web_config = get_web_page('{host}{name}'.format( host=self.web, name='download.cfg'), headers={'User-Agent': user_agent}) except ConnectionError: QtWidgets.QMessageBox.critical( self, translate('OpenLP.FirstTimeWizard', 'Network Error'), translate( 'OpenLP.FirstTimeWizard', 'There was a network error attempting ' 'to connect to retrieve initial configuration information' ), QtWidgets.QMessageBox.Ok) web_config = False if web_config: try: self.config.read_string(web_config) self.web = self.config.get('general', 'base url') self.songs_url = self.web + self.config.get( 'songs', 'directory') + '/' self.bibles_url = self.web + self.config.get( 'bibles', 'directory') + '/' self.themes_url = self.web + self.config.get( 'themes', 'directory') + '/' self.web_access = True except (NoSectionError, NoOptionError, MissingSectionHeaderError): log.debug( 'A problem occurred while parsing the downloaded config file' ) trace_error_handler(log) self.update_screen_list_combo() self.application.process_events() self.downloading = translate('OpenLP.FirstTimeWizard', 'Downloading {name}...') if self.has_run_wizard: self.songs_check_box.setChecked( self.plugin_manager.get_plugin_by_name('songs').is_active()) self.bible_check_box.setChecked( self.plugin_manager.get_plugin_by_name('bibles').is_active()) self.presentation_check_box.setChecked( self.plugin_manager.get_plugin_by_name( 'presentations').is_active()) self.image_check_box.setChecked( self.plugin_manager.get_plugin_by_name('images').is_active()) self.media_check_box.setChecked( self.plugin_manager.get_plugin_by_name('media').is_active()) self.custom_check_box.setChecked( self.plugin_manager.get_plugin_by_name('custom').is_active()) self.song_usage_check_box.setChecked( self.plugin_manager.get_plugin_by_name( 'songusage').is_active()) self.alert_check_box.setChecked( self.plugin_manager.get_plugin_by_name('alerts').is_active()) self.application.set_normal_cursor() # Sort out internet access for downloads if self.web_access: songs = self.config.get('songs', 'languages') songs = songs.split(',') for song in songs: self.application.process_events() title = self.config.get('songs_{song}'.format(song=song), 'title') filename = self.config.get('songs_{song}'.format(song=song), 'filename') sha256 = self.config.get('songs_{song}'.format(song=song), 'sha256', fallback='') item = QtWidgets.QListWidgetItem(title, self.songs_list_widget) item.setData(QtCore.Qt.UserRole, (filename, sha256)) item.setCheckState(QtCore.Qt.Unchecked) item.setFlags(item.flags() | QtCore.Qt.ItemIsUserCheckable) bible_languages = self.config.get('bibles', 'languages') bible_languages = bible_languages.split(',') for lang in bible_languages: self.application.process_events() language = self.config.get('bibles_{lang}'.format(lang=lang), 'title') lang_item = QtWidgets.QTreeWidgetItem(self.bibles_tree_widget, [language]) bibles = self.config.get('bibles_{lang}'.format(lang=lang), 'translations') bibles = bibles.split(',') for bible in bibles: self.application.process_events() title = self.config.get( 'bible_{bible}'.format(bible=bible), 'title') filename = self.config.get( 'bible_{bible}'.format(bible=bible), 'filename') sha256 = self.config.get( 'bible_{bible}'.format(bible=bible), 'sha256', fallback='') item = QtWidgets.QTreeWidgetItem(lang_item, [title]) item.setData(0, QtCore.Qt.UserRole, (filename, sha256)) item.setCheckState(0, QtCore.Qt.Unchecked) item.setFlags(item.flags() | QtCore.Qt.ItemIsUserCheckable) self.bibles_tree_widget.expandAll() self.application.process_events() # Download the theme screenshots themes = self.config.get('themes', 'files').split(',') for theme in themes: title = self.config.get('theme_{theme}'.format(theme=theme), 'title') filename = self.config.get('theme_{theme}'.format(theme=theme), 'filename') sha256 = self.config.get('theme_{theme}'.format(theme=theme), 'sha256', fallback='') screenshot = self.config.get( 'theme_{theme}'.format(theme=theme), 'screenshot') worker = ThemeScreenshotWorker(self.themes_url, title, filename, sha256, screenshot) worker.screenshot_downloaded.connect( self.on_screenshot_downloaded) thread_name = 'theme_screenshot_{title}'.format(title=title) run_thread(worker, thread_name) self.theme_screenshot_threads.append(thread_name) self.application.process_events()