def test_delete_folder_contents_case_5(self):
"""Test delete_folder_contents() when a folder path doesn't exist.
Case 5 consists in testing that
:meth:`~pyutils.genutils.delete_folder_contents` raises an
:exc:`OSError` exception when the folder to be cleared doesn't exist.
See Also
--------
populate_folder : populates a folder with text files and subdirectories.
"""
self.logger.warning("\n\n<color>test_delete_folder_contents_case_5()"
"</color>")
self.logger.info("Testing <color>case 5 of delete_folder_contents()"
"</color> when a folder path doesn't exist...")
try:
# Delete everything in the directory that doesn't exist
delete_folder_contents(folderpath=os.path.join(
self.sandbox_tmpdir, "fakedir"),
remove_subdirs=True,
delete_recursively=False)
except OSError as e:
self.logger.info("Raised an OSError exception as expected: "
"{}".format(get_error_msg(e)))
else:
self.fail("An OSError exception was not raised as expected")
def test_setup_logging_case_5(self):
"""Test setup_logging() when the logging config dict is missing an
important key.
Case 5 tests that :meth:`~pyutils.logutils.setup_logging` raises a
:exc:`KeyError` exception when the logging config dict is missing an
important key, i.e. a key that is needed in
:meth:`~pyutils.logutils.setup_logging`.
"""
self.logger.warning("\n\n<color>test_setup_logging_case_5()</color>")
self.logger.info("Testing <color>case 5 of setup_logging()</color>...")
# Remove a key from the logging config dict
corrupted_cfg = deepcopy(self.logging_cfg_dict)
expected_missing_key = 'handlers'
del corrupted_cfg[expected_missing_key]
# Setup logging with the corrupted config dict
with self.assertRaises(KeyError) as cm:
setup_logging_from_cfg(corrupted_cfg)
missing_key = cm.exception.args[0]
msg = "The actual missing key ('{}') is not the expected one " \
"('{}')".format(missing_key, expected_missing_key)
self.assertTrue(expected_missing_key == missing_key, msg)
self.logger.info("<color>Raised a KeyError exception as expected:"
"</color> {}".format(get_error_msg(cm.exception)))
def test_create_timestamped_dir_case_2(self):
"""Test create_timestamped_dir() with no permission to write in a
directory.
Case 2 consists in checking that the function
:meth:`~pyutils.genutils.create_create_timestamped_directory` raises a
:exc:`PermissionError` when we try to create a subdirectory in a
directory without the write permission.
"""
self.logger.warning("\n\n<color>test_create_timestamped_dir_case_2()"
"</color>")
self.logger.info("Testing <color>case 2 of create_timestamped_dir()"
"</color> with no write permission...")
test1_dirpath = os.path.join(self.sandbox_tmpdir, "testdir1")
create_dir(test1_dirpath)
# TODO: Only works on macOS/Linux:
os.chmod(test1_dirpath, 0o444)
with self.assertRaises(PermissionError) as cm:
create_timestamped_dir(os.path.join(test1_dirpath, "testdir2"))
self.logger.info(
"<color>Raised a PermissionError exception as expected:</color> "
"{}".format(get_error_msg(cm.exception)))
# Put back write permission to owner
os.chmod(test1_dirpath, 0o744)
def start_scraping(self):
"""Start the web scraping of lyrics websites.
This method iterates through each lyrics URL from the main config file
and delegates the important tasks (URL processing and scraping) to
separate methods (:meth:`_process_url` and :meth:`_scrape_webpage`).
Notes
-----
This method catches all exceptions that prevent a given URL of being
processed further, e.g. the webpage is not found (404 Error) or the URL
is not from a valid domain.
Any exception that is not caught here is redirected to the main script
calling this method. See for example the main script
:mod:`scripts.scraper`.
"""
# Process list of URLs to lyrics websites
for url in self.lyrics_urls:
skip_url = True
error = None
try:
webpage_filename = self._process_url(url)
self._scrape_webpage(url, webpage_filename)
except OSError as e:
logger.exception(e)
error = e
except urllib.error.URLError as e:
logger.exception(e)
logger.warning("The URL {} seems to be down!".format(url))
error = e
except (FileExistsError,
lyrics_scraping.exceptions.CurrentSessionURLError,
lyrics_scraping.exceptions.InvalidURLDomainError,
lyrics_scraping.exceptions.InvalidURLCategoryError,
lyrics_scraping.exceptions.MultipleLyricsURLError,
lyrics_scraping.exceptions.OverwriteSongError,
pyutils.exceptions.HTTP404Error,
pyutils.exceptions.SQLSanityCheckError) as e:
logger.error(e)
error = e
else:
skip_url = False
finally:
# Close db connection
self.db_conn.close()
# Add the URL as skipped or good
if skip_url:
self._add_skipped_url(url, get_error_msg(error))
else:
logger.debug("URL successfully processed: "
"{}".format(url))
self.good_urls.add(url)
def test_setup_logging_case_3(self):
"""Test setup_logging() when the logging config file doesn't exist.
Case 3 tests that :meth:`~pyutils.logutils.setup_logging` raises an
:exc:`OSError` exception when the logging config file doesn't exist.
"""
self.logger.warning("\n\n<color>test_setup_logging_case_3()</color>")
self.logger.info("Testing <color>case 3 of setup_logging()</color> "
"when a config file doesn't exist...")
with self.assertRaises(OSError) as cm:
setup_logging_from_cfg("bad_logging_config.yaml")
self.logger.info("<color>Raised an OSError exception as expected:"
"</color> {}".format(get_error_msg(cm.exception)))
def test_read_file(self):
"""Test read_file() when a file doesn't exist.
This test consists in checking that
:meth:`~pyutils.genutils.read_file()` raises an :exc:`OSError`
exception when a file doesn't exist.
"""
self.logger.warning("\n\n<color>test_read_file()</color>")
self.logger.info("Testing <color>read_file()</color> when a file "
"doesn't exist...")
# Write text to a file on disk
with self.assertRaises(OSError) as cm:
read_file("/bad/file/path.txt")
self.logger.info("<color>Raised an OSError exception as expected:"
"</color> {}".format(get_error_msg(cm.exception)))
def test_get_webpage_case_3(self):
"""Test get_webpage() when a URL raises a '404 Not found' error.
Test 3 consists in testing that
:meth:`~pyutils.webcache.WebCache.get_webpage` raises an
:exc:`pyutils.exceptions.HTTP404Error` when the URL refers to a
resource that doesn't exist but you can still connect to the server.
"""
self.logger.warning("\n\n<color>test_get_webpage_case_3()</color>")
self.logger.info("Testing <color>case 3 of get_webpage()</color> "
"when an URL is not found...")
bad_url = "https://en.wikipedia.org/wiki/bad_url"
self.logger.info("Retrieving the webpage: " + bad_url)
with self.assertRaises(pyutils.exceptions.HTTP404Error) as cm:
self.get_webpage(bad_url, test_from_cache=False)
self.logger.info("<color>Raised an HTTP404Error exception as expected:"
"</color> {}".format(get_error_msg(cm.exception)))
def test_get_error_msg(self):
"""Test that get_error_msg() returns an error message.
This function tests that :meth:`~pyutils.logutils.get_error_msg` returns
an error message from an exception.
"""
self.logger.warning("\n<color>test_get_error_msg()</color>")
self.logger.info("Testing <color>get_error_msg()</color>...")
exc = IOError("The file doesn't exist")
error_msg = get_error_msg(exc)
expected = "[OSError] The file doesn't exist"
msg = "The error message '{}' is different from the expected one " \
"'{}'".format(error_msg, expected)
self.assertTrue(error_msg == expected, msg)
self.logger.info(
"<color>The error message is the expected one:</color> "
"{}".format(error_msg))
def test_create_dir_case_2(self):
"""Test create_dir() when the directory already exists.
Case 2 consists in checking that the function
:meth:`~pyutils.genutils.create_dir` raises a :exc:`FileExistsError`
when we try to create a directory that already exists on disk.
"""
self.logger.warning("\n\n<color>test_create_dir_case_2()</color>")
self.logger.info("Testing <color>case 2 of create_dir()</color> when "
"a directory already exists...")
with self.assertRaises(FileExistsError) as cm:
dirpath = os.path.join(self.sandbox_tmpdir, "testdir")
create_dir(dirpath)
create_dir(dirpath)
self.logger.info(
"<color>Raised a FileExistsError exception as expected:</color> "
"{}".format(get_error_msg(cm.exception)))
def test_get_webpage_case_4(self):
"""Test get_webpage() when a URL doesn't exist.
Test 4 consists in testing that
:meth:`~pyutils.webcache.WebCache.get_webpage` raises an
:exc:`requests.exceptions.RequestException` when the URL doesn't exist,
i.e. no server could be reached.
"""
self.logger.warning("\n\n<color>test_get_webpage_case_4()</color>")
self.logger.info("Testing <color>case 4 of get_webpage()</color> "
"when an URL doesn't exist...")
bad_url = "https://thisurldoesntexistatall.com"
self.logger.info("Retrieving the webpage: " + bad_url)
with self.assertRaises(requests.exceptions.RequestException) as cm:
self.get_webpage(bad_url, test_from_cache=False)
self.logger.info(
"<color>Raised a 'requests.exceptions.RequestException' exception "
"as expected:</color> {}".format(get_error_msg(cm.exception)))
def _preprocess_msg(self, msg):
"""TODO
Parameters
----------
msg : str
Returns
-------
msg : str
"""
if isinstance(msg, Exception):
# If msg is an Exception, process the Exception to build the error
# message as a string
msg = get_error_msg(msg)
self._disabled = True if _disableColoring else self._disabled
if self._disabled: # coloring disabled
# Remove all tags
msg = self._remove_all_tags(msg)
return msg
def test_setup_logging_case_4(self):
"""Test setup_logging() when the logging config dict has an invalid
value.
Case 4 tests that :meth:`~pyutils.logutils.setup_logging` raises an
:exc:`ValueError` exception when the logging config dict is invalid,
e.g. a logging handler's class is written incorrectly.
"""
self.logger.warning("\n\n<color>test_setup_logging_case_4()</color>")
self.logger.info("Testing <color>case 4 of setup_logging()</color> "
"with an invalid config dict...")
# Corrupt a logging handler's class
# NOTE: if I use copy instead of deepcopy, logging_cfg will also
# reflect the corrupted handler's class
corrupted_cfg = deepcopy(self.logging_cfg_dict)
corrupted_cfg['handlers']['console']['class'] = 'bad.handler.class'
# Setup logging with the corrupted config dict
with self.assertRaises(ValueError) as cm:
setup_logging_from_cfg(corrupted_cfg)
self.logger.info("<color>Raised a ValueError exception as expected:"
"</color> {}".format(get_error_msg(cm.exception)))