def setUpClass(cls):
"""TODO
"""
super().setUpClass()
scraping._TESTING = True
# Setup logging for scraping module
format_ = "%(name)-35s: %(levelname)-8s %(message)s"
setup_basic_logger(name=scraping.logger.name,
add_console_handler=True,
add_file_handler=True,
log_filepath=cls.log_filepath,
remove_all_handlers=True,
console_format=format_,
file_format=format_)
def test_remove_everything_but(self):
"""TODO
"""
self.logger.warning("\n\n<color>test_remove_everything_but()</color>")
self.logger.info("Testing <color>_remove_everything_but()</color> "
"(StreamHandler)...")
# Setup test logger
log_filepath = os.path.join(self.sandbox_tmpdir, 'test.log')
remove_logger = setup_basic_logger(name="test_remove_everything_but",
add_console_handler=True,
add_file_handler=True,
log_filepath=log_filepath)
# Remove all handlers but the console handler
remove_logger._remove_everything_but(
handlers_to_keep=[logging.StreamHandler])
# Assert that there is only one console handler in the test logger
nb_handlers = len(remove_logger.handlers)
msg = "There should be one handler in keep_logger but there are " \
"{} handlers".format(nb_handlers)
self.assertTrue(nb_handlers == 1, msg)
# Make sure that the handler left is a console handler
h = remove_logger.handlers[0]
msg = "The remaining handler '{}' is not of the expected type " \
"'{}'".format(h, logging.StreamHandler)
self.assertIsInstance(h, logging.StreamHandler, msg)
self.logger.info("Only the StreamHandler is left in the logger")
def main():
"""TODO
Returns
-------
"""
# Setup argument parser
parser = argparse.ArgumentParser(
description="Create SQLite database")
parser.add_argument("-o", "--overwrite", action="store_true",
dest="overwrite", default=False,
help="Overwrite the database file")
parser.add_argument("-d", "--database", default="database.sqlite",
help="Path to the SQLite database file")
parser.add_argument("-s", "--schema", required=True,
help="Path to the schema file")
parser.add_argument("-sleep", type=int, default=2, help=argparse.SUPPRESS)
parser.add_argument("-dc", "--disable_color", action="store_true",
default=False, help=argparse.SUPPRESS)
# TODO: add verbose and quiet options
# Process command-line arguments
args = parser.parse_args()
# Setup logging for script
setup_basic_logger(
name=__name__,
add_console_handler=True,
handlers_to_remove=[logging.NullHandler])
# Setup logging for create_db
setup_basic_logger(
name=dbutils.__name__,
add_console_handler=True,
handlers_to_remove=[logging.NullHandler]
)
dbutils.SLEEP = args.sleep
if args.disable_color:
# Disable coloring log messages
dbutils.logger.disable_color()
logger.disable_color()
# Create database
return create_db(args.database, args.schema, args.overwrite)
def setUpClass(cls):
"""TODO
"""
# Setup temporary directories
cls.setup_tmp_dirs()
# Setup logging for TestFunctions
cls.log_filepath = os.path.join(cls.data_tmpdir, cls.LOGGING_FILENAME)
cls.logger = setup_basic_logger(name=__name__,
add_console_handler=True,
add_file_handler=cls.ADD_FILE_HANDLER,
log_filepath=cls.log_filepath,
remove_all_handlers=True)
# IMPORTANT: no printing before
# Print name of module to be tested
line_equals = "{}".format("=" * 92)
line_name = "{}<color>{}</color>".format(" " * 37,
cls.test_module_name)
cls.logger.info("\n# {} #".format(line_equals))
cls.logger.info(line_name)
cls.logger.info("# {} #".format(line_equals))
cls.logger.info("<color>Setting up {} tests...</color>".format(
cls.test_module_name))
# Print info about directories created
cls.logger.info("Main temporary directory created: " +
cls._main_tmpdir)
cls.logger.info("Sandbox directory created: " + cls.sandbox_tmpdir)
cls.logger.info("Data directory created: " + cls.data_tmpdir)
# Create SQLite db
if cls.CREATE_TEST_DATABASE:
cls.db_filepath = os.path.join(cls.data_tmpdir, "db.sqlite")
# NOTE: the logging is silenced for create_db
create_db(cls.db_filepath, cls.schema_filepath)
cls.logger.warning("<color>SQLite database created:</color> "
"{}".format(cls.db_filepath))
else:
cls.logger.warning("<color>Database not used</color>")
cls.logger.warning("ADD_FILE_HANDLER: <color>{}"
"</color>".format(cls.ADD_FILE_HANDLER))
cls.logger.warning("SHOW_FIRST_CHARS_IN_LOG: <color>{}"
"</color>".format(cls.SHOW_FIRST_CHARS_IN_LOG))
cls.logger.warning("Testing in the <color>{}</color> "
"environment".format(cls.logger._env))
def test_add_handlers_back(self):
"""TODO
"""
self.logger.warning("\n\n<color>test_add_handlers_back()</color>")
self.logger.info("Testing <color>_add_removed_handlers()</color>...")
# Setup test logger
add_logger = setup_basic_logger(name="test_add")
# Setup console handler without adding it to the test logger
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
# Add the handler to the logger's list of removed handlers
add_logger._removed_handlers.append(ch)
# Add the handler to the logger from the logger's list of removed handlers
add_logger._add_handlers_back()
# Assert that there is only one handler in the logger
nb_handlers = len(add_logger.handlers)
msg = "There should be only one handler but there are {} " \
"handlers".format(nb_handlers)
self.assertTrue(nb_handlers == 1, msg)
self.logger.info("The console handler was successfully added")
def test_remove_handler(self):
"""TODO
"""
self.logger.warning("\n\n<color>test_remove_handler()</color>")
self.logger.info("Testing <color>_remove_handler()</color>...")
# Setup test logger
log_filepath = os.path.join(self.sandbox_tmpdir, 'test.log')
remove_logger = setup_basic_logger(name="test_remove",
add_console_handler=True,
log_filepath=log_filepath)
msg = "There should only be one handler in remove_logger"
self.assertTrue(len(remove_logger.handlers) == 1, msg)
# Remove the handler from the test logger
ch = remove_logger.handlers[0]
remove_logger._remove_handler(ch)
# Assert that there should be no handler in the test logger
nb_handlers = len(remove_logger.handlers)
msg = "There should be no handler in remove_logger but there are " \
"{} handlers".format(nb_handlers)
self.assertTrue(nb_handlers == 0, msg)
self.logger.info("The console handler was successfully removed")
def main():
"""Main entry-point to the script.
Some of the actions that the script can perform:
- Fix documents extensions, or
- Group documents into folders,
- Modify filenames according to a template.
Notes
-----
Only one action at a time can be performed.
"""
# TODO: explain code
# TODO: global logger?
args = setup_argparser()
# ==============
# Logging config
# ==============
# NOTE: if quiet and verbose are both activated, only quiet will have an
# effect
if args.quiet: # Logging disabled
# Reset logger by removing all handlers and add a null handler
logger = setup_basic_logger(__name__, remove_all_initial_handlers=True)
logger.addHandler(NullHandler())
else: # Logging enabled
if args.no_color: # Color disabled
uninstall_colored_logger()
# Setup a basic CONSOLE logger with DEBUG as log level
logger = setup_basic_logger(
name=__name__,
add_console_handler=True,
remove_all_initial_handlers=True)
if args.verbose:
logger.setLevel(logging.DEBUG)
else:
logger.setLevel(logging.INFO)
if args.recursive:
logger.debug("<color>Recursive is ON</color>")
# =======
# Actions
# =======
retcode = 1
try:
# TODO: make it so that we can perform more than 1 action at a time
# NOTE: only one action at a time can be performed
if args.diff_dirpath:
retcode = diff_sets_of_docs(args.diff_dirpath[0],
args.diff_dirpath[1],
recursive=args.recursive)
elif args.fix_dirpath:
metadata = fix_extensions(args.fix_dirpath)
retcode = metadata.retcode
# TODO: comment if finished testing
# retcode = _undo_fix_extensions(metadata)
elif args.group_dirpath:
metadata = group_docs_into_folders(args.group_dirpath[0],
args.group_dirpath[1],
args.group_size)
retcode = metadata.retcode
# TODO: comment if finished testing
# retcode = _undo_group_docs_into_folders(metadata)
elif args.modify_dirpath:
retcode = modify_fnames(args.modify_dirpath)
elif args.show_dirpath:
retcode = show_results_about_docs(args.show_dirpath,
recursive=args.recursive)
else:
logger("No action selected")
except Exception as e:
# TODO: explain this line, check https://stackoverflow.com/a/4992124
# traceback.print_exc()
e = "<color>{}</color>".format(e)
if args.verbose:
logger.exception(e)
else:
logger.error(e)
finally:
return retcode
def setUpClass(cls):
"""TODO
"""
# TODO: explain
cls.meth_names = [k for k in cls.__dict__.keys() if k.startswith("test")]
cls.meth_names.sort()
# Setup temporary directories
cls.setup_tmp_dirs()
# Filepath where the log file will be saved
if cls.data_tmpdir:
cls.log_filepath = os.path.join(cls.data_tmpdir, cls.LOGGING_FILENAME)
# Setup logging for this TestBase
setup_basic_logger(
name=__name__,
add_console_handler=True,
# console_format= "%(name)-42s: %(levelname)-8s %(message)s",
add_file_handler=cls.ADD_FILE_HANDLER,
log_filepath=cls.log_filepath,
remove_all_initial_handlers=True)
# Setup logging for TestBase's child
setup_basic_logger(
name=cls.LOGGER_NAME,
add_console_handler=True,
# console_format="%(name)-42s: %(levelname)-8s %(message)s",
add_file_handler=cls.ADD_FILE_HANDLER,
log_filepath=cls.log_filepath,
remove_all_initial_handlers=True)
# IMPORTANT: no printing before
# Print name of module to be tested
line_equals = "{}".format("=" * 92)
line_name = "{}<color>{}</color>".format(" " * 37,
cls.TEST_MODULE_QUALNAME)
logger.info("\n# {} #".format(line_equals))
logger.info(line_name)
logger.info("# {} #".format(line_equals))
logger.info("<color>Setting up {} tests...</color>".format(
cls.TEST_MODULE_QUALNAME))
# Print info about directories created
if cls._main_tmpdir:
logger.info("Main temporary directory created: " + cls._main_tmpdir)
if cls.sandbox_tmpdir:
logger.info("Sandbox directory created: " + cls.sandbox_tmpdir)
if cls.data_tmpdir:
logger.info("Data directory created: " + cls.data_tmpdir)
else:
logger.warning("<color>No temporary directories created</color>")
# Create SQLite db
if cls.CREATE_TEST_DATABASE:
cls.test_db_filepath = os.path.join(cls.data_tmpdir, cls.TEST_DB_FILENAME)
# NOTE: the logging is silenced for create_db
create_db(cls.test_db_filepath, cls.SCHEMA_FILEPATH)
logger.warning("<color>SQLite database created:</color> "
"{}".format(cls.test_db_filepath))
else:
logger.warning("<color>Database not used</color>")
logger.warning("<color>ADD_FILE_HANDLER:</color> "
"{}".format(cls.ADD_FILE_HANDLER))
logger.warning("<color>SHOW_FIRST_CHARS_IN_LOG:</color> "
"{}".format(cls.SHOW_FIRST_CHARS_IN_LOG))
logger.warning("Testing in the <color>{}</color> "
"environment".format(cls.env_type))
def main():
"""Main entry-point to the script.
According to the user's choice of action, the script might:
- start the scraper,
- edit a configuration file, or
- reset/undo a configuration file.
Notes
-----
Only one action at a time can be performed.
"""
# TODO: explain
global _TESTING, logger
args = setup_argparser()
# ==============
# Logging config
# ==============
# NOTE: if quiet and verbose are both activated, only quiet will have an
# effect
if args.quiet: # Logging disabled
logger = setup_basic_logger(__name__, remove_all_handlers=True)
logger.addHandler(NullHandler())
else: # Logging enabled
if args.no_color: # Color disabled
uninstall_colored_logger()
if not _TESTING:
logger = setup_basic_logger(name=__name__,
add_console_handler=True,
remove_all_handlers=True)
if args.verbose:
logger.setLevel(logging.DEBUG)
else:
logger.setLevel(logging.INFO)
# =======
# Actions
# =======
retcode = 1
try:
# NOTE: only one action at a time can be performed
if args.edit:
retcode = edit_config(args.edit, args.app)
elif args.reset:
retcode = reset_config(args.reset)
elif args.undo:
retcode = undo_config(args.undo)
elif args.start_scraper:
retcode = start_scraper()
else:
# TODO: default when no action given is to start scraping?
print("No action selected: edit (-e), reset (-r) or start the "
"scraper (-s)")
except (AssertionError, AttributeError, FileNotFoundError,
KeyboardInterrupt, OSError, sqlite3.Error) as e:
# TODO: explain this line
# traceback.print_exc()
e = "<color>{}</color>".format(e)
if args.verbose:
logger.exception(e)
else:
logger.error(e)
finally:
return retcode
