def setUpClass(cls): """TODO """ super().setUpClass() scraping._TESTING = True # Setup logging for scraping module format_ = "%(name)-35s: %(levelname)-8s %(message)s" setup_basic_logger(name=scraping.logger.name, add_console_handler=True, add_file_handler=True, log_filepath=cls.log_filepath, remove_all_handlers=True, console_format=format_, file_format=format_)
def test_remove_everything_but(self): """TODO """ self.logger.warning("\n\n<color>test_remove_everything_but()</color>") self.logger.info("Testing <color>_remove_everything_but()</color> " "(StreamHandler)...") # Setup test logger log_filepath = os.path.join(self.sandbox_tmpdir, 'test.log') remove_logger = setup_basic_logger(name="test_remove_everything_but", add_console_handler=True, add_file_handler=True, log_filepath=log_filepath) # Remove all handlers but the console handler remove_logger._remove_everything_but( handlers_to_keep=[logging.StreamHandler]) # Assert that there is only one console handler in the test logger nb_handlers = len(remove_logger.handlers) msg = "There should be one handler in keep_logger but there are " \ "{} handlers".format(nb_handlers) self.assertTrue(nb_handlers == 1, msg) # Make sure that the handler left is a console handler h = remove_logger.handlers[0] msg = "The remaining handler '{}' is not of the expected type " \ "'{}'".format(h, logging.StreamHandler) self.assertIsInstance(h, logging.StreamHandler, msg) self.logger.info("Only the StreamHandler is left in the logger")
def main(): """TODO Returns ------- """ # Setup argument parser parser = argparse.ArgumentParser( description="Create SQLite database") parser.add_argument("-o", "--overwrite", action="store_true", dest="overwrite", default=False, help="Overwrite the database file") parser.add_argument("-d", "--database", default="database.sqlite", help="Path to the SQLite database file") parser.add_argument("-s", "--schema", required=True, help="Path to the schema file") parser.add_argument("-sleep", type=int, default=2, help=argparse.SUPPRESS) parser.add_argument("-dc", "--disable_color", action="store_true", default=False, help=argparse.SUPPRESS) # TODO: add verbose and quiet options # Process command-line arguments args = parser.parse_args() # Setup logging for script setup_basic_logger( name=__name__, add_console_handler=True, handlers_to_remove=[logging.NullHandler]) # Setup logging for create_db setup_basic_logger( name=dbutils.__name__, add_console_handler=True, handlers_to_remove=[logging.NullHandler] ) dbutils.SLEEP = args.sleep if args.disable_color: # Disable coloring log messages dbutils.logger.disable_color() logger.disable_color() # Create database return create_db(args.database, args.schema, args.overwrite)
def setUpClass(cls): """TODO """ # Setup temporary directories cls.setup_tmp_dirs() # Setup logging for TestFunctions cls.log_filepath = os.path.join(cls.data_tmpdir, cls.LOGGING_FILENAME) cls.logger = setup_basic_logger(name=__name__, add_console_handler=True, add_file_handler=cls.ADD_FILE_HANDLER, log_filepath=cls.log_filepath, remove_all_handlers=True) # IMPORTANT: no printing before # Print name of module to be tested line_equals = "{}".format("=" * 92) line_name = "{}<color>{}</color>".format(" " * 37, cls.test_module_name) cls.logger.info("\n# {} #".format(line_equals)) cls.logger.info(line_name) cls.logger.info("# {} #".format(line_equals)) cls.logger.info("<color>Setting up {} tests...</color>".format( cls.test_module_name)) # Print info about directories created cls.logger.info("Main temporary directory created: " + cls._main_tmpdir) cls.logger.info("Sandbox directory created: " + cls.sandbox_tmpdir) cls.logger.info("Data directory created: " + cls.data_tmpdir) # Create SQLite db if cls.CREATE_TEST_DATABASE: cls.db_filepath = os.path.join(cls.data_tmpdir, "db.sqlite") # NOTE: the logging is silenced for create_db create_db(cls.db_filepath, cls.schema_filepath) cls.logger.warning("<color>SQLite database created:</color> " "{}".format(cls.db_filepath)) else: cls.logger.warning("<color>Database not used</color>") cls.logger.warning("ADD_FILE_HANDLER: <color>{}" "</color>".format(cls.ADD_FILE_HANDLER)) cls.logger.warning("SHOW_FIRST_CHARS_IN_LOG: <color>{}" "</color>".format(cls.SHOW_FIRST_CHARS_IN_LOG)) cls.logger.warning("Testing in the <color>{}</color> " "environment".format(cls.logger._env))
def test_add_handlers_back(self): """TODO """ self.logger.warning("\n\n<color>test_add_handlers_back()</color>") self.logger.info("Testing <color>_add_removed_handlers()</color>...") # Setup test logger add_logger = setup_basic_logger(name="test_add") # Setup console handler without adding it to the test logger ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) # Add the handler to the logger's list of removed handlers add_logger._removed_handlers.append(ch) # Add the handler to the logger from the logger's list of removed handlers add_logger._add_handlers_back() # Assert that there is only one handler in the logger nb_handlers = len(add_logger.handlers) msg = "There should be only one handler but there are {} " \ "handlers".format(nb_handlers) self.assertTrue(nb_handlers == 1, msg) self.logger.info("The console handler was successfully added")
def test_remove_handler(self): """TODO """ self.logger.warning("\n\n<color>test_remove_handler()</color>") self.logger.info("Testing <color>_remove_handler()</color>...") # Setup test logger log_filepath = os.path.join(self.sandbox_tmpdir, 'test.log') remove_logger = setup_basic_logger(name="test_remove", add_console_handler=True, log_filepath=log_filepath) msg = "There should only be one handler in remove_logger" self.assertTrue(len(remove_logger.handlers) == 1, msg) # Remove the handler from the test logger ch = remove_logger.handlers[0] remove_logger._remove_handler(ch) # Assert that there should be no handler in the test logger nb_handlers = len(remove_logger.handlers) msg = "There should be no handler in remove_logger but there are " \ "{} handlers".format(nb_handlers) self.assertTrue(nb_handlers == 0, msg) self.logger.info("The console handler was successfully removed")
def main(): """Main entry-point to the script. Some of the actions that the script can perform: - Fix documents extensions, or - Group documents into folders, - Modify filenames according to a template. Notes ----- Only one action at a time can be performed. """ # TODO: explain code # TODO: global logger? args = setup_argparser() # ============== # Logging config # ============== # NOTE: if quiet and verbose are both activated, only quiet will have an # effect if args.quiet: # Logging disabled # Reset logger by removing all handlers and add a null handler logger = setup_basic_logger(__name__, remove_all_initial_handlers=True) logger.addHandler(NullHandler()) else: # Logging enabled if args.no_color: # Color disabled uninstall_colored_logger() # Setup a basic CONSOLE logger with DEBUG as log level logger = setup_basic_logger( name=__name__, add_console_handler=True, remove_all_initial_handlers=True) if args.verbose: logger.setLevel(logging.DEBUG) else: logger.setLevel(logging.INFO) if args.recursive: logger.debug("<color>Recursive is ON</color>") # ======= # Actions # ======= retcode = 1 try: # TODO: make it so that we can perform more than 1 action at a time # NOTE: only one action at a time can be performed if args.diff_dirpath: retcode = diff_sets_of_docs(args.diff_dirpath[0], args.diff_dirpath[1], recursive=args.recursive) elif args.fix_dirpath: metadata = fix_extensions(args.fix_dirpath) retcode = metadata.retcode # TODO: comment if finished testing # retcode = _undo_fix_extensions(metadata) elif args.group_dirpath: metadata = group_docs_into_folders(args.group_dirpath[0], args.group_dirpath[1], args.group_size) retcode = metadata.retcode # TODO: comment if finished testing # retcode = _undo_group_docs_into_folders(metadata) elif args.modify_dirpath: retcode = modify_fnames(args.modify_dirpath) elif args.show_dirpath: retcode = show_results_about_docs(args.show_dirpath, recursive=args.recursive) else: logger("No action selected") except Exception as e: # TODO: explain this line, check https://stackoverflow.com/a/4992124 # traceback.print_exc() e = "<color>{}</color>".format(e) if args.verbose: logger.exception(e) else: logger.error(e) finally: return retcode
def setUpClass(cls): """TODO """ # TODO: explain cls.meth_names = [k for k in cls.__dict__.keys() if k.startswith("test")] cls.meth_names.sort() # Setup temporary directories cls.setup_tmp_dirs() # Filepath where the log file will be saved if cls.data_tmpdir: cls.log_filepath = os.path.join(cls.data_tmpdir, cls.LOGGING_FILENAME) # Setup logging for this TestBase setup_basic_logger( name=__name__, add_console_handler=True, # console_format= "%(name)-42s: %(levelname)-8s %(message)s", add_file_handler=cls.ADD_FILE_HANDLER, log_filepath=cls.log_filepath, remove_all_initial_handlers=True) # Setup logging for TestBase's child setup_basic_logger( name=cls.LOGGER_NAME, add_console_handler=True, # console_format="%(name)-42s: %(levelname)-8s %(message)s", add_file_handler=cls.ADD_FILE_HANDLER, log_filepath=cls.log_filepath, remove_all_initial_handlers=True) # IMPORTANT: no printing before # Print name of module to be tested line_equals = "{}".format("=" * 92) line_name = "{}<color>{}</color>".format(" " * 37, cls.TEST_MODULE_QUALNAME) logger.info("\n# {} #".format(line_equals)) logger.info(line_name) logger.info("# {} #".format(line_equals)) logger.info("<color>Setting up {} tests...</color>".format( cls.TEST_MODULE_QUALNAME)) # Print info about directories created if cls._main_tmpdir: logger.info("Main temporary directory created: " + cls._main_tmpdir) if cls.sandbox_tmpdir: logger.info("Sandbox directory created: " + cls.sandbox_tmpdir) if cls.data_tmpdir: logger.info("Data directory created: " + cls.data_tmpdir) else: logger.warning("<color>No temporary directories created</color>") # Create SQLite db if cls.CREATE_TEST_DATABASE: cls.test_db_filepath = os.path.join(cls.data_tmpdir, cls.TEST_DB_FILENAME) # NOTE: the logging is silenced for create_db create_db(cls.test_db_filepath, cls.SCHEMA_FILEPATH) logger.warning("<color>SQLite database created:</color> " "{}".format(cls.test_db_filepath)) else: logger.warning("<color>Database not used</color>") logger.warning("<color>ADD_FILE_HANDLER:</color> " "{}".format(cls.ADD_FILE_HANDLER)) logger.warning("<color>SHOW_FIRST_CHARS_IN_LOG:</color> " "{}".format(cls.SHOW_FIRST_CHARS_IN_LOG)) logger.warning("Testing in the <color>{}</color> " "environment".format(cls.env_type))
def main(): """Main entry-point to the script. According to the user's choice of action, the script might: - start the scraper, - edit a configuration file, or - reset/undo a configuration file. Notes ----- Only one action at a time can be performed. """ # TODO: explain global _TESTING, logger args = setup_argparser() # ============== # Logging config # ============== # NOTE: if quiet and verbose are both activated, only quiet will have an # effect if args.quiet: # Logging disabled logger = setup_basic_logger(__name__, remove_all_handlers=True) logger.addHandler(NullHandler()) else: # Logging enabled if args.no_color: # Color disabled uninstall_colored_logger() if not _TESTING: logger = setup_basic_logger(name=__name__, add_console_handler=True, remove_all_handlers=True) if args.verbose: logger.setLevel(logging.DEBUG) else: logger.setLevel(logging.INFO) # ======= # Actions # ======= retcode = 1 try: # NOTE: only one action at a time can be performed if args.edit: retcode = edit_config(args.edit, args.app) elif args.reset: retcode = reset_config(args.reset) elif args.undo: retcode = undo_config(args.undo) elif args.start_scraper: retcode = start_scraper() else: # TODO: default when no action given is to start scraping? print("No action selected: edit (-e), reset (-r) or start the " "scraper (-s)") except (AssertionError, AttributeError, FileNotFoundError, KeyboardInterrupt, OSError, sqlite3.Error) as e: # TODO: explain this line # traceback.print_exc() e = "<color>{}</color>".format(e) if args.verbose: logger.exception(e) else: logger.error(e) finally: return retcode