def setUpClass(cls):
"""TODO
"""
# Setup temporary directories
cls.setup_tmp_dirs()
# Setup logging for TestFunctions
cls.log_filepath = os.path.join(cls.data_tmpdir, cls.LOGGING_FILENAME)
cls.logger = setup_basic_logger(name=__name__,
add_console_handler=True,
add_file_handler=cls.ADD_FILE_HANDLER,
log_filepath=cls.log_filepath,
remove_all_handlers=True)
# IMPORTANT: no printing before
# Print name of module to be tested
line_equals = "{}".format("=" * 92)
line_name = "{}<color>{}</color>".format(" " * 37,
cls.test_module_name)
cls.logger.info("\n# {} #".format(line_equals))
cls.logger.info(line_name)
cls.logger.info("# {} #".format(line_equals))
cls.logger.info("<color>Setting up {} tests...</color>".format(
cls.test_module_name))
# Print info about directories created
cls.logger.info("Main temporary directory created: " +
cls._main_tmpdir)
cls.logger.info("Sandbox directory created: " + cls.sandbox_tmpdir)
cls.logger.info("Data directory created: " + cls.data_tmpdir)
# Create SQLite db
if cls.CREATE_TEST_DATABASE:
cls.db_filepath = os.path.join(cls.data_tmpdir, "db.sqlite")
# NOTE: the logging is silenced for create_db
create_db(cls.db_filepath, cls.schema_filepath)
cls.logger.warning("<color>SQLite database created:</color> "
"{}".format(cls.db_filepath))
else:
cls.logger.warning("<color>Database not used</color>")
cls.logger.warning("ADD_FILE_HANDLER: <color>{}"
"</color>".format(cls.ADD_FILE_HANDLER))
cls.logger.warning("SHOW_FIRST_CHARS_IN_LOG: <color>{}"
"</color>".format(cls.SHOW_FIRST_CHARS_IN_LOG))
cls.logger.warning("Testing in the <color>{}</color> "
"environment".format(cls.logger._env))
def overwrite_test_db(self, overwrite_db):
"""Create a test database and try to overwrite it.
This function tests that :meth:`~pyutils.dbutils.create_db` that if
`overwrite_db` is True, the test database will be overwritten.
Otherwise, the database will not be overwritten.
This function asserts two things:
- the test database is created and
- the validity of the return code obtained after trying to overwrite
the test database.
Parameters
----------
overwrite_db : bool, optional
Whether the test database will be overwritten.
Notes
-----
This function is used by :meth:`test_create_db_case_2` and
:meth:`test_create_db_case_3:.
"""
db_filepath = os.path.join(self.sandbox_tmpdir, "db.sqlite")
retcode1 = create_db(db_filepath, self.schema_filepath)
msg = "The test database couldn't be created"
self.assertTrue(retcode1 == 0, msg)
retcode2 = create_db(db_filepath,
self.schema_filepath,
overwrite_db=overwrite_db,
pause=0)
msg = "The option 'overwrite_db' didn't have any effect when " \
"creating the database"
expected = 0 if overwrite_db else 1
self.assertTrue(retcode2 == expected, msg)
def main():
"""TODO
Returns
-------
"""
# Setup argument parser
parser = argparse.ArgumentParser(
description="Create SQLite database")
parser.add_argument("-o", "--overwrite", action="store_true",
dest="overwrite", default=False,
help="Overwrite the database file")
parser.add_argument("-d", "--database", default="database.sqlite",
help="Path to the SQLite database file")
parser.add_argument("-s", "--schema", required=True,
help="Path to the schema file")
parser.add_argument("-sleep", type=int, default=2, help=argparse.SUPPRESS)
parser.add_argument("-dc", "--disable_color", action="store_true",
default=False, help=argparse.SUPPRESS)
# TODO: add verbose and quiet options
# Process command-line arguments
args = parser.parse_args()
# Setup logging for script
setup_basic_logger(
name=__name__,
add_console_handler=True,
handlers_to_remove=[logging.NullHandler])
# Setup logging for create_db
setup_basic_logger(
name=dbutils.__name__,
add_console_handler=True,
handlers_to_remove=[logging.NullHandler]
)
dbutils.SLEEP = args.sleep
if args.disable_color:
# Disable coloring log messages
dbutils.logger.disable_color()
logger.disable_color()
# Create database
return create_db(args.database, args.schema, args.overwrite)
def setUpClass(cls):
"""TODO
"""
# TODO: explain
cls.meth_names = [k for k in cls.__dict__.keys() if k.startswith("test")]
cls.meth_names.sort()
# Setup temporary directories
cls.setup_tmp_dirs()
# Filepath where the log file will be saved
if cls.data_tmpdir:
cls.log_filepath = os.path.join(cls.data_tmpdir, cls.LOGGING_FILENAME)
# Setup logging for this TestBase
setup_basic_logger(
name=__name__,
add_console_handler=True,
# console_format= "%(name)-42s: %(levelname)-8s %(message)s",
add_file_handler=cls.ADD_FILE_HANDLER,
log_filepath=cls.log_filepath,
remove_all_initial_handlers=True)
# Setup logging for TestBase's child
setup_basic_logger(
name=cls.LOGGER_NAME,
add_console_handler=True,
# console_format="%(name)-42s: %(levelname)-8s %(message)s",
add_file_handler=cls.ADD_FILE_HANDLER,
log_filepath=cls.log_filepath,
remove_all_initial_handlers=True)
# IMPORTANT: no printing before
# Print name of module to be tested
line_equals = "{}".format("=" * 92)
line_name = "{}<color>{}</color>".format(" " * 37,
cls.TEST_MODULE_QUALNAME)
logger.info("\n# {} #".format(line_equals))
logger.info(line_name)
logger.info("# {} #".format(line_equals))
logger.info("<color>Setting up {} tests...</color>".format(
cls.TEST_MODULE_QUALNAME))
# Print info about directories created
if cls._main_tmpdir:
logger.info("Main temporary directory created: " + cls._main_tmpdir)
if cls.sandbox_tmpdir:
logger.info("Sandbox directory created: " + cls.sandbox_tmpdir)
if cls.data_tmpdir:
logger.info("Data directory created: " + cls.data_tmpdir)
else:
logger.warning("<color>No temporary directories created</color>")
# Create SQLite db
if cls.CREATE_TEST_DATABASE:
cls.test_db_filepath = os.path.join(cls.data_tmpdir, cls.TEST_DB_FILENAME)
# NOTE: the logging is silenced for create_db
create_db(cls.test_db_filepath, cls.SCHEMA_FILEPATH)
logger.warning("<color>SQLite database created:</color> "
"{}".format(cls.test_db_filepath))
else:
logger.warning("<color>Database not used</color>")
logger.warning("<color>ADD_FILE_HANDLER:</color> "
"{}".format(cls.ADD_FILE_HANDLER))
logger.warning("<color>SHOW_FIRST_CHARS_IN_LOG:</color> "
"{}".format(cls.SHOW_FIRST_CHARS_IN_LOG))
logger.warning("Testing in the <color>{}</color> "
"environment".format(cls.env_type))
def __init__(self,
db_filepath="",
overwrite_db=False,
use_webcache=True,
webcache_dirpath="~/.cache/lyric_scraping/",
expire_after=25920000,
use_compute_cache=True,
ram_size=100,
http_get_timeout=5,
delay_between_requests=8,
headers=WebCache.HEADERS,
seed=123456,
interactive=False,
delay_interactive=30,
best_match=False,
simulate=False,
ignore_errors=False):
self.skipped_urls = {}
self.good_urls = set()
self.checked_urls = set()
# TODO: AssertionError are raised in both lines
self.logging_cfg_filepath = get_data_filepath(file_type='log')
self.schema_filepath = get_data_filepath(file_type='schema')
# ==============
# Logging config
# ==============
if _SETUP_LOGGING:
# Setup logging for all custom modules based on the default logging
# config file
logger.debug("<color>Setting up logging ...</color>")
setup_logging_from_cfg(self.logging_cfg_filepath)
logger.info("<color>Logging is setup</color>")
else:
logger.warning("<color>Logging was already setup</color>")
# ===============
# Database config
# ===============
self.overwrite_db = overwrite_db
self.db_filepath = os.path.expanduser(db_filepath)
# TODO: remove db_conn from everywhere
self.db_conn = None
if self.db_filepath:
logger.debug("<color>Setting up the music database ...</color>")
# Create music db if necessary
# TODO: IOError and sqlite3.OperationalError are raised
create_db(self.db_filepath, self.schema_filepath,
self.overwrite_db)
logger.info("<color>Music database is setup</color>")
else:
# No database to fbe used
logger.debug("<color>No music database used</color>")
# ================
# Web cache config
# ================
self.webcache_dirpath = os.path.expanduser(webcache_dirpath)
self.cache_name = os.path.join(self.webcache_dirpath, "cache")
self.use_webcache = use_webcache
self.expire_after = expire_after
self.http_get_timeout = http_get_timeout
self.delay_between_requests = delay_between_requests
self.headers = headers
if self.use_webcache:
logger.debug("<color>Setting up web-cache ...</color>")
logger.debug("<color>Creating the web-cache directory: "
"{}</color>".format(self.webcache_dirpath))
try:
# TODO: FileExistsError and PermissionError are raised
create_dir(self.webcache_dirpath, overwrite=False)
except FileExistsError as e:
logger.debug("<color>{}</color>".format(e))
logger.debug("<color>The webcache directory already exists: "
"{}</color>".format(self.webcache_dirpath))
self.webcache = WebCache(
cache_name=self.cache_name,
expire_after=self.expire_after,
http_get_timeout=self.http_get_timeout,
delay_between_requests=self.delay_between_requests,
headers=self.headers)
logger.info("<color>web-cache is setup</color>")
else:
self.webcache = None
logger.debug("<color>No web-cache used</color>")
# ====================
# Compute cache config
# ====================
self.use_compute_cache = use_compute_cache
self.ram_size = ram_size
if self.use_compute_cache:
logger.debug("<color>Setting up compute-cache ...</color>")
self.compute_cache = ComputeCache(self.schema_filepath,
self.ram_size)
logger.info("<color>compute-cache is setup</color>")
else:
self.compute_cache = None
logger.debug("<color>No compute-cache used</color>")
# ==============
# Scraper config
# ==============
self.seed = seed
random.seed(self.seed)
logger.info("<color>Random number generator initialized with seed={}"
"</color>".format(self.seed))
self.interactive = interactive
self.delay_interactive = delay_interactive
self.best_match = best_match
self.simulate = simulate
self.ignore_errors = ignore_errors
self.min_year = 1000