def test_get_next_step_with_restart(self):
test_config = ArchiveConfig(test_logger, "test5",
"opendoors/tests/test_output").config
test_config["Processing"]["done_steps"] = "01, 02"
next_step, done_steps = get_next_step(test_config, "02")
self.assertEqual("01", next_step, "next_step should be set to 01")
self.assertEqual([], done_steps, "done_steps should be reset to empty")
def test_continue_from_last(self):
test_config = ArchiveConfig(test_logger, "test1",
"opendoors/tests/test_output").config
continue_from_last(test_config, test_logger, test_sql, steps)
self.assertSetEqual(
set("01, 02".split(', ')),
set(test_config['Processing']['done_steps'].split(', ')),
"continue_from_last should update the done_steps config")
def test_update_done_steps(self):
test_config = ArchiveConfig(test_logger, "test2",
"opendoors/tests/test_output").config
test_config["Processing"]["done_steps"] = "01, 02"
done_steps = update_done_steps(test_config, ["01", "02"], "03")
self.assertEqual(
"01, 02, 03", done_steps,
"update_done_steps should return the done steps as a string")
def test_get_next_step_after_02(self):
test_config = ArchiveConfig(test_logger, "test4",
"opendoors/tests/test_output").config
test_config["Processing"]["done_steps"] = "01, 02"
next_step, done_steps = get_next_step(test_config, "02")
self.assertEqual("02", next_step, "next_step should be set to 02")
self.assertListEqual(
test_config["Processing"]["done_steps"].split(", "), done_steps,
"done_steps should be the value currently in the config")
class TestEFictionSimplified(TestCase):
test_config_no_defs = ArchiveConfig(test_logger, "efiction_no_defs",
"efiction/tests/test_data").config
efiction_with_defs = simplified.EFictionSimplified(test_config,
test_logger, test_sql)
efiction_no_defs = simplified.EFictionSimplified(test_config_no_defs,
test_logger, test_sql)
def test_remove_unwanted_tables(self):
result = self.efiction_no_defs._remove_unwanted_tables([
"use database;", "create table fanfiction_stats;",
"create table fanfiction_stories;"
])
self.assertTrue('fanfiction_stats' not in result,
"statements on unwanted tables should be removed")
self.assertTrue(
'stories' in result,
"statements on desired tables should be stripped of their prefix and kept"
)
class TestSqlDb(TestCase):
test_logger = Logger("test")
test_config: ConfigParser = ArchiveConfig(test_logger, "test", "opendoors/tests/test_data").config
test_sql = SqlDb(test_config, test_logger)
def tearDown(self) -> None:
""" Remove any files generated in test_output """
filtered = [f for f in glob.glob('opendoors/tests/test_output/*') if not re.match(r'\.keep', f)]
for file in filtered:
os.remove(file)
def test_dump_database(self):
self.test_sql.load_sql_file_into_db(get_full_path("opendoors/tests/test_data/test.sql"))
self.test_sql.dump_database("od_test_sql", get_full_path("opendoors/tests/test_output/test_output.sql"))
with open(get_full_path("opendoors/tests/test_output/test_output.sql")) as f:
result = f.readlines()
self.assertEqual("(1,'Name1\\'s \\\"stringé\\\"',NULL),\n",
result[14],
"all the SQL statements should be present")
class TestOriginal(TestCase):
test_config_no_defs = ArchiveConfig(test_logger, "efiction_no_defs", "efiction/tests/test_data").config
efiction_with_defs = EFictionOriginal(test_config, test_logger, test_sql)
efiction_no_defs = EFictionOriginal(test_config_no_defs, test_logger, test_sql)
def tearDown(self) -> None:
""" Remove files created during the tests """
remove_output_files('efiction/tests/test_output/*')
@patch('builtins.input', lambda *args: 'efiction/tests/test_data/efiction.sql')
@patch('efiction.original.add_create_database')
def test_load_original_file(self, mock_add_create_database):
self.efiction_with_defs.load_original_file("efiction/tests/test_output")
mock_add_create_database.assert_called_once()
test_sql.load_sql_file_into_db.arg_should_contain("efictiontestnodefs_efiction_original_edited.sql")
def test_check_for_table_defs_no_defs(self):
statements = group_by_table(
['INSERT INTO fanfiction_authorinfo VALUES ("0","2","http://example.com")',
'INSERT INTO fanfiction_authorprefs VALUES ("5","BillBob","2","0","1","77")',
'INSERT INTO fanfiction_authors VALUES ("1","Author1","Author1","*****@*****.**")']
)
has_defs = EFictionOriginal._contains_table_defs(statements)
self.assertFalse(has_defs)
def test_check_for_table_defs_with_defs(self):
statements = group_by_table(
['CREATE TABLE fanfiction_authorinfo (`uid` int(11) NOT NULL AUTO_INCREMENT, `name` varchar(200))',
'INSERT INTO fanfiction_authorinfo VALUES ("5","BillBob","2","0","1","77")',
'INSERT INTO fanfiction_authors VALUES ("1","Author1","Author1","*****@*****.**")']
)
has_defs = self.efiction_with_defs._contains_table_defs(statements)
self.assertTrue(has_defs)
def test_add_table_definitions(self):
statements = ['INSERT INTO fanfiction_authorinfo VALUES ("0","2","http://example.com")',
"INSERT INTO fanfiction_stories VALUES ('1', 'thing');"]
result = self.efiction_no_defs._add_table_definitions(statements)
self.assertEqual("DROP TABLE IF EXISTS `fanfiction_stories`;", result[3].strip(),
"should generate DROP TABLE statements for the given tables")
self.assertTrue(result[4].strip().startswith("CREATE TABLE `fanfiction_stories`"),
"should generate CREATE TABLE statements for the given tables")
from unittest import TestCase
from unittest.mock import MagicMock
from efiction import simplified
from opendoors.config import ArchiveConfig
test_logger = MagicMock()
test_sql = MagicMock()
test_config = ArchiveConfig(MagicMock(), "efiction",
"efiction/tests/test_data").config
class TestEFictionSimplified(TestCase):
test_config_no_defs = ArchiveConfig(test_logger, "efiction_no_defs",
"efiction/tests/test_data").config
efiction_with_defs = simplified.EFictionSimplified(test_config,
test_logger, test_sql)
efiction_no_defs = simplified.EFictionSimplified(test_config_no_defs,
test_logger, test_sql)
def test_remove_unwanted_tables(self):
result = self.efiction_no_defs._remove_unwanted_tables([
"use database;", "create table fanfiction_stats;",
"create table fanfiction_stories;"
])
self.assertTrue('fanfiction_stats' not in result,
"statements on unwanted tables should be removed")
self.assertTrue(
'stories' in result,
"statements on desired tables should be stripped of their prefix and kept"
)
import glob
import os
import re
import shutil
from pathlib import Path
from unittest import TestCase
from unittest.mock import MagicMock, patch
from opendoors.config import ArchiveConfig
from opendoors.step_base import StepInfo
from steps.step_01 import Step01
from steps.step_02 import Step02
test_logger = MagicMock()
test_sql = MagicMock()
test_config = ArchiveConfig(MagicMock(), "test",
"steps/tests/test_data").config
class TestStep02(TestCase):
def tearDown(self) -> None:
""" Remove any files generated in test_output """
filtered = [
f for f in glob.glob('steps/tests/test_output/*')
if not re.match(r'\.keep', f)
]
for file in filtered:
try:
if Path(file).is_dir():
shutil.rmtree(file)
else:
os.remove(file)
def test_get_next_step_after_01(self):
test_config = ArchiveConfig(test_logger, "test3",
"opendoors/tests/test_output").config
next_step, done_steps = get_next_step(test_config, "01")
self.assertEqual("01", next_step, "next_step should be set to 01")
self.assertEqual([], done_steps, "done_steps should be reset to empty")
@atexit.register
def save_config_and_exit():
print("Saving config...")
config.save()
if __name__ == "__main__":
if len(sys.argv) > 1:
code_name = sys.argv[1]
else:
code_name = input(
">> Please provide a short, lowercase code name with no spaces or punctuation for the archive "
"you are processing (and make a note of it as you'll need it in future!):"
)
banner_text = f"""Starting processing for archive "{code_name}"..."""
banner = make_banner('=', banner_text)
working_dir = sys.argv[2] if len(
sys.argv) > 2 else create_or_set_working_dir(code_name)
logger = Logging(working_dir, code_name).logger()
logger.info(banner)
config = ArchiveConfig(logger, code_name, working_dir)
archive_config = config.config
progress.continue_from_last(archive_config, logger)
def test_save(self, _mock_write, _mock_open):
config = ArchiveConfig(Logger("test"), "test", "working_dir")
self.assertEqual(
"test", config.config["Archive"]["code_name"],
"code_name config should be set to the provided short code for the archive"
)
import datetime
from unittest import TestCase
from unittest.mock import MagicMock
from efiction.metadata import EFictionMetadata
from efiction.tests.test_utils import load_fixtures
from opendoors.config import ArchiveConfig
from opendoors.mysql import SqlDb
from opendoors.utils import get_full_path, remove_output_files
test_logger = MagicMock()
test_config = ArchiveConfig(test_logger, "efiction",
"efiction/tests/test_data").config
test_sql = SqlDb(test_config, test_logger)
class TestEFictionConverter(TestCase):
converter_config = test_config
converter_config['Processing']['working_dir'] = get_full_path(
"efiction/tests/test_output")
efiction_converter = EFictionMetadata(converter_config, test_logger,
test_sql, "test_path")
def setUp(self) -> None:
""" Load test data and create the Open Doors tables """
load_fixtures(test_config, test_sql)
self.efiction_converter.create_open_doors_db("test_path")
def tearDown(self) -> None:
""" Remove files created during the tests """