def test_set_config(datafiles):
from sheetwork.core.config.config import ConfigLoader
from sheetwork.core.config.project import Project
from sheetwork.core.main import parser
flags = FlagParser(
parser,
test_sheet_name="df_dropper",
project_dir=str(datafiles),
sheet_config_dir=str(datafiles),
)
project = Project(flags)
config = ConfigLoader(flags, project)
flags2 = FlagParser(
parser,
test_sheet_name="no_cols",
project_dir=str(datafiles),
sheet_config_dir=str(datafiles),
)
project2 = Project(flags2)
config2 = ConfigLoader(flags2, project2)
assert config.sheet_config == EXPECTED_CONFIG
assert config2.sheet_config == NO_COLS_EXPECTED_CONFIG
def test_decide_object_creation(monkeypatch, datafiles, project_name):
from sheetwork.core.flags import FlagParser
from sheetwork.core.config.project import Project
from sheetwork.core.main import parser
monkeypatch.setattr(Project, "PROJECT_FILENAME", f"{project_name}.yml")
monkeypatch.setattr(Project, "IS_TEST", True)
expected_object_creation_dict = {
"create_table": True,
"create_schema": True,
}
if project_name == "sheetwork_project_all_create":
expected_object_creation_dict = {
val: True
for val in expected_object_creation_dict
}
if project_name == "sheetwork_project_deprecated":
expected_object_creation_dict.update({"create_schema": False})
flags = FlagParser(parser, project_dir=str(datafiles))
project = Project(flags)
project.decide_object_creation()
assert project.object_creation_dct == expected_object_creation_dict
assert project.destructive_create_table is True
def test_load_project_from_yaml(datafiles):
from sheetwork.core.flags import FlagParser
from sheetwork.core.config.project import Project
from sheetwork.core.main import parser
flags = FlagParser(parser, project_dir=str(datafiles))
project = Project(flags)
project.load_project_from_yaml()
assert project.project_dict == EXPECTED_SHEETWORK_PROJECT
def test_make_df_from_worksheet(datafiles, monkeypatch, empty_workbook):
from sheetwork.core.clients.google import GoogleSpreadsheet
from sheetwork.core.config.profile import Profile
from sheetwork.core.flags import FlagParser
from sheetwork.core.config.project import Project
from sheetwork.core.main import parser
from sheetwork.core.exceptions import SheetLoadingError, NoWorkbookLoadedError
flags = FlagParser(parser,
project_dir=str(datafiles),
profile_dir=str(datafiles))
project = Project(flags)
profile = Profile(project)
def mock__check_google_creds_exist(self):
return True, Path()
monkeypatch.setattr(GoogleSpreadsheet, "_check_google_creds_exist",
mock__check_google_creds_exist)
g = GoogleSpreadsheet(profile)
if empty_workbook:
with pytest.raises(NoWorkbookLoadedError):
g.workbook = None
g.make_df_from_worksheet()
else:
g.workbook = "dummy_non_empty_content"
with pytest.raises(SheetLoadingError):
g.make_df_from_worksheet()
def handle(parser: argparse.ArgumentParser) -> Union[InitTask, SheetBag, None]:
"""Pure orchestrator function.
Calls pipeline based on the command asked for. It also sets up log levels and calls for CLI arg
parsing JIT!.
Args:
parser (argparse.ArgumentParser): parser module to use for CLI parsing
Returns:
Union[InitTask, SheetBag, None]: Ran object of type Task (need to rework TODO)
"""
flag_parser = FlagParser(parser)
flag_parser.consume_cli_arguments()
# set up traceback override
SheetworkTracebackManager(flag_parser)
if flag_parser.log_level == "debug":
log_manager.set_debug()
if flag_parser.args.command == "init":
task: Union[init_task.InitTask, upload_task.SheetBag] = init_task.InitTask(flag_parser)
return task.run()
if flag_parser.args.command == "upload":
project = Project(flag_parser)
config = ConfigLoader(flag_parser, project)
profile = Profile(project)
task = upload_task.SheetBag(config, flag_parser, profile)
return task.run()
raise NotImplementedError(f"{flag_parser.args.command} is not supported")
def test_load_plugins(datafiles):
from sheetwork.core.adapters.base.connection import BaseConnection, BaseCredentials
from sheetwork.core.adapters.base.impl import BaseSQLAdapter
from sheetwork.core.adapters.factory import AdapterContainer
from sheetwork.core.config.profile import Profile
from sheetwork.core.config.project import Project
from sheetwork.core.flags import FlagParser
from sheetwork.core.main import parser
flags = FlagParser(parser,
project_dir=str(datafiles),
profile_dir=str(datafiles))
project = Project(flags)
profile = Profile(project, "dev")
profile.read_profile()
a_c = AdapterContainer()
a_c.register_adapter(profile)
a_c.load_plugins()
allowed_adaptors = {
"sql_adapter": BaseSQLAdapter,
"connection_adapter": BaseConnection,
"credentials_adapter": BaseCredentials,
}
for plugin_type, plugin_class in allowed_adaptors.items():
c_ada = getattr(a_c, plugin_type)
assert issubclass(c_ada, plugin_class)
def test__override_gspread_default_creds(datafiles, monkeypatch):
import gspread
from sheetwork.core.clients.google import GoogleSpreadsheet
from sheetwork.core.config.profile import Profile
from sheetwork.core.config.project import Project
from sheetwork.core.flags import FlagParser
from sheetwork.core.main import parser
# mock check for credentials so that we don't have to have a creds file in GH actions
def mock__check_google_creds_exist(self) -> Tuple[bool, Path]:
return True, Path("dummy")
monkeypatch.setattr(GoogleSpreadsheet, "_check_google_creds_exist",
mock__check_google_creds_exist)
# Patch gspread functions so they don't attempt to do stuff
def mock_load_credentials(self) -> None:
# noop
pass
def mock_store_credentials(self) -> None:
# noop
pass
monkeypatch.setattr(gspread.auth, "load_credentials",
mock_load_credentials)
monkeypatch.setattr(gspread.auth, "store_credentials",
mock_store_credentials)
flags = FlagParser(parser,
project_dir=str(datafiles),
profile_dir=str(datafiles))
project = Project(flags)
profile = Profile(project, "end_user")
gsheet = GoogleSpreadsheet(profile)
# testing this also because who knows these might become deprecated
assert hasattr(gspread.auth, "DEFAULT_CONFIG_DIR")
assert hasattr(gspread.auth, "DEFAULT_CREDENTIALS_FILENAME")
assert hasattr(gspread.auth, "DEFAULT_AUTHORIZED_USER_FILENAME")
assert hasattr(gspread.auth, "DEFAULT_SERVICE_ACCOUNT_FILENAME")
# now make the replacements
gsheet._override_gspread_default_creds()
g_creds_dir = profile.google_credentials_dir
project_name = "sheetwork_test"
assert gspread.auth.DEFAULT_CONFIG_DIR == g_creds_dir
assert gspread.auth.DEFAULT_CREDENTIALS_FILENAME == g_creds_dir.joinpath(
f"{project_name}.json")
assert gspread.auth.DEFAULT_AUTHORIZED_USER_FILENAME == g_creds_dir.joinpath(
f"{project_name}_authorised_user.json")
assert gspread.auth.DEFAULT_SERVICE_ACCOUNT_FILENAME == g_creds_dir.joinpath(
f"{project_name}_service_account.json")
def test_override_object_creation_from_flags(monkeypatch, datafiles,
project_name):
from sheetwork.core.flags import FlagParser
from sheetwork.core.config.profile import Project
from sheetwork.core.main import parser
def mock_consume_cli_arguments(self):
self.create_table = True
self.destructive_create_table = True
monkeypatch.setattr(FlagParser, "consume_cli_arguments",
mock_consume_cli_arguments)
flags = FlagParser(parser, project_dir=str(datafiles))
flags.consume_cli_arguments()
project = Project(flags)
project.decide_object_creation()
assert project.object_creation_dct["create_table"] is True
assert project.object_creation_dct["create_schema"] is True
assert project.destructive_create_table is True
def test__override_cli_args(datafiles):
from sheetwork.core.config.config import ConfigLoader
from sheetwork.core.config.project import Project
from sheetwork.core.main import parser
flags = FlagParser(
parser,
test_sheet_name="sheet_with_no_schema",
project_dir=str(datafiles),
sheet_config_dir=str(datafiles),
)
project = Project(flags)
config = ConfigLoader(flags, project)
assert config.target_schema == project.target_schema
def test_read_profile(datafiles):
from sheetwork.core.config.profile import Profile
from sheetwork.core.config.project import Project
from sheetwork.core.flags import FlagParser
from sheetwork.core.main import parser
flags = FlagParser(parser,
project_dir=str(datafiles),
profile_dir=str(datafiles))
project = Project(flags)
profile = Profile(project, "dev")
profile.read_profile()
assert profile.profile_dict == EXPECTED_DEV_TEST_PROFILE
def test_exclude_columns(datafiles):
from sheetwork.core.sheetwork import SheetBag
from sheetwork.core.main import parser
flags = FlagParser(
parser,
test_sheet_name="df_dropper",
project_dir=str(datafiles),
sheet_config_dir=str(datafiles),
profile_dir=str(datafiles),
)
project = Project(flags)
profile = Profile(project)
config = ConfigLoader(flags, project)
df = generate_test_df(DROP_COL_DF)
excluded_df = SheetBag(config, flags, profile).exclude_columns(df)
assert excluded_df.columns.tolist() == EXCLUDED_DF_COLS
def test_rename_columns(datafiles):
from sheetwork.core.sheetwork import SheetBag
from sheetwork.core.main import parser
flags = FlagParser(
parser,
test_sheet_name="df_renamer",
project_dir=str(datafiles),
sheet_config_dir=str(datafiles),
profile_dir=str(datafiles),
)
project = Project(flags)
profile = Profile(project)
config = ConfigLoader(flags, project)
df = generate_test_df(DIRTY_DF)
renamed_df = SheetBag(config, flags, profile).rename_columns(df)
assert renamed_df.columns.tolist() == RENAMED_COLS
def test_load_sheet(datafiles):
from sheetwork.core.sheetwork import SheetBag
from sheetwork.core.main import parser
flags = FlagParser(
parser,
test_sheet_name="df_renamer",
project_dir=str(datafiles),
sheet_config_dir=str(datafiles),
profile_dir=str(datafiles),
)
project = Project(flags)
config = ConfigLoader(flags, project)
profile = Profile(project)
with mock.patch.object(SheetBag,
"_obtain_googlesheet",
return_value=generate_test_df(DIRTY_DF)):
sheetbag = SheetBag(config, flags, profile)
sheetbag.load_sheet()
target_df = generate_test_df(RENAMED_DF)
assert target_df.equals(sheetbag.sheet_df)