def test_sqlite_dispatch(self):
with mock.patch(
"piicatcher.explorer.sqlite.SqliteExplorer.scan", autospec=True
) as mock_scan_method:
with mock.patch(
"piicatcher.explorer.sqlite.SqliteExplorer.get_tabular", autospec=True
) as mock_tabular_method:
with mock.patch(
"piicatcher.explorer.explorer.tableprint", autospec=True
) as mock_table_print:
SqliteExplorer.dispatch(
Namespace(
path="connection",
list_all=None,
catalog={"format": "ascii_table"},
scan_type=None,
include_schema=(),
exclude_schema=(),
include_table=(),
exclude_table=(),
)
)
mock_scan_method.assert_called_once()
mock_tabular_method.assert_called_once()
mock_table_print.table.assert_called_once()
def test_sqlite(self):
self.assertEqual(
"select c1,c2 from t1",
SqliteExplorer._get_select_query(
self.schema,
self.schema.get_children()[0],
self.schema.get_children()[0].get_children()))
def test_sqlite_dispatch(self):
with mock.patch('piicatcher.explorer.sqlite.SqliteExplorer.scan',
autospec=True) as mock_scan_method:
with mock.patch(
'piicatcher.explorer.sqlite.SqliteExplorer.get_tabular',
autospec=True) as mock_tabular_method:
with mock.patch('piicatcher.explorer.explorer.tableprint',
autospec=True) as MockTablePrint:
SqliteExplorer.dispatch(
Namespace(path='connection',
list_all=None,
output_format='ascii_table',
scan_type=None,
catalog=None))
mock_scan_method.assert_called_once()
mock_tabular_method.assert_called_once()
MockTablePrint.table.assert_called_once()
def test_sqlite(self):
self.assertEqual(
'select "c1","c2" from t1',
SqliteExplorer._get_select_query(
self.schema,
self.schema.get_children()[0],
self.schema.get_children()[0].get_children(),
),
)
class TestCreateTables(TestCase):
sqlite_conn = 'file::memory:?cache=shared'
def setUp(self):
init_test(str(self.sqlite_conn))
self.explorer = SqliteExplorer(str(self.sqlite_conn))
def tearDown(self):
self.explorer.close_connection()
model_db_close()
def test_schemas_exist(self):
self.assertEqual(1, len(self.explorer.get_schemas()))
def test_tables_exist(self):
schema = self.explorer.get_schemas()[0]
self.assertEqual(["dbcolumns", "dbschemas", "dbtables"],
[t.get_name() for t in schema.get_tables()])
def test_sqlite_dispatch(self):
with mock.patch('piicatcher.explorer.sqlite.SqliteExplorer.scan', autospec=True) \
as mock_scan_method:
with mock.patch(
'piicatcher.explorer.sqlite.SqliteExplorer.get_tabular',
autospec=True) as mock_tabular_method:
with mock.patch('piicatcher.explorer.explorer.tableprint', autospec=True) \
as mock_table_print:
SqliteExplorer.dispatch(
Namespace(path='connection',
list_all=None,
catalog={'format': 'ascii_table'},
scan_type=None,
include_schema=(),
exclude_schema=(),
include_table=(),
exclude_table=()))
mock_scan_method.assert_called_once()
mock_tabular_method.assert_called_once()
mock_table_print.table.assert_called_once()
def temp_sqlite(request, tmpdir_factory):
temp_dir = tmpdir_factory.mktemp("sqlite_test")
sqlite_path = temp_dir.join("sqldb")
explorer = SqliteExplorer(Namespace(path=sqlite_path, catalog=None))
request.cls.explorer = explorer
request.cls.path = str(sqlite_path)
def finalizer():
explorer.get_connection().close()
rmtree(temp_dir)
logging.info("Deleted {}".format(str(temp_dir)))
request.addfinalizer(finalizer)
def temp_sqlite(request, tmpdir_factory):
temp_dir = tmpdir_factory.mktemp("sqlite_test")
sqlite_path = temp_dir.join("sqldb")
explorer = SqliteExplorer(
Namespace(
path=sqlite_path,
catalog=None,
include_schema=(),
exclude_schema=(),
include_table=(),
exclude_table=(),
))
if request.cls is not None:
request.cls.explorer = explorer
request.cls.path = str(sqlite_path)
yield str(sqlite_path)
explorer.connection.close()
rmtree(temp_dir)
logging.info("Deleted {}", str(temp_dir))
def scan_database(
connection: Any,
connection_type: str,
scan_type: str = "shallow",
include_schema: Tuple = (),
exclude_schema: Tuple = (),
include_table: Tuple = (),
exclude_table: Tuple = (),
) -> Dict[Any, Any]:
"""
Args:
connection (connection): Connection object to a database
connection_type (str): Database type. Can be one of sqlite, snowflake, athena, redshift, postgres, mysql or oracle
scan_type (str): Choose deep(scan data) or shallow(scan column names only)
include_schema (List[str]): Scan only schemas matching any pattern; When this option is not specified, all
non-system schemas in the target database will be scanned. Also, the pattern is
interpreted as a regular expression, so multiple schemas can also be selected
by writing wildcard characters in the pattern.
exclude_schema (List[str]): List of patterns. Do not scan any schemas matching any pattern. The pattern is
interpreted according to the same rules as include_schema. When both include_schema
and exclude_schema are given, the behavior is to dump just the schemas that
match at least one include_schema pattern but no exclude_schema patterns. If only
exclude_schema is specified, then matching schemas matching are excluded.
include_table (List[str]): List of patterns to match table. Similar in behaviour to include_schema.
exclude_table (List[str]): List of patterns to exclude matching table. Similar in behaviour to exclude_schema
Returns:
dict: A dictionary of schemata, tables and columns
"""
scanner: Explorer
if connection_type == "sqlite":
args = Namespace(
path=None,
scan_type=scan_type,
list_all=None,
catalog=None,
include_schema=include_schema,
exclude_schema=exclude_schema,
include_table=include_table,
exclude_table=exclude_table,
)
scanner = SqliteExplorer(args)
elif connection_type == "athena":
args = Namespace(
access_key=None,
secret_key=None,
staging_dir=None,
region=None,
scan_type=scan_type,
list_all=None,
include_schema=include_schema,
exclude_schema=exclude_schema,
include_table=include_table,
exclude_table=exclude_table,
catalog=None,
)
scanner = AthenaExplorer(args)
# elif connection_type == "snowflake":
# args = Namespace(
# account=None,
# warehouse=None,
# database=None,
# user=None,
# password=None,
# authenticator=None,
# okta_account_name=None,
# oauth_token=None,
# oauth_host=None,
# scan_type=scan_type,
# list_all=None,
# catalog=None,
# include_schema=include_schema,
# exclude_schema=exclude_schema,
# include_table=include_table,
# exclude_table=exclude_table,
# )
# scanner = SnowflakeExplorer(args)
elif (connection_type == "mysql" or connection_type == "postgres"
or connection_type == "redshift" or connection_type == "oracle"):
ns = Namespace(
host=None,
port=None,
user=None,
password=None,
database=None,
connection_type=connection_type,
scan_type=scan_type,
list_all=None,
catalog=None,
include_schema=include_schema,
exclude_schema=exclude_schema,
include_table=include_table,
exclude_table=exclude_table,
)
if ns.connection_type == "mysql":
scanner = MySQLExplorer(ns)
elif ns.connection_type == "postgres":
scanner = PostgreSQLExplorer(ns)
elif ns.connection_type == "redshift":
scanner = RedshiftExplorer(ns)
elif ns.connection_type == "oracle":
scanner = OracleExplorer(ns)
else:
raise AttributeError(
"Unknown connection type: {}".format(connection_type))
scanner.connection = connection
return _scan_db(scanner, scan_type)
def setUp(self):
init_test(str(self.sqlite_conn))
self.explorer = SqliteExplorer(str(self.sqlite_conn))