def main():
"""
Args / Config / invoke
"""
arg_object = parse_args()
if arg_object.verbose:
loggingsetup.init(logging.DEBUG)
else:
loggingsetup.init(logging.INFO)
config = load_config()
db_conn = build_db_conn(host=config["postgres_host"],
port=config["postgres_port"],
database=config["postgres_database"],
username=config["postgres_username"],
password=config["postgres_password"])
if arg_object.dry_run:
db_conn.set_dry_run(True)
if arg_object.do_snowflake:
db_conn.execute(f"truncate table {config['snowflake_table']}")
snowflake_writer = build_snowflake_buffered_writer(
db_conn=db_conn, snowflake_table=config['snowflake_table'])
process_snowflake_file(filepath=config['snowflake_file'],
snowflake_writer=snowflake_writer)
elif arg_object.do_sqlserver:
db_conn.execute(f"truncate table {config['sqlserver_table']}")
sqlserver_writer = build_sqlserver_buffered_writer(
db_conn=db_conn, sqlserver_table=config['sqlserver_table'])
process_sqlserver_file(filepath=config['sqlserver_file'],
sqlserver_writer=sqlserver_writer)
elif arg_object.do_merge:
db_conn.execute(f"truncate table {config['mapping_table']}")
merger = Merger(db_conn=db_conn,
snowflake_table=config['snowflake_table'],
sqlserver_table=config['sqlserver_table'],
mapping_table=config['mapping_table'])
merger.run_batches()
import datetime
import logging
import pytest
from digible.parser.snowflake_table_parser import SnowflakeTableParser
import digible.loggingsetup as loggingsetup
loggingsetup.init(loglevel=logging.DEBUG)
@pytest.fixture()
def snowflake_parser():
stp = SnowflakeTableParser()
return stp
def test_parse_line(snowflake_parser):
# line 1
line_dict = {
"AVAILABLE_UNITS": "10.0",
"APT_NAME": "Club at Highland Park",
"ADDRESS": "11402 Evans St, Omaha, NE 68164",
"ZIP": "68164",
"CITY": "Omaha",
"STATE": "NE",
"DATE": "2019-06-21 22:13:10.652224"
}
result = snowflake_parser.process_line(line_dict)
expected = {
"address_hash": b'\x85\x08?^\xcaUz\xe7\x14\xf7\xd45\x9b\xa0\xb9\xe3',
def test_global_init():
loggingsetup.init(loglevel=logging.DEBUG)