def test_insert_value_into(): md = get_mock_table_md() gen = SQLGenerator(md) query = gen.insert_values_into( ["test", "test", "2020-12-08 20:03:16.759617"]) expected = "INSERT INTO test.test_table_delta VALUES ('test','test','2020-12-08 20:03:16.759617');" assert query.strip() == expected.strip()
def test_table_md(): md = get_mock_table_md() # note - we could also use Marshmallow to define a schema but this is overkill at the moment assert md.table_name == "test_table_delta" assert md.schema_name == "test" assert md.columns == [ { "name": "id", "type": "varchar", "length": 300 }, { "name": "event_type", "type": "varchar", "length": 100 }, { "name": "event_ts", "type": "timestamp" }, ] assert md.delimiter == "," assert md.load_prefix == "test" assert md.filter_key == "event_ts" assert md.delta_params == { "master_table": "test_table", "delta_key": "id", }
def test_create_table_query(): md = get_mock_table_md() gen = SQLGenerator(md) query = gen.create_table_query() expected = """ CREATE TABLE IF NOT EXISTS test.test_table_delta(id varchar(300),event_type varchar(100),event_ts timestamp); """ assert query.strip() == expected.strip()
def test_drop_table_query(): md = get_mock_table_md() gen = SQLGenerator(md) query = gen.drop_table() expected = """ DROP TABLE IF EXISTS test.test_table_delta; """ assert query.strip() == expected.strip()
def test_copy_query(): md = get_mock_table_md() gen = SQLGenerator(md) query = gen.copy_query() expected = """COPY test.test_table_delta (id,event_type,event_ts) FROM STDIN WITH DELIMITER ',' CSV HEADER """ # comparing exact match of string because spaces can cause unexpected assertion failures assert query.replace(" ", "") == expected.replace(" ", "")
def test_upsert_on_id_query(): md = get_mock_table_md() gen = SQLGenerator(md) query = gen.upsert_on_id() expected = """ CREATE TABLE IF NOT EXISTS test.test_table (LIKE test.test_table_delta); DELETE FROM test.test_table WHERE id IN (SELECT id FROM test.test_table_delta); INSERT INTO test.test_table SELECT * FROM test.test_table_delta; """ # comparing exact match of string because spaces can cause unexpected assertion failures assert query.replace(" ", "") == expected.replace(" ", "")
def test_consumer_consume_events(publish_for_consumption): table_md = get_mock_table_md() pg_hook = PgHook() consumer = Consumer(host=RABBIT_MQ_HOST, queue=TEST_QUEUE, table_md=table_md) consumer_thread = threading.Thread(target=consumer.consume_events, daemon=True) consumer_thread.start() sleep(4) expected = [ ("foo", "created", datetime(2020, 12, 8, 20, 3, 16, 759617)), ("bar", "created", datetime(2014, 12, 8, 20, 3, 16, 759617)), ] with pg_hook.get_conn() as conn: with conn.cursor() as cur: cur.execute( f"SELECT * FROM {table_md.schema_name}.{table_md.table_name};") assert cur.fetchall() == expected consumer_thread.__running = False