def setUp(self): presto_catlog = "blackhole" presto_schema = "default" mysql_database = "mysql" mysql_user = "******" mysql_password = "******" self.airflow_api = AirflowAPI() self.minikube_ip = str(self.airflow_api.get_minikube_ip()) self.db_util = DBUtil() self.airflow_api.add_presto_connection("presto-conn", presto_catlog, presto_schema) self.airflow_api.add_mysql_connection("mysql-conn", mysql_database, mysql_user, mysql_password) self.mysql_conn = mysql.connector.connect(user=mysql_user, password=mysql_password, host=self.minikube_ip, port=MYSQL_DB_PORT, database=mysql_database, use_pure=False) self.prest_conn = prestodb.dbapi.connect( host=self.minikube_ip, port=PRESTO_DB_PORT, user='******', catalog=presto_catlog, schema=presto_schema, ) create_mysql_table_sql = """ CREATE TABLE IF NOT EXISTS mysql_region ( name VARCHAR(50),count int(10) ); """ self.db_util.create_table(self.mysql_conn, create_mysql_table_sql) create_presto_table_sql = """ CREATE TABLE region ( name varchar ) WITH ( split_count = 1, pages_per_split = 1, rows_per_page = 1, page_processing_delay = '5s' )""" self.db_util.create_table(self.prest_conn, create_presto_table_sql) insert_query_1 = "insert into region values('INDIA')" self.db_util.insert_into_table(self.prest_conn, insert_query_1)
class TestHelloWorldDag(unittest.TestCase): """Integration test for Hello world DAG""" def setUp(self): self.airflow_api = AirflowAPI() def test_hello_world(self): """helloword dag should run successfully""" execution_date = "2019-01-11T12:00:00+00:00" dag_id = "hello_world" self.airflow_api.trigger_dag(dag_id, execution_date) is_running = True while is_running: is_running = self.airflow_api.is_dag_running( dag_id, execution_date) self.assertEqual(is_running, False) self.assertEqual( self.airflow_api.get_dag_status(dag_id, execution_date), "success")
class TestPrestoToMySqlDag(unittest.TestCase): """Integration test for presto to mysql transfer""" def setUp(self): self.airflow_api = AirflowAPI() self.airflow_api.add_presto_connection("presto-conn", "tpch", "sf1") self.airflow_api.add_mysql_connection("mysql-conn", "mysql", "mysql", "mysql") def test_presto_to_mysql_transfer(self): """should transfer data from presto to mysql""" execution_date = "2019-01-12T12:00:00+00:00" dag_id = "presto_to_mysql" self.airflow_api.trigger_dag(dag_id, execution_date) is_running = True while is_running: is_running = self.airflow_api.is_dag_running( dag_id, execution_date) self.assertEqual(is_running, False) self.assertEqual( self.airflow_api.get_dag_status(dag_id, execution_date), "success")
def setUp(self): self.airflow_api = AirflowAPI() self.airflow_api.add_presto_connection("presto-conn", "tpch", "sf1") self.airflow_api.add_mysql_connection("mysql-conn", "mysql", "mysql", "mysql")
class TestPrestoToMySqlDag(unittest.TestCase): """Integration test for presto to mysql transfer""" mysql_conn = None prest_conn = None def setUp(self): presto_catlog = "blackhole" presto_schema = "default" mysql_database = "mysql" mysql_user = "******" mysql_password = "******" self.airflow_api = AirflowAPI() self.minikube_ip = str(self.airflow_api.get_minikube_ip()) self.db_util = DBUtil() self.airflow_api.add_presto_connection("presto-conn", presto_catlog, presto_schema) self.airflow_api.add_mysql_connection("mysql-conn", mysql_database, mysql_user, mysql_password) self.mysql_conn = mysql.connector.connect(user=mysql_user, password=mysql_password, host=self.minikube_ip, port=MYSQL_DB_PORT, database=mysql_database, use_pure=False) self.prest_conn = prestodb.dbapi.connect( host=self.minikube_ip, port=PRESTO_DB_PORT, user='******', catalog=presto_catlog, schema=presto_schema, ) create_mysql_table_sql = """ CREATE TABLE IF NOT EXISTS mysql_region ( name VARCHAR(50),count int(10) ); """ self.db_util.create_table(self.mysql_conn, create_mysql_table_sql) create_presto_table_sql = """ CREATE TABLE region ( name varchar ) WITH ( split_count = 1, pages_per_split = 1, rows_per_page = 1, page_processing_delay = '5s' )""" self.db_util.create_table(self.prest_conn, create_presto_table_sql) insert_query_1 = "insert into region values('INDIA')" self.db_util.insert_into_table(self.prest_conn, insert_query_1) def test_presto_to_mysql_transfer(self): """should transfer data from presto to mysql""" execution_date = "2019-05-12T14:00:00+00:00" dag_id = "presto_to_mysql" self.airflow_api.trigger_dag(dag_id, execution_date) is_running = True while is_running: is_running = self.airflow_api.is_dag_running( dag_id, execution_date) self.assertEqual(is_running, False) self.assertEqual( self.airflow_api.get_dag_status(dag_id, execution_date), "success") mysql_select_query = "SELECT name FROM mysql_region" row_count = self.db_util.get_row_count(self.mysql_conn, mysql_select_query) self.assertEqual(1, len(row_count)) def tearDown(self): drop_mysql_table = "drop table mysql_region" drop_presto_table = "drop table region" self.db_util.drop_table(self.mysql_conn, drop_mysql_table) self.db_util.drop_table(self.prest_conn, drop_presto_table) self.mysql_conn.close() self.prest_conn.close()
def setUp(self): self.airflow_api = AirflowAPI()