def create_phoenix_table(create_sql): phoenix_app = create_app(get_config('develop')) phoenix_app.app_context() database_url = phoenix_app.config['DATABASE_URL'] conn = phoenixdb.connect(database_url, autocommit=True) with conn.cursor() as cursor: cursor.execute(create_sql) conn.close()
def drop_table(table_name): phoenix_app = create_app(get_config('develop')) phoenix_app.app_context() database_url = phoenix_app.config['DATABASE_URL'] conn = phoenixdb.connect(database_url, autocommit=True) with conn.cursor(cursor_factory=phoenixdb.cursor.DictCursor) as cursor: drop_sql = "drop table \"" + table_name + "\"" cursor.execute(drop_sql) conn.close()
def insert_sqoop_meta(job_id): phoenix_app = create_app(get_config('develop')) phoenix_app.app_context() database_url = phoenix_app.config['DATABASE_URL'] conn = phoenixdb.connect(database_url, autocommit=True) with conn.cursor() as cursor: now_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') cursor.execute(insert_sqoop_meta_sql, (job_id, now_time, now_time)) conn.close()
def query_dp_data(query_sql): phoenix_app = create_app(get_config('develop')) phoenix_app.app_context() database_url = phoenix_app.config['DATABASE_URL'] conn = phoenixdb.connect(database_url, autocommit=True) with conn.cursor() as cursor: cursor.execute(query_sql) fetchall = cursor.fetchall() conn.close() return fetchall
def insert_phoenix(table_name, columns_str, data_list): phoenix_app = create_app(get_config('develop')) phoenix_app.app_context() database_url = phoenix_app.config['DATABASE_URL'] conn = phoenixdb.connect(database_url, autocommit=True) with conn.cursor() as cursor: columns_list = columns_str.split("^") insert_sql = "UPSERT INTO \"" + table_name + "\" VALUES (?" for i in range(len(columns_list) - 1): insert_sql += ", ?" insert_sql += ")" for clu in data_list: try: clu = list(clu) clu.insert(0, uuid1().hex) cursor.execute(insert_sql, clu) except ValueError as e: print(e) conn.close()
def data_join_clu(table0, table1, join_column0, join_column1, join_type, final_table_uuid, tmp_table): # 判断是否为临时表或最终表 if final_table_uuid is None: tmp_table_uuid = uuid1().hex else: tmp_table_uuid = final_table_uuid # 从元数据表获取join后需要建表的字段 table0_metadata = query_metadata(table0) table0_columns = table0_metadata[4] table1_metadata = query_metadata(table1) table1_columns = table1_metadata[4] original_columns_str = '{0}^{1}'.format(table0_columns, table1_columns) # 处理重名字段 columns_list, columns_str_meta, same_name = handle_same( original_columns_str, tmp_table_uuid) # 获取phoenix表连接 data_processing_app = create_app(get_config('develop')) data_processing_app.app_context() database_url = data_processing_app.config['DATABASE_URL'] conn = phoenixdb.connect(database_url, autocommit=True) # 查询join数据 fetchall_list = query_join(conn, columns_list, table0, table1, join_column0, join_column1, join_type, table1_columns) # 将查询插入 insert_join_data(columns_str_meta, conn, fetchall_list, tmp_table, tmp_table_uuid) conn.close() return '{0}^{1}'.format(tmp_table_uuid, same_name)
import os from ebidp import create_app from ebidp.configuration import get_config app_env = os.getenv('APP_ENV') app = create_app(get_config(app_env)) if __name__ == '__main__': app.run(host='0.0.0.0', port=80, debug=True)