def create_phoenix_table(create_sql):
phoenix_app = create_app(get_config('develop'))
phoenix_app.app_context()
database_url = phoenix_app.config['DATABASE_URL']
conn = phoenixdb.connect(database_url, autocommit=True)
with conn.cursor() as cursor:
cursor.execute(create_sql)
conn.close()
def drop_table(table_name):
phoenix_app = create_app(get_config('develop'))
phoenix_app.app_context()
database_url = phoenix_app.config['DATABASE_URL']
conn = phoenixdb.connect(database_url, autocommit=True)
with conn.cursor(cursor_factory=phoenixdb.cursor.DictCursor) as cursor:
drop_sql = "drop table \"" + table_name + "\""
cursor.execute(drop_sql)
conn.close()
def insert_sqoop_meta(job_id):
phoenix_app = create_app(get_config('develop'))
phoenix_app.app_context()
database_url = phoenix_app.config['DATABASE_URL']
conn = phoenixdb.connect(database_url, autocommit=True)
with conn.cursor() as cursor:
now_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
cursor.execute(insert_sqoop_meta_sql, (job_id, now_time, now_time))
conn.close()
def query_dp_data(query_sql):
phoenix_app = create_app(get_config('develop'))
phoenix_app.app_context()
database_url = phoenix_app.config['DATABASE_URL']
conn = phoenixdb.connect(database_url, autocommit=True)
with conn.cursor() as cursor:
cursor.execute(query_sql)
fetchall = cursor.fetchall()
conn.close()
return fetchall
def insert_phoenix(table_name, columns_str, data_list):
phoenix_app = create_app(get_config('develop'))
phoenix_app.app_context()
database_url = phoenix_app.config['DATABASE_URL']
conn = phoenixdb.connect(database_url, autocommit=True)
with conn.cursor() as cursor:
columns_list = columns_str.split("^")
insert_sql = "UPSERT INTO \"" + table_name + "\" VALUES (?"
for i in range(len(columns_list) - 1):
insert_sql += ", ?"
insert_sql += ")"
for clu in data_list:
try:
clu = list(clu)
clu.insert(0, uuid1().hex)
cursor.execute(insert_sql, clu)
except ValueError as e:
print(e)
conn.close()
def data_join_clu(table0, table1, join_column0, join_column1,
join_type, final_table_uuid, tmp_table):
# 判断是否为临时表或最终表
if final_table_uuid is None:
tmp_table_uuid = uuid1().hex
else:
tmp_table_uuid = final_table_uuid
# 从元数据表获取join后需要建表的字段
table0_metadata = query_metadata(table0)
table0_columns = table0_metadata[4]
table1_metadata = query_metadata(table1)
table1_columns = table1_metadata[4]
original_columns_str = '{0}^{1}'.format(table0_columns, table1_columns)
# 处理重名字段
columns_list, columns_str_meta, same_name = handle_same(
original_columns_str, tmp_table_uuid)
# 获取phoenix表连接
data_processing_app = create_app(get_config('develop'))
data_processing_app.app_context()
database_url = data_processing_app.config['DATABASE_URL']
conn = phoenixdb.connect(database_url, autocommit=True)
# 查询join数据
fetchall_list = query_join(conn, columns_list, table0, table1,
join_column0, join_column1,
join_type, table1_columns)
# 将查询插入
insert_join_data(columns_str_meta, conn, fetchall_list, tmp_table,
tmp_table_uuid)
conn.close()
return '{0}^{1}'.format(tmp_table_uuid, same_name)
import os
from ebidp import create_app
from ebidp.configuration import get_config
app_env = os.getenv('APP_ENV')
app = create_app(get_config(app_env))
if __name__ == '__main__':
app.run(host='0.0.0.0', port=80, debug=True)