def __init__(self, data_config: dict, databses: str):
self.db_type = data_config['db_type'].lower()
if self.db_type == 'mysql':
self.db = MysqlUtil(host=data_config['host'],
user=data_config['username'],
password=data_config['password'],
database=databses)
elif self.database_type == "mongo":
pass
def save_modifyed_columns_and_job_id(columns_str, job_id):
db_util = MysqlUtil(JOB_DB_Configure.mysql_host,
JOB_DB_Configure.mysql_user,
JOB_DB_Configure.mysql_password,
JOB_DB_Configure.mysql_database)
table = 'source_dupli_columns'
data_dict = {
"job_id":job_id,
"columns_str":columns_str,
}
return db_util.add_data_to_table(table, data_dict)
def save_key_and_job_id(key, job_id):
db_util = MysqlUtil(JOB_DB_Configure.mysql_host,
JOB_DB_Configure.mysql_user,
JOB_DB_Configure.mysql_password,
JOB_DB_Configure.mysql_database)
table = 'store_enc_keys'
data_dict = {
"enc_key":key,
"job_id":job_id,
}
return db_util.add_data_to_table(table, data_dict)
def obtain_job_conf_by_id(id):
db_util = MysqlUtil(JOB_DB_Configure.mysql_host,
JOB_DB_Configure.mysql_user,
JOB_DB_Configure.mysql_password,
JOB_DB_Configure.mysql_database)
table = 'mission'
selected_column = '*'
print '====== table {} perform data cleaning ======'.format(table)
print 'query data from mysql database...'
condation = ['id = {}'.format(id)]
job_conf_df = db_util.query_from_table_to_dataframe(table, selected_column, condation)
return job_conf_df
def get_modified_columns_str_by_job_id(job_id):
db_util = MysqlUtil(JOB_DB_Configure.mysql_host,
JOB_DB_Configure.mysql_user,
JOB_DB_Configure.mysql_password,
JOB_DB_Configure.mysql_database)
table = 'source_dupli_columns'
selected_column = 'columns_str'
print '====== table {} perform data cleaning ======'.format(table)
print 'query data from mysql database...'
condation = ['job_id = {}'.format(job_id)]
data_df = db_util.query_from_table_to_dataframe(table, selected_column, condation)
return data_df
class DatabaseOper:
def __init__(self, data_config: dict, databses: str):
self.db_type = data_config['db_type'].lower()
if self.db_type == 'mysql':
self.db = MysqlUtil(host=data_config['host'],
user=data_config['username'],
password=data_config['password'],
database=databses)
elif self.database_type == "mongo":
pass
def run_sql(self, sql: str):
"""
根据已连接的数据库，执行对应的 sql
:param database_type: 数据库类型
:param sql: 执行的sql
"""
if self.db_type == 'mysql':
return self.db.execute(sql)
elif self.database_type == 'mongo':
# TODO 只有需要可以继承mongo数据库
pass
def obtain_mysql_columns(job_id=None):
# 得到任务配置信息
job_conf_df = obtain_job_conf_by_id(job_id)
# 得到源配置信息
souce_conf_df = obtain_source_conf_by_id(int(job_conf_df['source_id']))
db_util = MysqlUtil(souce_conf_df['url'].values[0],
souce_conf_df['user_name'].values[0],
souce_conf_df['pass_word'].values[0],
souce_conf_df['db_name'].values[0])
table = souce_conf_df['table_name'].values[0]
selected_column = '*'
mysql_date_df = db_util.query_from_table_to_dataframe(
table, selected_column, [])
returnModel = {}
returnModel['datum'] = str(list(set(
mysql_date_df.columns.tolist()))).replace("u'",
"'").decode("unicode_escape")
response = jsonify(json.dumps(returnModel, ensure_ascii=False))
response.headers.add('Access-Control-Allow-Origin', '*')
return response