def table_list(desc): if request.method == "GET": db_id = request.json.get("dmp_database_id") conn = auto_connect(db_id=db_id) res = conn.tables_list current_app.logger.info(res) return resp_hanlder(result=res)
def update_chart_data(chart_id): import json from dmp import app with app.app_context(): from flask import current_app from dmp.models import Chart from dmp.utils.engine import auto_connect if Chart.exist_item_by_id(chart_id): current_chart = Chart.get(chart_id) data_table_id = current_chart.dmp_data_table_id query_string = current_chart.query_string conn = auto_connect(table_id=data_table_id) if current_chart.dmp_data_table.dmp_database_type in [1, 2, 4]: request_json = {"sql": query_string.get("sql")} _data = conn.exec_query(**request_json) if type(_data) == list: field = query_string.get("fields") current_chart.chart_data = [ dict(zip(field, d)) for d in _data ] current_chart.put() return True, "OK" else: return False, "item not exist"
def retrieve(desc): if request.method == "GET": retrieve_info = request.json dmp_data_table_id = retrieve_info.get("dmp_data_table_id") dmp_data_table = DataTable.get(dmp_data_table_id) database_id = dmp_data_table.dmp_database_id db_table_name = dmp_data_table.db_table_name conn = auto_connect(db_id=database_id) data = conn.retrieve(table_name=db_table_name) # current_app.logger.info(data) return resp_hanlder(result=data)
def retrieve(desc): """ SQL查询 --- tags: - SQL parameters: - name: dmp_data_table_id in: type: int required: True description: 数据表ID - name: sql in: type: string required: false description: SQL 语句,mysql,hive ,kylin 必填 - name: collection in: type: string required: false description: mongodb 参数 集合名(表名) responses: 0: description: OK """ request_json = request.json if request.json else {} data_table_id = request_json.get("dmp_data_table_id") if data_table_id: try: conn = auto_connect(table_id=data_table_id) result = conn.exec_query(**request_json) return resp_hanlder(code=0, result=result) except Exception as e: return resp_hanlder(code=999, msg=str(e)) else: return resp_hanlder(code=999, msg="缺少dmp_data_table_id")
def db_table_name(desc): if request.method == "GET": try: dmp_database_id = request.json.get("dmp_database_id") db_table_name_ = request.json.get("db_table_name") from dmp.utils.engine import auto_connect db_ = auto_connect(dmp_database_id) tbl = db_.tables_list from dmp.models import FormMigrate, FormUpload count1 = FormUpload.query.filter_by( destination_db_table_name=db_table_name_).count() count2 = FormMigrate.query.filter_by( new_table_name=db_table_name_).count() if db_table_name_ in tbl or count1 > 0 or count2 > 0: return resp_hanlder(result={ "exist": True, }) else: return resp_hanlder(result={ "exist": False, }) except Exception as err: return resp_hanlder(err=err)
def column(desc): if request.method == "GET": try: dmp_data_table_id = request.json.get("dmp_data_table_id") if DataTable.exist_item_by_id(dmp_data_table_id): data_table_info = DataTable.get(dmp_data_table_id) else: return resp_hanlder(code=999, msg="数据表不存在或已被删除") db_table_name = data_table_info.db_table_name colums4sdb = DataTableColumn.query.filter_by( dmp_data_table_id=dmp_data_table_id) column4sdb_array = [] current_app.logger.info(colums4sdb.count()) if colums4sdb.count() > 0: column4sdb_array = [col.__json__() for col in colums4sdb.all()] columns4db = [] columns4db = auto_connect( db_id=data_table_info.dmp_database_id).columns(db_table_name) current_app.logger.info(column4sdb_array) columns = [] for i in columns4db: mark = True for j in column4sdb_array: if j.get("dmp_data_table_column_name") == i.get( "dmp_data_table_column_name"): columns.append(j) mark = False if mark: columns.append(i) current_app.logger.info(columns) return resp_hanlder(result=columns) except Exception as err: current_app.logger.error(err) return resp_hanlder(code=999, err=err)
def approve(desc): if request.method == "PUT": try: # 获取请求参数 approve_form_info = request.json auth_token = request.headers.get('Authorization') approve_user_id = Users.decode_auth_token(auth_token) form_type = approve_form_info.get("dmp_form_type", None) form_id = approve_form_info.get("dmp_form_id", None) approve_result = approve_form_info.get("approve_result", None) answer = approve_form_info.get("answer", None) # 保存表单审批信息 approve_form = Forms.get(form_id) approve_form.approve_dmp_user_id = approve_user_id approve_form.approve_result = approve_result approve_form.approve_on = datetime.datetime.now() approve_form.answer = answer approve_form.put() # 执行审批动作 if form_type == 1: # 从数据库添加数据表单 try: if approve_result == 1: post( dmp_data_table_name=approve_form.info_form. dmp_data_table_name, db_table_name=approve_form.info_form.db_table_name, dmp_user_id=approve_form.submit_dmp_user_id, dmp_database_id=approve_form.info_form. dmp_database_id, dmp_case_id=approve_form.info_form.dmp_case_id, description=approve_form.description) approve_form.result = "SUCCEED!" except Exception as err: approve_form.result = "ASSOCIATION FAILED,ERROR_MSG:%s" % str( err) approve_form.finish = True elif form_type == 2: # 文件上传添加数据表单 upload_path = current_app.config.get("UPLOADED_PATH") file_path = os.path.join(upload_path, approve_form.info_form.filepath) file_type = approve_form.info_form.filetype column_line = approve_form.info_form.column_line column = approve_form.info_form.column.split(",") if type( approve_form.info_form.column) == str else [] destination_dmp_database_id = approve_form.info_form.destination_dmp_database_id destination_db_table_name = approve_form.info_form.destination_db_table_name dmp_data_table_name = approve_form.info_form.dmp_data_table_name method = approve_form.info_form.method description = approve_form.description submit_dmp_user_id = approve_form.submit_dmp_user_id dmp_case_id = approve_form.info_form.dmp_case_id destination_database = Database.get( destination_dmp_database_id) destination_database_type = destination_database.db_type destination_db_host = destination_database.db_host destination_db_port = destination_database.db_port destination_db_username = destination_database.db_username destination_db_passwd = destination_database.db_passwd destination_db_name = destination_database.db_name try: reader = [] text_column = [] csv_filepath = os.path.join( current_app.config.get("UPLOADED_PATH"), file_path) if file_type == 1 or file_type == 3: # csv、excel dt = pd.read_csv(csv_filepath, header=column_line) csv_column = [ "_".join(lazy_pinyin(d)) if is_contains_chinese(d) else d for d in list(dt.columns) ] text_column = column if column and len(column) == len( csv_column) else csv_column csv_column_d = [{ "index": i, "type": "string" } for i, cc in enumerate(text_column)] reader = textfile_reader(filepath=csv_filepath, column=csv_column_d) elif file_type == 2: # json dt = pd.read_csv(csv_filepath, header=0) csv_column = [ "_".join(lazy_pinyin(d)) if is_contains_chinese(d) else d for d in list(dt.columns) ] text_column = column if column and len(column) == len( csv_column) else csv_column csv_column_d = [{ "index": i, "type": "string" } for i, cc in enumerate(text_column)] reader = textfile_reader(filepath=csv_filepath, column=csv_column_d) pass writer = [] if destination_database_type == 1: # hive_writer hive_columns = [{ "name": col, "type": "string" } for col in text_column] hive_path = "/user/hive/warehouse/%s.db/%s" % ( destination_db_name, destination_db_table_name) hive_conn = auto_connect( db_id=destination_dmp_database_id) create_table_sql = create_table_query_handler( table_name=destination_db_table_name, fields=text_column, uniform_type="string", id_primary_key=False, semicolon=False, fieldDelimiter=",") print(create_table_sql) if method == 1: hive_conn.execsql(create_table_sql) elif method == 3: del_table_sql = "drop table {table_name}" hive_conn.execsql( del_table_sql.format( table_name=destination_db_table_name)) hive_conn.execsql(create_table_sql) else: pass writer = hive_writer( host=destination_db_host, port=8020, path=hive_path, filename=destination_db_table_name, column=hive_columns, fieldDelimiter=",", ) elif destination_database_type == 2: # mysql_writer create_table_sql = create_table_query_handler( table_name=destination_db_table_name, fields=text_column, uniform_type="text", id_primary_key=True, semicolon=True, fieldDelimiter=None) current_app.logger.info(create_table_sql) mysql_conn = auto_connect( db_id=destination_dmp_database_id) del_table_sql = "drop table {table_name};" if method == 1: mysql_conn.execsql(sql=create_table_sql) elif method == 2: pass elif method == 3: mysql_conn.execsql( del_table_sql.format( table_name=destination_db_table_name)) mysql_conn.execsql(create_table_sql) column = text_column writer = mysql_writer( model=1, username=destination_db_username, password=destination_db_passwd, column=column, host=destination_db_host, port=destination_db_port, db=destination_db_name, table=destination_db_table_name, ) elif destination_database_type == 3: # mongo_writer mongo_conn = auto_connect( db_id=destination_dmp_database_id) if method == 3: mongo_conn.del_table( table_name=destination_db_table_name) column = [{ "name": col, "type": "string" } for col in text_column] writer = mongodb_writer( host=destination_db_host, port=destination_db_port, username=destination_db_username, password=destination_db_passwd, db_name=destination_db_name, collection_name=destination_db_table_name, column=column, ) meta = { "form_id": approve_form.id, "csv_file_path": csv_filepath, "dmp_data_table_name": dmp_data_table_name, "db_table_name": destination_db_table_name, "dmp_user_id": submit_dmp_user_id, "dmp_database_id": destination_dmp_database_id, "dmp_case_id": dmp_case_id, "description": description, } job_hanlder.delay(reader=reader, writer=writer, channel=3, func=postfunc, meta=meta) approve_form.result = "SUCCEED!" except Exception as err: approve_form.result = "CREATE UPLOAD JOB FAILED,ERROR MESSAGE:%s" % str( err) approve_form.finish = True elif form_type == 3: # 数据迁移表单 if approve_result == 1: origin_data_table = DataTable.get( approve_form.info_form.origin_dmp_table_id) origin_database = Database.get( origin_data_table.dmp_database_id) origin_database_type = origin_database.db_type origin_db_host = origin_database.db_host origin_db_port = origin_database.db_port origin_db_username = origin_database.db_username origin_db_passwd = origin_database.db_passwd origin_db_name = origin_database.db_name origin_db_table_name = origin_data_table.db_table_name destination_database = Database.get( approve_form.info_form.destination_dmp_database_id) destination_database_type = destination_database.db_type destination_db_host = destination_database.db_host destination_db_port = destination_database.db_port destination_db_username = destination_database.db_username destination_db_passwd = destination_database.db_passwd destination_db_name = destination_database.db_name destination_db_table_name = approve_form.info_form.new_table_name try: base_column = auto_connect( db_id=origin_data_table.dmp_database_id).columns( origin_data_table.db_table_name) # current_app.logger.info(base_column) reader = [] if origin_database_type == 1: # hive_reader reader = hive_reader( host=origin_db_host, port=8020, path="/user/hive/warehouse/%s.db/%s" % (origin_db_name, origin_db_table_name), fileType="text", column=["*"], fieldDelimiter=',', encoding="utf-8") elif origin_database_type == 2: # mysql_reader column = [ col.get("dmp_data_table_column_name") for col in base_column ] reader = mysql_reader( username=origin_db_username, password=origin_db_passwd, column=column, host=origin_db_host, port=origin_db_port, db=origin_db_name, table=origin_db_table_name, where=None, ) elif origin_database_type == 3: # mongodb column = [{ "index": i + 1, "name": col.get("dmp_data_table_column_name"), "type": col.get("dmp_data_table_column_type") } for i, col in enumerate(base_column)] reader = mongodb_reader( host=origin_db_host, port=origin_db_port, db_name=origin_db_name, collection_name=origin_db_table_name, column=column, username=origin_db_username, password=origin_db_passwd) pass writer = [] if destination_database_type == 1: # hive_writer hive_col = [ col.get("dmp_data_table_column_name") for col in base_column ] hive_columns = [{ "name": col, "type": "string" } for col in hive_col] hive_path = "/user/hive/warehouse/%s.db/%s" % ( destination_db_name, destination_db_table_name) hive_conn = auto_connect( db_id=approve_form.info_form. destination_dmp_database_id) create_table_sql = create_table_query_handler( table_name=destination_db_table_name, fields=hive_col, uniform_type="string", id_primary_key=False, semicolon=False, fieldDelimiter=",") hive_conn.execsql(create_table_sql) writer = hive_writer( host=destination_db_host, port=8020, path=hive_path, filename=destination_db_table_name, column=hive_columns, fieldDelimiter=",", ) elif destination_database_type == 2: # mysql_writer column = [ col.get("dmp_data_table_column_name") for col in base_column ] create_table_sql = create_table_query_handler( table_name=destination_db_table_name, fields=column, uniform_type="text", id_primary_key=True, semicolon=True, fieldDelimiter=None) mysql_conn = auto_connect( db_id=approve_form.info_form. destination_dmp_database_id) mysql_conn.execsql(sql=create_table_sql) # preSQL = [] writer = mysql_writer( model=1, username=destination_db_username, password=destination_db_passwd, column=column, host=destination_db_host, port=destination_db_port, db=destination_db_name, table=destination_db_table_name, ) elif destination_database_type == 3: # mongo_writer column = [{ "name": col.get("dmp_data_table_column_name"), "type": "string" } for col in base_column] writer = mongodb_writer( host=destination_db_host, port=destination_db_port, username=destination_db_username if destination_db_username else None, password=destination_db_passwd if destination_db_passwd else None, db_name=destination_db_name, collection_name=destination_db_table_name, column=column, ) meta = {"form_id": approve_form.id} job_hanlder.delay(reader=reader, writer=writer, channel=3, func=job_finish, meta=meta) except Exception as err: approve_form.result = "CREATE MIGRATE JOB FAILED,ERROR MESSAGE:%s" % str( err) approve_form.finish = True raise err elif form_type == 4: # 数据导出表单 if approve_result == 1: origin_data_table = DataTable.get( approve_form.info_form.dmp_data_table_id) if origin_data_table: origin_database = Database.get( origin_data_table.dmp_database_id) origin_database_type = origin_database.db_type origin_db_host = origin_database.db_host origin_db_port = origin_database.db_port origin_db_username = origin_database.db_username origin_db_passwd = origin_database.db_passwd origin_db_name = origin_database.db_name origin_db_table_name = origin_data_table.db_table_name try: base_column = auto_connect( db_id=origin_data_table.dmp_database_id ).columns(origin_data_table.db_table_name) reader = [] if origin_database_type == 1: # hive_reader reader = hive_reader( host=origin_db_host, port=8020, path="/user/hive/warehouse/%s.db/%s" % (origin_db_name, origin_db_table_name), fileType="text", column=["*"], fieldDelimiter=',', encoding="utf-8") elif origin_database_type == 2: # mysql_reader column = [ col.get("dmp_data_table_column_name") for col in base_column ] reader = mysql_reader( username=origin_db_username, password=origin_db_passwd, column=column, host=origin_db_host, port=origin_db_port, db=origin_db_name, table=origin_db_table_name, where=None, ) elif origin_database_type == 3: # mongodb column = [{ "index": i + 1, "name": col.get("dmp_data_table_column_name"), "type": col.get("dmp_data_table_column_type") } for i, col in enumerate(base_column)] reader = mongodb_reader( host=origin_db_host, port=origin_db_port, db_name=origin_db_name, collection_name=origin_db_table_name, column=column, username=origin_db_username, password=origin_db_passwd) pass writer = [] download_path = os.path.join( current_app.config.get("DOWNLOAD_PATH"), approve_form.submit_dmp_username) file_name = origin_db_table_name + uuid_str( ) + ".csv" finally_name = origin_db_table_name + "-" + uuid_str( ) + ".csv" headers = [ col.get("dmp_data_table_column_name") for col in base_column ] writer = textfile_writer(filepath=download_path, filename=file_name, header=headers) ip = socket.gethostbyname(socket.gethostname()) ftp_url = "ftp://%s:21/%s" % ( str(ip), str( os.path.join( approve_form.submit_dmp_username, finally_name))) meta = { "form_id": approve_form.id, "file_name": file_name, "finally_name": finally_name, "download_path": download_path, "ftp_url": ftp_url, } job_hanlder.delay(reader=reader, writer=writer, channel=1, func=dlfunc, meta=meta) except Exception as err: approve_form.result = "CREATE DOWNLOAD JOB FAILED,ERROR MESSAGE:%s" % str( err) approve_form.finish = True else: approve_form.result = "The original data sheet information is not obtained" approve_form.finish = True approve_form.put() return resp_hanlder(result="OK!") except Exception as err: return resp_hanlder(code=999, err=err, msg=str(err))
def data_count(self): from dmp.utils.engine import auto_connect conn = auto_connect(db_id=self.dmp_database_id) count = conn.count(self.db_table_name) self.db_data_count = count self.put()
def get_data_by_data_service(api): """ 数据服务调用 --- tags: - DataService parameters: - name: api type: string required: true description: 数据接口路径 - name: secret_key type: string required: true description: 密钥 - name: page_num type: int required: true description: 数据页码 - name: 其他参数 type: string required: false description: 数据服务的筛选参数 responses: 0: description: 正常响应 """ try: api = str(api) print(api) request_params = request.json code = 100 msg = None result = None secret_key = request_params.get("secret_key", None) if secret_key: # user_id = Users.encode_auth_token(secret_key) user_id = Users.decode_auth_token(secret_key) if isinstance(user_id, str): return resp_hanlder(code=999, msg='缺少秘钥或秘钥已失效') else: code = 8101 msg = "缺少秘钥或秘钥已失效" return resp_hanlder(code=code, msg=msg, result=result) if DataService.exsit_data_service_by_apipath(api): current_data_sevice = DataService.query.filter_by( api_path=api).first() else: code = 8104 msg = "未匹配到改数据服务" return resp_hanlder(code=code, msg=msg, result=result) if current_data_sevice.state: # dataservice_log = UserDataService( # dmp_user_id=user_id, # dmp_data_service_id=current_data_sevice.id, # ) # dataservice_log.save() db_type = current_data_sevice.database_type missing, query_params = parse_query_params(request_params, current_data_sevice) else: code = 8108 msg = "当前数据服务未开启" return resp_hanlder(code=code, msg=msg, result=result) if missing == None: source_dmp_data_table_id = current_data_sevice.source_dmp_data_table_id else: code = 8105 msg = "缺少%d个必要参数:%s" % (len(missing), ",".join(missing)) return resp_hanlder(code=code, msg=msg, result=result) if source_dmp_data_table_id: page_num = request_params.get("page_num", 1) page_size = request_params.get("page_size", 100) conn = auto_connect(table_id=source_dmp_data_table_id) qp = {} if db_type in [1, 2]: query_sql_tmp = current_data_sevice.query_sql query_sql = format_sql( query_sql_tmp, query_params) if query_params else query_sql_tmp qp["sql"] = query_sql print(qp) data = conn.exec_query(**qp) result = data return resp_hanlder(code=code, msg=msg, result=result) elif db_type == 3: query_sql_tmp = current_data_sevice.query_params query_sql = eval(query_sql_tmp) query_sql[filter] = query_params qp = query_sql print(qp) data = conn.exec_query(**qp) result = data return resp_hanlder(code=code, msg=msg, result=result) else: code = 8107 msg = "源数据库异常" return resp_hanlder(code=code, msg=msg, result=result) else: msg = "源数据库异常" code = 8106 return resp_hanlder(code=code, msg=msg, result=result) except Exception as err: # raise err return resp_hanlder(code=999, msg=str(err))
def ds_retrieve(desc): """ 图表查询 --- tags: - SQL parameters: - name: source_dmp_data_table_id in: type: int required: True description: 数据表ID - name: query_sql_tmp in: type: string required: True description: 要调试的查询语句 - name: parameters in: type: dict required: true description: 参数,{"参数名":"参数值"} responses: 0: description: OK """ from .dataservice import format_sql try: request_json = request.json if request.json else {} data_table_id = request_json.get("source_dmp_data_table_id") query_sql_tmp = request_json.get("query_sql_tmp", None) parameters = request_json.get("parameters", {}) code = 0 if all([data_table_id, query_sql_tmp]): if DataTable.exist_item_by_id(data_table_id): current_data_table = DataTable.get(data_table_id) else: code = 999 msg = "数据表不存在" return resp_hanlder(code=code, msg=msg) db_type = current_data_table.dmp_database_type qp = {} conn = auto_connect(table_id=data_table_id) if db_type in [1, 2]: query_sql = format_sql( query_sql_tmp, parameters) if parameters else query_sql_tmp qp["sql"] = query_sql print(qp) data = conn.exec_query(**qp) result = data return resp_hanlder(code=code, result=result) elif db_type == 3: query_sql_tmp = current_data_sevice.query_params query_sql = eval(query_sql_tmp) query_sql[filter] = parameters qp = query_sql print(qp) data = conn.exec_query(**qp) result = data return resp_hanlder(code=code, result=result) else: code = 8107 msg = "源数据库异常" return resp_hanlder(code=code, msg=msg) else: code = 201 msg = "参数异常" return resp_hanlder(code=code, msg=msg) except Exception as err: code = 999 msg = str(err) return resp_hanlder(code=code, msg=msg)
def chart_retrieve(desc): """ 图表查询 --- tags: - SQL parameters: - name: dmp_data_table_id in: type: int required: True description: 数据表ID - name: dimension in: type: array required: True description: 维度,格式[{"name":"xxx"},{"name":"xxxxx"}] - name: measure in: type: array required: true description: 度量,格式[{"name":"xxx", "method":"count/sum/avg"},{......}] responses: 0: description: OK """ request_json = request.json if request.json else {} data_table_id = request_json.get("dmp_data_table_id") dimension = request_json.get("dimension") measure = request_json.get("measure") limit = request_json.get("limit") if data_table_id and DataTable.exist_item_by_id(data_table_id): data_table = DataTable.get(data_table_id) data_table_name = data_table.db_table_name db_type = data_table.dmp_database_type dimension_names = [d.get("name") for d in dimension] measure_names = [ m.get("name") if m.get("method") == None else "%s(%s)" % (m.get("method"), m.get("name")) for m in measure ] measure_names_methods = [ m.get("name") if m.get("method") == None else "%s_%s" % (m.get("name"), m.get("method")) for m in measure ] groupby = bool( sum([True if m.get("method") else False for m in measure])) sql = "select {p1} from {p2} {p3} {p4}".format( p1=",".join(dimension_names + measure_names), p2=data_table_name, p3="group by " + ",".join(dimension_names) if groupby else "", p4=";" if db_type == 2 else "") request_json["sql"] = sql try: conn = auto_connect(table_id=data_table_id) _data = conn.exec_query(**request_json) func2f = lambda x: round(float(x), 2) if type( x) in [float, Decimal] else x if type(_data) == list or type(_data) == tuple: result = {} result["data"] = [ dict( zip(dimension_names + measure_names_methods, map(func2f, d))) for d in _data ] result["query_string"] = { "sql": sql, "fields": dimension_names + measure_names_methods } return resp_hanlder(code=0, result=result) else: return resp_hanlder(code=999, msg="查询出错 你的查询SQl:%s" % sql) except Exception as e: return resp_hanlder(code=999, msg=str(e)) else: return resp_hanlder(code=999, msg="缺少dmp_data_table_id")