Ejemplo n.º 1
0
def table_list(desc):
    if request.method == "GET":
        db_id = request.json.get("dmp_database_id")
        conn = auto_connect(db_id=db_id)
        res = conn.tables_list
        current_app.logger.info(res)
        return resp_hanlder(result=res)
Ejemplo n.º 2
0
def update_chart_data(chart_id):
    import json
    from dmp import app
    with app.app_context():
        from flask import current_app
        from dmp.models import Chart
        from dmp.utils.engine import auto_connect
        if Chart.exist_item_by_id(chart_id):
            current_chart = Chart.get(chart_id)
            data_table_id = current_chart.dmp_data_table_id
            query_string = current_chart.query_string
            conn = auto_connect(table_id=data_table_id)
            if current_chart.dmp_data_table.dmp_database_type in [1, 2, 4]:
                request_json = {"sql": query_string.get("sql")}
                _data = conn.exec_query(**request_json)
                if type(_data) == list:
                    field = query_string.get("fields")

                    current_chart.chart_data = [
                        dict(zip(field, d)) for d in _data
                    ]
                    current_chart.put()
                return True, "OK"
        else:
            return False, "item not exist"
Ejemplo n.º 3
0
def retrieve(desc):
    if request.method == "GET":
        retrieve_info = request.json
        dmp_data_table_id = retrieve_info.get("dmp_data_table_id")
        dmp_data_table = DataTable.get(dmp_data_table_id)
        database_id = dmp_data_table.dmp_database_id
        db_table_name = dmp_data_table.db_table_name
        conn = auto_connect(db_id=database_id)
        data = conn.retrieve(table_name=db_table_name)
        # current_app.logger.info(data)
        return resp_hanlder(result=data)
Ejemplo n.º 4
0
def retrieve(desc):
    """
    SQL查询

    ---
    tags:
      - SQL
    parameters:
      - name: dmp_data_table_id
        in:
        type: int
        required: True
        description: 数据表ID
      - name: sql
        in:
        type: string
        required: false
        description: SQL 语句,mysql,hive ,kylin 必填
      - name: collection
        in:
        type: string
        required: false
        description: mongodb 参数  集合名(表名)

    responses:
      0:
        description: OK
    """
    request_json = request.json if request.json else {}
    data_table_id = request_json.get("dmp_data_table_id")
    if data_table_id:
        try:
            conn = auto_connect(table_id=data_table_id)
            result = conn.exec_query(**request_json)
            return resp_hanlder(code=0, result=result)
        except Exception as e:
            return resp_hanlder(code=999, msg=str(e))

    else:
        return resp_hanlder(code=999, msg="缺少dmp_data_table_id")
Ejemplo n.º 5
0
def db_table_name(desc):
    if request.method == "GET":
        try:
            dmp_database_id = request.json.get("dmp_database_id")
            db_table_name_ = request.json.get("db_table_name")
            from dmp.utils.engine import auto_connect
            db_ = auto_connect(dmp_database_id)
            tbl = db_.tables_list
            from dmp.models import FormMigrate, FormUpload
            count1 = FormUpload.query.filter_by(
                destination_db_table_name=db_table_name_).count()
            count2 = FormMigrate.query.filter_by(
                new_table_name=db_table_name_).count()
            if db_table_name_ in tbl or count1 > 0 or count2 > 0:
                return resp_hanlder(result={
                    "exist": True,
                })
            else:
                return resp_hanlder(result={
                    "exist": False,
                })
        except Exception as err:
            return resp_hanlder(err=err)
Ejemplo n.º 6
0
def column(desc):
    if request.method == "GET":
        try:
            dmp_data_table_id = request.json.get("dmp_data_table_id")
            if DataTable.exist_item_by_id(dmp_data_table_id):

                data_table_info = DataTable.get(dmp_data_table_id)
            else:
                return resp_hanlder(code=999, msg="数据表不存在或已被删除")
            db_table_name = data_table_info.db_table_name

            colums4sdb = DataTableColumn.query.filter_by(
                dmp_data_table_id=dmp_data_table_id)
            column4sdb_array = []
            current_app.logger.info(colums4sdb.count())
            if colums4sdb.count() > 0:
                column4sdb_array = [col.__json__() for col in colums4sdb.all()]
            columns4db = []
            columns4db = auto_connect(
                db_id=data_table_info.dmp_database_id).columns(db_table_name)
            current_app.logger.info(column4sdb_array)
            columns = []
            for i in columns4db:
                mark = True
                for j in column4sdb_array:
                    if j.get("dmp_data_table_column_name") == i.get(
                            "dmp_data_table_column_name"):
                        columns.append(j)
                        mark = False
                if mark:
                    columns.append(i)
            current_app.logger.info(columns)
            return resp_hanlder(result=columns)
        except Exception as err:
            current_app.logger.error(err)
            return resp_hanlder(code=999, err=err)
Ejemplo n.º 7
0
def approve(desc):
    if request.method == "PUT":
        try:
            # 获取请求参数
            approve_form_info = request.json
            auth_token = request.headers.get('Authorization')
            approve_user_id = Users.decode_auth_token(auth_token)
            form_type = approve_form_info.get("dmp_form_type", None)
            form_id = approve_form_info.get("dmp_form_id", None)
            approve_result = approve_form_info.get("approve_result", None)
            answer = approve_form_info.get("answer", None)
            # 保存表单审批信息
            approve_form = Forms.get(form_id)
            approve_form.approve_dmp_user_id = approve_user_id
            approve_form.approve_result = approve_result
            approve_form.approve_on = datetime.datetime.now()
            approve_form.answer = answer
            approve_form.put()
            # 执行审批动作
            if form_type == 1:
                # 从数据库添加数据表单
                try:
                    if approve_result == 1:
                        post(
                            dmp_data_table_name=approve_form.info_form.
                            dmp_data_table_name,
                            db_table_name=approve_form.info_form.db_table_name,
                            dmp_user_id=approve_form.submit_dmp_user_id,
                            dmp_database_id=approve_form.info_form.
                            dmp_database_id,
                            dmp_case_id=approve_form.info_form.dmp_case_id,
                            description=approve_form.description)
                    approve_form.result = "SUCCEED!"
                except Exception as err:
                    approve_form.result = "ASSOCIATION FAILED,ERROR_MSG:%s" % str(
                        err)
                approve_form.finish = True

            elif form_type == 2:
                # 文件上传添加数据表单
                upload_path = current_app.config.get("UPLOADED_PATH")
                file_path = os.path.join(upload_path,
                                         approve_form.info_form.filepath)
                file_type = approve_form.info_form.filetype
                column_line = approve_form.info_form.column_line
                column = approve_form.info_form.column.split(",") if type(
                    approve_form.info_form.column) == str else []
                destination_dmp_database_id = approve_form.info_form.destination_dmp_database_id
                destination_db_table_name = approve_form.info_form.destination_db_table_name
                dmp_data_table_name = approve_form.info_form.dmp_data_table_name
                method = approve_form.info_form.method
                description = approve_form.description
                submit_dmp_user_id = approve_form.submit_dmp_user_id
                dmp_case_id = approve_form.info_form.dmp_case_id

                destination_database = Database.get(
                    destination_dmp_database_id)
                destination_database_type = destination_database.db_type
                destination_db_host = destination_database.db_host
                destination_db_port = destination_database.db_port
                destination_db_username = destination_database.db_username
                destination_db_passwd = destination_database.db_passwd
                destination_db_name = destination_database.db_name
                try:
                    reader = []
                    text_column = []
                    csv_filepath = os.path.join(
                        current_app.config.get("UPLOADED_PATH"), file_path)
                    if file_type == 1 or file_type == 3:
                        # csv、excel
                        dt = pd.read_csv(csv_filepath, header=column_line)
                        csv_column = [
                            "_".join(lazy_pinyin(d))
                            if is_contains_chinese(d) else d
                            for d in list(dt.columns)
                        ]

                        text_column = column if column and len(column) == len(
                            csv_column) else csv_column
                        csv_column_d = [{
                            "index": i,
                            "type": "string"
                        } for i, cc in enumerate(text_column)]
                        reader = textfile_reader(filepath=csv_filepath,
                                                 column=csv_column_d)
                    elif file_type == 2:
                        # json
                        dt = pd.read_csv(csv_filepath, header=0)
                        csv_column = [
                            "_".join(lazy_pinyin(d))
                            if is_contains_chinese(d) else d
                            for d in list(dt.columns)
                        ]
                        text_column = column if column and len(column) == len(
                            csv_column) else csv_column
                        csv_column_d = [{
                            "index": i,
                            "type": "string"
                        } for i, cc in enumerate(text_column)]
                        reader = textfile_reader(filepath=csv_filepath,
                                                 column=csv_column_d)
                        pass
                    writer = []
                    if destination_database_type == 1:
                        # hive_writer
                        hive_columns = [{
                            "name": col,
                            "type": "string"
                        } for col in text_column]
                        hive_path = "/user/hive/warehouse/%s.db/%s" % (
                            destination_db_name, destination_db_table_name)
                        hive_conn = auto_connect(
                            db_id=destination_dmp_database_id)
                        create_table_sql = create_table_query_handler(
                            table_name=destination_db_table_name,
                            fields=text_column,
                            uniform_type="string",
                            id_primary_key=False,
                            semicolon=False,
                            fieldDelimiter=",")

                        print(create_table_sql)
                        if method == 1:
                            hive_conn.execsql(create_table_sql)
                        elif method == 3:
                            del_table_sql = "drop table {table_name}"
                            hive_conn.execsql(
                                del_table_sql.format(
                                    table_name=destination_db_table_name))
                            hive_conn.execsql(create_table_sql)
                        else:
                            pass
                        writer = hive_writer(
                            host=destination_db_host,
                            port=8020,
                            path=hive_path,
                            filename=destination_db_table_name,
                            column=hive_columns,
                            fieldDelimiter=",",
                        )

                    elif destination_database_type == 2:
                        # mysql_writer
                        create_table_sql = create_table_query_handler(
                            table_name=destination_db_table_name,
                            fields=text_column,
                            uniform_type="text",
                            id_primary_key=True,
                            semicolon=True,
                            fieldDelimiter=None)
                        current_app.logger.info(create_table_sql)
                        mysql_conn = auto_connect(
                            db_id=destination_dmp_database_id)
                        del_table_sql = "drop table {table_name};"
                        if method == 1:
                            mysql_conn.execsql(sql=create_table_sql)
                        elif method == 2:
                            pass
                        elif method == 3:
                            mysql_conn.execsql(
                                del_table_sql.format(
                                    table_name=destination_db_table_name))
                            mysql_conn.execsql(create_table_sql)
                        column = text_column
                        writer = mysql_writer(
                            model=1,
                            username=destination_db_username,
                            password=destination_db_passwd,
                            column=column,
                            host=destination_db_host,
                            port=destination_db_port,
                            db=destination_db_name,
                            table=destination_db_table_name,
                        )
                    elif destination_database_type == 3:
                        # mongo_writer
                        mongo_conn = auto_connect(
                            db_id=destination_dmp_database_id)
                        if method == 3:
                            mongo_conn.del_table(
                                table_name=destination_db_table_name)
                        column = [{
                            "name": col,
                            "type": "string"
                        } for col in text_column]
                        writer = mongodb_writer(
                            host=destination_db_host,
                            port=destination_db_port,
                            username=destination_db_username,
                            password=destination_db_passwd,
                            db_name=destination_db_name,
                            collection_name=destination_db_table_name,
                            column=column,
                        )
                    meta = {
                        "form_id": approve_form.id,
                        "csv_file_path": csv_filepath,
                        "dmp_data_table_name": dmp_data_table_name,
                        "db_table_name": destination_db_table_name,
                        "dmp_user_id": submit_dmp_user_id,
                        "dmp_database_id": destination_dmp_database_id,
                        "dmp_case_id": dmp_case_id,
                        "description": description,
                    }
                    job_hanlder.delay(reader=reader,
                                      writer=writer,
                                      channel=3,
                                      func=postfunc,
                                      meta=meta)
                    approve_form.result = "SUCCEED!"
                except Exception as err:
                    approve_form.result = "CREATE UPLOAD JOB FAILED,ERROR MESSAGE:%s" % str(
                        err)
                    approve_form.finish = True

            elif form_type == 3:
                # 数据迁移表单

                if approve_result == 1:
                    origin_data_table = DataTable.get(
                        approve_form.info_form.origin_dmp_table_id)
                    origin_database = Database.get(
                        origin_data_table.dmp_database_id)
                    origin_database_type = origin_database.db_type
                    origin_db_host = origin_database.db_host
                    origin_db_port = origin_database.db_port
                    origin_db_username = origin_database.db_username
                    origin_db_passwd = origin_database.db_passwd
                    origin_db_name = origin_database.db_name
                    origin_db_table_name = origin_data_table.db_table_name

                    destination_database = Database.get(
                        approve_form.info_form.destination_dmp_database_id)
                    destination_database_type = destination_database.db_type
                    destination_db_host = destination_database.db_host
                    destination_db_port = destination_database.db_port
                    destination_db_username = destination_database.db_username
                    destination_db_passwd = destination_database.db_passwd
                    destination_db_name = destination_database.db_name
                    destination_db_table_name = approve_form.info_form.new_table_name

                    try:
                        base_column = auto_connect(
                            db_id=origin_data_table.dmp_database_id).columns(
                                origin_data_table.db_table_name)
                        # current_app.logger.info(base_column)
                        reader = []
                        if origin_database_type == 1:
                            # hive_reader
                            reader = hive_reader(
                                host=origin_db_host,
                                port=8020,
                                path="/user/hive/warehouse/%s.db/%s" %
                                (origin_db_name, origin_db_table_name),
                                fileType="text",
                                column=["*"],
                                fieldDelimiter=',',
                                encoding="utf-8")
                        elif origin_database_type == 2:
                            # mysql_reader
                            column = [
                                col.get("dmp_data_table_column_name")
                                for col in base_column
                            ]
                            reader = mysql_reader(
                                username=origin_db_username,
                                password=origin_db_passwd,
                                column=column,
                                host=origin_db_host,
                                port=origin_db_port,
                                db=origin_db_name,
                                table=origin_db_table_name,
                                where=None,
                            )

                        elif origin_database_type == 3:
                            # mongodb
                            column = [{
                                "index":
                                i + 1,
                                "name":
                                col.get("dmp_data_table_column_name"),
                                "type":
                                col.get("dmp_data_table_column_type")
                            } for i, col in enumerate(base_column)]
                            reader = mongodb_reader(
                                host=origin_db_host,
                                port=origin_db_port,
                                db_name=origin_db_name,
                                collection_name=origin_db_table_name,
                                column=column,
                                username=origin_db_username,
                                password=origin_db_passwd)
                            pass
                        writer = []
                        if destination_database_type == 1:
                            # hive_writer
                            hive_col = [
                                col.get("dmp_data_table_column_name")
                                for col in base_column
                            ]
                            hive_columns = [{
                                "name": col,
                                "type": "string"
                            } for col in hive_col]
                            hive_path = "/user/hive/warehouse/%s.db/%s" % (
                                destination_db_name, destination_db_table_name)
                            hive_conn = auto_connect(
                                db_id=approve_form.info_form.
                                destination_dmp_database_id)
                            create_table_sql = create_table_query_handler(
                                table_name=destination_db_table_name,
                                fields=hive_col,
                                uniform_type="string",
                                id_primary_key=False,
                                semicolon=False,
                                fieldDelimiter=",")

                            hive_conn.execsql(create_table_sql)

                            writer = hive_writer(
                                host=destination_db_host,
                                port=8020,
                                path=hive_path,
                                filename=destination_db_table_name,
                                column=hive_columns,
                                fieldDelimiter=",",
                            )
                        elif destination_database_type == 2:
                            # mysql_writer
                            column = [
                                col.get("dmp_data_table_column_name")
                                for col in base_column
                            ]
                            create_table_sql = create_table_query_handler(
                                table_name=destination_db_table_name,
                                fields=column,
                                uniform_type="text",
                                id_primary_key=True,
                                semicolon=True,
                                fieldDelimiter=None)
                            mysql_conn = auto_connect(
                                db_id=approve_form.info_form.
                                destination_dmp_database_id)
                            mysql_conn.execsql(sql=create_table_sql)
                            # preSQL = []
                            writer = mysql_writer(
                                model=1,
                                username=destination_db_username,
                                password=destination_db_passwd,
                                column=column,
                                host=destination_db_host,
                                port=destination_db_port,
                                db=destination_db_name,
                                table=destination_db_table_name,
                            )
                        elif destination_database_type == 3:
                            # mongo_writer
                            column = [{
                                "name":
                                col.get("dmp_data_table_column_name"),
                                "type":
                                "string"
                            } for col in base_column]
                            writer = mongodb_writer(
                                host=destination_db_host,
                                port=destination_db_port,
                                username=destination_db_username
                                if destination_db_username else None,
                                password=destination_db_passwd
                                if destination_db_passwd else None,
                                db_name=destination_db_name,
                                collection_name=destination_db_table_name,
                                column=column,
                            )

                        meta = {"form_id": approve_form.id}
                        job_hanlder.delay(reader=reader,
                                          writer=writer,
                                          channel=3,
                                          func=job_finish,
                                          meta=meta)
                    except Exception as err:
                        approve_form.result = "CREATE MIGRATE JOB FAILED,ERROR MESSAGE:%s" % str(
                            err)
                        approve_form.finish = True
                        raise err

            elif form_type == 4:
                # 数据导出表单
                if approve_result == 1:
                    origin_data_table = DataTable.get(
                        approve_form.info_form.dmp_data_table_id)
                    if origin_data_table:
                        origin_database = Database.get(
                            origin_data_table.dmp_database_id)
                        origin_database_type = origin_database.db_type
                        origin_db_host = origin_database.db_host
                        origin_db_port = origin_database.db_port
                        origin_db_username = origin_database.db_username
                        origin_db_passwd = origin_database.db_passwd
                        origin_db_name = origin_database.db_name
                        origin_db_table_name = origin_data_table.db_table_name
                        try:
                            base_column = auto_connect(
                                db_id=origin_data_table.dmp_database_id
                            ).columns(origin_data_table.db_table_name)

                            reader = []
                            if origin_database_type == 1:
                                # hive_reader
                                reader = hive_reader(
                                    host=origin_db_host,
                                    port=8020,
                                    path="/user/hive/warehouse/%s.db/%s" %
                                    (origin_db_name, origin_db_table_name),
                                    fileType="text",
                                    column=["*"],
                                    fieldDelimiter=',',
                                    encoding="utf-8")
                            elif origin_database_type == 2:
                                # mysql_reader
                                column = [
                                    col.get("dmp_data_table_column_name")
                                    for col in base_column
                                ]
                                reader = mysql_reader(
                                    username=origin_db_username,
                                    password=origin_db_passwd,
                                    column=column,
                                    host=origin_db_host,
                                    port=origin_db_port,
                                    db=origin_db_name,
                                    table=origin_db_table_name,
                                    where=None,
                                )

                            elif origin_database_type == 3:
                                # mongodb
                                column = [{
                                    "index":
                                    i + 1,
                                    "name":
                                    col.get("dmp_data_table_column_name"),
                                    "type":
                                    col.get("dmp_data_table_column_type")
                                } for i, col in enumerate(base_column)]
                                reader = mongodb_reader(
                                    host=origin_db_host,
                                    port=origin_db_port,
                                    db_name=origin_db_name,
                                    collection_name=origin_db_table_name,
                                    column=column,
                                    username=origin_db_username,
                                    password=origin_db_passwd)
                                pass
                            writer = []
                            download_path = os.path.join(
                                current_app.config.get("DOWNLOAD_PATH"),
                                approve_form.submit_dmp_username)
                            file_name = origin_db_table_name + uuid_str(
                            ) + ".csv"
                            finally_name = origin_db_table_name + "-" + uuid_str(
                            ) + ".csv"
                            headers = [
                                col.get("dmp_data_table_column_name")
                                for col in base_column
                            ]
                            writer = textfile_writer(filepath=download_path,
                                                     filename=file_name,
                                                     header=headers)
                            ip = socket.gethostbyname(socket.gethostname())
                            ftp_url = "ftp://%s:21/%s" % (
                                str(ip),
                                str(
                                    os.path.join(
                                        approve_form.submit_dmp_username,
                                        finally_name)))

                            meta = {
                                "form_id": approve_form.id,
                                "file_name": file_name,
                                "finally_name": finally_name,
                                "download_path": download_path,
                                "ftp_url": ftp_url,
                            }
                            job_hanlder.delay(reader=reader,
                                              writer=writer,
                                              channel=1,
                                              func=dlfunc,
                                              meta=meta)
                        except Exception as err:
                            approve_form.result = "CREATE DOWNLOAD JOB FAILED,ERROR MESSAGE:%s" % str(
                                err)
                            approve_form.finish = True
                    else:
                        approve_form.result = "The original data sheet information is not obtained"
                        approve_form.finish = True
            approve_form.put()
            return resp_hanlder(result="OK!")
        except Exception as err:
            return resp_hanlder(code=999, err=err, msg=str(err))
Ejemplo n.º 8
0
 def data_count(self):
     from dmp.utils.engine import auto_connect
     conn = auto_connect(db_id=self.dmp_database_id)
     count = conn.count(self.db_table_name)
     self.db_data_count = count
     self.put()
Ejemplo n.º 9
0
def get_data_by_data_service(api):
    """
    数据服务调用

    ---
    tags:
      - DataService
    parameters:
      - name: api
        type: string
        required: true
        description: 数据接口路径
      - name: secret_key
        type: string
        required: true
        description: 密钥
      - name: page_num
        type: int
        required: true
        description: 数据页码
      - name: 其他参数
        type: string
        required: false
        description: 数据服务的筛选参数
    responses:
      0:
        description: 正常响应
	"""
    try:
        api = str(api)
        print(api)
        request_params = request.json
        code = 100
        msg = None
        result = None
        secret_key = request_params.get("secret_key", None)
        if secret_key:
            # user_id = Users.encode_auth_token(secret_key)
            user_id = Users.decode_auth_token(secret_key)
            if isinstance(user_id, str):
                return resp_hanlder(code=999, msg='缺少秘钥或秘钥已失效')
        else:
            code = 8101
            msg = "缺少秘钥或秘钥已失效"
            return resp_hanlder(code=code, msg=msg, result=result)

        if DataService.exsit_data_service_by_apipath(api):
            current_data_sevice = DataService.query.filter_by(
                api_path=api).first()
        else:
            code = 8104
            msg = "未匹配到改数据服务"
            return resp_hanlder(code=code, msg=msg, result=result)
        if current_data_sevice.state:
            # dataservice_log = UserDataService(
            # dmp_user_id=user_id,
            # dmp_data_service_id=current_data_sevice.id,
            # )
            # dataservice_log.save()
            db_type = current_data_sevice.database_type
            missing, query_params = parse_query_params(request_params,
                                                       current_data_sevice)
        else:
            code = 8108
            msg = "当前数据服务未开启"
            return resp_hanlder(code=code, msg=msg, result=result)
        if missing == None:
            source_dmp_data_table_id = current_data_sevice.source_dmp_data_table_id
        else:
            code = 8105
            msg = "缺少%d个必要参数:%s" % (len(missing), ",".join(missing))
            return resp_hanlder(code=code, msg=msg, result=result)
        if source_dmp_data_table_id:
            page_num = request_params.get("page_num", 1)
            page_size = request_params.get("page_size", 100)
            conn = auto_connect(table_id=source_dmp_data_table_id)
            qp = {}
            if db_type in [1, 2]:
                query_sql_tmp = current_data_sevice.query_sql
                query_sql = format_sql(
                    query_sql_tmp,
                    query_params) if query_params else query_sql_tmp
                qp["sql"] = query_sql
                print(qp)
                data = conn.exec_query(**qp)
                result = data
                return resp_hanlder(code=code, msg=msg, result=result)
            elif db_type == 3:
                query_sql_tmp = current_data_sevice.query_params
                query_sql = eval(query_sql_tmp)
                query_sql[filter] = query_params
                qp = query_sql
                print(qp)
                data = conn.exec_query(**qp)
                result = data
                return resp_hanlder(code=code, msg=msg, result=result)
            else:
                code = 8107
                msg = "源数据库异常"
                return resp_hanlder(code=code, msg=msg, result=result)

        else:
            msg = "源数据库异常"
            code = 8106
            return resp_hanlder(code=code, msg=msg, result=result)
    except Exception as err:
        # raise err
        return resp_hanlder(code=999, msg=str(err))
Ejemplo n.º 10
0
def ds_retrieve(desc):
    """
    图表查询

    ---
    tags:
      - SQL
    parameters:
      - name: source_dmp_data_table_id
        in:
        type: int
        required: True
        description: 数据表ID
      - name: query_sql_tmp
        in:
        type: string
        required: True
        description: 要调试的查询语句
      - name: parameters
        in:
        type: dict
        required: true
        description: 参数,{"参数名":"参数值"}

    responses:
      0:
        description: OK
    """
    from .dataservice import format_sql
    try:
        request_json = request.json if request.json else {}
        data_table_id = request_json.get("source_dmp_data_table_id")
        query_sql_tmp = request_json.get("query_sql_tmp", None)
        parameters = request_json.get("parameters", {})
        code = 0
        if all([data_table_id, query_sql_tmp]):
            if DataTable.exist_item_by_id(data_table_id):
                current_data_table = DataTable.get(data_table_id)
            else:
                code = 999
                msg = "数据表不存在"
                return resp_hanlder(code=code, msg=msg)
            db_type = current_data_table.dmp_database_type
            qp = {}
            conn = auto_connect(table_id=data_table_id)
            if db_type in [1, 2]:
                query_sql = format_sql(
                    query_sql_tmp, parameters) if parameters else query_sql_tmp
                qp["sql"] = query_sql
                print(qp)
                data = conn.exec_query(**qp)
                result = data
                return resp_hanlder(code=code, result=result)
            elif db_type == 3:
                query_sql_tmp = current_data_sevice.query_params
                query_sql = eval(query_sql_tmp)
                query_sql[filter] = parameters
                qp = query_sql
                print(qp)
                data = conn.exec_query(**qp)
                result = data
                return resp_hanlder(code=code, result=result)
            else:
                code = 8107
                msg = "源数据库异常"
                return resp_hanlder(code=code, msg=msg)
        else:
            code = 201
            msg = "参数异常"
            return resp_hanlder(code=code, msg=msg)
    except Exception as err:
        code = 999
        msg = str(err)
        return resp_hanlder(code=code, msg=msg)
Ejemplo n.º 11
0
def chart_retrieve(desc):
    """
    图表查询

    ---
    tags:
      - SQL
    parameters:
      - name: dmp_data_table_id
        in:
        type: int
        required: True
        description: 数据表ID
      - name: dimension
        in:
        type: array
        required: True
        description: 维度,格式[{"name":"xxx"},{"name":"xxxxx"}]
      - name: measure
        in:
        type: array
        required: true
        description: 度量,格式[{"name":"xxx", "method":"count/sum/avg"},{......}]

    responses:
      0:
        description: OK
    """
    request_json = request.json if request.json else {}
    data_table_id = request_json.get("dmp_data_table_id")
    dimension = request_json.get("dimension")
    measure = request_json.get("measure")
    limit = request_json.get("limit")
    if data_table_id and DataTable.exist_item_by_id(data_table_id):
        data_table = DataTable.get(data_table_id)
        data_table_name = data_table.db_table_name
        db_type = data_table.dmp_database_type
        dimension_names = [d.get("name") for d in dimension]
        measure_names = [
            m.get("name") if m.get("method") == None else "%s(%s)" %
            (m.get("method"), m.get("name")) for m in measure
        ]
        measure_names_methods = [
            m.get("name") if m.get("method") == None else "%s_%s" %
            (m.get("name"), m.get("method")) for m in measure
        ]

        groupby = bool(
            sum([True if m.get("method") else False for m in measure]))
        sql = "select {p1} from {p2} {p3} {p4}".format(
            p1=",".join(dimension_names + measure_names),
            p2=data_table_name,
            p3="group by " + ",".join(dimension_names) if groupby else "",
            p4=";" if db_type == 2 else "")
        request_json["sql"] = sql
        try:
            conn = auto_connect(table_id=data_table_id)
            _data = conn.exec_query(**request_json)
            func2f = lambda x: round(float(x), 2) if type(
                x) in [float, Decimal] else x
            if type(_data) == list or type(_data) == tuple:

                result = {}
                result["data"] = [
                    dict(
                        zip(dimension_names + measure_names_methods,
                            map(func2f, d))) for d in _data
                ]
                result["query_string"] = {
                    "sql": sql,
                    "fields": dimension_names + measure_names_methods
                }

                return resp_hanlder(code=0, result=result)
            else:
                return resp_hanlder(code=999, msg="查询出错 你的查询SQl:%s" % sql)

        except Exception as e:
            return resp_hanlder(code=999, msg=str(e))

    else:
        return resp_hanlder(code=999, msg="缺少dmp_data_table_id")