Example #1
0
    def execute_check(self, db_name=None, sql=''):
        """上线单执行前的检查, 返回Review set"""
        config = SysConfig()
        # 进行Inception检查,获取检测结果
        if not config.get('inception'):
            try:
                inception_engine = GoInceptionEngine()
                inc_check_result = inception_engine.execute_check(
                    instance=self.instance, db_name=db_name, sql=sql)
            except Exception as e:
                logger.debug(f"goInception检测语句报错:错误信息{traceback.format_exc()}")
                raise RuntimeError(
                    f"goInception检测语句报错,请注意检查系统配置中goInception配置,错误信息:\n{e}")
        else:
            try:
                inception_engine = InceptionEngine()
                inc_check_result = inception_engine.execute_check(
                    instance=self.instance, db_name=db_name, sql=sql)
            except Exception as e:
                logger.debug(f"Inception检测语句报错:错误信息{traceback.format_exc()}")
                raise RuntimeError(
                    f"Inception检测语句报错,请注意检查系统配置中Inception配置,错误信息:\n{e}")
        # 判断Inception检测结果
        if inc_check_result.error:
            logger.debug(f"Inception检测语句报错:错误信息{inc_check_result.error}")
            raise RuntimeError(
                f"Inception检测语句报错,错误信息:\n{inc_check_result.error}")

        # 禁用/高危语句检查
        check_critical_result = ReviewSet(full_sql=sql)
        line = 1
        critical_ddl_regex = config.get('critical_ddl_regex', '')
        p = re.compile(critical_ddl_regex)
        check_critical_result.syntax_type = 2  # TODO 工单类型 0、其他 1、DDL,2、DML

        for row in inc_check_result.rows:
            statement = row.sql
            # 去除注释
            statement = remove_comments(statement, db_type='mysql')
            # 禁用语句
            if re.match(r"^select", statement.lower()):
                check_critical_result.is_critical = True
                result = ReviewResult(
                    id=line,
                    errlevel=2,
                    stagestatus='驳回不支持语句',
                    errormessage='仅支持DML和DDL语句,查询语句请使用SQL查询功能!',
                    sql=statement)
            # 高危语句
            elif critical_ddl_regex and p.match(statement.strip().lower()):
                check_critical_result.is_critical = True
                result = ReviewResult(id=line,
                                      errlevel=2,
                                      stagestatus='驳回高危SQL',
                                      errormessage='禁止提交匹配' +
                                      critical_ddl_regex + '条件的语句!',
                                      sql=statement)
            # 正常语句
            else:
                result = ReviewResult(
                    id=line,
                    errlevel=0,
                    stagestatus='Audit completed',
                    errormessage='None',
                    sql=statement,
                    affected_rows=0,
                    execute_time=0,
                )

            # 没有找出DDL语句的才继续执行此判断
            if check_critical_result.syntax_type == 2:
                if get_syntax_type(statement, parser=False,
                                   db_type='mysql') == 'DDL':
                    check_critical_result.syntax_type = 1
            check_critical_result.rows += [result]

            # 遇到禁用和高危语句直接返回
            if check_critical_result.is_critical:
                check_critical_result.error_count += 1
                return check_critical_result
            line += 1
        return inc_check_result
Example #2
0
    def execute_check(self, db_name=None, sql=''):
        """上线单执行前的检查, 返回Review set"""
        sql = sqlparse.format(sql, strip_comments=True)
        sql_list = sqlparse.split(sql)

        # 禁用/高危语句检查
        check_result = ReviewSet(full_sql=sql)
        line = 1
        critical_ddl_regex = self.config.get('critical_ddl_regex', '')
        p = re.compile(critical_ddl_regex)
        check_result.syntax_type = 2  # TODO 工单类型 0、其他 1、DDL,2、DML

        for statement in sql_list:
            statement = statement.rstrip(';')
            # 禁用语句
            if re.match(r"^select|^show", statement.lower()):
                check_result.is_critical = True
                result = ReviewResult(
                    id=line,
                    errlevel=2,
                    stagestatus='驳回不支持语句',
                    errormessage='仅支持DML和DDL语句,查询语句请使用SQL查询功能!',
                    sql=statement)
            # 高危语句
            elif critical_ddl_regex and p.match(statement.strip().lower()):
                check_result.is_critical = True
                result = ReviewResult(id=line,
                                      errlevel=2,
                                      stagestatus='驳回高危SQL',
                                      errormessage='禁止提交匹配' +
                                      critical_ddl_regex + '条件的语句!',
                                      sql=statement)
            # alter语句
            elif re.match(r"^alter", statement.lower()):
                # alter table语句
                if re.match(r"^alter\s+table\s+(.+?)\s+", statement.lower()):
                    table_name = re.match(r"^alter\s+table\s+(.+?)\s+",
                                          statement.lower(), re.M).group(1)
                    if '.' not in table_name:
                        table_name = f"{db_name}.{table_name}"
                    table_engine = self.get_table_engine(table_name)['engine']
                    table_exist = self.get_table_engine(table_name)['status']
                    if table_exist == 1:
                        if not table_engine.endswith(
                                'MergeTree') and table_engine not in (
                                    'Merge', 'Distributed'):
                            check_result.is_critical = True
                            result = ReviewResult(
                                id=line,
                                errlevel=2,
                                stagestatus='驳回不支持SQL',
                                errormessage=
                                'ALTER TABLE仅支持*MergeTree,Merge以及Distributed等引擎表!',
                                sql=statement)
                        else:
                            # delete与update语句,实际是alter语句的变种
                            if re.match(
                                    r"^alter\s+table\s+(.+?)\s+(delete|update)\s+",
                                    statement.lower()):
                                if not table_engine.endswith('MergeTree'):
                                    check_result.is_critical = True
                                    result = ReviewResult(
                                        id=line,
                                        errlevel=2,
                                        stagestatus='驳回不支持SQL',
                                        errormessage=
                                        'DELETE与UPDATE仅支持*MergeTree引擎表!',
                                        sql=statement)
                                else:
                                    result = self.explain_check(
                                        check_result, db_name, line, statement)
                            else:
                                result = self.explain_check(
                                    check_result, db_name, line, statement)
                    else:
                        check_result.is_critical = True
                        result = ReviewResult(
                            id=line,
                            errlevel=2,
                            stagestatus='表不存在',
                            errormessage=f'表 {table_name} 不存在!',
                            sql=statement)
                # 其他alter语句
                else:
                    result = self.explain_check(check_result, db_name, line,
                                                statement)
            # truncate语句
            elif re.match(r"^truncate\s+table\s+(.+?)(\s|$)",
                          statement.lower()):
                table_name = re.match(r"^truncate\s+table\s+(.+?)(\s|$)",
                                      statement.lower(), re.M).group(1)
                if '.' not in table_name:
                    table_name = f"{db_name}.{table_name}"
                table_engine = self.get_table_engine(table_name)['engine']
                table_exist = self.get_table_engine(table_name)['status']
                if table_exist == 1:
                    if table_engine in ('View', 'File,', 'URL', 'Buffer',
                                        'Null'):
                        check_result.is_critical = True
                        result = ReviewResult(
                            id=line,
                            errlevel=2,
                            stagestatus='驳回不支持SQL',
                            errormessage=
                            'TRUNCATE不支持View,File,URL,Buffer和Null表引擎!',
                            sql=statement)
                    else:
                        result = self.explain_check(check_result, db_name,
                                                    line, statement)
                else:
                    check_result.is_critical = True
                    result = ReviewResult(id=line,
                                          errlevel=2,
                                          stagestatus='表不存在',
                                          errormessage=f'表 {table_name} 不存在!',
                                          sql=statement)
            # insert语句,explain无法正确判断,暂时只做表存在性检查与简单关键字匹配
            elif re.match(r"^insert", statement.lower()):
                if re.match(
                        r"^insert\s+into\s+(.+?)(\s+|\s*\(.+?)(values|format|select)(\s+|\()",
                        statement.lower()):
                    table_name = re.match(
                        r"^insert\s+into\s+(.+?)(\s+|\s*\(.+?)(values|format|select)(\s+|\()",
                        statement.lower(), re.M).group(1)
                    if '.' not in table_name:
                        table_name = f"{db_name}.{table_name}"
                    table_exist = self.get_table_engine(table_name)['status']
                    if table_exist == 1:
                        result = ReviewResult(
                            id=line,
                            errlevel=0,
                            stagestatus='Audit completed',
                            errormessage='None',
                            sql=statement,
                            affected_rows=0,
                            execute_time=0,
                        )
                    else:
                        check_result.is_critical = True
                        result = ReviewResult(
                            id=line,
                            errlevel=2,
                            stagestatus='表不存在',
                            errormessage=f'表 {table_name} 不存在!',
                            sql=statement)
                else:
                    check_result.is_critical = True
                    result = ReviewResult(id=line,
                                          errlevel=2,
                                          stagestatus='驳回不支持SQL',
                                          errormessage='INSERT语法不正确!',
                                          sql=statement)
            # 其他语句使用explain ast简单检查
            else:
                result = self.explain_check(check_result, db_name, line,
                                            statement)

            # 没有找出DDL语句的才继续执行此判断
            if check_result.syntax_type == 2:
                if get_syntax_type(statement, parser=False,
                                   db_type='mysql') == 'DDL':
                    check_result.syntax_type = 1
            check_result.rows += [result]

            # 遇到禁用和高危语句直接返回
            if check_result.is_critical:
                check_result.error_count += 1
                return check_result
            line += 1
        return check_result
Example #3
0
    def execute_check(self, db_name=None, sql='', close_conn=True):
        """
        上线单执行前的检查, 返回Review set
        update by Jan.song 20200302
        使用explain对数据修改预计进行检测
        """
        config = SysConfig()
        check_result = ReviewSet(full_sql=sql)
        # explain支持的语法
        explain_re = r"^merge|^update|^delete|^insert|^create\s+table|^create\s+index|^create\s+unique\s+index"
        # 禁用/高危语句检查
        line = 1
        # 保存SQL中的新建对象
        object_name_list = set()
        critical_ddl_regex = config.get('critical_ddl_regex', '')
        p = re.compile(critical_ddl_regex)
        check_result.syntax_type = 2  # TODO 工单类型 0、其他 1、DDL,2、DML
        try:
            sqlitemList = get_full_sqlitem_list(sql, db_name)
            for sqlitem in sqlitemList:
                sql_lower = sqlitem.statement.lower().rstrip(';')
                # 禁用语句
                if re.match(r"^select|^with|^explain", sql_lower):
                    check_result.is_critical = True
                    result = ReviewResult(
                        id=line,
                        errlevel=2,
                        stagestatus='驳回不支持语句',
                        errormessage='仅支持DML和DDL语句,查询语句请使用SQL查询功能!',
                        sql=sqlitem.statement)
                # 高危语句
                elif critical_ddl_regex and p.match(sql_lower.strip()):
                    check_result.is_critical = True
                    result = ReviewResult(id=line,
                                          errlevel=2,
                                          stagestatus='驳回高危SQL',
                                          errormessage='禁止提交匹配' +
                                          critical_ddl_regex + '条件的语句!',
                                          sql=sqlitem.statement)
                # 驳回未带where数据修改语句,如确实需做全部删除或更新,显示的带上where 1=1
                elif re.match(r"^update((?!where).)*$|^delete((?!where).)*$",
                              sql_lower):
                    check_result.is_critical = True
                    result = ReviewResult(id=line,
                                          errlevel=2,
                                          stagestatus='驳回未带where数据修改',
                                          errormessage='数据修改需带where条件!',
                                          sql=sqlitem.statement)
                # 驳回事务控制,会话控制SQL
                elif re.match(r"^set|^rollback|^exit", sql_lower):
                    check_result.is_critical = True
                    result = ReviewResult(
                        id=line,
                        errlevel=2,
                        stagestatus='SQL中不能包含^set|^rollback|^exit',
                        errormessage='SQL中不能包含^set|^rollback|^exit',
                        sql=sqlitem.statement)

                # 通过explain对SQL做语法语义检查
                elif re.match(explain_re,
                              sql_lower) and sqlitem.stmt_type == 'SQL':
                    if self.check_create_index_table(
                            db_name=db_name,
                            sql=sql_lower,
                            object_name_list=object_name_list):
                        object_name = self.get_sql_first_object_name(
                            sql=sql_lower)
                        if '.' in object_name:
                            object_name = object_name
                        else:
                            object_name = f"""{db_name}.{object_name}"""
                        object_name_list.add(object_name)
                        result = ReviewResult(
                            id=line,
                            errlevel=1,
                            stagestatus='WARNING:新建表的新建索引语句暂无法检测!',
                            errormessage='WARNING:新建表的新建索引语句暂无法检测!',
                            stmt_type=sqlitem.stmt_type,
                            object_owner=sqlitem.object_owner,
                            object_type=sqlitem.object_type,
                            object_name=sqlitem.object_name,
                            sql=sqlitem.statement)
                    elif len(object_name_list) > 0 and self.get_dml_table(
                            db_name=db_name,
                            sql=sql_lower,
                            object_name_list=object_name_list):
                        result = ReviewResult(
                            id=line,
                            errlevel=1,
                            stagestatus='WARNING:新建表的数据修改暂无法检测!',
                            errormessage='WARNING:新建表的数据修改暂无法检测!',
                            stmt_type=sqlitem.stmt_type,
                            object_owner=sqlitem.object_owner,
                            object_type=sqlitem.object_type,
                            object_name=sqlitem.object_name,
                            sql=sqlitem.statement)
                    else:
                        result_set = self.explain_check(db_name=db_name,
                                                        sql=sqlitem.statement,
                                                        close_conn=False)
                        if result_set['msg']:
                            check_result.is_critical = True
                            result = ReviewResult(
                                id=line,
                                errlevel=2,
                                stagestatus='explain语法检查未通过!',
                                errormessage=result_set['msg'],
                                sql=sqlitem.statement)
                        else:
                            # 对create table\create index\create unique index语法做对象存在性检测
                            if re.match(
                                    r"^create\s+table|^create\s+index|^create\s+unique\s+index",
                                    sql_lower):
                                object_name = self.get_sql_first_object_name(
                                    sql=sql_lower)
                                # 保存create对象对后续SQL做存在性判断
                                if '.' in object_name:
                                    object_name = object_name
                                else:
                                    object_name = f"""{db_name}.{object_name}"""
                                if self.object_name_check(
                                        db_name=db_name,
                                        object_name=object_name
                                ) or object_name in object_name_list:
                                    check_result.is_critical = True
                                    result = ReviewResult(
                                        id=line,
                                        errlevel=2,
                                        stagestatus=f"""{object_name}对象已经存在!""",
                                        errormessage=
                                        f"""{object_name}对象已经存在!""",
                                        sql=sqlitem.statement)
                                else:
                                    object_name_list.add(object_name)
                                    if result_set['rows'] > 1000:
                                        result = ReviewResult(
                                            id=line,
                                            errlevel=1,
                                            stagestatus='影响行数大于1000,请关注',
                                            errormessage='影响行数大于1000,请关注',
                                            sql=sqlitem.statement,
                                            stmt_type=sqlitem.stmt_type,
                                            object_owner=sqlitem.object_owner,
                                            object_type=sqlitem.object_type,
                                            object_name=sqlitem.object_name,
                                            affected_rows=result_set['rows'],
                                            execute_time=0,
                                        )
                                    else:
                                        result = ReviewResult(
                                            id=line,
                                            errlevel=0,
                                            stagestatus='Audit completed',
                                            errormessage='None',
                                            sql=sqlitem.statement,
                                            stmt_type=sqlitem.stmt_type,
                                            object_owner=sqlitem.object_owner,
                                            object_type=sqlitem.object_type,
                                            object_name=sqlitem.object_name,
                                            affected_rows=result_set['rows'],
                                            execute_time=0,
                                        )
                            else:
                                if result_set['rows'] > 1000:
                                    result = ReviewResult(
                                        id=line,
                                        errlevel=1,
                                        stagestatus='影响行数大于1000,请关注',
                                        errormessage='影响行数大于1000,请关注',
                                        sql=sqlitem.statement,
                                        stmt_type=sqlitem.stmt_type,
                                        object_owner=sqlitem.object_owner,
                                        object_type=sqlitem.object_type,
                                        object_name=sqlitem.object_name,
                                        affected_rows=result_set['rows'],
                                        execute_time=0,
                                    )
                                else:
                                    result = ReviewResult(
                                        id=line,
                                        errlevel=0,
                                        stagestatus='Audit completed',
                                        errormessage='None',
                                        sql=sqlitem.statement,
                                        stmt_type=sqlitem.stmt_type,
                                        object_owner=sqlitem.object_owner,
                                        object_type=sqlitem.object_type,
                                        object_name=sqlitem.object_name,
                                        affected_rows=result_set['rows'],
                                        execute_time=0,
                                    )
                # 其它无法用explain判断的语句
                else:
                    # 对alter table做对象存在性检查
                    if re.match(r"^alter\s+table\s", sql_lower):
                        object_name = self.get_sql_first_object_name(
                            sql=sql_lower)
                        if '.' in object_name:
                            object_name = object_name
                        else:
                            object_name = f"""{db_name}.{object_name}"""
                        if not self.object_name_check(
                                db_name=db_name, object_name=object_name
                        ) and object_name not in object_name_list:
                            check_result.is_critical = True
                            result = ReviewResult(
                                id=line,
                                errlevel=2,
                                stagestatus=f"""{object_name}对象不存在!""",
                                errormessage=f"""{object_name}对象不存在!""",
                                sql=sqlitem.statement)
                        else:
                            result = ReviewResult(
                                id=line,
                                errlevel=1,
                                stagestatus='当前平台,此语法不支持审核!',
                                errormessage='当前平台,此语法不支持审核!',
                                sql=sqlitem.statement,
                                stmt_type=sqlitem.stmt_type,
                                object_owner=sqlitem.object_owner,
                                object_type=sqlitem.object_type,
                                object_name=sqlitem.object_name,
                                affected_rows=0,
                                execute_time=0,
                            )
                    # 对create做对象存在性检查
                    elif re.match(r"^create", sql_lower):
                        object_name = self.get_sql_first_object_name(
                            sql=sql_lower)
                        if '.' in object_name:
                            object_name = object_name
                        else:
                            object_name = f"""{db_name}.{object_name}"""
                        if self.object_name_check(
                                db_name=db_name, object_name=object_name
                        ) or object_name in object_name_list:
                            check_result.is_critical = True
                            result = ReviewResult(
                                id=line,
                                errlevel=2,
                                stagestatus=f"""{object_name}对象已经存在!""",
                                errormessage=f"""{object_name}对象已经存在!""",
                                sql=sqlitem.statement)
                        else:
                            object_name_list.add(object_name)
                            result = ReviewResult(
                                id=line,
                                errlevel=1,
                                stagestatus='当前平台,此语法不支持审核!',
                                errormessage='当前平台,此语法不支持审核!',
                                sql=sqlitem.statement,
                                stmt_type=sqlitem.stmt_type,
                                object_owner=sqlitem.object_owner,
                                object_type=sqlitem.object_type,
                                object_name=sqlitem.object_name,
                                affected_rows=0,
                                execute_time=0,
                            )
                    else:
                        result = ReviewResult(
                            id=line,
                            errlevel=1,
                            stagestatus='当前平台,此语法不支持审核!',
                            errormessage='当前平台,此语法不支持审核!',
                            sql=sqlitem.statement,
                            stmt_type=sqlitem.stmt_type,
                            object_owner=sqlitem.object_owner,
                            object_type=sqlitem.object_type,
                            object_name=sqlitem.object_name,
                            affected_rows=0,
                            execute_time=0,
                        )
                # 判断工单类型
                if get_syntax_type(sql=sqlitem.statement,
                                   db_type='oracle') == 'DDL':
                    check_result.syntax_type = 1
                check_result.rows += [result]
                # 遇到禁用和高危语句直接返回,提高效率
                if check_result.is_critical:
                    check_result.error_count += 1
                    return check_result
                line += 1
        except Exception as e:
            logger.warning(
                f"Oracle 语句执行报错,第{line}个SQL:{sqlitem.statement},错误信息{traceback.format_exc()}"
            )
            check_result.error = str(e)
        finally:
            if close_conn:
                self.close()
        return check_result
Example #4
0
    def execute_check(self, db_name=None, sql=''):
        """上线单执行前的检查, 返回Review set"""
        config = SysConfig()
        check_result = ReviewSet(full_sql=sql)
        # 禁用/高危语句检查
        line = 1
        critical_ddl_regex = config.get('critical_ddl_regex', '')
        p = re.compile(critical_ddl_regex)
        check_result.syntax_type = 2  # TODO 工单类型 0、其他 1、DDL,2、DML
        for statement in sqlparse.split(sql):
            statement = sqlparse.format(statement, strip_comments=True)
            # 禁用语句
            if re.match(r"^select", statement.lower()):
                check_result.is_critical = False
                result = ReviewResult(id=line,
                                      errlevel=0,
                                      stagestatus='Select statements',
                                      errormessage='None',
                                      sql=statement)
                check_result.rows += [result]
                check_result.syntax_type = 0
                # check_result.error_count += 1
            # 高危语句
            elif critical_ddl_regex and p.match(statement.strip().lower()):
                check_result.is_critical = True
                result = ReviewResult(id=line,
                                      errlevel=2,
                                      stagestatus='驳回高危SQL',
                                      errormessage='禁止提交匹配' +
                                      critical_ddl_regex + '条件的语句!',
                                      sql=statement)
            # 正常语句
            else:
                result = ReviewResult(
                    id=line,
                    errlevel=0,
                    stagestatus='Audit completed',
                    errormessage='None',
                    sql=statement,
                    affected_rows=0,
                    execute_time=0,
                )
            # 判断工单类型
            # 没有找出DDL语句的才继续执行此判断
            if check_result.syntax_type == 2:
                if get_syntax_type(statement) == 'DDL':
                    check_result.syntax_type = 1
            check_result.rows += [result]

            # 遇到禁用和高危语句直接返回,提高效率
            if check_result.is_critical:
                check_result.error_count += 1
                return check_result
            line += 1
        # 通过检测的再进行inception检查
        if config.get('go_inception'):
            try:
                inception_engine = GoInceptionEngine()
                check_result = inception_engine.execute_check(
                    instance=self.instance, db_name=db_name, sql=sql)
            except Exception as e:
                logger.debug(f"Inception检测语句报错:错误信息{traceback.format_exc()}")
                raise RuntimeError(
                    f"Inception检测语句报错,请注意检查系统配置中Inception配置,错误信息:\n{e}")
        else:
            try:
                inception_engine = InceptionEngine()
                check_result = inception_engine.execute_check(
                    instance=self.instance, db_name=db_name, sql=sql)
            except Exception as e:
                logger.debug(f"Inception检测语句报错:错误信息{traceback.format_exc()}")
                raise RuntimeError(
                    f"Inception检测语句报错,请注意检查系统配置中Inception配置,错误信息:\n{e}")
        return check_result
Example #5
0
    def execute_check(self, db_name=None, sql=''):
        """上线单执行前的检查, 返回Review set"""
        config = SysConfig()
        check_result = ReviewSet(full_sql=sql)
        # 禁用/高危语句检查
        line = 1
        critical_ddl_regex = config.get('critical_ddl_regex', '')
        p = re.compile(critical_ddl_regex)
        check_result.syntax_type = 2  # TODO 工单类型 0、其他 1、DDL,2、DML

        # 把所有SQL转换成SqlItem List。 如有多行(内部有多个;)执行块,约定以delimiter $$作为开始, 以$$结束
        # 需要在函数里实现单条SQL做sqlparse.format(sql, strip_comments=True)
        sqlitemList = get_full_sqlitem_list(sql, db_name)

        for sqlitem in sqlitemList:
            # 禁用语句
            if re.match(r"^\s*select", sqlitem.statement.lower(), re.I):
                check_result.is_critical = True
                result = ReviewResult(
                    id=line,
                    errlevel=2,
                    stagestatus='驳回不支持语句',
                    errormessage='仅支持DML和DDL语句,查询语句请使用SQL查询功能!',
                    sql=sqlitem.statement)
            # 高危语句
            elif critical_ddl_regex and p.match(
                    sqlitem.statement.strip().lower()):
                check_result.is_critical = True
                result = ReviewResult(id=line,
                                      errlevel=2,
                                      stagestatus='驳回高危SQL',
                                      errormessage='禁止提交匹配' +
                                      critical_ddl_regex + '条件的语句!',
                                      sql=sqlitem.statement)

            # 正常语句
            else:
                result = ReviewResult(
                    id=line,
                    errlevel=0,
                    stagestatus='Audit completed',
                    errormessage='None',
                    sql=sqlitem.statement,
                    stmt_type=sqlitem.stmt_type,
                    object_owner=sqlitem.object_owner,
                    object_type=sqlitem.object_type,
                    object_name=sqlitem.object_name,
                    affected_rows=0,
                    execute_time=0,
                )
            # 判断工单类型
            if get_syntax_type(sqlitem.statement) == 'DDL':
                check_result.syntax_type = 1
            check_result.rows += [result]

            # 遇到禁用和高危语句直接返回,提高效率
            if check_result.is_critical:
                check_result.error_count += 1
                return check_result
            line += 1
        return check_result
Example #6
0
def submit(request):
    """正式提交SQL, 此处生成工单"""
    sql_content = request.POST['sql_content']
    workflow_title = request.POST['workflow_name']
    # 检查用户是否有权限涉及到资源组等, 比较复杂, 可以把检查权限改成一个独立的方法
    # 工单表中可以考虑不存储资源组相关信息
    # 工单和实例关联, 实例和资源组关联, 资源组和用户关联。
    group_name = request.POST['group_name']
    group_id = ResourceGroup.objects.get(group_name=group_name).group_id
    instance_name = request.POST['instance_name']
    instance = Instance.objects.get(instance_name=instance_name)
    db_name = request.POST.get('db_name')
    is_backup = request.POST['is_backup']
    notify_users = request.POST.getlist('notify_users')
    list_cc_addr = [
        email['email'] for email in Users.objects.filter(
            username__in=notify_users).values('email')
    ]

    # 服务器端参数验证
    if sql_content is None or workflow_title is None or instance_name is None or db_name is None or is_backup is None:
        context = {'errMsg': '页面提交参数可能为空'}
        return render(request, 'error.html', context)

    # 验证组权限(用户是否在该组、该组是否有指定实例)
    try:
        user_instances(request.user, type='master',
                       db_type='mysql').get(instance_name=instance_name)
    except Exception:
        context = {'errMsg': '你所在组未关联该实例!'}
        return render(request, 'error.html', context)

    sql_content = sql_content.strip()

    # 审核
    try:
        check_engine = get_engine(instance)
        check_result = check_engine.execute_check(db_name=db_name,
                                                  sql=sql_content)
    except Exception as msg:
        context = {'errMsg': msg}
        return render(request, 'error.html', context)

    if not check_result:
        context = {'errMsg': 'inception返回的结果集为空!可能是SQL语句有语法错误'}
        return render(request, 'error.html', context)

    # 遍历result,看是否有任何自动审核不通过的地方,并且按配置确定是标记审核不通过还是放行,放行的可以在工单内原生执行
    sys_config = SysConfig()
    is_manual = 0
    workflow_status = 'workflow_manreviewing'
    for row in check_result.rows:
        # 1表示警告,不影响执行
        if row.errlevel == 1 and sys_config.get('auto_review_wrong',
                                                '') == '1':
            workflow_status = 'workflow_autoreviewwrong'
            break
        # 2表示严重错误,或者inception不支持的语法,标记手工执行,可以跳过inception直接执行
        elif row.errlevel == 2:
            is_manual = 1
            if sys_config.get('auto_review_wrong', '') in ('', '1', '2'):
                workflow_status = 'workflow_autoreviewwrong'
            break
        elif re.match(r"\w*comments\w*", row.errormessage):
            is_manual = 1
            if sys_config.get('auto_review_wrong', '') in ('', '1', '2'):
                workflow_status = 'workflow_autoreviewwrong'
            break

    # 判断SQL是否包含DDL语句,SQL语法 1、DDL,2、DML
    syntax_type = 2
    for stmt in sqlparse.split(sql_content):
        if get_syntax_type(stmt) == 'DDL':
            syntax_type = 1
            break

    # 调用工作流生成工单
    # 使用事务保持数据一致性
    try:
        with transaction.atomic():
            # 存进数据库里
            sql_workflow = SqlWorkflow.objects.create(
                workflow_name=workflow_title,
                group_id=group_id,
                group_name=group_name,
                engineer=request.user.username,
                engineer_display=request.user.display,
                audit_auth_groups=Audit.settings(
                    group_id, WorkflowDict.workflow_type['sqlreview']),
                status=workflow_status,
                is_backup=is_backup,
                instance=instance,
                db_name=db_name,
                is_manual=is_manual,
                syntax_type=syntax_type,
                create_time=timezone.now())
            SqlWorkflowContent.objects.create(
                workflow=sql_workflow,
                sql_content=sql_content,
                review_content=check_result.json(),
                execute_result='')
            workflow_id = sql_workflow.id
            # 自动审核通过了,才调用工作流
            if workflow_status == 'workflow_manreviewing':
                # 调用工作流插入审核信息, 查询权限申请workflow_type=2
                Audit.add(WorkflowDict.workflow_type['sqlreview'], workflow_id)
    except Exception as msg:
        logger.error(traceback.format_exc())
        context = {'errMsg': msg}
        return render(request, 'error.html', context)
    else:
        # 自动审核通过了,才调用工作流,进行消息通知
        if workflow_status == 'workflow_manreviewing':
            # 再次获取审核信息
            audit_id = Audit.detail_by_workflow_id(
                workflow_id=workflow_id,
                workflow_type=WorkflowDict.workflow_type['sqlreview']).audit_id
            async_task(notify_for_audit,
                       audit_id=audit_id,
                       email_cc=list_cc_addr,
                       timeout=60)

    return HttpResponseRedirect(reverse('sql:detail', args=(workflow_id, )))