def extract_clinic_term(str_origin, separator='||'): """ 函数主要用于在特定位置加入分隔符(||) a. 连续数字被其他字符分隔 --> || b. ; --> || c. ? --> || :param str_origin 待处理的字符串 :param separator 分隔符 :return: """ logger = initiate_log(logger_name=Constants.log_name_default) logger.debug('原始的字符串:%s', str_origin) pattern = re.compile(r'(\d+)(\D+)') data = pattern.findall(str_origin) nums = [int(v[0]) for v in data if len(data) > 0] nums_new = is_element_increment(nums) logger.debug('连续递增的数字: %s', nums_new) if nums_new: for i in nums_new: regex = '' for j in i: regex += '({num})(\D+)'.format(num=j) pattern = re.compile(r'{regex}'.format(regex=regex)) str_origin = pattern.sub(regex_replace, str_origin) pattern = re.compile(r'(;|\?)+') str_origin = pattern.sub(r'{separator}'.format(separator=separator), str_origin) pattern = re.compile(r'\|\|{2,}') # 去除重复的 || return pattern.sub(separator, str_origin)
def __init__(self): """ 初始化 MySQL 连接参数 """ logger = initiate_log(__name__) try: self.config = { 'host': db_config.host, 'database': db_config.db, 'user': db_config.user, 'password': db_config.passwd, 'raise_on_warnings': False, 'charset': 'utf8', } self.cnx = connect(**self.config) except Error as err: if err.errno == errorcode.ER_ACCESS_DENIED_ERROR: err_msg = "Something is wrong with your user name or password" logger.error(err_msg) elif err.errno == errorcode.ER_BAD_DB_ERROR: err_msg = "Database does not exist" logger.error(err_msg) else: err_msg = err.msg logger.error(err_msg) raise DpException(ErrorConstants.ec_sys_error, ErrorConstants.error_code_message .get(ErrorConstants.ec_sys_error) + err_msg) finally: self.cursor = self.cnx.cursor(dictionary=True) logger.info("database connect success.")
def main(): logger = initiate_log(logger_name=Constants.log_name_brief) separator = '||' table_name_wip_cterm = 'wip_cterm' table_name_wip_cterm_element = 'wip_cterm_element' connector = MySQLConnector() sql = """ select * from {table_name} where origin_cd; """.format(table_name=table_name_wip_cterm) data = connector.query_data(sql=sql) for i, d in enumerate(data[:]): for k, v in d.items(): data[i] if k == 'str_new': str_ret = extract_clinic_term(v) data[i]['str_ret'] = str_ret ''' 构造 sql ''' length = len(data) sql_value_wip_cterm = sql_value_wip_cterm_element = '' for i, v in enumerate(data[:length]): str_ret = v['str_ret'].replace('"', '\\"') sql_value_wip_cterm += '("{origin_cd}", "{str_ret}")'.format( origin_cd=v['origin_cd'], str_ret=str_ret) if i != length - 1: sql_value_wip_cterm += ', ' for j, k in enumerate(str_ret.split(separator)): sql_value_wip_cterm_element += '("{origin_cd}", "{term}")'.format( origin_cd=v['origin_cd'], term=k) if j != len(str_ret.split(separator)) - 1: sql_value_wip_cterm_element += ', ' if i != length - 1: sql_value_wip_cterm_element += ', ' ''' wip_cterm 表插入数据 ''' logger.info('begin insert data into wip_cterm') sql = """ insert into {table_name}(`origin_cd`, `str_ret`) values{sql_value} on DUPLICATE key update str_ret = values(`str_ret`); """.format(table_name=table_name_wip_cterm, sql_value=sql_value_wip_cterm) connector = MySQLConnector() connector.manipulate_data(sql) logger.info('end insert data into wip_cterm') ''' wip_cterm_element 表插入数据 ''' logger.info('begin insert data into wip_cterm_element') sql = """ insert ignore into {TABLE_NAME}(`origin_cd`, `term`) values{sql_value} """.format(TABLE_NAME=table_name_wip_cterm_element, sql_value=sql_value_wip_cterm_element) connector = MySQLConnector() connector.manipulate_data(sql) logger.info('end insert data into wip_cterm_element')
class TestNltk(unittest.TestCase): """ nltk: natural language toolkit """ logger = initiate_log() def setUp(self): pass # nltk.download("all") def test_simple(self): sentence = """我最爱吃的东西是凤梨""" tokens = jieba.lcut(sentence) self.logger.debug(','.join(tokens)) jieba.add_word('爱吃') references_1 = jieba.lcut('我爱吃的东西是凤梨啊') self.logger.debug('references 1: %s', references_1) references_2 = jieba.lcut('他不爱吃苹果') self.logger.debug('references 2: %s', references_2) references_3 = jieba.lcut('我们都是中国人地地道道') self.logger.debug('references 3: %s', references_3) score = nltk.bleu([references_1], tokens) self.logger.debug('bleu score is %s', score) pass
class Jieba(unittest.TestCase): """ jieba 中文分词 """ logger = initiate_log() # def __init__(self): # self.logger = initiate_log() def get_split_line(self, topic): return '=' * 30 + topic + '=' * 30 def test_split_mode(self): """ 分词模式 """ topic = '分词模式' split_line = self.get_split_line(topic) self.logger.debug(split_line) """ 全模式:把句子中所有的金额已成词的词语都扫描出来,速度非常快,但不能解决歧义 """ seg_list = jieba.cut("中华人民共和国", cut_all=True) self.logger.info("{topic}_全模式: {msg}".format(topic=topic, msg="/ ".join(seg_list))) """ 精确模式:试图将句子最精确的切开,适合文本分析 """ seg_list = jieba.cut("中华人民共和国", cut_all=False) self.logger.info("{topic}_精确模式(默认): {msg}".format( topic=topic, msg="/".join(seg_list))) """ 搜索模式:在精确模式的基础上,对长词再次切分,提高召回率,适合用于搜索引擎分词 """ seg_list = jieba.cut_for_search("中华人民共和国") self.logger.info("{topic}_搜索模式: {msg}".format(topic=topic, msg=", ".join(seg_list))) def test_user_dict(self): """ 2. 自定义词典 """ topic = '添加自定义词典' split_line = self.get_split_line(topic) self.logger.debug(split_line) test_sent = """李小福是创新办主任也是云计算方面的专家; 什么是八一双鹿\n 例如我输入一个带“韩玉赏鉴”的标题,在自定义词库中也增加了此词为N类\n 台中」正確應該不會被切開。mac上可分出「石墨烯」;此時又可以分出來凱特琳了。 """ words = jieba.cut(test_sent) self.logger.debug('{topic}_原始: {msg}'.format(topic=topic, msg='/'.join(words))) """ 调整词典:动态修改词典 """ userdict_path = os.path.dirname(__file__) + "/jieba_dict/dict.txt" jieba.add_word('石墨烯') jieba.add_word('凱特琳') jieba.del_word('自定义词') jieba.load_userdict(userdict_path) words = jieba.cut(test_sent) self.logger.debug('{topic}_自定义字典分词:{msg}'.format(topic=topic, msg='/'.join(words))) self.logger.debug('test split words' + "=" * 40) terms = jieba.cut('easy_install is great') self.logger.debug('{topic}_字典分词: {msg}'.format(topic=topic, msg='/'.join(terms))) jieba.del_word('easy_install') terms = jieba.cut('easy_install is great') self.logger.debug('{topic}_删除单词: {msg}'.format(topic=topic, msg='/'.join(terms))) terms = jieba.cut('python 的正则表达式是好用的') self.logger.debug('{topic}_单词: {msg}'.format(topic=topic, msg='/'.join(terms))) self.logger.debug('test frequency tune' + "=" * 40) word = '这里中将应该被切开' self.logger.debug('{topic}_调低词频之前: {msg}'.format(topic=topic, msg='/'.join( jieba.cut(word)))) self.logger.debug('{topic}_调整词频: {msg}'.format( topic=topic, msg='before: {before}, after: {after}'.format( before=jieba.get_FREQ('中将'), after=jieba.suggest_freq(('中', '将'), True)))) self.logger.debug('{topic}_调低词频之后: {msg}'.format( topic=topic, msg='/'.join(jieba.cut(word, HMM=False)))) jieba.del_word('台中') word = '[台中]正确应该不会被切开' self.logger.debug('{topic}_调高词频之前: {msg}'.format(topic=topic, msg='/'.join( jieba.cut(word)))) self.logger.debug('{topic}_调整词频: {msg}'.format( topic=topic, msg='before: {before}, after: {after}'.format( before=jieba.get_FREQ('台中'), after=jieba.suggest_freq('台中', True)))) self.logger.debug('{topic}_调高词频之后: {msg}'.format( topic=topic, msg='/'.join(jieba.cut(word, HMM=False)))) def test_extract_tags(self): """ 3. 关键词抽取 """ topic = '关键词抽取' split_line = self.get_split_line(topic=topic) self.logger.info(split_line) term = '我们时人中国的可是is of super man' res = analyse.extract_tags(term) self.logger.info('{topic}_标准抽取: {term} -> {msg}'.format(topic=topic, term=term, msg=res)) # TODO: 自定义语料库运行有问题 user_idf_path = os.path.dirname(__file__) + '/jieba_dict/idf.txt.big' analyse.set_idf_path(user_idf_path) res = analyse.extract_tags(term) self.logger.info('{topic}_自定义逆向文件频率: {term} -> {msg}'.format( topic=topic, term=term, msg=res)) def test_tokenize(self): """ Tokenize: 返回词语在原文的位置 """ topic = '返回词语在原文的位置' split_line = self.get_split_line(topic) self.logger.debug(split_line) term = '我们都是炎黄子孙' res = jieba.tokenize(term) for tk in res: self.logger.debug('{topic}_默认模式_{term}: {msg}'.format( topic=topic, term=term, msg="word %s\t\t start: %d \t\t end:%d" % (tk[0], tk[1], tk[2]))) res = jieba.tokenize(term, mode='search') for tk in res: self.logger.debug('{topic}_搜索模式_{term}: {msg}'.format( topic=topic, term=term, msg="word %s\t\t start: %d \t\t end:%d" % (tk[0], tk[1], tk[2])))
# -*- coding: utf-8 -*- from kazoo.client import KazooClient from dataProcess.dap.modules.log_initiate import initiate_log from kazoo.client import KazooState from kazoo.protocol.states import WatchedEvent from kazoo.handlers.threading import KazooTimeoutError import unittest import json logger = initiate_log() hosts = "192.168.150.250:2181" node = "dcm/beta" nodeMysql = node + "/mysql" def connection_listener(state): if state == KazooState.SUSPENDED: # handle being disconnected from zookeeper logger.error("disconnect from zookeeper") def watcher_default(event: WatchedEvent): """ default watcher: A watch function passed to get() or exists() will be called when the data on the node changes or the node itself is deleted. and it is one-time watch events :param event: :return:
# -*- coding: utf-8 -*- from kazoo.client import KazooClient from kazoo.client import KazooState from kazoo.protocol.states import WatchedEvent from kazoo.handlers.threading import KazooTimeoutError from dataProcess.dap.confs.zk import zk_hosts from dataProcess.dap.modules.log_initiate import initiate_log from dataProcess.dap.modules.dp_exception import DpException from dataProcess.dap.modules.constants import ErrorConstants, Constants import json import dataProcess.dap.confs.db_config as db_config logger = initiate_log(__name__) def connection_listener(state): if state == KazooState.SUSPENDED: # handle being disconnected from zookeeper err_msg = "disconnect from zookeeper" logger.error(err_msg) raise DpException( ErrorConstants.ec_sys_error, ErrorConstants.error_code_message.get(ErrorConstants.ec_sys_error) + err_msg) class ZkConfig: def __init__(self): self.zk = KazooClient(hosts=zk_hosts) try: self.zk.start()