def excep_load_test(db_client, db, coll, log_path, file_regex, function_name):
excep_pattern = re.compile(r'[A-Za-z.]*Exception')
date_pattern = re.compile(
r'(\d{4}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}[,|.]\d+)')
file_dic = tool_util.get_file_info(log_path, file_regex)
now_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')
# logging.info(f'{file_dic}')
for file_name in file_dic.keys():
# logging.info(f'{file_name}')
file_list = open(file_name, 'r', encoding='utf-8', errors='ignore')
for exception_info in file_list:
if not exception_info.__contains__('duplicate'):
exception_info = exception_info.strip()
excep_type = excep_pattern.findall(exception_info)
time_info = date_pattern.findall(exception_info)
time_info = time_info if len(time_info) != 0 else now_time
now_time = time_info
if len(excep_type) != 0:
# logging.info(f'{time_info}:{excep_type}')
# logging.info(exception_info)
excep_dic = update_dic(time_info[0], excep_type[0],
exception_info, file_name,
function_name)
mongo_writer.conn_insertone(db_client, db, coll, excep_dic)
logging.debug(f'excep_dic is {excep_dic}')
password = '******'
mongos_host = '10.68.112.11'
mongos_port = 22001
db_name = 'aihc'
coll_name = 'stat_excep'
client = mongo_writer.auth(username, password, mongos_host, mongos_port)
stat_db = "aihc_stat"
stat_coll = "stat_excepstat_in"
path = '/data/mongo/docker_mnt/log_recv/'
regex = 'recv'
func_name = 'recv'
before_day = 1
# excep_load(client, db_name, coll_name, path, regex, func_name)
logger.info(f'exception_stat_load start, path is {path}, regex is {regex}')
file_info_dic_from_path = tool_util.get_file_info(path, regex, before_day)
deal_file_dic = tool_util.get_deal_file_dic(client, stat_db, stat_coll,
file_info_dic_from_path,
roboter.getNodeIp())
for name, num in deal_file_dic.items():
file_modify_time = file_info_dic_from_path.get(name)
stat_util.insert_stat(mongo_writer, client, stat_db, stat_coll, name,
file_modify_time, roboter.getNodeIp())
exceptinfo_list = tool_util.read_stat_info(name, num)
combine_excep_stat(client, db_name, coll_name, func_name,
exceptinfo_list, name)
stat_util.update_stat(mongo_writer, client, stat_db, stat_coll, name,
num + len(exceptinfo_list), file_modify_time,
roboter.getNodeIp())
result = {"recv_exception_in": "succeed"}
username = '******'
password = '******'
mongos_host = '10.19.85.33'
mongos_port = 34000
db_name = 'test'
coll_name = 'stat_recv'
client = mongo_writer.auth(username, password, mongos_host, mongos_port)
stat_db = "test"
stat_coll = "recv_stat_in"
# 读文件获取stat文件信息入库
path = '/Users/mtr/PycharmProjects/mongoQuery/resource/recv'
regex = 'recv'
date_before = 1
logging.info(f'recv_stat_load start, path is {path}, regex is {regex}')
file_info_dic_from_path = tool_util.get_file_info(path, regex, date_before)
file_dic = tool_util.get_deal_file_dic(client, stat_db, stat_coll, file_info_dic_from_path)
for name, num in file_dic.items():
file_modify_time = file_info_dic_from_path.get(name)
stat_util.insert_stat(mongo_writer, client, stat_db, stat_coll, name, file_modify_time)
statinfo_list = tool_util.read_stat_info(name, num)
for stat_str in statinfo_list:
stat_doc = combine_stat_doc(stat_str)
mongo_writer.conn_insertone(client, db_name, coll_name, stat_doc)
stat_util.update_stat(mongo_writer, client, stat_db, stat_coll,
name, num + len(statinfo_list), file_modify_time)
logging.info(f'recv_stat_load end')
# 测试read_stat_info函数
# name = '/Users/mtr/PycharmProjects/mongoQuery/resource/emit/xdrEmit_stat_emit33_4214_2020121619.txt'
# num = -1
