db_name = 'aihc'
coll_name = 'stat_excep'
client = mongo_writer.auth(username, password, mongos_host, mongos_port)
stat_db = "aihc_stat"
stat_coll = "stat_excepstat_in"
path = '/data/mongo/docker_mnt/log_recv/'
regex = 'recv'
func_name = 'recv'
before_day = 1
# excep_load(client, db_name, coll_name, path, regex, func_name)
logger.info(f'exception_stat_load start, path is {path}, regex is {regex}')
file_info_dic_from_path = tool_util.get_file_info(path, regex, before_day)
deal_file_dic = tool_util.get_deal_file_dic(client, stat_db, stat_coll,
file_info_dic_from_path,
roboter.getNodeIp())
for name, num in deal_file_dic.items():
file_modify_time = file_info_dic_from_path.get(name)
stat_util.insert_stat(mongo_writer, client, stat_db, stat_coll, name,
file_modify_time, roboter.getNodeIp())
exceptinfo_list = tool_util.read_stat_info(name, num)
combine_excep_stat(client, db_name, coll_name, func_name,
exceptinfo_list, name)
stat_util.update_stat(mongo_writer, client, stat_db, stat_coll, name,
num + len(exceptinfo_list), file_modify_time,
roboter.getNodeIp())
result = {"recv_exception_in": "succeed"}
roboter.setContent(JSON.dumps(result))
logger.info('recv_exception_in end')
# logging.debug(f'filename is {file_name}')
# if file_name is not None:
# file_info_dic.pop(file_name)
# logging.info(f'file_deal_dic is {file_info_dic}')
# return file_info_dic
if __name__ == '__main__':
# 数据库认证
username = '******'
password = '******'
mongos_host = '10.19.85.33'
mongos_port = 34000
db_name = 'test'
coll_name = 'stat_rollback'
client = mongo_writer.auth(username, password, mongos_host, mongos_port)
stat_db = "test"
stat_coll = "mongo_rollback_stat_in"
mongo_data_dir_name = '/Users/mtr/PycharmProjects/mongoQuery/resource'
rollback_dir_name = 'rollback'
bson_pattern = re.compile(r'.*\.bson$')
func_name = 'rollback'
file_info_dic_from_path = get_rollback_file_info(mongo_data_dir_name, rollback_dir_name, bson_pattern)
file_dic = tool_util.get_deal_file_dic(client, stat_db, stat_coll, file_info_dic_from_path)
logging.info(f'files_dic is {file_dic}')
for name in file_dic.keys():
file_modify_time = file_info_dic_from_path.get(name)
stat_util.insert_stat(mongo_writer, client, stat_db, stat_coll, name, file_modify_time)
combine_rollback_stat(client, db_name, coll_name, func_name, name)
stat_util.update_stat(mongo_writer, client, stat_db, stat_coll, name, 1, file_modify_time)
logging.info(f'mongo_rollback_load end')
# 数据库认证
username = '******'
password = '******'
mongos_host = '10.19.85.33'
mongos_port = 34000
db_name = 'test'
coll_name = 'stat_emit'
client = mongo_writer.auth(username, password, mongos_host, mongos_port)
stat_db = "test"
stat_coll = "stat_emit_in"
# 读文件获取stat文件信息入库
path = '/Users/mtr/PycharmProjects/mongoQuery/resource/emit'
regex = 'emit'
logging.info(f'start emit load, path is {path}')
before_day = 1
file_info_dic_from_path = tool_util.get_file_info(path, regex, before_day)
file_dic = tool_util.get_deal_file_dic(client, stat_db, stat_coll,
file_info_dic_from_path)
for name, num in file_dic.items():
file_modify_time = file_info_dic_from_path.get(name)
stat_util.insert_stat(mongo_writer, client, stat_db, stat_coll, name,
file_modify_time)
statinfo_list = tool_util.read_stat_info(name, num)
for stat_str in statinfo_list:
stat_doc = combine_stat_doc(stat_str)
mongo_writer.conn_insertone(client, db_name, coll_name, stat_doc)
stat_util.update_stat(mongo_writer, client, stat_db, stat_coll, name,
num + len(statinfo_list), file_modify_time)
"Ip": roboter.getNodeIp()})
mongo_writer.conn_insertone(db_client, db, coll, stat_dic)
if __name__ == '__main__':
# 数据库认证
username = '******'
password = '******'
mongos_host = '10.68.112.11'
mongos_port = 22001
db_name = 'aihc'
coll_name = 'stat_rollback'
client = mongo_writer.auth(username, password, mongos_host, mongos_port)
stat_db = "aihc_stat"
stat_coll = "stat_rollbackstat_in"
mongo_data_dir_name = '/db1/mongo/mongo_data1/'
rollback_dir_name = 'rollback'
bson_pattern = re.compile(r'.*\.bson$')
func_name = 'rollback'
file_info_dic_from_path = get_rollback_file_info(mongo_data_dir_name, rollback_dir_name, bson_pattern)
file_dic = tool_util.get_deal_file_dic(client, stat_db, stat_coll, file_info_dic_from_path,roboter.getNodeIp())
logger.info(f'files_dic is {file_dic}')
for name in file_dic.keys():
file_modify_time = file_info_dic_from_path.get(name)
stat_util.insert_stat(mongo_writer, client, stat_db, stat_coll, name, file_modify_time, roboter.getNodeIp())
combine_rollback_stat(client, db_name, coll_name, func_name, name)
stat_util.update_stat(mongo_writer, client, stat_db, stat_coll, name, 1, file_modify_time, roboter.getNodeIp())
result = {"rollback_stat_load": "succeed"}
roboter.setContent(JSON.dumps(result))
logger.info(f'rollback_stat_load end')