def start(split_date): """ 获取并拆分一天的日志 """ LOCAL_LOG_PATH_NAME_LST,OUT_PUT_PATH_LST = get_parse_path(split_date) # 本地打开 for _server_id in LOCAL_LOG_PATH_NAME_LST: read_file = LOCAL_LOG_PATH_NAME_LST[_server_id].format(cur_date=split_date) try: log_lines = open(read_file, 'r') print(split_date) #每个服务器初始状态 last_line_num = utility.read_file_last_line(read_file) print"this file last line num is:",last_line_num cur_line_num = 0 utility.global_log_lst = [] err_num = 0 out_put_file_path = OUT_PUT_PATH_LST[_server_id].format(cur_date=split_date,use_path="tables") if not os.path.exists(out_put_file_path): os.makedirs(out_put_file_path) os.chmod(out_put_file_path, stat.S_IRWXG + stat.S_IRWXO + stat.S_IRWXU) os.chdir(out_put_file_path) #开始读 if log_lines: start = time.time() #打开文件 file_path = open(FILE_NAME, 'w+') for _log_line in log_lines: cur_line_num += 1 _log_line = _log_line.strip() log_dict = log_parse(_log_line) if not log_dict: err_num += 1 continue for key, val in log_dict.items(): if key in EQUIP_ACTION_LST: dat = _insert_equip_change_log(log_dict['uid'], log_dict['log_time'], log_dict['platform_id'], log_dict['server_id'], log_dict['action'],log_dict['level'], key, val) if dat: utility.global_log_lst.extend(dat) # TOD:1.限制读取条数 utility.read_limit(file_path, last_line_num, cur_line_num) print 'err_num is: ', err_num print FILE_NAME, " loop_dump use time is: ", time.time() - start del utility.global_log_lst[:] # 快速删除大列表 # 关闭文件 file_path.close() # 装备产出 _output_CREATE_EQUIP() # 装备消耗 _output_CONSUME_EQUIP() except: pass
def start(split_date): """ 获取并拆分一天的日志 """ # 本地打开 LOCAL_LOG_PATH_NAME , OUT_PUT_PATH = get_parse_path(split_date) for _server_id in LOCAL_LOG_PATH_NAME: try: read_file = LOCAL_LOG_PATH_NAME[_server_id].format(cur_date=split_date) log_lines = open(read_file, 'r') print(split_date) print _server_id last_line_num = read_file_last_line(read_file) print "this file last line num is: ", last_line_num cur_line_num = 0 err_num = 0 _count = 0 # 目录 out_put_file_path = OUT_PUT_PATH[_server_id].format(cur_date=split_date, use_path="tables") if not os.path.exists(out_put_file_path): os.makedirs(out_put_file_path) os.chmod(out_put_file_path, stat.S_IRWXG + stat.S_IRWXO + stat.S_IRWXU) os.chdir(out_put_file_path) if log_lines: global cur_action_log_dict, stage_result_dict cur_action_log_dict = {} stage_result_dict = {} for _log_line in log_lines: cur_line_num += 1 _log_line = _log_line.strip() log_dict = log_parse(_log_line) if not log_dict: err_num += 1 continue if log_dict['action'] in CUR_ACTION_LST: _count += 1 action_id = log_dict['action'] action_str = game_define.EVENT_LOG_ACTION_SQL_NAME_DICT.get(action_id, 'Err') # 插入列表 用来输出文件 if action_str in cur_action_log_dict: cur_action_log_dict[action_str].append(log_dict) else: cur_action_log_dict[action_str] = [log_dict] if _count >= READ_LINES: # print "READ_LINES...... cur_line_num is: ", cur_line_num update_data(cur_action_log_dict, split_date) # 到达限制数量dump一次 cur_action_log_dict = {} _count = 0 elif _count > 0 and last_line_num == cur_line_num: print "last update_data...... last_line is: ", cur_line_num update_data(cur_action_log_dict, split_date) # 最后一次dump cur_action_log_dict = {} _count = 0 print 'err_num is: ', err_num #困难副本 英雄副本 _output_HARD_STAGE_CHALLENGE() except: pass
def start(split_date): """ 获取并拆分一天的日志 """ # 本地打开 LOCAL_LOG_PATH_NAME , OUT_PUT_PATH = get_parse_path(split_date) for _server_id in LOCAL_LOG_PATH_NAME: try: read_file = LOCAL_LOG_PATH_NAME[_server_id].format(cur_date=split_date) log_lines = open(read_file, 'r') print(split_date) print _server_id last_line_num = read_file_last_line(read_file) print "this file last line num is: ", last_line_num cur_line_num = 0 err_num = 0 _count = 0 # 目录 out_put_file_path = OUT_PUT_PATH[_server_id].format(cur_date=split_date, use_path="tables") if not os.path.exists(out_put_file_path): os.makedirs(out_put_file_path) os.chmod(out_put_file_path, stat.S_IRWXG + stat.S_IRWXO + stat.S_IRWXU) os.chdir(out_put_file_path) if log_lines: global cur_action_log_dict, stage_result_dict cur_action_log_dict = {} stage_result_dict = {} for _log_line in log_lines: cur_line_num += 1 _log_line = _log_line.strip() log_dict = log_parse(_log_line) if not log_dict: err_num += 1 continue if log_dict['action'] in CUR_ACTION_LST: action_id = log_dict['action'] action_str = game_define.EVENT_LOG_ACTION_SQL_NAME_DICT.get(action_id, 'Err') _count += 1 if action_str in cur_action_log_dict: cur_action_log_dict[action_str].append(log_dict) else: cur_action_log_dict[action_str] = [log_dict] if _count >= READ_LINES: # print "READ_LINES...... cur_line_num is: ", cur_line_num update_data(cur_action_log_dict, split_date) # 到达限制数量dump一次 cur_action_log_dict = {} _count = 0 elif _count > 0 and last_line_num == cur_line_num: print "last update_data...... last_line is: ", cur_line_num update_data(cur_action_log_dict, split_date) # 最后一次dump cur_action_log_dict = {} _count = 0 print 'err_num is: ', err_num # 普通副本 _output_NORMAL_STAGE_CHALLENGE() except: pass