def read_file_dict_with_filename(file_name, search_date, server_id=10003, folder='tables'):
dat_dict = dict()
# 注：用到通服的需要改
if server_id == -1:
for each_id in get_server_path(search_date).keys():
SERVER_LOGS_DATA_LST = get_server_path(search_date)
if os.path.exists(SERVER_LOGS_DATA_LST[each_id].format(cur_date=search_date, use_path=folder)):
file_path = SERVER_LOGS_DATA_LST[each_id].format(cur_date=search_date, use_path=folder) + file_name
print file_path
if os.path.exists(file_path):
out_put_file = open(file_path, 'r')
_tmp_dict = cPickle.load(out_put_file)
dat_dict = dict(dat_dict, **_tmp_dict) # 两个字典合并
out_put_file.close()
else:
SERVER_LOGS_DATA_LST = get_server_path(search_date)
if os.path.exists(SERVER_LOGS_DATA_LST[server_id].format(cur_date=search_date, use_path=folder)):
file_path = SERVER_LOGS_DATA_LST[server_id].format(cur_date=search_date, use_path=folder) + file_name
print file_path
if os.path.exists(file_path):
out_put_file = open(file_path, 'r')
dat_dict = cPickle.load(out_put_file)
out_put_file.close()
# print dat_dict
return dat_dict
def read_file_dict_with_filename(file_name,
search_date,
server_id=10003,
folder='tables'):
dat_dict = dict()
# 注：用到通服的需要改
if server_id == -1:
for each_id in get_server_path(search_date).keys():
SERVER_LOGS_DATA_LST = get_server_path(search_date)
if os.path.exists(SERVER_LOGS_DATA_LST[each_id].format(
cur_date=search_date, use_path=folder)):
file_path = SERVER_LOGS_DATA_LST[each_id].format(
cur_date=search_date, use_path=folder) + file_name
print file_path
if os.path.exists(file_path):
out_put_file = open(file_path, 'r')
_tmp_dict = cPickle.load(out_put_file)
dat_dict = dict(dat_dict, **_tmp_dict) # 两个字典合并
out_put_file.close()
else:
SERVER_LOGS_DATA_LST = get_server_path(search_date)
if os.path.exists(SERVER_LOGS_DATA_LST[server_id].format(
cur_date=search_date, use_path=folder)):
file_path = SERVER_LOGS_DATA_LST[server_id].format(
cur_date=search_date, use_path=folder) + file_name
print file_path
if os.path.exists(file_path):
out_put_file = open(file_path, 'r')
dat_dict = cPickle.load(out_put_file)
out_put_file.close()
# print dat_dict
return dat_dict
def read_file(action, from_date, to_date, server_id=10003, folder='all_action'):
dat_lst = []
total_days = (to_date - from_date).days + 1
for i in xrange(total_days):
# 每行的日期
dat_dict = dict()
cur_date = from_date + datetime.timedelta(days=i)
# 注：用到通服的需要改
if server_id == -1:
for each_id in get_server_path(cur_date).keys():
SERVER_LOGS_DATA_LST = get_server_path(cur_date)
file_path = SERVER_LOGS_DATA_LST[each_id].format(cur_date=cur_date, use_path=folder)+game_define.EVENT_LOG_ACTION_SQL_NAME_DICT[action]
print file_path
if os.path.exists(file_path):
out_put_file = open(file_path, 'r')
dat_dict = pickle.load(out_put_file)
dat_lst.extend(dat_dict)
out_put_file.close()
# print dat_dict
else:
SERVER_LOGS_DATA_LST = get_server_path(cur_date)
file_path = SERVER_LOGS_DATA_LST[server_id].format(cur_date=cur_date, use_path=folder)+game_define.EVENT_LOG_ACTION_SQL_NAME_DICT[action]
print file_path
if os.path.exists(file_path):
out_put_file = open(file_path, 'r')
dat_dict = pickle.load(out_put_file)
dat_lst.extend(dat_dict)
out_put_file.close()
return dat_lst
def read_file_with_user_get(file_name, search_start_date, user_id, server_id):
data_lst = []
OUT_PUT_PATH_LST = get_server_path(search_start_date)
if not user_id or not search_start_date:
return data_lst
if int(server_id) == -1:
return data_lst
else:
try:
log_path = OUT_PUT_PATH_LST[int(server_id)].format(
cur_date=search_start_date, use_path=file_name)
except:
return data_lst
if os.path.exists(log_path):
cur_date = search_start_date
file_path = log_path + "%s" % (user_id)
#print file_path
if not os.path.exists(file_path):
return data_lst
file = open(file_path, 'r')
for log_line in file.readlines():
data_lst.append(eval(log_line))
return data_lst
def read_file_double_lst(file_name,
from_date,
to_date,
server_id=10003,
folder='tables'):
dat_lst1 = []
dat_lst2 = []
total_days = (to_date - from_date).days + 1
for i in xrange(total_days):
tmp_lst = []
# 每行的日期
cur_date = from_date + datetime.timedelta(days=i)
SERVER_LOGS_DATA_LST = get_server_path(cur_date)
file_path = SERVER_LOGS_DATA_LST[server_id].format(
cur_date=cur_date, use_path=folder) + file_name
print file_path
if os.path.exists(file_path):
out_put_file = open(file_path, 'r')
tmp_lst = pickle.load(out_put_file)
dat_lst1.extend(tmp_lst[0])
dat_lst2.extend([tmp_lst[1]]) # 表头 只取一个最合适的
out_put_file.close()
# print tmp_lst
return dat_lst1, dat_lst2
def read_file_with_user_get(file_name,search_start_date,user_id,server_id):
data_lst = []
OUT_PUT_PATH_LST=get_server_path(search_start_date)
print OUT_PUT_PATH_LST
if not user_id or not search_start_date :
return data_lst
if int(server_id) == -1:
return data_lst
else:
try:
log_path = OUT_PUT_PATH_LST[int(server_id)].format(cur_date=search_start_date,use_path=file_name)
except:
return data_lst
print log_path
if os.path.exists(log_path):
cur_date = search_start_date
file_path = log_path+"%s" % (user_id)
print file_path
if not os.path.exists(file_path):
return data_lst
file=open(file_path,'r')
for log_line in file.readlines():
data_lst.append(eval(log_line))
return data_lst
def read_file_with_user_get(search_start_date,search_end_date,server_id): dat_lst=[] OUT_PUT_PATH_LST=get_server_path(search_start_date) #print server_id if int(server_id) == -1: return dat_lst # else: # try: # # log_path = OUT_PUT_PATH_LST[int(server_id)].format(cur_date=search_start_date,use_path='tables') # except: # return dat_lst if not search_start_date or not search_end_date: return dat_lst #print total_day total_day = (search_end_date-search_start_date).days+1 print total_day for i in xrange(total_day): cur_date = search_start_date+datetime.timedelta(days=i) path=OUT_PUT_PATH_LST[int(server_id)].format(cur_date=cur_date,use_path='tables') print path union_count_file_path=path + 'UNION_COUNT' print union_count_file_path #print union_count_file_path if os.path.exists(union_count_file_path): print 'do it ' f = open(union_count_file_path, 'r') for j in pickle.load(f): print j dat_lst.append(j) return dat_lst
def read_file_with_user_get(search_start_date, search_end_date, server_id):
dat_lst = []
OUT_PUT_PATH_LST = get_server_path(search_start_date)
#print server_id
if int(server_id) == -1:
return dat_lst
# else:
# try:
# # log_path = OUT_PUT_PATH_LST[int(server_id)].format(cur_date=search_start_date,use_path='tables')
# except:
# return dat_lst
if not search_start_date or not search_end_date:
return dat_lst
#print total_day
total_day = (search_end_date - search_start_date).days + 1
print total_day
for i in xrange(total_day):
cur_date = search_start_date + datetime.timedelta(days=i)
path = OUT_PUT_PATH_LST[int(server_id)].format(cur_date=cur_date,
use_path='tables')
print path
union_count_file_path = path + 'UNION_COUNT'
print union_count_file_path
#print union_count_file_path
if os.path.exists(union_count_file_path):
print 'do it '
f = open(union_count_file_path, 'r')
for j in pickle.load(f):
print j
dat_lst.append(j)
return dat_lst
def read_file_with_filename_dict(file_name, from_date, to_date, server_id=10003, folder='tables'):
dat_dict_all = dict()
total_days = (to_date - from_date).days + 1
for i in xrange(total_days):
# 每行的日期
dat_dict = dict()
cur_date = from_date + datetime.timedelta(days=i)
SERVER_LOGS_DATA_LST = get_server_path(cur_date)
if -1 == server_id:
for each_ser_id in SERVER_LOGS_DATA_LST.keys():
file_path = SERVER_LOGS_DATA_LST[each_ser_id].format(cur_date=cur_date, use_path=folder) + file_name
print file_path
if os.path.exists(file_path):
out_put_file = open(file_path, 'r')
dat_dict = pickle.load(out_put_file)
for key, value in dat_dict.items():
dat_dict_all.setdefault(key, []).extend(value)
out_put_file.close()
# print "ss",dat_dict
else:
file_path = SERVER_LOGS_DATA_LST[server_id].format(cur_date=cur_date, use_path=folder) + file_name
print file_path
if os.path.exists(file_path):
out_put_file = open(file_path, 'r')
dat_dict = pickle.load(out_put_file)
for key, value in dat_dict.items():
dat_dict_all.setdefault(key, []).extend(value)
out_put_file.close()
# print "ss",dat_dict
return dat_dict_all
def cpickle_load_one_day(sreach_data,dir_name,file_name,server_id):
dat_lst = []
SERVER_LOGS_DATA_LST = get_server_path(sreach_data)
action_file_abs_path = SERVER_LOGS_DATA_LST[int(server_id)].format(cur_date = sreach_data,use_path = dir_name) + file_name
print action_file_abs_path
if os.path.exists(action_file_abs_path):
out_put_file = open(action_file_abs_path, 'r')
dat_lst = cPickle.load(out_put_file)
return dat_lst
def cpickle_load_one_day(sreach_data, dir_name, file_name, server_id):
dat_lst = []
SERVER_LOGS_DATA_LST = get_server_path(sreach_data)
action_file_abs_path = SERVER_LOGS_DATA_LST[int(server_id)].format(
cur_date=sreach_data, use_path=dir_name) + file_name
print action_file_abs_path
if os.path.exists(action_file_abs_path):
out_put_file = open(action_file_abs_path, 'r')
dat_lst = cPickle.load(out_put_file)
return dat_lst
def read_file(action,
from_date,
to_date,
server_id=10003,
folder='all_action'):
dat_lst = []
total_days = (to_date - from_date).days + 1
for i in xrange(total_days):
# 每行的日期
dat_dict = dict()
cur_date = from_date + datetime.timedelta(days=i)
# 注：用到通服的需要改
if server_id == -1:
for each_id in get_server_path(cur_date).keys():
SERVER_LOGS_DATA_LST = get_server_path(cur_date)
file_path = SERVER_LOGS_DATA_LST[each_id].format(
cur_date=cur_date, use_path=folder
) + game_define.EVENT_LOG_ACTION_SQL_NAME_DICT[action]
print file_path
if os.path.exists(file_path):
out_put_file = open(file_path, 'r')
dat_dict = pickle.load(out_put_file)
dat_lst.extend(dat_dict)
out_put_file.close()
# print dat_dict
else:
SERVER_LOGS_DATA_LST = get_server_path(cur_date)
file_path = SERVER_LOGS_DATA_LST[server_id].format(
cur_date=cur_date, use_path=folder
) + game_define.EVENT_LOG_ACTION_SQL_NAME_DICT[action]
print file_path
if os.path.exists(file_path):
out_put_file = open(file_path, 'r')
dat_dict = pickle.load(out_put_file)
dat_lst.extend(dat_dict)
out_put_file.close()
return dat_lst
def read_action_single_file(file_name,uid,event,sreach_data,server_id,dir_name):
event_id = int(event.split('-')[0])
CATCH_LOGS_DAT_LST = get_server_path(sreach_data)
action_file = CATCH_LOGS_DAT_LST[server_id].format(cur_date = sreach_data,use_path = dir_name)
action_file = action_file +str(uid) + os.sep + game_define.EVENT_LOG_ACTION_SQL_NAME_DICT[event_id]
print action_file
if os.path.exists(action_file):
with open(action_file, 'r') as out_put_file:
try:
while out_put_file:
yield cPickle.load(out_put_file)
except EOFError,e:
print e
def read_file_with_user_get(cur_date,server_id): dat_lst = [] OUT_PUT_PATH_LST=get_server_path(cur_date) if int(server_id) == -1: return dat_lst if not cur_date: return dat_lst log_path=OUT_PUT_PATH_LST[int(server_id)].format(cur_date=cur_date,use_path='tables') if os.path.exists(log_path ): union_count_file_path=log_path +'UNION_SHOP' if os.path.exists(union_count_file_path): f = open(union_count_file_path, 'r') dat_lst = pickle.load(f) return dat_lst
def read_file_with_single_day(file_name, search_date, server_id=10003, folder='tables'):
# dat_lst = []
dat_dict = []
# 每行的日期
SERVER_LOGS_DATA_LST = get_server_path(search_date)
path = SERVER_LOGS_DATA_LST[server_id].format(cur_date=search_date, use_path=folder)
if os.path.exists(path):
file_path = path + file_name
print file_path
if os.path.exists(file_path):
out_put_file = open(file_path, 'r')
dat_dict = pickle.load(out_put_file)
# dat_lst.extend([dat_dict])
out_put_file.close()
print dat_dict
return dat_dict
def read_file_with_user_get(search_start_date,server_id):
dat_lst = []
OUT_PUT_PATH_LST=get_server_path(search_start_date)
if int(server_id) == -1:
return dat_lst
if not search_start_date :
return dat_lst
log_path=OUT_PUT_PATH_LST[int(server_id)].format(cur_date=search_start_date,use_path='tables')
if os.path.exists(log_path):
union_buy_reward_file_path = log_path+ 'UNION_STAGE'
if os.path.exists(union_buy_reward_file_path):
f = pickle.load(open(union_buy_reward_file_path,'r'))
for i in f:
dat_lst.append(i)
return dat_lst
def read_action_single_file(file_name, uid, event, sreach_data, server_id,
dir_name):
event_id = int(event.split('-')[0])
CATCH_LOGS_DAT_LST = get_server_path(sreach_data)
action_file = CATCH_LOGS_DAT_LST[server_id].format(cur_date=sreach_data,
use_path=dir_name)
action_file = action_file + str(
uid) + os.sep + game_define.EVENT_LOG_ACTION_SQL_NAME_DICT[event_id]
print action_file
if os.path.exists(action_file):
with open(action_file, 'r') as out_put_file:
try:
while out_put_file:
yield cPickle.load(out_put_file)
except EOFError, e:
print e
def read_file_with_user_get(cur_date, server_id):
dat_lst = []
OUT_PUT_PATH_LST = get_server_path(cur_date)
if int(server_id) == -1:
return dat_lst
if not cur_date:
return dat_lst
log_path = OUT_PUT_PATH_LST[int(server_id)].format(cur_date=cur_date,
use_path='tables')
if os.path.exists(log_path):
union_count_file_path = log_path + 'UNION_SHOP'
if os.path.exists(union_count_file_path):
f = open(union_count_file_path, 'r')
dat_lst = pickle.load(f)
return dat_lst
def read_file_with_filename(file_name, from_date, to_date, server_id=10003, folder='tables'):
dat_lst = []
total_days = (to_date - from_date).days + 1
for i in xrange(total_days):
dat_dict = dict()
# 每行的日期
cur_date = from_date + datetime.timedelta(days=i)
SERVER_LOGS_DATA_LST = get_server_path(cur_date)
file_path = SERVER_LOGS_DATA_LST[server_id].format(cur_date=cur_date, use_path=folder) + file_name
print file_path
if os.path.exists(file_path):
out_put_file = open(file_path, 'r')
dat_dict = pickle.load(out_put_file)
dat_lst.extend(dat_dict)
out_put_file.close()
# print "ss",dat_dict
return dat_lst
def read_file_with_user_get(file_name,search_start_date,search_end_date,server_id):
data_lst = []
if int(server_id) == -1:
return data_lst
total_day = (search_end_date-search_start_date).days+1
print total_day
for i in xrange(total_day):
cur_date = search_start_date+datetime.timedelta(days=i)
OUT_PUT_PATH_LST=get_server_path(cur_date)
log_path=OUT_PUT_PATH_LST[int(server_id)].format(cur_date=cur_date,use_path='tables')
file_path = log_path+ file_name
if not os.path.exists(file_path):
continue
try:
data_lst.append(pickle.load(open(file_path, 'r')))
except :
pass
print data_lst
return data_lst
def read_file_with_single_day(file_name,
search_date,
server_id=10003,
folder='tables'):
# dat_lst = []
dat_dict = []
# 每行的日期
SERVER_LOGS_DATA_LST = get_server_path(search_date)
path = SERVER_LOGS_DATA_LST[server_id].format(cur_date=search_date,
use_path=folder)
if os.path.exists(path):
file_path = path + file_name
print file_path
if os.path.exists(file_path):
out_put_file = open(file_path, 'r')
dat_dict = pickle.load(out_put_file)
# dat_lst.extend([dat_dict])
out_put_file.close()
print dat_dict
return dat_dict
def read_file_with_user_get(search_start_date, server_id):
dat_lst = []
OUT_PUT_PATH_LST = get_server_path(search_start_date)
if int(server_id) == -1:
return dat_lst
else:
try:
log_path = OUT_PUT_PATH_LST[int(server_id)].format(
cur_date=search_start_date, use_path='tables')
except:
return dat_lst
if not search_start_date:
return dat_lst
if os.path.exists(log_path):
union_buy_reward_file_path = log_path + "%s" % 'UNION_BUY_REWARD'
if os.path.exists(union_buy_reward_file_path):
for i in pickle.load(open(union_buy_reward_file_path, 'r')):
dat_lst.append(i)
return dat_lst
def read_file_double_lst(file_name, from_date, to_date, server_id=10003, folder='tables'):
dat_lst1 = []
dat_lst2 = []
total_days = (to_date - from_date).days + 1
for i in xrange(total_days):
tmp_lst = []
# 每行的日期
cur_date = from_date + datetime.timedelta(days=i)
SERVER_LOGS_DATA_LST = get_server_path(cur_date)
file_path = SERVER_LOGS_DATA_LST[server_id].format(cur_date=cur_date, use_path=folder) + file_name
print file_path
if os.path.exists(file_path):
out_put_file = open(file_path, 'r')
tmp_lst = pickle.load(out_put_file)
dat_lst1.extend(tmp_lst[0])
dat_lst2.extend([tmp_lst[1]]) # 表头 只取一个最合适的
out_put_file.close()
# print tmp_lst
return dat_lst1, dat_lst2
def read_file_with_user_get(search_start_date, search_end_date, server_id): dat_lst = [] if int(server_id) == -1: return dat_lst if not search_end_date or not search_start_date: return dat_lst total_day = (search_end_date-search_start_date).days+1 for i in xrange(total_day): cur_date = search_start_date+datetime.timedelta(days=i) OUT_PUT_PATH=get_server_path(cur_date) log_path=OUT_PUT_PATH[int(server_id)].format(cur_date=cur_date,use_path='tables') union_sign_file_path = log_path+ 'UNION_SIGN' if os.path.exists(union_sign_file_path): for j in pickle.load(open(union_sign_file_path, 'r')): a = j dat_lst.append(a) return dat_lst
def read_file_with_user_get(file_name, search_start_date, search_end_date,
server_id):
data_lst = []
if int(server_id) == -1:
return data_lst
total_day = (search_end_date - search_start_date).days + 1
print total_day
for i in xrange(total_day):
cur_date = search_start_date + datetime.timedelta(days=i)
OUT_PUT_PATH_LST = get_server_path(cur_date)
log_path = OUT_PUT_PATH_LST[int(server_id)].format(cur_date=cur_date,
use_path='tables')
file_path = log_path + file_name
if not os.path.exists(file_path):
continue
try:
data_lst.append(pickle.load(open(file_path, 'r')))
except:
pass
print data_lst
return data_lst
def read_file_with_filename(file_name,
from_date,
to_date,
server_id=10003,
folder='tables'):
dat_lst = []
total_days = (to_date - from_date).days + 1
for i in xrange(total_days):
dat_dict = dict()
# 每行的日期
cur_date = from_date + datetime.timedelta(days=i)
SERVER_LOGS_DATA_LST = get_server_path(cur_date)
file_path = SERVER_LOGS_DATA_LST[server_id].format(
cur_date=cur_date, use_path=folder) + file_name
print file_path
if os.path.exists(file_path):
out_put_file = open(file_path, 'r')
dat_dict = pickle.load(out_put_file)
dat_lst.extend(dat_dict)
out_put_file.close()
# print "ss",dat_dict
return dat_lst
def read_file_with_filename_dict(file_name,
from_date,
to_date,
server_id=10003,
folder='tables'):
dat_dict_all = dict()
total_days = (to_date - from_date).days + 1
for i in xrange(total_days):
# 每行的日期
dat_dict = dict()
cur_date = from_date + datetime.timedelta(days=i)
SERVER_LOGS_DATA_LST = get_server_path(cur_date)
if -1 == server_id:
for each_ser_id in SERVER_LOGS_DATA_LST.keys():
file_path = SERVER_LOGS_DATA_LST[each_ser_id].format(
cur_date=cur_date, use_path=folder) + file_name
print file_path
if os.path.exists(file_path):
out_put_file = open(file_path, 'r')
dat_dict = pickle.load(out_put_file)
for key, value in dat_dict.items():
dat_dict_all.setdefault(key, []).extend(value)
out_put_file.close()
# print "ss",dat_dict
else:
file_path = SERVER_LOGS_DATA_LST[server_id].format(
cur_date=cur_date, use_path=folder) + file_name
print file_path
if os.path.exists(file_path):
out_put_file = open(file_path, 'r')
dat_dict = pickle.load(out_put_file)
for key, value in dat_dict.items():
dat_dict_all.setdefault(key, []).extend(value)
out_put_file.close()
# print "ss",dat_dict
return dat_dict_all
