def save_recon_data():
# 用户有效阅读数据
read_ids = get_read_ids()
# 推荐内容数据
recon_dict: dict = get_recons()
# 评测后台用户数据
eval_user: dict = {}
def func(line):
info = json.loads(line)
udid, name = info.get('udid'), info.get('name')
eval_user[udid] = name
file_util.read_file('/tmp/eval_user.txt', func)
path = '/tmp/推荐内容.csv'
data_list = []
for udid in recon_dict:
if udid not in read_ids: continue
user_name = eval_user.get(udid) # 用户名
deal_ids = recon_dict.get(udid) # 有效阅读折扣id
percent = cate_percent(deal_ids) # 有效阅读分类百分比
data_list.append(build_data(udid, user_name, len(deal_ids), percent))
print(len(data_list))
csv_util.cover(path, title, data_list)
def __recent_valid_deals__():
"""
从本地读取折扣tag信息，如果没有，就从mysql读
:return 结构如下
{
100:{
"is_hot":True,
"tags":["lining","anta"]
},
200:{
"is_hot":True,
"tags":["lining","man"]
}
}
"""
if os.path.exists(__tmp_deal_info_path__()):
info_d = {}
def func(line):
js = json.loads(line)
info_d[int(js['id'])] = js['info']
file_util.read_file(__tmp_deal_info_path__(), func)
return __check_deal_info__(info_d)
info_d = __read_from_db__()
data = []
for deal_id, info in info_d.items():
d = {'id': deal_id, 'info': info}
l = my_json.dumps(d)
data.append(l)
file_util.cover(__tmp_deal_info_path__(), data)
return __check_deal_info__(info_d)
def get_recons():
d = {}
def func(line):
splts = line.split(":")
udid = splts[0]
deal_ids = json.loads(splts[1])
d[udid] = deal_ids
file_util.read_file(recon_deal_ids, func)
return d
def init_cate_name_cache():
d = {}
def func(line):
splt = line.split(":")
d[int(splt[0])] = splt[1].replace("\n", "")
file_util.read_file(cate_name_file, func)
global cate_name_cache
cate_name_cache = d
print(cate_name_cache)
def init_cache():
d = {}
def func(line):
splt = line.split(":")
d[int(splt[0])] = json.loads(splt[1].replace("\n", ""))
file_util.read_file(deal_cate_lv1, func)
global cache
cache = d
print(cache)
def __rel_tags__():
# 读取相关tag信息
rel_d, cur = {}, ""
def func(line):
rel_d.setdefault(cur, {})
splt = line.split("\t")
rel_d[cur][splt[0]] = splt[1].strip().split(';')
for filename in __word_files__():
cur = __get_name__(filename.split(".")[0])
path = __tmp_dir__() + filename
file_util.read_file(path, func)
return rel_d
def parse_logs(test_num):
log(f'Parsing logs for test: {test_num}')
test_logs_path = get_logs_root_path(test_num)
logs_dir = f'{logs_dir_path}/{test_logs_path}/'
for log_dir in os.listdir(logs_dir):
fn_name = log_dir
fn_logs_dir = logs_dir + fn_name
image_size_mb = get_fn_image_size(fn_name)
csv_results = []
for log_file in os.listdir(fn_logs_dir):
log(f'Parsing logs file {log_file} in dir {fn_logs_dir}')
fn_log = read_file(fn_logs_dir + '/' + log_file)
csv_results.append([
image_size_mb,
get_memory_size(fn_log),
get_init_duration(fn_log),
get_artificial_init_duration(fn_log),
get_duration(fn_log),
get_available_threads(fn_log)
])
full_path = results_dir_path + test_logs_path + fn_name + '.csv'
write_to_csv(csv_results, results_header, full_path)
write_average_init_duration(test_num, results_dir_path + test_logs_path)
def read_v2_config() -> Optional[dict]:
try:
# path = config.get_v2_config_path()
# file_util.touch(path)
# with open(path, encoding='utf-8') as f:
# return f.read()
content = file_util.read_file(__v2ray_conf_path)
return json.loads(content)
# conf_path = '/usr/local/etc/v2ray/'
# files: List[str] = file_util.list_files(conf_path)
# v2_config: dict = {}
# for file_name in files:
# content: str = file_util.read_file(f'{conf_path}{file_name}')
# for conf_key in V2_CONF_KEYS:
# if conf_key in file_name:
# conf: dict = json.loads(content)
# if conf_key in ['inbounds', 'outbounds']:
# conf.setdefault(conf_key, [])
# else:
# conf.setdefault(conf_key, {})
# v2_config[conf_key] = conf[conf_key]
# break
# return v2_config
except Exception as e:
logging.error(
'An error occurred while reading the v2ray configuration file: ' +
str(e))
return None
def cal_all_cates(): """ 计算出所有的分类顺序 """ cate_list = [] s = set() def func(line): info = json.loads(line) info = info['percent'] for k in info: s.add(k) file_util.read_file(recon_file_lv1, func) file_util.read_file(read_file_lv1, func) for x in s: cate_list.append(x) print(cate_list)
def init():
"""
初始化缓存
"""
if not os.path.exists(local_file):
print("nofile")
return
global cache
def func(line):
row = json.loads(line)
cache[int(row['id'])] = row
file_util.read_file(local_file, func)
print("一共[", len(cache), "]条分类信息")
def read_v2_config() -> Optional[dict]:
try:
content = file_util.read_file(__conf_path)
return json.loads(content)
except Exception as e:
logging.error(
f"An error occurred while reading the {__v2_cmd_name} configuration file: "
+ str(e))
return None
def raw_to_csv():
"""
返回本地文件中的数据
:return:
"""
recon_dict, read_dict = {}, {}
def recon_func(line):
info = json.loads(line)
recon_dict[info['udid']] = info['percent']
def read_func(line):
info = json.loads(line)
read_dict[info['udid']] = info['percent']
file_util.read_file(recon_file_lv1, recon_func)
file_util.read_file(read_file_lv1, read_func)
print(recon_dict)
print(read_dict)
return recon_dict, read_dict
def get_read_ids():
"""
提取用户有效阅读的数据
:return {
udid: [dealId1, dealId2]
}
"""
user_deal_ids = {}
def func(line):
if line.__contains__('+'): return
if line.__contains__('temp_deal_read.udid'): return
splts = line.split('|')
if splts[2].__contains__('NULL'): return
udid = splts[1].strip()
deal_id = int(splts[2].strip())
user_deal_ids.setdefault(udid, [])
user_deal_ids.get(udid).append(deal_id)
file_util.read_file(user_deal_read_file, func)
return user_deal_ids
def analysis():
all_deals = []
def func(line):
d = json.loads(line)
all_deals.append(d)
file_util.read_file(tmp_file, func)
no_category, nocategory_hot = 0, 0
# 折扣分类统计， 热门折扣分类统计
category_percent, category_hot_percent = {
"name": "有效折扣"
}, {
"name": "有效热门折扣"
}
hot_total = 0
for deal in all_deals:
# 统计总数
if int(deal['is_hotpick']) == 1: hot_total += 1
# 没有分类
if not deal.get('category'):
no_category += 1
if int(deal['is_hotpick']) == 1: nocategory_hot += 1
continue
has_category = False
for y in deal['category'].split(','):
category_id = top_category(int(y))
label_cn = category.get_attr_in_order(category_id)
if label_cn:
has_category = True
else:
continue
# 折扣统计
category_percent.setdefault(label_cn, 0)
category_percent[label_cn] += 1
# 热门折扣统计
if int(deal['is_hotpick']) == 1:
category_hot_percent.setdefault(label_cn, 0)
category_hot_percent[label_cn] += 1
if not has_category:
print("分类有问题", deal.get('id'), deal['category'])
no_category += 1
if int(deal['is_hotpick']) == 1:
nocategory_hot += 1
category_percent['无分类'] = no_category
category_hot_percent['无分类'] = nocategory_hot
category_percent['总数'] = len(all_deals)
category_hot_percent['总数'] = hot_total
title = ['name', '总数']
for k in category_percent:
if k not in title:
title.append(k)
print(category_percent)
print(category_hot_percent)
csv_util.cover(deal_analysis_file, title,
[category_percent, category_hot_percent])