def check_rank_rose(self, bv_id: str, rank_info: dict):
""" check rank rose """
if not self.check_rank_info(bv_id, rank_info):
return
idx, pts = rank_info["id"], rank_info["pts"]
b_id = bv_id + str(rank_info["day"]) + str(rank_info["type"])
if b_id not in self.rank["T"]:
self.rank["T"][b_id] = [idx // 10]
else:
self.rank["T"][b_id].append(idx // 10)
self.rank["L"][b_id] = idx
is_hot = self.is_hot(bv_id)
is_hot = "[热门]" if is_hot else ""
title = (self.bv_ids[bv_id]["title"].split("|", 1)[0]
if bv_id in self.bv_ids else "")
rank_str = "热榜{}(%s){}|{}Day List, Rank: {}, Score: {}".format(
is_hot, title, rank_info["day"], idx, rank_info["pts"])
if bv_id in self.bv_ids:
created = self.bv_ids[bv_id]["created"]
ts = get_time_str((time.time() - created) / 60)
ts_str = time_str(created) + "-" + time_str()
else:
ts, ts_str = "", ""
rank_context = rank_str % ts_str
rank_str = rank_str % ts
send_email(rank_context, rank_str)
def get_check(self):
''' check comment '''
self.load_av_lists()
av_id_list = [[ii['aid'], ii['comment']]
for ii in self.av_id_map.values()
if not re.findall(self.ignore_list, str(ii['aid']))]
av_map = {ii['aid']: ii for ii in self.av_id_map.values()}
self.comment_next = {ii: True for (ii, _) in av_id_list}
if self.av_id_list and len(
self.av_id_list) and len(self.av_id_list) != len(av_id_list):
new_av_id = [
ii for (ii, _) in av_id_list
if not ii in self.av_id_list and not ii in self.del_map
]
self.rank_map = {**self.rank_map, **{ii: [] for ii in new_av_id}}
echo(1, new_av_id)
for ii in new_av_id:
shell_str = 'nohup ipython3 bilibili/bsocket.py {} %d >> log.txt 2>&1 &'.format(
ii)
echo(0, shell_str)
os.system(shell_str % 1)
os.system(shell_str % 2)
email_str = '{} av:{} was releasing at {}!!! Please check the auto pipeline.'.format(
av_map[ii]['title'], ii, time_str(av_map[ii]['created']))
email_str2 = '{} {} is release at {}.\nPlease check the online & common program.\n\nBest wish for you\n--------\nSend from script by gunjianpan.'.format(
av_map[ii]['title'], time_str(av_map[ii]['created']),
self.BASIC_AV_URL % ii)
send_email(email_str2, email_str)
self.update_ini(ii)
self.public[ii] = [av_map[ii]['created'], av_map[ii]['mid']]
self.av_id_list = [ii for (ii, _) in av_id_list]
now_hour = int(time_str(time_format='%H'))
now_min = int(time_str(time_format='%M'))
now_time = now_hour + now_min / 60
if now_time > self.ignore_start and now_time < self.ignore_end:
return
if os.path.exists('{}comment.pkl'.format(comment_dir)):
with codecs.open('{}comment.pkl'.format(comment_dir), 'rb') as f:
self.comment = pickle.load(f)
if self.assign_up_mid == -1:
return
threading_list = []
for (ii, jj) in av_id_list:
if ii not in self.comment:
self.comment[ii] = {}
work = threading.Thread(target=self.comment_check_schedule,
args=(
ii,
jj,
))
threading_list.append(work)
for work in threading_list:
work.start()
for work in threading_list:
work.join()
with codecs.open('{}comment.pkl'.format(comment_dir), 'wb') as f:
pickle.dump(self.comment, f)
return av_id_list
def send_email_notification(list_completed_backups, list_notification_emails):
global g
body = ""
sep = ""
backup_completed_str = "Backup(s) completed"
for completed_backup in list_completed_backups:
folder_name = completed_backup[0]
url = completed_backup[1]
expiry_days = completed_backup[2]
body = (
body
+ sep
+ "Completed "
+ folder_name
+ " backup which is accessible at "
+ url
+ " for "
+ str(expiry_days)
+ " days."
)
sep = "\r\n\r\n"
if g.run_util_errors is not None and len(g.run_util_errors) > 0:
body = (
body
+ sep
+ "There were errors running the following utility(s): "
+ ", ".join(g.run_util_errors)
+ ". See messages_xxx.log in backup zip file for details."
)
backup_completed_str = backup_completed_str + " with errors"
util.send_email(list_notification_emails, backup_completed_str, body)
def have_places_once(self):
"""
have places
"""
url = 'http://elective.pku.edu.cn/elective2008/edu/pku/stu/elective/controller/supplement/refreshLimit.do'
if not os.path.exists('%scookie' % data_path):
print('Brush Cookie not exist!!!')
return
with open('%scookie' % data_path, 'r') as f:
cookie = f.readlines()
headers = {
'X-Requested-With': 'XMLHttpRequest',
'Cookie': '',
'Content-Type': get_content_type(),
'Accept': get_accept('xhr'),
"Origin": "http://elective.pku.edu.cn",
"Referer": "http://elective.pku.edu.cn/elective2008/edu/pku/stu/elective/controller/supplement/SupplyCancel.do",
}
headers['Cookie'] = cookie[0][:-1]
data = {
"index": '10',
"seq": 'yjkc20141100016542',
}
ca = proxy_req(url, 11, data, header=headers)
if not ca:
if round(time.time()) - self.laster_timestamp > 60:
send_email("Cookie failure", "Cookie failure")
return False
print(ca['electedNum'])
self.laster_timestamp = round(time.time())
return int(ca['electedNum']) < 120
def history_rank(self, time_gap: int, now_info: list, av_id: int):
echo(0, 'send history rank')
time_gap = round(time_gap / 10) * 10
history_map = {
ii: jj
for ii, jj in self.history_map[time_gap].items() if jj[1]
}
other_views = [int(ii[1]) for ii in history_map.values()]
other_views_len = len(other_views)
other_views.append(now_info[1])
ov_sort_idx = np.argsort(-np.array(other_views))
av_ids = list(history_map.keys())
now_sorted = [
jj for jj, ii in enumerate(ov_sort_idx) if ii == other_views_len
][0] + 1
other_result = [(jj + 1, av_ids[ii])
for jj, ii in enumerate(ov_sort_idx[:4])
if ii != other_views_len]
time_tt = self.get_time_str(time_gap)
email_title = 'av{}发布{}, 本年度排名No.{}/{}, 播放量: {}, 点赞: {}, 硬币: {}, 收藏: {}, 弹幕: {}'.format(
av_id, time_tt, now_sorted, len(other_views), now_info[1],
now_info[2], now_info[3], now_info[4], now_info[7])
email_title += self.get_history_rank(now_info)
context = '{}\n\n'.format(email_title)
for no, av in other_result[:3]:
data_info = history_map[av]
context += '{}, av{}, 本年度No.{}, 播放量: {}, 点赞: {}, 硬币: {}, 收藏: {}, 弹幕: {}{}, 发布时间: {}\n'.format(
self.av_id_map[av]['title'].split('|',
1)[0], av, no, data_info[1],
data_info[2], data_info[3], data_info[4], data_info[7],
self.get_history_rank(data_info),
time_str(self.av_id_map[av]['created']))
context += '\nBest wish for you\n--------\nSend from script by gunjianpan.'
send_email(context, email_title)
self.history_check_finish.append(round(time_gap / 10))
def have_bad_comment(self,
req_list: list,
av_id: int,
pn: int,
parent_rpid=None):
""" check comment and send warning email if error """
rpid, ctime, like, plat, current_level, uname, sex, content, sign, idx, sort = (
req_list)
ctimes = time_str(ctime, time_format=self.T_FORMAT)
ctime = time_str(ctime)
if not len(regex.findall(self.keyword, content)):
return True
rpid = "{}{}".format(
rpid, "" if not parent_rpid else "-{}".format(parent_rpid))
url = self.BASIC_AV_URL % av_id
rpid_str = "{}-{}".format(av_id, rpid)
if rpid in [kk for ii in self.ignore_rpid.values() for kk in ii]:
return True
if self.email_limit < 1 or (
rpid_str in self.email_send_time
and self.email_send_time[rpid_str] >= self.email_limit):
return True
if rpid_str in self.email_send_time:
self.email_send_time[rpid_str] += 1
else:
self.email_send_time[rpid_str] = 1
rank_info = [
r_info for bv_id, r_info in self.bv_ids.items()
if r_info["aid"] == av_id
][0]
title = rank_info["title"].split("|", 1)[0]
sort = "热门" if sort else "时间"
email_content = "Date: {}\nUrl: {}\nTitle: {},\nPage: {} #{}@{},\nUser: {},\nSex: {},\nsign: {}\nlike: {}\nplat: {}\nlevel:{}\nconetnt: {},\n".format(
ctime,
title,
url,
pn,
idx,
rpid,
uname,
sex,
sign,
like,
plat,
current_level,
content,
)
email_subject = "评论({}){}{}{}#{}".format(ctimes, title, sort, pn, idx)
echo("4|warning", email_content, email_subject)
send_email(email_content, email_subject, assign_rec=self.assign_rec)
def have_places_once(self):
"""
have places
"""
url = 'http://elective.pku.edu.cn/elective2008/edu/pku/stu/elective/controller/supplement/refreshLimit.do'
if not os.path.exists('%scookie' % data_path):
print('Brush Cookie not exist!!!')
return
with open('%scookie' % data_path, 'r') as f:
cookie = f.readlines()
headers = {
'pragma':
'no-cache',
'X-Requested-With':
'XMLHttpRequest',
'cache-control':
'no-cache',
'Cookie':
'',
'Content-Type':
'application/x-www-form-urlencoded;charset=UTF-8',
'Accept':
'application/json, text/javascript, */*; q=0.01',
"Accept-Encoding":
"",
"Accept-Language":
"zh-CN,zh;q=0.9",
"User-Agent":
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3682.0 Safari/537.36",
"Origin":
"http://elective.pku.edu.cn",
"Referer":
"http://elective.pku.edu.cn/elective2008/edu/pku/stu/elective/controller/supplement/SupplyCancel.do",
}
headers['Cookie'] = cookie[0][:-1]
data = {
"index": '10',
"seq": 'yjkc20141100016542',
}
ca = proxy_req(url, 11, data, header=headers)
if not ca:
if round(time.time()) - self.laster_timestamp > 60:
send_email("Cookie failure", "Cookie failure")
return False
print(ca['electedNum'])
self.laster_timestamp = round(time.time())
return int(ca['electedNum']) < 120
def send_email_notification(list_completed_backups, list_notification_emails):
global g
body = ''
sep = ''
backup_completed_str = 'Backup(s) completed'
for completed_backup in list_completed_backups:
folder_name = completed_backup[0]
url = completed_backup[1]
expiry_days = completed_backup[2]
body = body + sep + 'Completed ' + folder_name + ' backup which is accessible at ' + url + ' for ' + \
str(expiry_days) + ' days.'
sep = '\r\n\r\n'
body += '\r\n\r\nSent from local IP address ' + util.get_ip_address()
util.send_email(list_notification_emails, backup_completed_str, body)
def check_rank_rose(self, av_id: int, rank_list: list):
''' check rank rose '''
if not self.check_rank_list(av_id, rank_list):
return
rank, score = rank_list[:2]
av_id_id = int(av_id) * 10 + int(rank_list[-1])
if av_id_id not in self.rank:
self.rank[av_id_id] = [rank_list[0] // 10]
else:
self.rank[av_id_id].append(rank_list[0] // 10)
self.last_rank[av_id_id] = rank_list[0]
send_email(
'%d day List || Rank: %d Score: %d' %
(int(rank_list[-1]), rank, score),
'%d day List || Rank: %d Score: %d' %
(int(rank_list[-1]), rank, score))
def check_overdue(self):
def check_overdue_once(data: list) -> bool:
dif_time = time_stamp(data[-2]) - time_stamp()
return dif_time > 0 and dif_time <= self.ONE_HOURS * self.ONE_DAY
overdue_article = [(article_id, article_list[4]) for article_id, ii in self.tpwd_db_map.items() for article_list in ii.values() if check_overdue_once(article_list)]
overdue_id = set([article_id for article_id, _ in overdue_article])
overdue_list = [(article_id, len([1 for a_id, tpwd in overdue_article if article_id == a_id])) for article_id in overdue_id]
if not len(overdue_list):
return
title = '链接需要更新#{}#篇'.format(len(overdue_list))
content = title + '\n \n'
for article_id, num in overdue_list:
content += '{}, 需要更新{}个链接，{}\n'.format(self.share2article[article_id][2], num, self.NOTE_URL % article_id)
content += '\n\nPlease update within 6 hours, Thx!'
echo('2|debug', title, content)
send_email(content, title)
def email_update_result(self, article_id: str, r_log: list, r_num: int):
p = self.share2article[article_id][-2].split("/")[-1]
article_info = self.list_recent[p]
name = article_info["name"].replace(".note", "")
subject = "更新({}){}/{}条[{}]".format(
time_str(time_format=self.T_FORMAT), r_num, len(r_log), article_info["name"]
)
content = "\n".join(
[
"Title: {}".format(article_info["name"]),
"Time: {}".format(time_str()),
"Update Num: {}/{}条".format(r_num, len(r_log)),
"",
*r_log,
]
)
send_email(content, subject, assign_rec=self.assign_rec)
def load_rank(self):
''' load rank '''
assign_1 = self.load_rank_index(1, 1)
assign_2 = self.load_rank_index(1, 3)
have_assign = assign_1 or assign_2
print(assign_1, assign_2, have_assign)
if self.have_assign and not have_assign:
send_email('No rank.....No Rank......No Rank.....',
'No rank.....No Rank......No Rank.....')
self.have_assign = have_assign
print('Rank_map_len:', len(self.rank_map.keys()), 'Empty:',
len([1 for ii in self.rank_map.values() if not len(ii)]))
youshan = [
','.join([str(kk) for kk in [ii, *jj]])
for ii, jj in self.rank_map.items()
]
with codecs.open(data_dir + 'youshang', 'w', encoding='utf-8') as f:
f.write('\n'.join(youshan))
def load_rank(self):
""" load rank """
self.load_rank_index(1, 3)
self.load_rank_index(1, 1)
assign_1, assign_2 = self.have_assign[1], self.have_assign[3]
have_assign = assign_1 + assign_2
echo("4|debug", assign_1, assign_2, have_assign)
not_ranks = [
ii for ii in self.have_assign["T"] if not ii in have_assign
]
self.have_assign = have_assign
echo(
"4|debug",
"Rank_map_len:",
len(self.rank_map.keys()),
"Empty:",
len([1 for ii in self.rank_map.values() if not len(ii)]),
)
youshan = [
",".join([str(kk) for kk in [ii, *list(jj.values())]])
for ii, jj in self.rank_map.items()
]
with codecs.open(data_dir + "youshang", "w", encoding="utf-8") as f:
f.write("\n".join(youshan))
if not len(not_ranks):
return
for bv_id in not_ranks:
title = (self.bv_ids[bv_id]["title"].split("|", 1)[0]
if bv_id in self.bv_ids else "")
no_rank_warning = "下榜({}){},{}".format(
time_str(time_format=self.T_FORMAT), title,
self.NO_RANK_CONSTANT)
send_email(no_rank_warning, no_rank_warning,
self.special_info_email)
time.sleep(pow(np.pi, 2))
send_email(no_rank_warning, no_rank_warning,
self.special_info_email)
echo("4|error", no_rank_warning)
def have_places(self):
"""
brush class
"""
version = begin_time()
have_places = False
while not have_places:
if self.have_places_once():
send_email('大数据专题', '大数据专题 有名额啦 有名额啦')
send_email('大数据专题', '大数据专题 有名额啦 有名额啦')
send_email('大数据专题', '大数据专题 有名额啦 有名额啦')
have_places = True
time.sleep(random.randint(10, 20))
end_time(version)
def have_bad_comment(self,
req_list: list,
av_id: int,
pn: int,
parent_rpid=None):
''' check comment and send warning email if error '''
rpid, ctime, like, _, _, uname, sex, content, sign, idx = req_list
if not len(re.findall(self.keyword, content)):
return True
rpid = '{}{}'.format(rpid,
'' if not parent_rpid else '-{}'.format(rpid))
url = self.BASIC_AV_URL % av_id
rpid_str = '{}-{}'.format(av_id, rpid)
if str(av_id) in self.ignore_rpid and rpid in self.ignore_rpid[str(
av_id)]:
return True
if self.email_limit < 1 or (
rpid_str in self.email_send_time
and self.email_send_time[rpid_str] > self.email_limit):
return True
email_content = '%s\nUrl: %s Page: %d #%d@%s,\nUser: %s,\nSex: %s,\nconetnt: %s,\nsign: %s\nlike: %d' % (
ctime, url, pn, idx, rpid, uname, sex, content, sign, like)
email_subject = '(%s)av_id: %s || #%s Comment Warning !!!' % (
ctime, av_id, rpid)
print(email_content, email_subject)
send_email_time = 0
send_email_result = False
while not send_email_result and send_email_time < 4:
send_email_result = send_email(email_content, email_subject)
send_email_time += 1
if rpid_str in self.email_send_time:
self.email_send_time[rpid_str] += 1
else:
self.email_send_time[rpid_str] = 0
def get_check(self):
""" check comment """
self.delay_load_history_data()
bv_list = [[ii["bvid"], ii["aid"], ii["comment"]]
for ii in self.bv_ids.values()
if not regex.findall(self.ignore_list, str(ii["aid"]))]
bv_map = {ii["bvid"]: ii for ii in self.bv_ids.values()}
if self.bv_list and len(
self.bv_list) and len(self.bv_list) != len(bv_list):
new_bv_list = [(ii, jj) for ii, jj, _ in bv_list
if not ii in self.bv_list and not ii in self.del_map
]
self.rank_map = {
**self.rank_map,
**{ii: {}
for ii, _ in new_bv_list}
}
echo("1|error", "New Bv av ids:", new_bv_list)
for bv_id, av_id in new_bv_list:
rank_info = bv_map[bv_id]
shell_str = "nohup python3 bilibili/bsocket.py {} %d >> log.txt 2>&1 &".format(
av_id)
echo("0|error", "Shell str:", shell_str)
os.system(shell_str % 1)
os.system(shell_str % 2)
email_str = "发布({}){}#{} {}".format(
time_str(rank_info["created"], time_format=self.T_FORMAT),
rank_info["title"],
bv_id,
av_id,
)
email_str2 = "{} {} is release at {}.\nPlease check the online & common program.".format(
rank_info["title"],
time_str(rank_info["created"]),
self.BASIC_BV_URL % bv_id,
)
send_email(email_str2, email_str, self.special_info_email)
self.update_ini(bv_id, av_id)
self.public["T"][bv_id] = [
rank_info["created"], rank_info["mid"]
]
self.last_check[bv_id] = int(time_stamp())
self.bv_list = [ii for (ii, _, _) in bv_list]
now_hour = int(time_str(time_format="%H"))
now_min = int(time_str(time_format="%M"))
now_time = now_hour + now_min / 60
if now_time > self.ignore_start and now_time < self.ignore_end:
return
if self.assign_mid == -1:
return
# threads = [self.pool.submit(self.check_type_req, bv_id) for bv_id in rank_map.keys()]
# list(as_completed(threads))
threading_list = []
for (_, ii, jj) in bv_list:
work = threading.Thread(target=self.comment_check_schedule,
args=(ii, jj))
threading_list.append(work)
for work in threading_list:
work.start()
for work in threading_list:
work.join()
return bv_list
def history_rank(self, time_gap: int, now_info: dict, bv_id: int):
echo("0|info", "send history rank")
time_gap = int(time_gap / 10) * 10
history_map = {
ii: jj
for ii, jj in self.history_map[time_gap].items() if jj[1]
}
if len(history_map) < 5:
self.load_history_data()
other_views = [int(ii[1]) for ii in history_map.values()]
other_views_len = len(other_views)
other_views.append(now_info["view"])
ov_sort_idx = np.argsort(-np.array(other_views))
av_ids = list(history_map.keys())
now_sorted = [
jj for jj, ii in enumerate(ov_sort_idx) if ii == other_views_len
][0] + 1
other_result = [(jj + 1, av_ids[ii])
for jj, ii in enumerate(ov_sort_idx[:4])
if ii != other_views_len]
time_tt = get_time_str(time_gap)
rank_info = self.bv_ids[bv_id]
title = rank_info["title"].split("|", 1)[0]
title_email = "排名(发布{}){}本年度排名No.{}/{}, 播放量: {}, 点赞: {}, 硬币: {}, 收藏: {}, 弹幕: {}".format(
time_tt,
title,
now_sorted,
len(other_views),
now_info["view"],
now_info["like"],
now_info["coin"],
now_info["favorite"],
now_info["danmaku"],
)
email_title = "bv{}发布{}, 本年度排名No.{}/{}, 播放量: {}, 点赞: {}, 硬币: {}, 收藏: {}, 弹幕: {}".format(
bv_id,
time_tt,
now_sorted,
len(other_views),
now_info["view"],
now_info["like"],
now_info["coin"],
now_info["favorite"],
now_info["danmaku"],
)
email_title += self.get_history_rank(now_info)
context = "{}\n\n".format(email_title)
for no, bv in other_result[:3]:
data_info = history_map[bv]
context += "{}, bv{}, 本年度No.{}, 播放量: {}, 点赞: {}, 硬币: {}, 收藏: {}, 弹幕: {}, 累计播放: {}{}, 发布时间: {}\n".format(
self.bv_ids[bv]["title"].split("|", 1)[0],
bv,
no,
data_info[1],
data_info[2],
data_info[3],
data_info[4],
data_info[7],
self.bv_ids[bv]["play"],
self.get_history_rank(data_info),
time_str(self.bv_ids[bv]["created"]),
)
send_email(context, title_email)
self.check_done[bv_id].append(round(time_gap / 10))
if not os.path.isfile("tmp/predictions_%s.csv" % today):
sys.stdout.write("Predictions for %s haven't been made yet..." % today)
sys.exit(1)
# read in and prepare prediction/metrics dfs
project_dir = os.environ["PROJECT_DIR"]
predictions = pd.read_csv(project_dir + "tmp/predictions_%s.csv" % today)
predictions["date"] = pd.to_datetime(predictions["date"])
predictions = predictions.sort_values(by="date").reset_index(drop=True)
metrics = pd.read_csv(project_dir + "data/metrics.csv")
# send email with predictions/historical metrics
send_from = "*****@*****.**"
send_to = ["*****@*****.**"
] #, "*****@*****.**"]
subject = "NBA Predictions %s" % datetime.datetime.strptime(
today, "%Y%m%d").strftime("%m-%d-%Y")
text = "Predictions for games happening on %s (all times Eastern): <br><br>" % datetime.datetime.strptime(
today, "%Y%m%d").strftime("%m-%d-%Y")
text += predictions.to_html()
text += "<br> Notes:"
text += "<ul><li> Spread is from the perspective of the home team, e.g., -4 means the home team is predicted to lose by 4.</li>"
text += "<li> A '1' in the moneyline column means that the home team is predicted to win; '0' means they are predicted to lose.</li></ul>"
text += "10-fold cross validation scores:<br><br>"
text += metrics.to_html()
text += "<br>Disclaimers:"
text += "<ul><li>This doesn't take into account injuries, roster moves, or schedule oddities (e.g., back-to-back, long road trips, etc.).</li>"
text += "<li>Use this only as a rough guide and a supplement to personal research.</li></ul>"
send_email(send_from=send_from, send_to=send_to, subject=subject, text=text)
