def get_check(self): ''' check comment ''' self.load_av_lists() av_id_list = [[ii['aid'], ii['comment']] for ii in self.av_id_map.values() if not re.findall(self.ignore_list, str(ii['aid']))] av_map = {ii['aid']: ii for ii in self.av_id_map.values()} self.comment_next = {ii: True for (ii, _) in av_id_list} if self.av_id_list and len( self.av_id_list) and len(self.av_id_list) != len(av_id_list): new_av_id = [ ii for (ii, _) in av_id_list if not ii in self.av_id_list and not ii in self.del_map ] self.rank_map = {**self.rank_map, **{ii: [] for ii in new_av_id}} echo(1, new_av_id) for ii in new_av_id: shell_str = 'nohup ipython3 bilibili/bsocket.py {} %d >> log.txt 2>&1 &'.format( ii) echo(0, shell_str) os.system(shell_str % 1) os.system(shell_str % 2) email_str = '{} av:{} was releasing at {}!!! Please check the auto pipeline.'.format( av_map[ii]['title'], ii, time_str(av_map[ii]['created'])) email_str2 = '{} {} is release at {}.\nPlease check the online & common program.\n\nBest wish for you\n--------\nSend from script by gunjianpan.'.format( av_map[ii]['title'], time_str(av_map[ii]['created']), self.BASIC_AV_URL % ii) send_email(email_str2, email_str) self.update_ini(ii) self.public[ii] = [av_map[ii]['created'], av_map[ii]['mid']] self.av_id_list = [ii for (ii, _) in av_id_list] now_hour = int(time_str(time_format='%H')) now_min = int(time_str(time_format='%M')) now_time = now_hour + now_min / 60 if now_time > self.ignore_start and now_time < self.ignore_end: return if os.path.exists('{}comment.pkl'.format(comment_dir)): with codecs.open('{}comment.pkl'.format(comment_dir), 'rb') as f: self.comment = pickle.load(f) if self.assign_up_mid == -1: return threading_list = [] for (ii, jj) in av_id_list: if ii not in self.comment: self.comment[ii] = {} work = threading.Thread(target=self.comment_check_schedule, args=( ii, jj, )) threading_list.append(work) for work in threading_list: work.start() for work in threading_list: work.join() with codecs.open('{}comment.pkl'.format(comment_dir), 'wb') as f: pickle.dump(self.comment, f) return av_id_list
def check_rank_rose(self, bv_id: str, rank_info: dict): """ check rank rose """ if not self.check_rank_info(bv_id, rank_info): return idx, pts = rank_info["id"], rank_info["pts"] b_id = bv_id + str(rank_info["day"]) + str(rank_info["type"]) if b_id not in self.rank["T"]: self.rank["T"][b_id] = [idx // 10] else: self.rank["T"][b_id].append(idx // 10) self.rank["L"][b_id] = idx is_hot = self.is_hot(bv_id) is_hot = "[热门]" if is_hot else "" title = (self.bv_ids[bv_id]["title"].split("|", 1)[0] if bv_id in self.bv_ids else "") rank_str = "热榜{}(%s){}|{}Day List, Rank: {}, Score: {}".format( is_hot, title, rank_info["day"], idx, rank_info["pts"]) if bv_id in self.bv_ids: created = self.bv_ids[bv_id]["created"] ts = get_time_str((time.time() - created) / 60) ts_str = time_str(created) + "-" + time_str() else: ts, ts_str = "", "" rank_context = rank_str % ts_str rank_str = rank_str % ts send_email(rank_context, rank_str)
def check_rank(self, av_id: int, times=0): rank_list = self.rank_map[av_id] if av_id in self.rank_map else [] changeHeaders({'Referer': self.BASIC_AV_URL % av_id}) url = self.ARCHIVE_STAT_URL % av_id json_req = proxy_req(url, 1) if not self.have_error(json_req): if (av_id not in self.av_id_list and times < 3) or (av_id in self.av_id_list and times < 6): self.check_rank(av_id, times + 1) return json_req = json_req['data'] need = [ 'view', 'like', 'coin', 'favorite', 'reply', 'share', 'danmaku' ] data = [json_req[index] for index in need] if not self.check_view(av_id, data[0]): if times < 3: self.check_rank(av_id, times + 1) return if len(rank_list): data = [time_str(), *data, *rank_list[:2], *rank_list[-2:]] else: data = [time_str(), *data] with codecs.open('%s%d.csv' % (history_dir, av_id), 'a', encoding='utf-8') as f: f.write(','.join([str(index) for index in data]) + '\n') if av_id in self.last_check and int( time.time()) - self.last_check[av_id] > one_day: self.del_map[av_id] = 1 del self.rank_map[av_id] if av_id == self.basic_av_id: clean_csv(av_id) elif av_id not in self.last_check and int( time.time()) > one_day + self.begin_timestamp: self.del_map[av_id] = 1 del self.rank_map[av_id] if av_id == self.basic_av_id: clean_csv(av_id) self.last_view[av_id] = data[1] now_time = time.time() echo(0, av_id, av_id == self.basic_av_id, av_id in self.public, (now_time - self.public[av_id][0]) < 3.1 * one_day * 60, self.public[av_id]) if av_id == self.basic_av_id and av_id in self.public and ( now_time - self.public[av_id][0]) < 3.1 * one_day * 60: time_gap = (now_time - self.public[av_id][0]) / 60 echo(3, 'Time Gap:', round(time_gap / 10)) if round(time_gap / 10) in self.history_check_list and round( time_gap / 10) not in self.history_check_finish: self.history_rank(time_gap, data, av_id)
def have_bad_comment(self, req_list: list, av_id: int, pn: int, parent_rpid=None): """ check comment and send warning email if error """ rpid, ctime, like, plat, current_level, uname, sex, content, sign, idx, sort = ( req_list) ctimes = time_str(ctime, time_format=self.T_FORMAT) ctime = time_str(ctime) if not len(regex.findall(self.keyword, content)): return True rpid = "{}{}".format( rpid, "" if not parent_rpid else "-{}".format(parent_rpid)) url = self.BASIC_AV_URL % av_id rpid_str = "{}-{}".format(av_id, rpid) if rpid in [kk for ii in self.ignore_rpid.values() for kk in ii]: return True if self.email_limit < 1 or ( rpid_str in self.email_send_time and self.email_send_time[rpid_str] >= self.email_limit): return True if rpid_str in self.email_send_time: self.email_send_time[rpid_str] += 1 else: self.email_send_time[rpid_str] = 1 rank_info = [ r_info for bv_id, r_info in self.bv_ids.items() if r_info["aid"] == av_id ][0] title = rank_info["title"].split("|", 1)[0] sort = "热门" if sort else "时间" email_content = "Date: {}\nUrl: {}\nTitle: {},\nPage: {} #{}@{},\nUser: {},\nSex: {},\nsign: {}\nlike: {}\nplat: {}\nlevel:{}\nconetnt: {},\n".format( ctime, title, url, pn, idx, rpid, uname, sex, sign, like, plat, current_level, content, ) email_subject = "评论({}){}{}{}#{}".format(ctimes, title, sort, pn, idx) echo("4|warning", email_content, email_subject) send_email(email_content, email_subject, assign_rec=self.assign_rec)
def history_rank(self, time_gap: int, now_info: list, av_id: int): echo(0, 'send history rank') time_gap = round(time_gap / 10) * 10 history_map = { ii: jj for ii, jj in self.history_map[time_gap].items() if jj[1] } other_views = [int(ii[1]) for ii in history_map.values()] other_views_len = len(other_views) other_views.append(now_info[1]) ov_sort_idx = np.argsort(-np.array(other_views)) av_ids = list(history_map.keys()) now_sorted = [ jj for jj, ii in enumerate(ov_sort_idx) if ii == other_views_len ][0] + 1 other_result = [(jj + 1, av_ids[ii]) for jj, ii in enumerate(ov_sort_idx[:4]) if ii != other_views_len] time_tt = self.get_time_str(time_gap) email_title = 'av{}发布{}, 本年度排名No.{}/{}, 播放量: {}, 点赞: {}, 硬币: {}, 收藏: {}, 弹幕: {}'.format( av_id, time_tt, now_sorted, len(other_views), now_info[1], now_info[2], now_info[3], now_info[4], now_info[7]) email_title += self.get_history_rank(now_info) context = '{}\n\n'.format(email_title) for no, av in other_result[:3]: data_info = history_map[av] context += '{}, av{}, 本年度No.{}, 播放量: {}, 点赞: {}, 硬币: {}, 收藏: {}, 弹幕: {}{}, 发布时间: {}\n'.format( self.av_id_map[av]['title'].split('|', 1)[0], av, no, data_info[1], data_info[2], data_info[3], data_info[4], data_info[7], self.get_history_rank(data_info), time_str(self.av_id_map[av]['created'])) context += '\nBest wish for you\n--------\nSend from script by gunjianpan.' send_email(context, email_title) self.history_check_finish.append(round(time_gap / 10))
def public_monitor(self, av_id: int, times: int): ''' a monitor ''' self.public_list.append(av_id) data_time, mid = self.public[av_id] self.get_star_num(mid, 0) self.check_rank_v2(av_id, 0) time.sleep(5) follower = self.star[mid] if mid in self.star else 0 origin_data = self.data_v2[av_id] if av_id in self.data_v2 else [] sleep_time = data_time + one_day - int(time.time()) if sleep_time < 0: return print('Monitor Begin %d' % (av_id)) time.sleep(sleep_time) self.get_star_num(mid, 0) self.check_rank_v2(av_id, 0) time.sleep(5) follower_2 = self.star[mid] if mid in self.star else 0 one_day_data = self.data_v2[av_id] if av_id in self.data_v2 else [] data = [ time_str(data_time), av_id, follower, follower_2, *origin_data, *one_day_data ] with codecs.open(data_dir + 'public.csv', 'a', encoding='utf-8') as f: f.write(','.join([str(ii) for ii in data]) + '\n')
def get_star_num(self, mid: int, times: int, load_disk=False): ''' get star num''' url = self.RELATION_STAT_URL % mid header = { **headers, **{ 'Origin': self.BILIBILI_URL, 'Referer': self.AV_URL } } if 'Host' in header: del header['Host'] req = proxy_req(url, 2, header=header) if req is None or req.status_code != 200 or len( req.text) < 8 or not '{' in req.text: if times < 3: self.get_star_num(mid, times + 1, load_disk) return try: json_req = json.loads(req.text[7:-1]) self.star[mid] = json_req['data']['follower'] if load_disk and self.check_star(mid, self.star[mid]): self.last_star[mid] = self.star[mid] with open('{}star.csv'.format(data_dir), 'a') as f: f.write('%s,%d\n' % (time_str(), self.star[mid])) except: pass
def public_monitor(self, bv_id: str): """ a monitor """ self.public["L"].append(bv_id) created, mid = self.public["T"][bv_id] self.get_star_num(mid) self.check_rank_v2(bv_id) time.sleep(5) follower = self.star["T"][mid] if mid in self.star["T"] else 0 data1 = self.data_v2[bv_id] if bv_id in self.data_v2 else {} sleep_time = created + one_day - int(time_stamp()) if sleep_time < 0: return echo("4|debug", "Monitor Begin %s" % (bv_id)) time.sleep(sleep_time) self.get_star_num(mid) self.check_rank_v2(bv_id) time.sleep(5) follower_2 = self.star["T"][mid] if mid in self.star["T"] else 0 data2 = self.data_v2[bv_id] if bv_id in self.data_v2 else [] data = [ time_str(created), bv_id, follower, follower_2, *list(data.values()), *list(data2.values()), ] with codecs.open(data_dir + "public.csv", "a", encoding="utf-8") as f: f.write(",".join([str(ii) for ii in data]) + "\n")
def prepare_req(self, hotel_id: int = 4889292, city_id: int = 2, startDate: str = time_str(-1, '%Y-%m-%d'), depDate: str = time_str(int(time.time() + one_day), '%Y-%m-%d')): referer_url = HOTEL_DETAIL_URL % hotel_id changeHeaders({'Referer': referer_url}) data = { 'city': city_id, 'checkin': startDate, 'cjeckout': depDate, 'defalutVal': None } return basic_req(AJAX_PROMOTION_URL, 11, data=data)
def generate_other_params(self, hotel_id: int = 4889292, city_id: int = 2, startDate: str = time_str(-1, '%Y-%m-%d'), depDate: str = time_str( int(time.time() + one_day), '%Y-%m-%d')): ''' generate other params ''' params = { 'psid': None, 'MasterHotelID': hotel_id, 'hotel': hotel_id, 'EDM': 'F', 'roomId': None, 'IncludeRoom': None, 'city': city_id, 'showspothotel': 'T', 'supplier': None, 'IsDecoupleSpotHotelAndGroup': 'F', 'contrast': 0, 'brand': 776, 'startDate': startDate, 'depDate': depDate, 'IsFlash': 'F', 'RequestTravelMoney': 'F', 'hsids': None, 'IsJustConfirm': None, 'contyped': 0, 'priceInfo': -1, 'equip': None, 'filter': None, 'productcode': None, 'couponList': None, 'abForHuaZhu': None, 'defaultLoad': 'T', 'esfiltertag': None, 'estagid': None, 'Currency': None, 'Exchange': None, 'minRoomId': 0, 'maskDiscount': 0, 'TmFromList': 'F', 'th': 119, 'RoomGuestCount': '1,1,0', 'promotionf': None, 'allpoint': None, } return params
def email_update_result(self, article_id: str, r_log: list, r_num: int): p = self.share2article[article_id][-2].split("/")[-1] article_info = self.list_recent[p] name = article_info["name"].replace(".note", "") subject = "更新({}){}/{}条[{}]".format( time_str(time_format=self.T_FORMAT), r_num, len(r_log), article_info["name"] ) content = "\n".join( [ "Title: {}".format(article_info["name"]), "Time: {}".format(time_str()), "Update Num: {}/{}条".format(r_num, len(r_log)), "", *r_log, ] ) send_email(content, subject, assign_rec=self.assign_rec)
def check_rank_v2(self, bv_id: str): rank_info = self.rank_map[bv_id] if bv_id in self.rank_map else {} stat = self.get_view_detail(bv_id) if stat is None or "stat" not in stat: return stat = stat["stat"] data = {**stat, "time": time_str(), **rank_info} self.data_v2[bv_id] = data
def check_rank(self, bv_id: str): rank_info = self.rank_map[bv_id] if bv_id in self.rank_map else {} stat = self.get_view_detail(bv_id) if stat is None or "stat" not in stat: return stat = stat["stat"] data = {**stat, "time": time_str(), **rank_info} need = [ "time", "view", "like", "coin", "favorite", "reply", "share", "danmaku", "id", "pts", "type", "day", ] output = [str(data[ii]) for ii in need if ii in data] output = output + [str(v) for k, v in data.items() if k not in need] with codecs.open("{}{}.csv".format(history_dir, bv_id), "a", encoding="utf-8") as f: f.write(",".join(output) + "\n") if (bv_id in self.last_check and int(time_stamp()) - self.last_check[bv_id] > one_day / 2): self.del_map[bv_id] = 1 del self.rank_map[bv_id] if bv_id == self.basic_bv_id: clean_csv(bv_id) elif (bv_id not in self.last_check and int(time_stamp()) > one_day + self.begin_timestamp): self.del_map[bv_id] = 1 del self.rank_map[bv_id] if bv_id == self.basic_bv_id: clean_csv(bv_id) self.last_view[bv_id] = data["view"] now_time = time_stamp() if not bv_id in self.public["T"] or bv_id not in self.assign_ids: return time_gap = (now_time - self.public["T"][bv_id][0]) / 60 echo("0|debug", bv_id, time_gap < (4.5 * one_day / 60), self.public["T"][bv_id]) if time_gap >= (4.5 * one_day / 60): return if not bv_id in self.check_done: self.check_done[bv_id] = [] echo("3|info", "Time Gap:", int(time_gap / 10)) if (int(time_gap / 10) in self.history_check_list and int(time_gap / 10) not in self.check_done[bv_id]): self.history_rank(time_gap, data, bv_id)
def check_rank_v2(self, av_id: int, times=0): rank_list = self.rank_map[av_id] if av_id in self.rank_map else [] changeHeaders({'Referer': self.BASIC_AV_URL % av_id}) url = self.ARCHIVE_STAT_URL % av_id json_req = proxy_req(url, 1) if not self.have_error(json_req): if times < 3: self.check_rank_v2(av_id, times + 1) return json_req = json_req['data'] need = [ 'view', 'like', 'coin', 'favorite', 'reply', 'share', 'danmaku' ] data = [json_req[index] for index in need] if len(rank_list): data = [time_str(), *data, *rank_list[:2], *rank_list[-2:]] else: data = [time_str(), *data] self.data_v2[av_id] = data
def get_star_num(self, mid: int, load_disk: bool = False): """ get star num""" url = self.RELATION_STAT_URL % mid star_json = self.get_api_req(url, self.basic_bv_id, 1) if star_json is None: return self.star["T"][mid] = star_json["follower"] if not load_disk or not self.check_star(mid, star_json["follower"]): return self.star["L"][mid] = star_json["follower"] with open("{}star.csv".format(data_dir), "a") as f: f.write("{},{}\n".format(time_str(), star_json["follower"]))
def decoder_tpwd_once(self, article_id: str, tpwd: str, mode: int = 0): req = self.decoder_tpwd(tpwd) if req is None or not len(req): return temp_map = {ii: req[ii] for ii in self.NEED_KEY} if temp_map["validDate"] == self.ZERO_STAMP or "-" in temp_map["validDate"]: temp_map["validDate"] = 1500000000 else: temp_map["validDate"] = ( time_stamp(time_format="%d天%H小时%M分%S秒", time_str=req["validDate"]) - self.BASIC_STAMP + time_stamp() ) temp_map["validDate"] = time_str(temp_map["validDate"]) temp_map["url"] = temp_map["url"].strip() if article_id not in self.tpwd_map: self.tpwd_map[article_id] = {} self.tpwd_map[article_id][tpwd] = temp_map if not mode: self.decoder_tpwd_url(article_id, tpwd)
def load_rank(self): """ load rank """ self.load_rank_index(1, 3) self.load_rank_index(1, 1) assign_1, assign_2 = self.have_assign[1], self.have_assign[3] have_assign = assign_1 + assign_2 echo("4|debug", assign_1, assign_2, have_assign) not_ranks = [ ii for ii in self.have_assign["T"] if not ii in have_assign ] self.have_assign = have_assign echo( "4|debug", "Rank_map_len:", len(self.rank_map.keys()), "Empty:", len([1 for ii in self.rank_map.values() if not len(ii)]), ) youshan = [ ",".join([str(kk) for kk in [ii, *list(jj.values())]]) for ii, jj in self.rank_map.items() ] with codecs.open(data_dir + "youshang", "w", encoding="utf-8") as f: f.write("\n".join(youshan)) if not len(not_ranks): return for bv_id in not_ranks: title = (self.bv_ids[bv_id]["title"].split("|", 1)[0] if bv_id in self.bv_ids else "") no_rank_warning = "下榜({}){},{}".format( time_str(time_format=self.T_FORMAT), title, self.NO_RANK_CONSTANT) send_email(no_rank_warning, no_rank_warning, self.special_info_email) time.sleep(pow(np.pi, 2)) send_email(no_rank_warning, no_rank_warning, self.special_info_email) echo("4|error", no_rank_warning)
def get_comment_detail(self, comment: dict, av_id: int, pn: int, parent_rpid=None) -> List: ''' get comment detail ''' # print(comment) ctime = time_str(comment['ctime']) wait_list = ['rpid', 'member', 'content', 'like', 'idx'] wait_list_mem = ['uname', 'sex', 'sign', 'level_info'] wait_list_content = ['message', 'plat'] rpid, member, content, like, idx = [comment[ii] for ii in wait_list] uname, sex, sign, level = [member[ii] for ii in wait_list_mem] current_level = level['current_level'] content, plat = [content[ii] for ii in wait_list_content] req_list = [ rpid, ctime, like, plat, current_level, uname, sex, content, sign, idx ] self.have_bad_comment(req_list, av_id, pn, parent_rpid) req_list[-2] = req_list[-2].replace(',', ' ').replace('\n', ' ') req_list[-3] = req_list[-3].replace(',', ' ').replace('\n', ' ') return req_list
def get_danmaku(self, av_id: int): self.dm_map = {} self.dm_exec = ThreadPoolExecutor(max_workers=100) mkdir(dm_dir) output_path = "{}{}_dm.csv".format(dm_dir, av_id) view_data = self.get_view_detail(av_id) if view_data is None: return cid_list = [ii["cid"] for ii in view_data["pages"]] dm_map = self.dm_map[av_id] if av_id in self.dm_map else {} cid_list = [ ii for ii in cid_list if ii not in dm_map or len(dm_map[ii]) == 0 ] dm_thread = [ self.dm_exec.submit(self.get_danmaku_once, ii) for ii in cid_list ] need_dm = view_data["stat"]["danmaku"] need_p = len(view_data["pages"]) echo(2, "Begin {} p thread, need {} dm".format(need_p, need_dm)) dm_list = list(as_completed(dm_thread)) dm_list = [ii.result() for ii in as_completed(dm_thread)] dm_list = [ii for ii in dm_list if ii is not None] dm_map = {**dm_map, **{jj: ii for ii, jj in dm_list}} dm_num = sum([len(ii) for ii in dm_map.values()]) p_num = len(dm_map) self.dm_map[av_id] = dm_map title = "{} {} Total {} p {} dm, Actual {} p {} dm".format( view_data["title"], self.BASIC_AV_URL % av_id, need_p, need_dm, p_num, dm_num, ) result = [title, ""] for cid in view_data["pages"]: if cid["cid"] not in dm_map: continue dm = dm_map[cid["cid"]] dm = [[ float(ii.split(",")[0]), time_str(time_stamp=int(ii.split(",")[4])), jj, ] for ii, jj in dm] dm = sorted(dm, key=lambda i: i[0]) dm = [",".join([get_min_s(str(ii)), jj, kk]) for ii, jj, kk in dm] p_title = "p{} {} Total {} dm".format(cid["page"], cid["part"], len(dm)) result.extend([p_title, *dm, ""]) with open(output_path, "w") as f: f.write("\n".join(result)) print_str = "Load {} p {} dm to {}, except {} p {} m".format( output_path, len(dm_list), dm_num, need_p, need_dm) if need_dm == dm_num: echo(1, print_str, "success") else: echo(0, print_str, "error")
def get_data(self, origin_data: list) -> str: ''' get data ''' return ','.join(str(ii) for ii in [time_str(), *origin_data]) + '\n'
def history_rank(self, time_gap: int, now_info: dict, bv_id: int): echo("0|info", "send history rank") time_gap = int(time_gap / 10) * 10 history_map = { ii: jj for ii, jj in self.history_map[time_gap].items() if jj[1] } if len(history_map) < 5: self.load_history_data() other_views = [int(ii[1]) for ii in history_map.values()] other_views_len = len(other_views) other_views.append(now_info["view"]) ov_sort_idx = np.argsort(-np.array(other_views)) av_ids = list(history_map.keys()) now_sorted = [ jj for jj, ii in enumerate(ov_sort_idx) if ii == other_views_len ][0] + 1 other_result = [(jj + 1, av_ids[ii]) for jj, ii in enumerate(ov_sort_idx[:4]) if ii != other_views_len] time_tt = get_time_str(time_gap) rank_info = self.bv_ids[bv_id] title = rank_info["title"].split("|", 1)[0] title_email = "排名(发布{}){}本年度排名No.{}/{}, 播放量: {}, 点赞: {}, 硬币: {}, 收藏: {}, 弹幕: {}".format( time_tt, title, now_sorted, len(other_views), now_info["view"], now_info["like"], now_info["coin"], now_info["favorite"], now_info["danmaku"], ) email_title = "bv{}发布{}, 本年度排名No.{}/{}, 播放量: {}, 点赞: {}, 硬币: {}, 收藏: {}, 弹幕: {}".format( bv_id, time_tt, now_sorted, len(other_views), now_info["view"], now_info["like"], now_info["coin"], now_info["favorite"], now_info["danmaku"], ) email_title += self.get_history_rank(now_info) context = "{}\n\n".format(email_title) for no, bv in other_result[:3]: data_info = history_map[bv] context += "{}, bv{}, 本年度No.{}, 播放量: {}, 点赞: {}, 硬币: {}, 收藏: {}, 弹幕: {}, 累计播放: {}{}, 发布时间: {}\n".format( self.bv_ids[bv]["title"].split("|", 1)[0], bv, no, data_info[1], data_info[2], data_info[3], data_info[4], data_info[7], self.bv_ids[bv]["play"], self.get_history_rank(data_info), time_str(self.bv_ids[bv]["created"]), ) send_email(context, title_email) self.check_done[bv_id].append(round(time_gap / 10))
def get_check(self): """ check comment """ self.delay_load_history_data() bv_list = [[ii["bvid"], ii["aid"], ii["comment"]] for ii in self.bv_ids.values() if not regex.findall(self.ignore_list, str(ii["aid"]))] bv_map = {ii["bvid"]: ii for ii in self.bv_ids.values()} if self.bv_list and len( self.bv_list) and len(self.bv_list) != len(bv_list): new_bv_list = [(ii, jj) for ii, jj, _ in bv_list if not ii in self.bv_list and not ii in self.del_map ] self.rank_map = { **self.rank_map, **{ii: {} for ii, _ in new_bv_list} } echo("1|error", "New Bv av ids:", new_bv_list) for bv_id, av_id in new_bv_list: rank_info = bv_map[bv_id] shell_str = "nohup python3 bilibili/bsocket.py {} %d >> log.txt 2>&1 &".format( av_id) echo("0|error", "Shell str:", shell_str) os.system(shell_str % 1) os.system(shell_str % 2) email_str = "发布({}){}#{} {}".format( time_str(rank_info["created"], time_format=self.T_FORMAT), rank_info["title"], bv_id, av_id, ) email_str2 = "{} {} is release at {}.\nPlease check the online & common program.".format( rank_info["title"], time_str(rank_info["created"]), self.BASIC_BV_URL % bv_id, ) send_email(email_str2, email_str, self.special_info_email) self.update_ini(bv_id, av_id) self.public["T"][bv_id] = [ rank_info["created"], rank_info["mid"] ] self.last_check[bv_id] = int(time_stamp()) self.bv_list = [ii for (ii, _, _) in bv_list] now_hour = int(time_str(time_format="%H")) now_min = int(time_str(time_format="%M")) now_time = now_hour + now_min / 60 if now_time > self.ignore_start and now_time < self.ignore_end: return if self.assign_mid == -1: return # threads = [self.pool.submit(self.check_type_req, bv_id) for bv_id in rank_map.keys()] # list(as_completed(threads)) threading_list = [] for (_, ii, jj) in bv_list: work = threading.Thread(target=self.comment_check_schedule, args=(ii, jj)) threading_list.append(work) for work in threading_list: work.start() for work in threading_list: work.join() return bv_list
def log_write(self, url): """ failure log """ with codecs.open("proxy.log", 'a', encoding='utf-8') as f: f.write(time_str() + url + '\n')
def get_data(self, origin_data: list) -> str: """ get data """ return ",".join(str(ii) for ii in [time_str(), *origin_data]) + "\n"