def _getYoutube_m3u8_sync(youtubeLink, isLog=True): out, err, errcode = None, None, -1 tmp_retryTime = 0 while tmp_retryTime < 2: out, err, errcode = __runCMDSync( 'youtube-dl --no-check-certificate -j {}'.format(youtubeLink), isLog) out = out.decode('utf-8') if isinstance(out, (bytes, bytearray)) else out if errcode == 0: try: vDict = json.loads(out) except Exception: vDict = None if vDict: if vDict.get('is_live') != True: return out, None, err, 999 #mean this is not a live title = vDict.get('uploader', '') + '_' + vDict.get( 'title', '') url = vDict.get('url', '') if url.endswith('.m3u8'): return url, title, err, errcode else: tmp_retryTime += 1 time.sleep(30) utitls.myLogger("_getYoutube_m3u8_sync SOURCE:{} ERROR:{}".format( youtubeLink, out)) return out, None, err, errcode
def _forwardStreamCMD_sync(title, subscribe_obj, inputStreamLink, outputRTMP): os.makedirs('archive_videos', exist_ok=True) os.makedirs('temp_videos', exist_ok=True) utitls.myLogger("_forwardStream_sync LOG:%s, %s" % (inputStreamLink, outputRTMP)) title = title.replace('https', '') title = title.replace('http', '') title = title.replace('hlsvariant', '') reserved_list = [ '/', '\\', ':', '?', '%', '*', '|', '"', '.', ' ', '<', '>' ] for val in reserved_list: title = title.replace(val, '_') out, err, errcode = None, None, None recordFileName_nosuffix = datetime.datetime.now().strftime( "%Y-%m-%dT%H-%M-%S") + "_" + utitls.remove_emoji(title.strip()) recordFilePath = os.path.join(os.getcwd(), 'temp_videos', recordFileName_nosuffix + '.flv') # tmp_input = 'ffmpeg -loglevel error -i "{}"'.format(inputStreamLink) tmp_input = 'streamlink --retry-streams 6 --retry-max 10 --retry-open 10 --hls-timeout 120 --hls-playlist-reload-attempts 10 -O {} best|ffmpeg -loglevel error -i pipe:0'.format( inputStreamLink) tmp_out_rtmp = '-f flv "{}"'.format(outputRTMP) tmp_out_file = '-y -f flv "{}"'.format(recordFilePath) tmp_encode = '-vcodec copy -acodec aac -strict -2 -ac 2 -bsf:a aac_adtstoasc -flags +global_header' cmd_list = [tmp_input, tmp_encode, tmp_out_rtmp] tmp_should_record = None if subscribe_obj: tmp_should_record = subscribe_obj.get('is_should_record', None) if tmp_should_record == None: if utitls.configJson().get('is_auto_record', False): tmp_should_record = True if tmp_should_record == True: cmd_list.append( '-vcodec copy -acodec aac -strict -2 -ac 2 -bsf:a aac_adtstoasc') cmd_list.append(tmp_out_file) cmd = '' for val in cmd_list: cmd += val + ' ' cmd = cmd.strip() #strip the last ' ' out, err, errcode = __runCMDSync(cmd) try: if os.path.exists(recordFilePath): # move the video to archive floder and change container to mp4 archive_path = os.path.join(os.getcwd(), 'archive_videos', recordFileName_nosuffix + '.mp4') __runCMDSync( 'ffmpeg -loglevel error -i {} -codec copy -y {}'.format( recordFilePath, archive_path)) os.remove(recordFilePath) except Exception: utitls.myLogger(traceback.format_exc()) return out, err, errcode
def _forwardToBilibili_Sync(channelId, link, room_title, area_id=None, isSubscribeQuest=True): resloveURLOK = False tmp_retryTime = 10 while tmp_retryTime > 0: if 'youtube.com/' in link or 'youtu.be/' in link: m3u8Link, err, errcode = _getYotube_m3u8_sync(link) if errcode == 0: link = m3u8Link resloveURLOK = True break else: tmp_retryTime -= 1 time.sleep(60) else: utitls.myLogger( '_forwardToBilibili_Sync LOG: Unsupport ForwardLink:' + link) return if resloveURLOK: b, t_room_id, rtmp_link = bilibiliStartLive(channelId, room_title, area_id) if rtmp_link: tmp_quest = questInfo._getObjWithRTMPLink(rtmp_link) if tmp_quest != None: try: os.kill(tmp_quest.get('pid', None), signal.SIGKILL) except Exception: utitls.myLogger(traceback.format_exc()) questInfo.removeQuest(rtmp_link) # force stream _forwardStream_sync(link, rtmp_link, isSubscribeQuest)
def removeQuest(rtmpLink): quest = _getObjWithRTMPLink(rtmpLink) if quest: utitls.myLogger('RemoveQuest LOG:\n Removed QUEST:%s' % quest) tmp_quest_list = _getQuestList() tmp_quest_list.remove(quest) utitls.myLogger('Current Quest List:\n,{}'.format(tmp_quest_list)) _saveQuestList(tmp_quest_list)
def _getYotube_m3u8_sync(youtubeLink): out, err, errcode = __runCMDSync('youtube-dl --no-check-certificate -g {}'.format(youtubeLink)) out = out.decode('utf-8') if isinstance(out, (bytes, bytearray)) else out out = out.strip('\n').strip() if not out.endswith('.m3u8'): errcode = -1 utitls.myLogger("_getYotube_m3u8_sync ERROR:%s" % out) return out, err, errcode
def startWebServer(): # ip = utitls.configJson().get('serverIP') port = utitls.configJson().get('serverPort') server = ThreadedHTTPServer(('', int(port)), RequestHandler) utitls.myLogger('WebServerStarted, Listening on localhost:%s' % port) # sys.stderr = open('logfile.txt', 'a+', 1) server.serve_forever() return server.shutdown()
def resolveStreamToM3u8(streamLink, isLog=True): out, title, err, errcode = None, streamLink, None, -1 tmp_retryTime = 0 while tmp_retryTime < 2: out, err, errcode = __runCMDSync( 'streamlink -j "{}" best'.format(streamLink), isLog) out = out.decode('utf-8') if isinstance(out, (bytes, bytearray)) else out if errcode == 0: try: vDict = json.loads(out) except Exception: vDict = None if vDict: streamM3U8 = vDict.get('url', None) if streamM3U8 == None: return out, title, err, 999 #mean this is not a live tmp_title = streamLink tmp_uploader = "" # just pack the title videoId = streamLink.partition('/watch?v=')[2] if 'youtu.be/' in streamLink: videoId = streamLink.partition('youtu.be/')[2] videoId = videoId.split('/')[0] c_l = re.findall(r"channel/(.*)/live", streamLink) if len(c_l) > 0: # find the channelID channelID = c_l[0] item = myRequests.getYoutubeLiveVideoInfoFromChannelID( channelID) if item == None: item = myRequests.getYoutubeLiveVideoInfoFromChannelID( channelID, 'upcoming') if item: snippet = item.get('snippet', {}) tmp_title = snippet.get('title', streamLink) tmp_uploader = snippet.get('channelTitle', channelID) elif videoId != '': item = myRequests.getYoutubeLiveStreamInfo(videoId) if item: snippet = item.get('snippet', {}) tmp_title = snippet.get('title', streamLink) tmp_uploader = snippet.get('channelTitle', "") title = "{}_{}".format(tmp_uploader, tmp_title) m3u8Link = streamLink return m3u8Link, title, err, errcode else: tmp_retryTime += 1 time.sleep(30) if isLog: utitls.myLogger("resolveStreamToM3u8 SOURCE:{} ERROR:{}".format( streamLink, out)) return out, title, err, errcode
def login(username, password): browser = None try: if utitls.configJson().get("driver_type", "chrome") == "firefox": firefox_option = webdriver.FirefoxOptions() firefox_option.headless = True browser = webdriver.Firefox(firefox_options=firefox_option) else: chrome_options = webdriver.ChromeOptions() chrome_options.headless = True browser = webdriver.Chrome(chrome_options=chrome_options) browser.get('https://passport.bilibili.com/login') Wait(browser, 60).until( Expect.visibility_of_element_located((By.CLASS_NAME, "gt_slider"))) username_input = browser.find_element_by_id("login-username") username_input.send_keys(username) password_input = browser.find_element_by_id("login-passwd") password_input.send_keys(password) retry_times = 0 max_retry_times = utitls.configJson().get("login_retry_times", 3) while retry_times < max_retry_times: do_captcha(browser) Wait(browser, 20).until( Expect.visibility_of_element_located( (By.CLASS_NAME, "gt_info_tip"))) if Expect.visibility_of_element_located((By.CLASS_NAME, "gt_success")) \ or Expect.visibility_of_element_located((By.ID, "banner_link")): break retry_times += 1 Wait(browser, 10).until( Expect.invisibility_of_element_located( (By.CLASS_NAME, "gt_fail"))) time.sleep(1) if retry_times >= max_retry_times: return "" #check is login Success Wait(browser, 60).until( Expect.visibility_of_element_located((By.ID, "banner_link"))) browser.get('https://link.bilibili.com/p/center/index') Wait(browser, 10).until( Expect.visibility_of_element_located((By.CLASS_NAME, "user"))) time.sleep(5) #wait for the cookies cookies = browser.get_cookies() cookies_str_array = [] for cookie in cookies: cookies_str_array.append(cookie["name"] + "=" + cookie["value"]) browser.quit() return ";".join(cookies_str_array) except Exception as e: utitls.myLogger(traceback.format_exc()) utitls.myLogger(str(e)) if browser is not None: browser.quit() return ""
def getAccInfo(self): resDict = self._baseGet( 'https://api.bilibili.com/x/member/web/account') if resDict: if resDict['code'] == 0: return resDict['data'] else: myLogger('ERROR: Account no login') return None
def stopLive(self, room_id): resDict = self._basePost( 'https://api.live.bilibili.com/room/v1/Room/stopLive', { 'room_id': room_id, 'platform': 'pc', 'csrf_token': self.csrf_token }) if resDict: if resDict['code'] != 0: myLogger('ERROR: StopLive Failed')
def updateRoomTitle(self, room_id, title): resDict = self._basePost( 'https://api.live.bilibili.com/room/v1/Room/update', { 'room_id': room_id, 'title': title, 'csrf_token': self.csrf_token }) if resDict: if resDict['code'] != 0: myLogger('ERROR: update room title Failed')
def main(): try: #init the quest list questInfo.initQuestList() Async_subscribeTheList(True) startWebServer() except Exception as e: utitls.myLogger(traceback.format_exc()) utitls.myLogger(str(e))
def main(): try: AutoOperate.preparingAllAccountsCookies() AutoOperate.clearOldQuests() AutoOperate.perparingAllComingVideos() except BaseException as e: utitls.myLogger(str(e)) utitls.myLogger(traceback.format_exc()) startWebServer() pass
def __init__(): global g_main_scheduler g_main_scheduler = BackgroundScheduler(timezone=utc) g_main_scheduler.add_jobstore('sqlalchemy', url='sqlite:///jobs.sqlite') g_main_scheduler.start() log_jobs() import threading myLogger("g_main_scheduler in this thread:{}".format( threading.current_thread()))
def addQuest(forwardLinkOrign, rtmpLink, isSubscribeQuest=False): forwardLinkOrign = str(forwardLinkOrign) rtmpLink = str(rtmpLink) questDict = { 'forwardLinkOrign': forwardLinkOrign, 'rtmpLink': rtmpLink, 'isSubscribeQuest': isSubscribeQuest } utitls.myLogger('AddQuest LOG:\n AddQuest QUEST:%s' % questDict) tmp_quest_list = _getQuestList() tmp_quest_list.append(questDict) utitls.myLogger('Current Quest List:\n,{}'.format(tmp_quest_list)) _saveQuestList(tmp_quest_list)
def startLive(self, room_id, area_id): resDict = self._basePost( 'https://api.live.bilibili.com/room/v1/Room/startLive', { 'room_id': room_id, 'platform': 'pc', 'area_v2': area_id, 'csrf_token': self.csrf_token }) if resDict: if resDict['code'] == 0: rtmp_link = resDict['data']['rtmp']['addr'] + resDict['data'][ 'rtmp']['code'] myLogger("Current RTMP_LINK:%s" % rtmp_link) return rtmp_link else: return None
def removeQuest(rtmpLink, isSubscribeQuest=False, questAcc=None): tmp_quest_list = _getQuestList() for quest in tmp_quest_list: tmp_can_remove = False if isSubscribeQuest: if quest.get('mark') == questAcc: tmp_can_remove = True else: if rtmpLink and quest.get('rtmpLink', "").split('/')[-1] == str(rtmpLink).split('/')[-1]: tmp_can_remove = True if tmp_can_remove: utitls.myLogger('RemoveQuest LOG:\n Removed QUEST:%s' % quest) tmp_quest_list.remove(quest) utitls.myLogger('Current Quest List:\n,{}'.format(tmp_quest_list)) _saveQuestList(tmp_quest_list) return
def _basePost(url, data): try: myLogger("POST URL:%s--" % url) myLogger("DATA :%s--" % data) response = requests.post(url, data=data, timeout=30) except Exception as e: myLogger(str(e)) myLogger(traceback.format_exc()) return None return _baseRequestProcess(response)
def addQuest(forwardLinkOrign, rtmpLink, isSubscribeQuest=False, questAcc=None): if checkIfInQuest(rtmpLink, isSubscribeQuest, questAcc): return forwardLinkOrign = str(forwardLinkOrign) rtmpLink = str(rtmpLink) questDict = { 'isDead': False, 'forwardLinkOrign': forwardLinkOrign, 'rtmpLink': rtmpLink, 'isSubscribeQuest': isSubscribeQuest, 'title': None } utitls.myLogger('AddQuest LOG:\n AddQuest QUEST:%s' % questDict) tmp_quest_list = _getQuestList() tmp_quest_list.append(questDict) utitls.myLogger('Current Quest List:\n,{}'.format(tmp_quest_list)) _saveQuestList(tmp_quest_list)
def __runCMDSync(cmd, isLog=True): try: p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pid = p.pid if isLog: utitls.myLogger("CMD RUN START with PID:{}\nCMD: {}".format( pid, cmd)) try: rtmpLink = cmd.partition('-f flv "')[2].partition('"')[ 0] #get the first -f link if rtmpLink.startswith('rtmp://'): questInfo.updateQuestInfo('pid', pid, rtmpLink) except Exception: pass out, err = p.communicate() errcode = p.returncode if isLog: utitls.myLogger( "CMD RUN END with PID:{}\nCMD: {}\nOUT: {}\nERR: {}\nERRCODE: {}" .format(pid, cmd, out, err, errcode)) except Exception as e: out, err, errcode = None, e, -1 utitls.myLogger(traceback.format_exc()) return out, err, errcode
def _baseGet(url): try: myLogger("GET URL:%s--" % url) response = requests.get(url, timeout=30) except Exception as e: myLogger(str(e)) myLogger(traceback.format_exc()) return None return _baseRequestProcess(response)
def _baseRequestProcess(response): if response == None: return None try: myLogger("Request URL:%s-----------------" % response.request.url) myLogger("Method:%s, Code:%s,\n text:%s" % (response.request.method, str( response.status_code), response.text)) if response.status_code == 200: try: return response.json() except Exception: return None else: return None except Exception as e: myLogger("request Exception:%s" % str(e)) myLogger(traceback.format_exc()) return None
def _forwardStreamCMD_sync(inputM3U8, outputRTMP): utitls.myLogger("_forwardStream_sync LOG:%s, %s" % (inputM3U8, outputRTMP)) out, err, errcode = None, None, None tmp_retryTime = 0 tmp_cmdStartTime = time.time() while tmp_retryTime <= 30: # must be <= out, err, errcode = __runCMDSync('ffmpeg -loglevel error -i "{}" -vcodec copy -acodec aac -strict -2 -ar 44100 -ab 128k -ac 2 -bsf:a aac_adtstoasc -bufsize 3000k -flags +global_header -f flv "{}"'.format(inputM3U8, outputRTMP)) if errcode == -9: utitls.myLogger("_forwardStreamCMD_sync LOG: Kill Current procces by rtmp:%s" % outputRTMP) break # maybe can ignore the error if ran after 2min? if time.time() - tmp_cmdStartTime < 120: tmp_retryTime += 1 # make it can exit else: tmp_retryTime = 0 # let every Connect success reset the retrytime tmp_cmdStartTime = time.time() #import should not miss it. time.sleep(10) #rtmp buffer can hold 3 secounds or less utitls.myLogger('_forwardStreamCMD_sync LOG: CURRENT RETRY TIME:%s' % tmp_retryTime) utitls.myLogger("_forwardStream_sync LOG RETRYING___________THIS:\ninputM3U8:%s, \noutputRTMP:%s" % (inputM3U8, outputRTMP)) return out, err, errcode
def __runCMDSync(cmd): try: p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pid = p.pid utitls.myLogger("CMD RUN START with PID:{}\nCMD: {}".format(pid, cmd)) try: rtmpLink = cmd.split(' ')[-1].strip('"') if rtmpLink.startswith('rtmp://'): questInfo.updateQuestWithPID(pid, rtmpLink) except Exception: pass out, err = p.communicate() errcode = p.returncode utitls.myLogger("CMD RUN END with PID:{}\nCMD: {}\nOUT: {}\nERR: {}\nERRCODE: {}".format(pid, cmd, out, err, errcode)) except Exception as e: out, err, errcode = None, e, -1 utitls.myLogger(traceback.format_exc()) return out, err, errcode
def _forwardStream_sync(link, outputRTMP, isSubscribeQuest): if questInfo.checkIfInQuest(outputRTMP, isSubscribeQuest): utitls.myLogger("_forwardStream_sync ERROR: rtmp already in quest!!!!\n link:%s, \n rtmpLink:%s" % (link, outputRTMP)) return questInfo.addQuest(link, outputRTMP, isSubscribeQuest) if outputRTMP.startswith('rtmp://'): if 'youtube.com/' in link or 'youtu.be/' in link: m3u8Link, err, errcode = _getYotube_m3u8_sync(link) if errcode == 0: link = m3u8Link if link.endswith('.m3u8') or '.m3u8' in link: _forwardStreamCMD_sync(link, outputRTMP) else: utitls.myLogger("_forwardStream_sync ERROR: Unsupport forwardLink:%s" % link) else: utitls.myLogger("_forwardStream_sync ERROR: Invalid outputRTMP:%s" % outputRTMP) questInfo.removeQuest(outputRTMP)
port = '' if port == '80' else ':' + port tmp_callback_url = 'http://' + ip + port + '/subscribe' if isSubscribe: time.sleep(1) #wait the server start preparing subscribe(tmp_callback_url, tmp_subscribeId) else: unsubscribe(tmp_callback_url, tmp_subscribeId) def main(): try: #init the quest list questInfo.initQuestList() Async_subscribeTheList(True) startWebServer() except Exception as e: utitls.myLogger(traceback.format_exc()) utitls.myLogger(str(e)) if __name__ == "__main__": try: while True: main() utitls.myLogger('RESTART WERSERVER') time.sleep(5) except KeyboardInterrupt: # subscribeTheList_sync(False) utitls.myLogger('Running END-------------------------\n')
def do_GET(self): request_path = self.path rc = 404 rb = None params = parse_qs(urlsplit(request_path).query) try: if request_path.startswith('/web/'): fname, ext = os.path.splitext(request_path) if ext in (".html", ".css", ".js"): tmp_filePath = os.path.join(os.getcwd()) for v in request_path.split('/'): tmp_filePath = os.path.join(tmp_filePath, v) print(tmp_filePath) with open(tmp_filePath, 'r', encoding='utf-8') as f: self.send_response(200) self.send_header('Content-type', types_map[ext]) self.end_headers() self.wfile.write(f.read().encode('utf-8')) return except Exception: utitls.myLogger(traceback.format_exc()) self.send_error(404) self.end_headers() return if request_path.startswith('/subscribe?'): hub_challenge_list = params.get('hub.challenge', None) if None != hub_challenge_list: rc = 202 rb = hub_challenge_list[0] elif request_path.startswith('/questlist'): rc = 200 rb = json.dumps(getQuestList_AddStarts()) elif request_path.startswith('/get_manual_json'): rc = 200 rb = json.dumps(utitls.manualJson()) elif request_path.startswith('/addRestreamSrc'): rc = 200 srcNote_list = params.get('srcNote', None) srcLink_list = params.get('srcLink', None) if srcNote_list and srcLink_list: tmp_srcNote = srcNote_list[0].strip() tmp_srcLink = srcLink_list[0].strip() utitls.addManualSrc(tmp_srcNote, tmp_srcLink) rb = json.dumps({"code": 0, "msg": "添加成功"}) elif request_path.startswith('/addRtmpDes'): rc = 200 rtmpNote_list = params.get('rtmpNote', None) rtmpLink_list = params.get('rtmpLink', None) if rtmpNote_list and rtmpLink_list: tmp_rtmpNote = rtmpNote_list[0].strip() tmp_rtmpLink = rtmpLink_list[0].strip() utitls.addManualDes(tmp_rtmpNote, tmp_rtmpLink) rb = json.dumps({"code": 0, "msg": "添加成功"}) elif request_path.startswith('/kill_quest?'): rc = 200 tmp_rtmpLink_list = params.get('rtmpLink', None) if tmp_rtmpLink_list: tmp_rtmpLink = tmp_rtmpLink_list[0].strip() tmp_quest = _getObjWithRTMPLink(tmp_rtmpLink) if tmp_quest != None: try: os.kill(tmp_quest.get('pid', None), signal.SIGKILL) rb = json.dumps({"code": 0, "msg": "操作成功"}) except Exception: utitls.myLogger(traceback.format_exc()) rb = json.dumps({"code": -2, "msg": "错误PID,操作失败!!"}) else: rb = json.dumps({ "code": -1, "msg": "查找不到对应的任务:{},操作失败!!".format(tmp_rtmpLink) }) elif request_path.startswith('/live_restream?'): forwardLink_list = params.get('forwardLink', None) restreamRtmpLink_list = params.get('restreamRtmpLink', None) if forwardLink_list and restreamRtmpLink_list: tmp_forwardLink = forwardLink_list[0].strip() tmp_rtmpLink = restreamRtmpLink_list[0].strip() isForwardLinkFormateOK = True if 'rtmp://' in tmp_rtmpLink: if 'twitcasting.tv/' in tmp_forwardLink: #('https://www.', 'twitcasting.tv/', 're2_takatsuki/fwer/aeqwet') tmp_twitcasID = tmp_forwardLink.partition( 'twitcasting.tv/')[2] tmp_twitcasID = tmp_twitcasID.split('/')[0] tmp_forwardLink = 'http://twitcasting.tv/{}/metastream.m3u8/?video=1'.format( tmp_twitcasID) elif '.m3u8' in tmp_forwardLink \ or 'youtube.com/' in tmp_forwardLink or 'youtu.be/' in tmp_forwardLink: tmp_forwardLink = tmp_forwardLink else: isForwardLinkFormateOK = False rc = 200 if isForwardLinkFormateOK: if checkIfInQuest(tmp_rtmpLink) == False: #try to restream async_forwardStream(tmp_forwardLink, tmp_rtmpLink, False) rb = json.dumps({ "code": 0, "msg": "请求成功。请等待大概30秒,网络不好时程序会自动重试30次。也可以查看任务状态看是否添加成功。\ \nRequesting ForwardLink: {},\nRequesting RestreamRtmpLink: {}\n\n当前任务:\n{}" .format(tmp_forwardLink, tmp_rtmpLink, getQuestListStr()) }) else: rb = json.dumps({ "code": 1, "msg": "当前推流已经在任务中. \nRequesting ForwardLink: {},\nRequesting RestreamRtmpLink: {}\n\n\n-----------------CurrentQuests:\n{}" .format(tmp_forwardLink, tmp_rtmpLink, getQuestListStr()) }) else: rb = json.dumps({ "code": -3, "msg": "来源地址格式错误, 请查看上面支持的格式" }) else: rc = 200 rb = json.dumps({ "code": -4, "msg": "RTMPLink格式错误!!! bilibili的RTMPLink格式是两串合起来。\nEXAMPLE:rtmp://XXXXXX.acg.tv/live-js/?streamname=live_XXXXXXX&key=XXXXXXXXXX" }) self.send_response(rc) self.send_header('Content-type', 'text/html; charset=utf-8') self.end_headers() if None != rb: try: self.wfile.write(rb.encode('utf-8')) except Exception: print(traceback.format_exc())
def login(username, password): browser = None try: if utitls.configJson().get("driver_type", "chrome") == "firefox": firefox_option = webdriver.FirefoxOptions() firefox_option.headless = True browser = webdriver.Firefox(firefox_options=firefox_option) else: chrome_options = webdriver.ChromeOptions() chrome_options.headless = True chrome_options.add_argument('--no-sandbox') chrome_options.add_argument('--disable-dev-shm-usage') chrome_options.add_argument('--disable-sync') chrome_options.add_argument('--disable-plugins') chrome_options.add_argument('--disable-extensions') chrome_options.add_argument('--disable-translate') browser = webdriver.Chrome(chrome_options=chrome_options) browser.get( 'https://link.bilibili.com/p/center/index' ) # if no login, it will go to the login page. The main bilibil.com have too much images it will take a lot memory Wait(browser, 60).until( Expect.visibility_of_element_located((By.CLASS_NAME, "gt_slider"))) username_input = browser.find_element_by_id("login-username") username_input.send_keys(username) password_input = browser.find_element_by_id("login-passwd") password_input.send_keys(password) utitls.myLogger( 'Inputing the username and passwd, username:{}'.format(username)) retry_times = 0 max_retry_times = utitls.configJson().get("login_retry_times", 3) while retry_times < max_retry_times: do_captcha(browser) Wait(browser, 20).until( Expect.visibility_of_element_located( (By.CLASS_NAME, "gt_info_tip"))) if Expect.visibility_of_element_located((By.CLASS_NAME, "gt_success")) \ or Expect.visibility_of_element_located((By.ID, "banner_link")): utitls.myLogger('Login Success~') break retry_times += 1 Wait(browser, 10).until( Expect.invisibility_of_element_located( (By.CLASS_NAME, "gt_fail"))) utitls.myLogger('Login FAIL~') time.sleep(1) if retry_times >= max_retry_times: utitls.myLogger('Retrying MAX') return "" #check is login Success time.sleep(5) #wait for the cookies browser.get('https://link.bilibili.com/p/center/index') time.sleep(5) #wait for the cookies cookies = browser.get_cookies() utitls.myLogger('Setting the Cookies:{}'.format(cookies)) cookies_str_array = [] for cookie in cookies: cookies_str_array.append(cookie["name"] + "=" + cookie["value"]) browser.quit() return ";".join(cookies_str_array) except Exception as e: utitls.myLogger(traceback.format_exc()) utitls.myLogger(str(e)) if browser is not None: browser.quit() return ""
def do_POST(self): request_path = self.path rc = 404 utitls.myLogger("\n----- Request POST Start ----->\n") utitls.myLogger(request_path) content_length = int(self.headers['Content-Length']) post_data = self.rfile.read(content_length) utitls.myLogger(self.headers) utitls.myLogger(post_data) utitls.myLogger("<----- Request POST End -----\n") if '/subscribe' in request_path: # check the secert secert = self.headers.get('X-Hub-Signature', '').split('=')[1] if utitls.verifySecert(secert, post_data): try: tree = ET.ElementTree(ET.fromstring(post_data.decode())) except Exception: utitls.myLogger(traceback.format_exc()) self.send_response(rc) self.end_headers() return ns = { 'dfns': 'http://www.w3.org/2005/Atom', 'yt': 'http://www.youtube.com/xml/schemas/2015', 'at': 'http://purl.org/atompub/tombstones/1.0' } root = tree.getroot() if root.find('dfns:title', ns) != None: tmp_feedTitle = root.find('dfns:title', ns).text tmp_feedUpadatedTime = root.find('dfns:updated', ns).text try: entry = root.findall('dfns:entry', ns)[0] #maybe more than one? tmp_entry_title = entry.find('dfns:title', ns).text tmp_entry_videoId = entry.find('yt:videoId', ns).text tmp_entry_channelId = entry.find('yt:channelId', ns).text tmp_entry_link = entry.find('dfns:link', ns).attrib.get('href') tmp_entry_publishedTime = entry.find( 'dfns:published', ns).text tmp_entry_updatedTime = entry.find('dfns:updated', ns).text utitls.myLogger("%s, %s" % (tmp_feedTitle, tmp_feedUpadatedTime)) utitls.myLogger( "%s, %s, %s, %s, %s, %s " % (tmp_entry_title, tmp_entry_videoId, tmp_entry_channelId, tmp_entry_link, tmp_entry_publishedTime, tmp_entry_updatedTime)) #try to restream Async_forwardToBilibili( tmp_entry_channelId, tmp_entry_link, tmp_entry_title, utitls.configJson().get('area_id')) except Exception: utitls.myLogger(traceback.format_exc()) self.send_response(rc) self.end_headers() return rc = 204 else: utitls.myLogger("verifySecert Failed with:" + secert) self.send_response(rc) self.end_headers() else: self.send_response(rc) self.end_headers()
def log_jobs(): global g_main_scheduler for v in g_main_scheduler.get_jobs(): myLogger("jobId:{}, jobName:{}, jobNextTime{}".format( v.id, v.name, v.next_run_time))