def fetch_games(url: str) -> dict: """ Gets all games from the NHL API """ tprint(f"Looking up games..") tprint(f"@ {url}", debug_only=True) return requests.get(url).json()
def _create_segments(game_id: int, marks: Iterator[str]) -> int: filename: str = f"{game_id}_raw.mkv" tprint("Creating segments", debug_only=True) seg: int = 0 procs: List[subprocess.Popen] = [] for mark in marks: if mark == "end": break next_mark = next(marks) if next_mark != "end": seg += 1 length = float(next_mark) - float(mark) procs.append( split_video_into_cuts(filename, game_id, mark, seg, length)) else: seg += 1 procs.append(split_video_into_cuts(filename, game_id, mark, seg)) ret_codes = [p.wait() for p in procs] if not all(i == 0 for i in ret_codes): failed_procs = [p for p in procs if p.returncode != 0] print([i.stdout.readlines() for i in failed_procs]) raise ExternalProgramError("Segment creation failed") return seg
def obfuscate(download: Download) -> None: """ Pads the end of the video with 100 minutes of black """ game_tracking.update_game_status(download.game_id, GameStatus.obfuscating) input_file: str = f"{download.game_id}_silent.mkv" obfuscate_concat_content = _create_obfuscation_concat_content(input_file) concat_list_file = f"{download.game_id}/obfuscate_concat_list.txt" write_lines_to_file(obfuscate_concat_content, concat_list_file) tprint("Obfuscating end time of video..") output_file: str = f"{download.game_id}_obfuscated.mkv" concat_video(concat_list_file, output_file) os.remove(input_file) cut_to_closest_hour(download.game_id) game_tracking.update_game_status(download.game_id, GameStatus.moving) move_file_to_download_folder(download) game_tracking.update_game_status(download.game_id, GameStatus.completed) game_tracking.download_finished(download.game_id)
def _merge_cuts_to_silent_video(game_id: int) -> None: tprint( "Merging segments back to single video and saving: " + f"{game_id}_silent.mkv", debug_only=True, ) concat_video(f"{game_id}/concat_list.txt", f"{game_id}_silent.mkv")
def _download_individual_video_files(download: Download, num_of_hashes: int) -> None: tprint("Starting download of individual video files", debug_only=True) command = "aria2c -i %s/download_file.txt -j 10 %s" % ( download.game_id, _get_download_options(download.game_id), ) proc, plines = call_subprocess_and_get_stdout_iterator(command) game_tracking.increment_download_attempts(download.game_id) # Track progress and print progress bar progress = 0 for line in plines: if (b"Download complete" in line and b".ts\n" in line and progress < num_of_hashes): progress += 1 print_progress_bar(progress, num_of_hashes, prefix="Downloading:") game_tracking.update_progress(download.game_id, progress, num_of_hashes) proc.wait() if proc.returncode != 0: stdout = proc.stdout.readlines() dump_pickle_if_debug_enabled(stdout) new_dl_filename = _get_dllog_filename(download.game_id, 1) move(f"{download.game_id}_dl.log", new_dl_filename) tprint("Failed to download at least one chunk, attempting to retry..") _retry_failed_files(download, new_dl_filename, 2) game_tracking.clear_progress(download.game_id)
def downloadStream(self, stream_url, outputFile): tprint('Downloading the stream...') command = 'bash ./nhldl.sh "' + stream_url + '" ' + self.quality print(command) p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) pi = iter(p.stdout.readline, b'') downloadFile = None downloadDirectory = None outfile = open(outputFile + '.log', 'w') for line in pi: outfile.write(line) if('Fetching master m3u8 fh' in line): fh = re.search(r'.*/NHL_GAME_VIDEO_(.*)/.*', line, re.M | re.I) downloadFile = 'NHL_GAME_VIDEO_' + fh.group(1) + '.mp4' downloadDirectory = 'NHL_GAME_VIDEO_' + fh.group(1) # Wait for it to finish p.wait() outfile.close() tprint("Stream downloaded. Cleaning up!") # Rename the output fh command = 'mv ' + downloadFile + ' ' + outputFile subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True).wait() # Remove the old directory command = 'rm -rf ' + downloadDirectory subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True).wait()
def get_best_stream(game: Game) -> dict: best_stream: dict = {} best_score: int = -1 for stream in game.streams: score: int = 0 if stream.get("callLetters", "") in get_preferred_streams(): score += 1000 if stream["language"] == "eng": score += 100 if stream_matches_home_away(game, stream["mediaFeedType"]): score += 50 if score > best_score: best_score = score best_stream = stream best_call = best_stream.get("callLetters", "N/A") all_calls = [i.get("callLetters", "N/A") for i in game.streams] tprint( f"Stream {best_call} was selected for {game.game_id} from {all_calls}", debug_only=True, ) # if our preferred stream cannot be downloaded, set the best stream to {} # and say this game is not yet ready to be downloaded if best_stream.get("mediaState", "") != "MEDIA_ARCHIVE": tprint(f"Stream was found for game {game.game_id} that is " f"not archived yet, waiting..") best_stream = {} return best_stream
def login_and_save_cookie() -> None: """ Logs in to NHLTV.com and saves the auth cookie for later use """ user = _get_username_and_password() authorization = get_auth_cookie_value() if authorization: HEADERS.update({"Authorization": authorization}) login_data = { "nhlCredentials": { "email": user.username, "password": user.password } } tprint("Logging in to NHL.com..") req = requests.post( LOGIN_URL, headers={ **HEADERS, "Authorization": authorization }, json=login_data, ) verify_request_200(req) save_cookie(req.cookies)
def _merge_fragments_to_single_video(game_id: int) -> None: tprint("Merge to a single video", debug_only=True) concat_video( _get_concat_file_name(game_id), _get_raw_file_name(game_id), extra_args="-bsf:a aac_adtstoasc", )
def verify_cmd_exists_in_path(cmd: str) -> None: """ Verifies that *cmd* exists by running `which {cmd}` and ensuring rc is 0 """ tprint(f"Checking for {cmd}..", debug_only=True) if not call_subprocess_and_report_rc(f"which {cmd}"): raise CommandMissing(f"{cmd} is missing, please install it") tprint(f"{cmd} exists", debug_only=True)
def _shorten_video(game_id: int) -> None: tprint("Shortening download to 100 files") command = "mv %s/download_file.txt %s/download_file_orig.txt;" % ( game_id, game_id, ) command += ("head -100 %s/download_file_orig.txt > %s/download_file.txt;" % (game_id, game_id)) command += "rm -f %s/download_file_orig.txt;" % game_id call_subprocess_and_raise_on_error(command)
def _verify_nhltv_request_status_succeeded(nhltv_json: dict) -> None: """ Takes a response from the session key URL and raises AuthenticationFailed if authentication failed """ # Expecting negative values to always be bad i.e.: # -3500 is Sign-on restriction: # Too many usage attempts if nhltv_json["status_code"] < 0: tprint(nhltv_json["status_message"]) raise AuthenticationFailed(nhltv_json["status_message"])
def loop() -> None: get_and_download_games() check_interval = get_checkinterval() tprint(f"No games to download, waiting {check_interval} minutes " f"before checking again..") sleep(check_interval * 60) # check if we need to refresh the login (auth cookie) cookie_expiration = get_auth_cookie_expires_in_minutes() if cookie_expiration is None or cookie_expiration < 30: login_and_save_cookie()
def verify_request_200(req: Any) -> None: """ Validates that the request was successful (200) or raises appropriate Exception """ if req.status_code != 200: tprint("There was an error with the request") if req.status_code == 401: msg = "Your username and password is likely incorrect" tprint(msg) raise AuthenticationFailed(msg) raise RequestFailed
def _verify_game_is_not_blacked_out(nhltv_json: dict) -> None: """ Takes a response from the session key URL and raises BlackoutRestriction if the game is blacked out """ if nhltv_json["status_code"] == 1 and ( nhltv_json["user_verified_event"][0]["user_verified_content"][0] ["user_verified_media_item"][0]["blackout_status"]["status"] == "BlackedOutStatus"): msg = "This game is affected by blackout restrictions." tprint(msg) raise BlackoutRestriction(msg)
def checkForNewGame(self, startDate="YYYY-MM-DD", endDate="YYYY-MM-DD"): """ Fetches game schedule between two dates and returns it as a json source """ tprint('Checking for new game between ' + startDate + " and " + endDate) url = 'http://statsapi.web.nhl.com/api/v1/schedule?expand=schedule.teams,schedule.linescore,schedule.scoringplays,schedule.game.content.media.epg&startDate=' url += startDate + '&endDate=' + endDate + '&site=en_nhl&platform=playstation' tprint('Looking up games @ ' + url) # url = 'http://statsapi.web.nhl.com/api/v1/schedule?expand=schedule.teams,schedule.linescore,schedule.scoringplays,schedule.game.content.media.epg&startDate=2016-04-10&endDate=2016-04-10&site=en_nhl&platform=playstation' req = urllib2.Request(url) req.add_header('Connection', 'close') req.add_header('User-Agent', UA_PS4) response = urllib2.urlopen(req) return json.load(response)
def checkForNewGame(self, startDate="YYYY-MM-DD", endDate="YYYY-MM-DD"): """ Fetches game schedule between two dates and returns it as a json source """ tprint('Checking for new game between ' + startDate + " and " + endDate) url = 'http://statsapi.web.nhl.com/api/v1/schedule?expand=schedule.teams,schedule.linescore,schedule.scoringplays,schedule.game.content.media.epg&startDate=' url += startDate + '&endDate=' + endDate + '&site=en_nhl&platform=playstation' tprint('Looking up games @ ' + url) # url = 'http://statsapi.web.nhl.com/api/v1/schedule?expand=schedule.teams,schedule.linescore,schedule.scoringplays,schedule.game.content.media.epg&startDate=2016-04-10&endDate=2016-04-10&site=en_nhl&platform=playstation' req = urllib2.Request(url) req.add_header('Connection', 'close') req.add_header('User-Agent', UA_PS4) response = urllib2.urlopen(req) return json.load(response)
def getGameId(self): current_time = datetime.now() startDate = (current_time.date() - timedelta(days=4)).isoformat() endDate = current_time.date().isoformat() json_source = self.checkForNewGame(startDate, endDate) # Go through all games in the file and look for the next game gameToGet, favTeamHomeAway = self.lookForTheNextGameToGet(json_source) bestScore = -1 bestEpg = None for epg in gameToGet['content']['media']['epg'][0]['items']: score = 0 if (epg['language'] == 'eng'): score = score + 100 if (epg['mediaFeedType'] == favTeamHomeAway): score = score + 50 if (score > bestScore): bestScore = score bestEpg = epg # If there isn't a bestEpg then treat it like an archive case if bestEpg is None: bestEpg = {} bestEpg['mediaState'] = '' # If the feed is good to go then return the info if (bestEpg['mediaState'] == 'MEDIA_ARCHIVE'): gameID = gameToGet['gamePk'] contentID = str(bestEpg['mediaPlaybackId']) eventID = str(bestEpg['eventId']) tprint("Found a game: " + str(gameID)) waitTimeInMin = 0 return gameID, contentID, eventID, waitTimeInMin # If it is not then figure out how long to wait and wait # If the game hasn't started then wait until 3 hours after the game has started startDateTime = datetime.strptime(gameToGet['gameDate'], '%Y-%m-%dT%H:%M:%SZ') if (startDateTime > datetime.utcnow()): waitUntil = startDateTime + timedelta(minutes=150) waitTimeInMin = ( (waitUntil - datetime.utcnow()).total_seconds()) / 60 tprint("Game scheduled for " + gameToGet['gameDate'] + " hasn't started yet") return None, None, None, waitTimeInMin raise (self.NoGameFound)
def getSessionKey(self, game_id, event_id, content_id, authorization): session_key = str(getSetting(sid="session_key", tid=self.teamID)) if session_key == '': tprint("need to fetch new session key") epoch_time_now = str(int(round(time.time() * 1000))) url = 'https://mf.svc.nhl.com/ws/media/mf/v2.4/stream?eventId=' url += event_id + '&format=json&platform=WEB_MEDIAPLAYER&subject=NHLTV&_=' url += epoch_time_now req = urllib2.Request(url) req.add_header("Accept", "application/json") req.add_header("Accept-Encoding", "deflate") req.add_header("Accept-Language", "en-US,en;q=0.8") req.add_header("Connection", "keep-alive") req.add_header("Authorization", authorization) req.add_header("User-Agent", UA_PC) req.add_header("Origin", "https://www.nhl.com") req.add_header( "Referer", "https://www.nhl.com/tv/" + game_id + "/" + event_id + "/" + content_id) response = urllib2.urlopen(req) json_source = json.load(response) response.close() tprint("status_code" + str(json_source['status_code'])) # Expecting - values to always be bad i.e.: -3500 is Sign-on restriction: Too many usage attempts if json_source['status_code'] < 0: tprint(json_source['status_message']) # can't handle this at the moment lest get out of here return 'error' tprint("REQUESTED SESSION KEY") if json_source['status_code'] == 1: if json_source['user_verified_event'][0][ 'user_verified_content'][0][ 'user_verified_media_item'][0]['blackout_status'][ 'status'] == 'BlackedOutStatus': msg = "You do not have access to view this content. To watch live games and learn more about blackout restrictions, please visit NHL.TV" tprint(msg) return 'blackout' session_key = str(json_source['session_key']) setSetting(sid='session_key', value=session_key, tid=self.teamID) return session_key
def cut_to_closest_hour(game_id: int) -> None: """ Cuts video to the closest hour, rounding down, minimum 1 """ input_file = f"{game_id}_obfuscated.mkv" video_length: int = get_video_length(input_file) desired_len_in_seconds = _get_desired_length_after_obfuscation( video_length) tprint("Cutting video to closest hour", debug_only=True) output_file: str = f"{game_id}_ready.mkv" cut_video(input_file, output_file, desired_len_in_seconds) os.remove(input_file)
def getGameId(self): current_time = datetime.now() startDate = (current_time.date() - timedelta(days=4)).isoformat() endDate = current_time.date().isoformat() json_source = self.checkForNewGame(startDate, endDate) # Go through all games in the file and look for the next game gameToGet, favTeamHomeAway = self.lookForTheNextGameToGet(json_source) bestScore = -1 bestEpg = None for epg in gameToGet['content']['media']['epg'][0]['items']: score = 0 if(epg['language'] == 'eng'): score = score + 100 if(epg['mediaFeedType'] == favTeamHomeAway): score = score + 50 if(score > bestScore): bestScore = score bestEpg = epg # If there isn't a bestEpg then treat it like an archive case if bestEpg is None: bestEpg = {} bestEpg['mediaState'] = '' # If the feed is good to go then return the info if(bestEpg['mediaState'] == 'MEDIA_ARCHIVE'): gameID = gameToGet['gamePk'] contentID = str(bestEpg['mediaPlaybackId']) eventID = str(bestEpg['eventId']) tprint("Found a game: " + str(gameID)) waitTimeInMin = 0 return gameID, contentID, eventID, waitTimeInMin # If it is not then figure out how long to wait and wait # If the game hasn't started then wait until 3 hours after the game has started startDateTime = datetime.strptime(gameToGet['gameDate'], '%Y-%m-%dT%H:%M:%SZ') if(startDateTime > datetime.utcnow()): waitUntil = startDateTime + timedelta(minutes=150) waitTimeInMin = ((waitUntil - datetime.utcnow()).total_seconds()) / 60 tprint("Game scheduled for " + gameToGet['gameDate'] + " hasn't started yet") return None, None, None, waitTimeInMin raise(self.NoGameFound)
def redo_broken_downloads(self, outFile): DOWNLOAD_OPTIONS = " --load-cookies=" + COOKIES_TXT_FILE + " --log='" + outFile + "_download.log' --log-level=notice --quiet=true --retry-wait=1 --max-file-not-found=5 --max-tries=5 --header='Accept: */*' --header='Accept-Language: en-US,en;q=0.8' --header='Origin: https://www.nhl.com' -U='Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.97 Safari/537.36' --enable-http-pipelining=true --auto-file-renaming=false --allow-overwrite=true " logFileName = outFile + '_download.log' # Set counters lastErrorCount = 0 lastLineNumber = 0 while (True): # Loop through log file looking for errors logFile = open(logFileName, "r") errors = [] curLineNumber = 0 for line in logFile: curLineNumber = curLineNumber + 1 if (curLineNumber > lastLineNumber): # Is line an error? if ('[ERROR]' in line): error_match = re.search(r'/.*K/(.*)', line, re.M | re.I).group(1) errors.append(error_match) lastLineNumber = curLineNumber logFile.close() if (len(errors) > 0): tprint('Found ' + str(len(errors)) + ' download errors.') if (lastErrorCount == len(errors)): wait( reason= "Same number of errrors as last time so waiting 10 minutes", minutes=10) self.remove_lines_without_errors(errors) tprint('Trying to download the erroneous files again...') # User aria2 to download the list command = 'aria2c -i ./temp/download_file.txt -j 20 ' + DOWNLOAD_OPTIONS _ = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True).wait() lastErrorCount = len(errors)
def getSessionKey(self, game_id, event_id, content_id, authorization): session_key = str(getSetting(sid="session_key")) if session_key == '': tprint("need to fetch new session key") epoch_time_now = str(int(round(time.time() * 1000))) url = 'https://mf.svc.nhl.com/ws/media/mf/v2.4/stream?eventId=' url += event_id + '&format=json&platform=WEB_MEDIAPLAYER&subject=NHLTV&_=' url += epoch_time_now req = urllib2.Request(url) req.add_header("Accept", "application/json") req.add_header("Accept-Encoding", "deflate") req.add_header("Accept-Language", "en-US,en;q=0.8") req.add_header("Connection", "keep-alive") req.add_header("Authorization", authorization) req.add_header("User-Agent", UA_PC) req.add_header("Origin", "https://www.nhl.com") req.add_header("Referer", "https://www.nhl.com/tv/" + game_id + "/" + event_id + "/" + content_id) response = urllib2.urlopen(req) json_source = json.load(response) response.close() tprint("status_code" + str(json_source['status_code'])) # Expecting - values to always be bad i.e.: -3500 is Sign-on restriction: Too many usage attempts if json_source['status_code'] < 0: tprint(json_source['status_message']) # can't handle this at the moment lest get out of here return 'error' tprint("REQUESTED SESSION KEY") if json_source['status_code'] == 1: if json_source['user_verified_event'][0]['user_verified_content'][0]['user_verified_media_item'][0]['blackout_status']['status'] == 'BlackedOutStatus': msg = "You do not have access to view this content. To watch live games and learn more about blackout restrictions, please visit NHL.TV" tprint(msg) return 'blackout' session_key = str(json_source['session_key']) setSetting(sid='session_key', value=session_key) return session_key
def main(): """ Find the gameID or wait until one is ready """ createMandatoryFiles() gameID = None waitTimeInMin = 60 while (gameID is None) or (waitTimeInMin > 0): try: gameID, contentID, eventID, waitTimeInMin = dl.getGameId() except dl.NoGameFound: wait(reason="No new game.", minutes=24 * 60) continue except dl.GameStartedButNotAvailableYet: wait(reason="Game has started but isn't available yet", minutes=10) continue if waitTimeInMin > 0: wait(reason="Game hasn't started yet.", minutes=waitTimeInMin) continue if gameID is None: wait(reason="Did not find a gameID.", minutes=waitTimeInMin) # When one is found then fetch the stream and save the cookies for it tprint('Fetching the stream URL') while True: try: stream_url, _, game_info = dl.fetchStream(gameID, contentID, eventID) break except dl.BlackoutRestriction: wait(reason="Game is effected by NHL Game Center blackout restrictions.", minutes=12 * 60) saveCookiesAsText() tprint("Downloading stream_url") outputFile = str(gameID) + '_raw.mkv' dl.download_nhl(stream_url, outputFile) # Update the settings to reflect that the game was downloaded setSetting('lastGameID', gameID) # Remove silence tprint("Removing silence...") newFileName = DOWNLOAD_FOLDER + game_info + "_" + str(gameID) + '.mkv' silenceSkip(outputFile, newFileName) if MOBILE_VIDEO is True: tprint("Re-encoding for phone...") reEncode(newFileName, str(gameID) + '_phone.mkv')
def download_game(stream: Stream) -> Download: download = _get_download_from_stream(stream) clean_up_download(download.game_id) _create_download_folder(download.game_id) tprint( f"Starting download of game {download.game_id} ({download.game_info})") game_tracking.update_game_status(download.game_id, GameStatus.downloading) game_tracking.download_started(download.game_id) game_tracking.set_game_info(download.game_id, download.game_info) _download_master_file(download) _download_quality_file(download.game_id, _get_quality_url(download)) download_file_contents, decode_hashes = _parse_quality_file(download) write_lines_to_file(download_file_contents, f"{download.game_id}/download_file.txt") # for testing only shorten it to 100 if get_shorten_video(): _shorten_video(download.game_id) decode_hashes = decode_hashes[:45] _download_individual_video_files(download, len(decode_hashes)) concat_file_content = _decode_video_and_get_concat_file_content( download, decode_hashes) write_lines_to_file(concat_file_content, _get_concat_file_name(download.game_id)) _merge_fragments_to_single_video(download.game_id) _remove_ts_files(download.game_id) return download
def downloadStream(self, stream_url, outputFile): tprint('Downloading the stream...') command = 'bash ./nhldl.sh "' + stream_url + '" ' + self.quality print(command) p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) pi = iter(p.stdout.readline, b'') downloadFile = None downloadDirectory = None outfile = open(outputFile + '.log', 'w') for line in pi: outfile.write(line) if ('Fetching master m3u8 fh' in line): fh = re.search(r'.*/NHL_GAME_VIDEO_(.*)/.*', line, re.M | re.I) downloadFile = 'NHL_GAME_VIDEO_' + fh.group(1) + '.mp4' downloadDirectory = 'NHL_GAME_VIDEO_' + fh.group(1) # Wait for it to finish p.wait() outfile.close() tprint("Stream downloaded. Cleaning up!") # Rename the output fh command = 'mv ' + downloadFile + ' ' + outputFile subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True).wait() # Remove the old directory command = 'rm -rf ' + downloadDirectory subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True).wait()
def reEncode(inputFile, outputFile): # command_pass1 = 'ffmpeg -y -nostats -i ' + inputFile + ' -r 30 -vf scale=640x360 -c:v libx265 -preset fast -crf 24 -pass 1 -codec:a copy -f mp4 /dev/null' command_pass2 = 'ffmpeg -y -nostats -i ' + inputFile + ' -r 30 -vf scale=640x360 -c:v libx265 -preset slow -x265-params bframes=0:crf=24:b-adapt=0 -codec:a opus -b:a 48k ' + outputFile + '.mkv' # subprocess.Popen(command_pass1, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True).wait() # tprint('Pass 1 Complete!') subprocess.Popen(command_pass2, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True).wait() tprint('Pass 2 Complete!') tprint('Splitting...') # Create first hour command = 'ffmpeg -y -t 3600 -nostats -i ' + outputFile + '.mkv -c:v copy -codec:a copy ' + outputFile + '_start.mkv' subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True).wait() # Create rest command = 'ffmpeg -y -ss 3600 -nostats -i ' + outputFile + '.mkv -c:v copy -codec:a copy ' + outputFile + '_end.mkv' subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True).wait() # Clean up command = 'rm ffmpeg2pass-0.log' subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) command = 'rm ' + outputFile + '.mkv' subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
def _decode_video_and_get_concat_file_content(download: Download, decode_hashes: List) -> List: tprint("Decode video files", debug_only=True) game_tracking.update_game_status(download.game_id, GameStatus.decoding) procs: List[Any] = [] grouped = [ list(g) for k, g in groupby(decode_hashes, lambda s: s["key_number"]) ] pool = Pool() procs = [ pool.apply_async(_decode_sublist, (download, sublist)) for sublist in grouped ] concat_file_content = [p.get() for p in procs] flat = [i for s in concat_file_content for i in s] return flat
def redo_broken_downloads(self, outFile): DOWNLOAD_OPTIONS = " --load-cookies=" + COOKIES_TXT_FILE + " --log='" + outFile + "_download.log' --log-level=notice --quiet=true --retry-wait=1 --max-file-not-found=5 --max-tries=5 --header='Accept: */*' --header='Accept-Language: en-US,en;q=0.8' --header='Origin: https://www.nhl.com' -U='Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.97 Safari/537.36' --enable-http-pipelining=true --auto-file-renaming=false --allow-overwrite=true " logFileName = outFile + '_download.log' # Set counters lastErrorCount = 0 lastLineNumber = 0 while(True): # Loop through log file looking for errors logFile = open(logFileName, "r") errors = [] curLineNumber = 0 for line in logFile: curLineNumber = curLineNumber + 1 if(curLineNumber > lastLineNumber): # Is line an error? if('[ERROR]' in line): error_match = re.search(r'/.*K/(.*)', line, re.M | re.I).group(1) errors.append(error_match) lastLineNumber = curLineNumber logFile.close() if(len(errors) > 0): tprint('Found ' + str(len(errors)) + ' download errors.') if(lastErrorCount == len(errors)): wait(reason="Same number of errrors as last time so waiting 10 minutes", minutes=10) self.remove_lines_without_errors(errors) tprint('Trying to download the erroneous files again...') # User aria2 to download the list command = 'aria2c -i ./temp/download_file.txt -j 20 ' + DOWNLOAD_OPTIONS _ = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True).wait() lastErrorCount = len(errors)
def get_games_to_download() -> Tuple[Game, ...]: """ Gets all the games that are available to be downloaded and matches the criteria (eg. correct team, not already downloaded etc.) """ start_date: str = get_start_date() end_date: str = get_end_date() all_games: dict = fetch_games( get_schedule_url_between_dates(start_date, end_date)) dump_json_if_debug_enabled(all_games) filtered_games = tuple(filter_games(all_games)) games_objects: Iterable[Game] = create_game_objects(filtered_games) add_games_to_tracking(filtered_games) games_list: Tuple[Game, ...] = tuple(games_objects) game_ids: List[int] = [i.game_id for i in games_list] tprint(f"Found games {game_ids}", debug_only=True) return games_list
def main(): """ Find the game ID or wait until one is ready """ createMandatoryFiles() game_id = None wait_time_in_min = 60 while game_id is None or wait_time_in_min > 0: try: game_id, content_id, event_id, wait_time_in_min = dl.get_game_id() except dl.NoGameFound: wait(reason="No new game.", minutes=24 * 60) continue except dl.GameStartedButNotAvailableYet: wait(reason="Game has started but isn't available yet", minutes=10) continue if wait_time_in_min > 0: wait(reason="Game hasn't started yet.", minutes=wait_time_in_min) continue if game_id is None: wait(reason="Did not find a game_id.", minutes=wait_time_in_min) # When one is found then fetch the stream and save the cookies for it logging.debug('Fetching the stream URL') while True: try: stream_url, _, game_info = dl.fetchStream(game_id, content_id, event_id) break except dl.BlackoutRestriction: wait(reason= "Game is effected by NHL Game Center blackout restrictions.", minutes=12 * 60) saveCookiesAsText() tprint("Downloading stream_url") outputFile = str(game_id) + '_raw.mkv' dl.download_nhl(stream_url, outputFile) # Update the settings to reflect that the game was downloaded setSetting('lastGameID', game_id) # Remove silence tprint("Removing silence...") newFileName = DOWNLOAD_FOLDER + game_info + "_" + str(game_id) + '.mkv' silenceSkip(outputFile, newFileName) if MOBILE_VIDEO is True: tprint("Re-encoding for phone...") reEncode(newFileName, str(game_id) + '_phone.mkv')
def _retry_failed_files( download: Download, last_dllog_filename: str, attempt: int, max_attempts: int = 5, ): # scan log and save urls dllog_content = _get_dllog_contents(last_dllog_filename) failed_urls = [] for line in dllog_content: if "[ERROR]" in line: failed_urls.append(line.split("URI=")[-1]) if len(failed_urls) > 100: tprint( f"Too many failed chunks to retry, failed chunks: {len(failed_urls)} " ) raise DownloadError() tprint( f"Retrying download of {len(failed_urls)} chunks, attempt {attempt} of {max_attempts}" ) dlfile_contents = _get_downloadfile_contents(download.game_id) new_dlfile_contents = [] for idx, line in enumerate(dlfile_contents): if line in failed_urls: new_dlfile_contents.append(line) new_dlfile_contents.append(dlfile_contents[idx + 1]) write_lines_to_file(new_dlfile_contents, f"{download.game_id}/download_file.txt") command = "aria2c -i %s/download_file.txt -j 10 %s" % ( download.game_id, _get_download_options(download.game_id), ) proc, plines = call_subprocess_and_get_stdout_iterator(command) proc.wait() if proc.returncode == 0: return new_log_filename = _get_dllog_filename(download.game_id, attempt) move(f"{download.game_id}_dl.log", new_log_filename) if attempt >= max_attempts: tprint(f"Downloading game {download.game_id} failed") raise DownloadError() _retry_failed_files(download, new_log_filename, attempt + 1)
def test_tprint(mocker, mock_datetime): mp = mocker.patch("builtins.print") tprint("boo") mp.assert_called_once_with( f"{mock_datetime.strftime('%b %-d %H:%M:%S')} - boo")
url = 'https://gateway.web.nhl.com/ws/subscription/flow/nhlPurchase.login' login_data = '{"nhlCredentials":{"email":"' + self.userName + '","password":"******"}}' req = urllib2.Request(url, data=login_data, headers={ "Accept": "*/*", "Accept-Encoding": "gzip, deflate", "Accept-Language": "en-US,en;q=0.8", "Content-Type": "application/json", "Origin": "https://www.nhl.com", "Authorization": authorization, "Connection": "keep-alive", "User-Agent": UA_PC}) try: response = opener.open(req) except HTTPError as e: tprint('The server couldn\'t fulfill the request.') tprint('Error code: ', e.code) tprint(url) # Error 401 for invalid login if e.code == 401: msg = "Please check that your username and password are correct" tprint(msg) response.close() cj.save(ignore_discard=True) def logout(self, display_msg=None): cj = cookielib.LWPCookieJar(COOKIES_LWP_FILE) try: cj.load(COOKIES_LWP_FILE, ignore_discard=True)
def download_nhl(self, url, outFile, retry_errored=False): logFile = outFile + "_download.log" DOWNLOAD_OPTIONS = " --load-cookies=" + COOKIES_TXT_FILE + " --log='" + logFile + "' --log-level=notice --quiet=true --retry-wait=1 --max-file-not-found=5 --max-tries=5 --header='Accept: */*' --header='Accept-Language: en-US,en;q=0.8' --header='Origin: https://www.nhl.com' -U='Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.97 Safari/537.36' --enable-http-pipelining=true --auto-file-renaming=false --allow-overwrite=true " tprint("Starting Download: " + url) # Pull url_root url_root = re.match('(.*)master_tablet60.m3u8', url, re.M | re.I).group(1) # Create the temp and keys directory if not os.path.exists('./temp/keys'): os.makedirs('./temp/keys') # Get the master m3u8 masterFile = "temp/master.m3u8" self.downloadWebPage(url, masterFile, logFile) quality_url = url_root + self.getQualityUrlFromMaster_m3u8(masterFile) # Get the m3u8 for the quality inputFile = "temp/input.m3u8" self.downloadWebPage(quality_url, inputFile, logFile) # Parse m3u8 # Create files download_file = "./temp/download_file.txt" decode_hashes = self.createDownloadFile(inputFile, download_file, quality_url) # for testing only shorten it to 100 # tprint("shorting to 100 files for testing") # command = 'mv ./temp/download_file.txt ./temp/download_file_orig.txt;' # command += 'head -100 ./temp/download_file_orig.txt > ./temp/download_file.txt;' # command += 'rm -f ./temp/download_file_orig.txt;' # p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True).wait() # User aria2 to download the list tprint("starting download of individual video files") command = 'aria2c -i ./temp/download_file.txt -j 20 ' + DOWNLOAD_OPTIONS p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True).wait() # Repair broken downloads if necessary if retry_errored is True: self.redo_broken_downloads(outFile) # Create the concat file concat_file = open("./temp/concat.txt", "w") # Iterate through the decode_hashes and run the decoder function tprint("Decode video files") for dH in decode_hashes: cur_key = 'blank' key_val = '' # If the cur_key isn't the one from the has then refresh the key_val if(cur_key != dH['key_number']): # Extract the key value command = 'xxd -p ./temp/keys/' + dH['key_number'] p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) pi = iter(p.stdout.readline, b'') for line in pi: key_val = line.strip('\n') cur_key = dH['key_number'] p.wait() # Decode TS command = 'openssl enc -aes-128-cbc -in "./temp/' + dH['ts_number'] + '.ts" -out "./temp/' + dH['ts_number'] + '.ts.dec" -d -K ' + key_val + ' -iv ' + dH['iv'] subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True).wait() # Move decoded files over old files command = 'mv ./temp/' + dH['ts_number'] + '.ts.dec ./temp/' + dH['ts_number'] + '.ts' subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True).wait() # Add to concat file concat_file.write('file ' + dH['ts_number'] + '.ts\n') # close concat file concat_file.close() # merge to single command = 'ffmpeg -y -nostats -loglevel 0 -f concat -i ./temp/concat.txt -c copy -bsf:a aac_adtstoasc ' + outFile subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True).wait() # delete the old directory command = 'rm -rf ./temp' subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True).wait()
def fetchStream(self, game_id, content_id, event_id): stream_url = '' media_auth = '' authorization = self.getAuthCookie() if authorization == '': self.login() authorization = self.getAuthCookie() if authorization == '': return stream_url, media_auth, "" cj = cookielib.LWPCookieJar(COOKIES_LWP_FILE) cj.load(COOKIES_LWP_FILE, ignore_discard=True) tprint("Fetching session_key") session_key = self.getSessionKey(game_id, event_id, content_id, authorization) opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj)) tprint("Checking session key") if session_key == '': return stream_url, media_auth, "" # Org url = 'https://mf.svc.nhl.com/ws/media/mf/v2.4/stream?contentId=' url += str(content_id) + '&playbackScenario=HTTP_CLOUD_TABLET_60&platform=IPAD&sessionKey=' url += urllib.quote_plus(session_key) req = urllib2.Request(url) req.add_header("Accept", "*/*") req.add_header("Accept-Encoding", "deflate") req.add_header("Accept-Language", "en-US,en;q=0.8") req.add_header("Connection", "keep-alive") req.add_header("Authorization", authorization) req.add_header("User-Agent", UA_NHL) req.add_header("Proxy-Connection", "keep-alive") response = opener.open(req) json_source = json.load(response) response.close() # Pulling out game_info in formated like "2017-03-06_VAN-ANA" for file name prefix game_info = self.getGameInfo(json_source) tprint("game info=" + game_info) # Expecting - values to always be bad i.e.: -3500 is Sign-on restriction: Too many usage attempts if json_source['status_code'] < 0: tprint(json_source['status_message']) # can't handle this at the moment lest get out of here exit(1) if json_source['status_code'] == 1: if json_source['user_verified_event'][0]['user_verified_content'][0]['user_verified_media_item'][0]['blackout_status']['status'] == 'BlackedOutStatus': msg = "You do not have access to view this content. To watch live games and learn more about blackout restrictions, please visit NHL.TV" tprint(msg) raise self.BlackoutRestriction stream_url = json_source['user_verified_event'][0]['user_verified_content'][0]['user_verified_media_item'][0]['url'] media_auth = str(json_source['session_info']['sessionAttributes'][0]['attributeName']) + "=" + str(json_source['session_info']['sessionAttributes'][0]['attributeValue']) session_key = json_source['session_key'] setSetting(sid='media_auth', value=media_auth) # Update Session Key setSetting(sid='session_key', value=session_key) # Add media_auth cookie ck = cookielib.Cookie(version=0, name='mediaAuth', value="" + media_auth.replace('mediaAuth=', '') + "", port=None, port_specified=False, domain='.nhl.com', domain_specified=True, domain_initial_dot=True, path='/', path_specified=True, secure=False, expires=(int(time.time()) + 7500), discard=False, comment=None, comment_url=None, rest={}, rfc2109=False) cj = cookielib.LWPCookieJar(COOKIES_LWP_FILE) cj.load(COOKIES_LWP_FILE, ignore_discard=True) cj.set_cookie(ck) cj.save(ignore_discard=False) return stream_url, media_auth, game_info
def silenceSkip(inputFile, outputFile): tprint("Analyzing " + inputFile + " for silence.") command = "ffmpeg -y -nostats -i " + inputFile + " -af silencedetect=n=-50dB:d=10 -c:v copy -c:a libmp3lame -f mp4 /dev/null" p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) pi = iter(p.stdout.readline, b'') marks = [] marks.append('0') for line in pi: if('silencedetect' in line): start_match = re.search(r'.*silence_start: (.*)', line, re.M | re.I) end_match = re.search(r'.*silence_end: (.*) \|.*', line, re.M | re.I) if((start_match is not None) and (start_match.lastindex == 1)): marks.append(start_match.group(1)) # tprint("Start: " + start_match.group(1)) if((end_match is not None) and end_match.lastindex == 1): marks.append(end_match.group(1)) # tprint("End: " + end_match.group(1)) # If it is not an even number of segments then add the end point. If the last silence goes # to the endpoint then it will be an even number. if(len(marks) % 2 == 1): marks.append('end') # Make a temp dir command = 'mkdir ./temp' subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True).wait() tprint("Creating segments.") seg = 0 # Create segments for i in range(0, len(marks)): if(i % 2 == 0): if marks[i + 1] is not 'end': seg = seg + 1 length = float(marks[i + 1]) - float(marks[i]) command = 'ffmpeg -y -nostats -i ' + inputFile + ' -ss ' + str(marks[i]) + ' -t ' + str(length) + ' -c:v copy -c:a copy ./temp/cut' + str(seg) + '.mp4' else: seg = seg + 1 command = 'ffmpeg -y -nostats -i ' + inputFile + ' -ss ' + str(marks[i]) + ' -c:v copy -c:a copy ./temp/cut' + str(seg) + '.mp4' subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True).wait() # Create file list fh = open("./temp/concat_list.txt", "w") for i in range(1, seg + 1): fh.write("file\t" + 'cut' + str(i) + '.mp4\n') fh.close() # Create the download directory if required command = 'mkdir -p $(dirname ' + outputFile + ')' subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True).wait() command = 'ffmpeg -y -nostats -f concat -i ./temp/concat_list.txt -c copy ' + outputFile subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True).wait() tprint("Merging segments back to single video and saving: " + outputFile) # Erase temp command = 'rm -rf ./temp' subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) # Erase orig file command = 'rm ' + inputFile subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
def fetchStream(self, game_id, content_id, event_id): stream_url = '' media_auth = '' authorization = self.getAuthCookie() if authorization == '': self.login() authorization = self.getAuthCookie() if authorization == '': return stream_url, media_auth, "" cj = cookielib.LWPCookieJar(COOKIES_LWP_FILE) cj.load(COOKIES_LWP_FILE, ignore_discard=True) tprint("Fetching session_key") session_key = self.getSessionKey(game_id, event_id, content_id, authorization) opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj)) tprint("Checking session key") if session_key == '': return stream_url, media_auth, "" # Org url = 'https://mf.svc.nhl.com/ws/media/mf/v2.4/stream?contentId=' url += str( content_id ) + '&playbackScenario=HTTP_CLOUD_TABLET_60&platform=IPAD&sessionKey=' url += urllib.quote_plus(session_key) req = urllib2.Request(url) req.add_header("Accept", "*/*") req.add_header("Accept-Encoding", "deflate") req.add_header("Accept-Language", "en-US,en;q=0.8") req.add_header("Connection", "keep-alive") req.add_header("Authorization", authorization) req.add_header("User-Agent", UA_NHL) req.add_header("Proxy-Connection", "keep-alive") response = opener.open(req) json_source = json.load(response) response.close() # Pulling out game_info in formated like "2017-03-06_VAN-ANA" for file name prefix game_info = self.getGameInfo(json_source) tprint("game info=" + game_info) # Expecting - values to always be bad i.e.: -3500 is Sign-on restriction: Too many usage attempts if json_source['status_code'] < 0: tprint(json_source['status_message']) # can't handle this at the moment lest get out of here exit(1) if json_source['status_code'] == 1: if json_source['user_verified_event'][0]['user_verified_content'][ 0]['user_verified_media_item'][0]['blackout_status'][ 'status'] == 'BlackedOutStatus': msg = "You do not have access to view this content. To watch live games and learn more about blackout restrictions, please visit NHL.TV" tprint(msg) raise self.BlackoutRestriction stream_url = json_source['user_verified_event'][0][ 'user_verified_content'][0]['user_verified_media_item'][0]['url'] media_auth = str(json_source['session_info']['sessionAttributes'][0] ['attributeName']) + "=" + str( json_source['session_info']['sessionAttributes'] [0]['attributeValue']) session_key = json_source['session_key'] setSetting(sid='media_auth', value=media_auth, tid=self.teamID) # Update Session Key setSetting(sid='session_key', value=session_key, tid=self.teamID) # Add media_auth cookie ck = cookielib.Cookie(version=0, name='mediaAuth', value="" + media_auth.replace('mediaAuth=', '') + "", port=None, port_specified=False, domain='.nhl.com', domain_specified=True, domain_initial_dot=True, path='/', path_specified=True, secure=False, expires=(int(time.time()) + 7500), discard=False, comment=None, comment_url=None, rest={}, rfc2109=False) cj = cookielib.LWPCookieJar(COOKIES_LWP_FILE) cj.load(COOKIES_LWP_FILE, ignore_discard=True) cj.set_cookie(ck) cj.save(ignore_discard=False) return stream_url, media_auth, game_info
def test_tprint_debug_off(mocker, mock_datetime, mock_debug_dumps_enabled, mock_print): mock_debug_dumps_enabled.return_value = False tprint("boo", True) mock_print.assert_not_called()
def test_tprint_debug_on(mocker, mock_datetime, mock_debug_dumps_enabled, mock_print): mock_debug_dumps_enabled.return_value = True tprint("boo", True) mock_print.assert_called_once_with( f"{mock_datetime.strftime('%b %-d %H:%M:%S')} - boo")
def parse_args(): global DOWNLOAD_FOLDER global RETRY_ERRORED_DOWNLOADS global MOBILE_VIDEO if which("ffmpeg") is False: print ("Missing ffmpeg command please install or check PATH exiting...") exit(1) if which("aria2c") is False: print ("Missing aria2c command please install or check PATH exiting...") exit(1) parser = argparse.ArgumentParser(description='%(prog)s: Download NHL TV') parser.add_argument( "-t", "--team", dest="TEAMID", help="Team ID i.e. 17 or DET or Detroit", required=True) parser.add_argument( "-u", "--username", dest="USERNAME", help="User name of your NHLTV account") parser.add_argument( "-p", "--password", dest="PASSWORD", help="Password of your NHL TV account ") parser.add_argument( "-q", "--quality", dest="QUALITY", help="is highest by default you can set it to 5000, 3500, 1500, 900") parser.add_argument( "-d", "--download_folder", dest="DOWNLOAD_FOLDER", help="Output folder where you want to store your final file like $HOME/Desktop/NHL/") parser.set_defaults(feature=True) parser.add_argument( "-r", "--retry", dest="RETRY_ERRORED_DOWNLOADS", action='store_true', help="Usually works fine without, Use this flag if you want it perfect") parser.add_argument( "-m", "--mobile_video", dest="MOBILE_VIDEO", action='store_true', help="Set this to also encode video for mobile devices") args = parser.parse_args() if args.TEAMID: teams = Teams() team = teams.getTeam(args.TEAMID) dl.teamID = team.id if args.USERNAME: dl.userName = args.USERNAME setSetting("USERNAME", args.USERNAME) else: dl.userName = getSetting("USERNAME") if args.PASSWORD: dl.passWord = args.PASSWORD setSetting("PASSWORD", args.PASSWORD) else: dl.passWord = getSetting("PASSWORD") if args.QUALITY: dl.quality = str(args.QUALITY) if args.DOWNLOAD_FOLDER: DOWNLOAD_FOLDER = args.DOWNLOAD_FOLDER setSetting("DOWNLOAD_FOLDER", DOWNLOAD_FOLDER) else: DOWNLOAD_FOLDER = getSetting("DOWNLOAD_FOLDER") tprint("DOWNLOAD_FOLDER got set to " + DOWNLOAD_FOLDER) if args.RETRY_ERRORED_DOWNLOADS: RETRY_ERRORED_DOWNLOADS = args.RETRY_ERRORED_DOWNLOADS if args.MOBILE_VIDEO: MOBILE_VIDEO = args.MOBILE_VIDEO while(True): main()