def download(video_id, max_workers, format='mkv', **kwargs): video_id = parse_video_id(video_id) print("Looking up video...") video = twitch.get_video(video_id) print("Fetching access token...") access_token = twitch.get_access_token(video_id) print("Fetching playlists...") playlists = twitch.get_playlists(video_id, access_token) quality, playlist_url = _select_quality(playlists) print("\nFetching playlist...") base_url, filenames = twitch.get_playlist_urls(playlist_url) # Create a temp dir to store downloads if it doesn't exist directory = '{}/twitch-dl/{}/{}'.format(tempfile.gettempdir(), video_id, quality) pathlib.Path(directory).mkdir(parents=True, exist_ok=True) print("Download dir: {}".format(directory)) print("Downloading VODs with {} workers...".format(max_workers)) paths = _download_files(base_url, directory, filenames, max_workers) print("\n\nJoining files...") target = _video_target_filename(video, format) _join_vods(directory, paths, target) print("\nDeleting vods...") for path in paths: os.unlink(path) print("\nDownloaded: {}".format(target))
def test_get_videos(): videos = twitch.get_channel_videos(TEST_CHANNEL, 3, "time") assert videos["pageInfo"] assert len(videos["edges"]) > 0 video_id = videos["edges"][0]["node"]["id"] video = twitch.get_video(video_id) assert video["id"] == video_id
def _download_video(video_id, max_workers, format='mp4', start=None, end=None, keep=False, **kwargs): if start and end and end <= start: raise ConsoleError("End time must be greater than start time") _log(video_id, "Recherche la video ") video = twitch.get_video(video_id) _log(video_id, "Informations sur {}".format(video['title'])) print_out("Trouvé: <blue>{}</blue> by <yellow>{}</yellow>".format( video['title'], video['channel']['display_name'])) access_token = twitch.get_access_token(video_id) _log(video_id, "Obtention de la liste des fichiers...") playlists = twitch.get_playlists(video_id, access_token) parsed = m3u8.loads(playlists) selected = _select_quality(parsed.playlists) # print_out("\nListe...") response = requests.get(selected.uri) response.raise_for_status() playlist = m3u8.loads(response.text) base_uri = re.sub("/[^/]+$", "/", selected.uri) target_dir = _crete_temp_dir(base_uri) filenames = list(_get_files(playlist, start, end)) # Save playlists for debugging purposes with open(target_dir + "playlists.m3u8", "w") as f: f.write(playlists) with open(target_dir + "playlist.m3u8", "w") as f: f.write(response.text) # print_out("\nTélécharge {} VODs avec {} threads dans {}".format( # len(filenames), max_workers, target_dir)) file_paths = download_files(video_id, base_uri, target_dir, filenames, max_workers) target = _video_target_filename(video, format) print_out("\nCible: {}".format(target)) _join_vods(target_dir, file_paths, "videos/Download/{}".format(target)) if keep: print_out("\nTemporary files not deleted: {}".format(target_dir)) else: # print_out("\nSupprime le fichier temporaire...") shutil.rmtree(target_dir) print_out("Fichier téléchargé: {}".format(target)) _log(video_id, "Terminé {}".format(target))
def download(video_id, max_workers, format='mkv', start=None, end=None, keep=False, **kwargs): video_id = _parse_video_id(video_id) if start and end and end <= start: raise ConsoleError("End time must be greater than start time") print_out("Looking up video...") video = twitch.get_video(video_id) print_out("Found: <blue>{}</blue> by <yellow>{}</yellow>".format( video['title'], video['channel']['display_name'])) print_out("Fetching access token...") access_token = twitch.get_access_token(video_id) print_out("Fetching playlists...") playlists = twitch.get_playlists(video_id, access_token) parsed = m3u8.loads(playlists) selected = _select_quality(parsed.playlists) print_out("\nFetching playlist...") response = requests.get(selected.uri) response.raise_for_status() playlist = m3u8.loads(response.text) base_uri = re.sub("/[^/]+$", "/", selected.uri) target_dir = _crete_temp_dir(base_uri) filenames = list(_get_files(playlist, start, end)) # Save playlists for debugging purposes with open(target_dir + "playlists.m3u8", "w") as f: f.write(playlists) with open(target_dir + "playlist.m3u8", "w") as f: f.write(response.text) print_out("\nDownloading {} VODs using {} workers to {}".format( len(filenames), max_workers, target_dir)) _download_files(base_uri, target_dir, filenames, max_workers) print_out("\n\nJoining files...") target = _video_target_filename(video, format) _join_vods(target_dir, filenames, target) if keep: print_out("\nTemporary files not deleted: {}".format(target_dir)) else: print_out("\nDeleting temporary files...") shutil.rmtree(target_dir) print_out("Downloaded: {}".format(target))
def download(video_id, max_workers, format='mkv', start=None, end=None, **kwargs): video_id = parse_video_id(video_id) if start and end and end <= start: raise ConsoleError("End time must be greater than start time") print_out("Looking up video...") video = twitch.get_video(video_id) print_out("Found: <blue>{}</blue> by <yellow>{}</yellow>".format( video['title'], video['channel']['display_name'])) print_out("Fetching access token...") access_token = twitch.get_access_token(video_id) print_out("Fetching playlists...") playlists = twitch.get_playlists(video_id, access_token) quality, playlist_url = _select_quality(playlists) print_out("\nFetching playlist...") base_url, filenames = twitch.get_playlist_urls(playlist_url, start, end) if not filenames: raise ConsoleError("No vods matched, check your start and end times") # Create a temp dir to store downloads if it doesn't exist directory = '{}/twitch-dl/{}/{}'.format(tempfile.gettempdir(), video_id, quality) pathlib.Path(directory).mkdir(parents=True, exist_ok=True) print_out("Download dir: {}".format(directory)) print_out("Downloading {} VODs using {} workers...".format( len(filenames), max_workers)) paths = _download_files(base_url, directory, filenames, max_workers) print_out("\n\nJoining files...") target = _video_target_filename(video, format) _join_vods(directory, paths, target) print_out("\nDeleting vods...") for path in paths: os.unlink(path) print_out("\nDownloaded: {}".format(target))
def sort_meeting_data(): """ sorts meeting data into training and test data based on the person who uploaded it :return: """ start_path = "Data/Meeting Identifier/Crude Data/" files = os.listdir(start_path) player_names = dict() for file in files: if file == ".DS_Store": continue # split the filepath on the dash items = file.split("-") video_id = items[3] if video_id in player_names: # if we already have the player name cached, use it player_name = player_names[video_id] else: # otherwise, get the player name from the video ID video = twitch.get_video(video_id) player_name = video['creator']['displayName'] # cache the player name player_names[video_id] = player_name if is_name_test(player_name): data_set = "Test Data" else: data_set = "Training Data" output = os.path.join("Data", "Meeting Identifier", data_set, file) os.rename(start_path + file, output)
def sort_classifier_crude_data(): """ sorts through "crude" data and puts it into Test data :return: None """ start_path = "Data/Game Classifier/Crude Data/" files = os.listdir(start_path) player_names = dict() for file in files: # split the filepath on the dash items = file.split("-") image_kind = items[0] video_id = items[1] if video_id in player_names: # if we already have the player name cached, use it player_name = player_names[video_id] else: # otherwise, get the player name from the video ID video = twitch.get_video(video_id) player_name = video['channel']['display_name'] # cache the player name player_names[video_id] = player_name if is_name_test(player_name): data_set = "Test Data" else: data_set = "Training Data" output = os.path.join("Data", "Game Classifier", data_set, image_kind, file) os.rename(start_path + file, output)
def download(video_id, max_workers, format='mkv', **kwargs): print("Looking up video...") video = twitch.get_video(video_id) print("Fetching access token...") access_token = twitch.get_access_token(video_id) print("Fetching playlists...") playlists = twitch.get_playlists(video_id, access_token) playlist_url = _select_playlist_by_quality(playlists) print("\nFetching playlist...") base_url, filenames = twitch.get_playlist_urls(playlist_url) target = _video_target_filename(video, format) with tempfile.TemporaryDirectory() as directory: print("Downloading...") _download_files(base_url, directory, filenames, max_workers) print("\n\nJoining files...") _join_vods(directory, filenames, target) print("\nDownloaded: {}".format(target))
def info(args): video_id = utils.parse_video_identifier(args.identifier) if video_id: print_log("Fetching video...") video = twitch.get_video(video_id) if not video: raise ConsoleError("Video {} not found".format(video_id)) print_log("Fetching access token...") access_token = twitch.get_access_token(video_id) print_log("Fetching playlists...") playlists = twitch.get_playlists(video_id, access_token) if video: if args.json: video_json(video, playlists) else: video_info(video, playlists) return clip_slug = utils.parse_clip_identifier(args.identifier) if clip_slug: print_log("Fetching clip...") clip = twitch.get_clip(clip_slug) if not clip: raise ConsoleError("Clip {} not found".format(clip_slug)) if args.json: print_json(clip) else: clip_info(clip) return raise ConsoleError("Invalid input: {}".format(args.video))
def _download_video(video_id, args): if args.start and args.end and args.end <= args.start: raise ConsoleError("End time must be greater than start time") if os.path.isfile(str(Path.home()) + "/.twitchdownloads/" + video_id): print("File already downloaded") return print_out("<dim>Looking up video...</dim>") video = twitch.get_video(video_id) save_json_video(video_id, video, _video_target_filename(video, "json")) print_out("Found: <blue>{}</blue> by <yellow>{}</yellow>".format( video['title'], video['channel']['display_name'])) print_out("<dim>Fetching access token...</dim>") access_token = twitch.get_access_token(video_id) print_out("<dim>Fetching playlists...</dim>") playlists_m3u8 = twitch.get_playlists(video_id, access_token) playlists = list(_parse_playlists(playlists_m3u8)) playlist_uri = (_get_playlist_by_name(playlists, args.quality) if args.quality else _select_playlist_interactive(playlists)) print_out("<dim>Fetching playlist...</dim>") response = requests.get(playlist_uri) response.raise_for_status() playlist = m3u8.loads(response.text) base_uri = re.sub("/[^/]+$", "/", playlist_uri) target_dir = _crete_temp_dir(base_uri) vod_paths = _get_vod_paths(playlist, args.start, args.end) # Save playlists for debugging purposes with open(path.join(target_dir, "playlists.m3u8"), "w") as f: f.write(playlists_m3u8) with open(path.join(target_dir, "playlist.m3u8"), "w") as f: f.write(response.text) print_out("\nDownloading {} VODs using {} workers to {}".format( len(vod_paths), args.max_workers, target_dir)) path_map = download_files(base_uri, target_dir, vod_paths, args.max_workers) # Make a modified playlist which references downloaded VODs # Keep only the downloaded segments and skip the rest org_segments = playlist.segments.copy() playlist.segments.clear() for segment in org_segments: if segment.uri in path_map: segment.uri = path_map[segment.uri] playlist.segments.append(segment) playlist_path = path.join(target_dir, "playlist_downloaded.m3u8") playlist.dump(playlist_path) print_out("\n\nJoining files...") target = _video_target_filename(video, args.format) _join_vods(playlist_path, target) if args.keep: print_out( "\n<dim>Temporary files not deleted: {}</dim>".format(target_dir)) else: print_out("\n<dim>Deleting temporary files...</dim>") shutil.rmtree(target_dir) print_out("\nDownloaded: <green>{}</green>".format(target))
def _download_video(video_id, args): if args.start and args.end and args.end <= args.start: raise ConsoleError("End time must be greater than start time") print_out("<dim>Looking up video...</dim>") video = twitch.get_video(video_id) if not video: raise ConsoleError("Video {} not found".format(video_id)) print_out("Found: <blue>{}</blue> by <yellow>{}</yellow>".format( video['title'], video['creator']['displayName'])) target = _video_target_filename(video, args) print_out("Output: <blue>{}</blue>".format(target)) if not args.overwrite and path.exists(target): response = input("File exists. Overwrite? [Y/n]: ") if response.lower().strip() not in ["", "y"]: raise ConsoleError("Aborted") args.overwrite = True print_out("<dim>Fetching access token...</dim>") access_token = twitch.get_access_token(video_id) print_out("<dim>Fetching playlists...</dim>") playlists_m3u8 = twitch.get_playlists(video_id, access_token) playlists = list(_parse_playlists(playlists_m3u8)) playlist_uri = (_get_playlist_by_name(playlists, args.quality) if args.quality else _select_playlist_interactive(playlists)) print_out("<dim>Fetching playlist...</dim>") response = requests.get(playlist_uri) response.raise_for_status() playlist = m3u8.loads(response.text) base_uri = re.sub("/[^/]+$", "/", playlist_uri) target_dir = _crete_temp_dir(base_uri) vod_paths = _get_vod_paths(playlist, args.start, args.end) # Save playlists for debugging purposes with open(path.join(target_dir, "playlists.m3u8"), "w") as f: f.write(playlists_m3u8) with open(path.join(target_dir, "playlist.m3u8"), "w") as f: f.write(response.text) print_out("\nDownloading {} VODs using {} workers to {}".format( len(vod_paths), args.max_workers, target_dir)) path_map = download_files(base_uri, target_dir, vod_paths, args.max_workers) # Make a modified playlist which references downloaded VODs # Keep only the downloaded segments and skip the rest org_segments = playlist.segments.copy() playlist.segments.clear() for segment in org_segments: if segment.uri in path_map: segment.uri = path_map[segment.uri] playlist.segments.append(segment) playlist_path = path.join(target_dir, "playlist_downloaded.m3u8") playlist.dump(playlist_path) if args.no_join: print_out("\n\n<dim>Skipping joining files...</dim>") print_out("VODs downloaded to:\n<blue>{}</blue>".format(target_dir)) return print_out("\n\nJoining files...") _join_vods(playlist_path, target, args.overwrite, video) if args.keep: print_out( "\n<dim>Temporary files not deleted: {}</dim>".format(target_dir)) else: print_out("\n<dim>Deleting temporary files...</dim>") shutil.rmtree(target_dir) print_out("\nDownloaded: <green>{}</green>".format(target))