def download(video_id, max_workers, format='mkv', **kwargs):
video_id = parse_video_id(video_id)
print("Looking up video...")
video = twitch.get_video(video_id)
print("Fetching access token...")
access_token = twitch.get_access_token(video_id)
print("Fetching playlists...")
playlists = twitch.get_playlists(video_id, access_token)
quality, playlist_url = _select_quality(playlists)
print("\nFetching playlist...")
base_url, filenames = twitch.get_playlist_urls(playlist_url)
# Create a temp dir to store downloads if it doesn't exist
directory = '{}/twitch-dl/{}/{}'.format(tempfile.gettempdir(), video_id,
quality)
pathlib.Path(directory).mkdir(parents=True, exist_ok=True)
print("Download dir: {}".format(directory))
print("Downloading VODs with {} workers...".format(max_workers))
paths = _download_files(base_url, directory, filenames, max_workers)
print("\n\nJoining files...")
target = _video_target_filename(video, format)
_join_vods(directory, paths, target)
print("\nDeleting vods...")
for path in paths:
os.unlink(path)
print("\nDownloaded: {}".format(target))
def test_get_videos():
videos = twitch.get_channel_videos(TEST_CHANNEL, 3, "time")
assert videos["pageInfo"]
assert len(videos["edges"]) > 0
video_id = videos["edges"][0]["node"]["id"]
video = twitch.get_video(video_id)
assert video["id"] == video_id
def _download_video(video_id,
max_workers,
format='mp4',
start=None,
end=None,
keep=False,
**kwargs):
if start and end and end <= start:
raise ConsoleError("End time must be greater than start time")
_log(video_id, "Recherche la video ")
video = twitch.get_video(video_id)
_log(video_id, "Informations sur {}".format(video['title']))
print_out("Trouvé: <blue>{}</blue> by <yellow>{}</yellow>".format(
video['title'], video['channel']['display_name']))
access_token = twitch.get_access_token(video_id)
_log(video_id, "Obtention de la liste des fichiers...")
playlists = twitch.get_playlists(video_id, access_token)
parsed = m3u8.loads(playlists)
selected = _select_quality(parsed.playlists)
# print_out("\nListe...")
response = requests.get(selected.uri)
response.raise_for_status()
playlist = m3u8.loads(response.text)
base_uri = re.sub("/[^/]+$", "/", selected.uri)
target_dir = _crete_temp_dir(base_uri)
filenames = list(_get_files(playlist, start, end))
# Save playlists for debugging purposes
with open(target_dir + "playlists.m3u8", "w") as f:
f.write(playlists)
with open(target_dir + "playlist.m3u8", "w") as f:
f.write(response.text)
# print_out("\nTélécharge {} VODs avec {} threads dans {}".format(
# len(filenames), max_workers, target_dir))
file_paths = download_files(video_id, base_uri, target_dir, filenames,
max_workers)
target = _video_target_filename(video, format)
print_out("\nCible: {}".format(target))
_join_vods(target_dir, file_paths, "videos/Download/{}".format(target))
if keep:
print_out("\nTemporary files not deleted: {}".format(target_dir))
else:
# print_out("\nSupprime le fichier temporaire...")
shutil.rmtree(target_dir)
print_out("Fichier téléchargé: {}".format(target))
_log(video_id, "Terminé {}".format(target))
def download(video_id,
max_workers,
format='mkv',
start=None,
end=None,
keep=False,
**kwargs):
video_id = _parse_video_id(video_id)
if start and end and end <= start:
raise ConsoleError("End time must be greater than start time")
print_out("Looking up video...")
video = twitch.get_video(video_id)
print_out("Found: <blue>{}</blue> by <yellow>{}</yellow>".format(
video['title'], video['channel']['display_name']))
print_out("Fetching access token...")
access_token = twitch.get_access_token(video_id)
print_out("Fetching playlists...")
playlists = twitch.get_playlists(video_id, access_token)
parsed = m3u8.loads(playlists)
selected = _select_quality(parsed.playlists)
print_out("\nFetching playlist...")
response = requests.get(selected.uri)
response.raise_for_status()
playlist = m3u8.loads(response.text)
base_uri = re.sub("/[^/]+$", "/", selected.uri)
target_dir = _crete_temp_dir(base_uri)
filenames = list(_get_files(playlist, start, end))
# Save playlists for debugging purposes
with open(target_dir + "playlists.m3u8", "w") as f:
f.write(playlists)
with open(target_dir + "playlist.m3u8", "w") as f:
f.write(response.text)
print_out("\nDownloading {} VODs using {} workers to {}".format(
len(filenames), max_workers, target_dir))
_download_files(base_uri, target_dir, filenames, max_workers)
print_out("\n\nJoining files...")
target = _video_target_filename(video, format)
_join_vods(target_dir, filenames, target)
if keep:
print_out("\nTemporary files not deleted: {}".format(target_dir))
else:
print_out("\nDeleting temporary files...")
shutil.rmtree(target_dir)
print_out("Downloaded: {}".format(target))
def download(video_id,
max_workers,
format='mkv',
start=None,
end=None,
**kwargs):
video_id = parse_video_id(video_id)
if start and end and end <= start:
raise ConsoleError("End time must be greater than start time")
print_out("Looking up video...")
video = twitch.get_video(video_id)
print_out("Found: <blue>{}</blue> by <yellow>{}</yellow>".format(
video['title'], video['channel']['display_name']))
print_out("Fetching access token...")
access_token = twitch.get_access_token(video_id)
print_out("Fetching playlists...")
playlists = twitch.get_playlists(video_id, access_token)
quality, playlist_url = _select_quality(playlists)
print_out("\nFetching playlist...")
base_url, filenames = twitch.get_playlist_urls(playlist_url, start, end)
if not filenames:
raise ConsoleError("No vods matched, check your start and end times")
# Create a temp dir to store downloads if it doesn't exist
directory = '{}/twitch-dl/{}/{}'.format(tempfile.gettempdir(), video_id,
quality)
pathlib.Path(directory).mkdir(parents=True, exist_ok=True)
print_out("Download dir: {}".format(directory))
print_out("Downloading {} VODs using {} workers...".format(
len(filenames), max_workers))
paths = _download_files(base_url, directory, filenames, max_workers)
print_out("\n\nJoining files...")
target = _video_target_filename(video, format)
_join_vods(directory, paths, target)
print_out("\nDeleting vods...")
for path in paths:
os.unlink(path)
print_out("\nDownloaded: {}".format(target))
def sort_meeting_data():
"""
sorts meeting data into training and test data based on the person who
uploaded it
:return:
"""
start_path = "Data/Meeting Identifier/Crude Data/"
files = os.listdir(start_path)
player_names = dict()
for file in files:
if file == ".DS_Store":
continue
# split the filepath on the dash
items = file.split("-")
video_id = items[3]
if video_id in player_names:
# if we already have the player name cached, use it
player_name = player_names[video_id]
else:
# otherwise, get the player name from the video ID
video = twitch.get_video(video_id)
player_name = video['creator']['displayName']
# cache the player name
player_names[video_id] = player_name
if is_name_test(player_name):
data_set = "Test Data"
else:
data_set = "Training Data"
output = os.path.join("Data", "Meeting Identifier", data_set, file)
os.rename(start_path + file, output)
def sort_classifier_crude_data():
"""
sorts through "crude" data and puts it into Test data
:return: None
"""
start_path = "Data/Game Classifier/Crude Data/"
files = os.listdir(start_path)
player_names = dict()
for file in files:
# split the filepath on the dash
items = file.split("-")
image_kind = items[0]
video_id = items[1]
if video_id in player_names:
# if we already have the player name cached, use it
player_name = player_names[video_id]
else:
# otherwise, get the player name from the video ID
video = twitch.get_video(video_id)
player_name = video['channel']['display_name']
# cache the player name
player_names[video_id] = player_name
if is_name_test(player_name):
data_set = "Test Data"
else:
data_set = "Training Data"
output = os.path.join("Data", "Game Classifier", data_set, image_kind,
file)
os.rename(start_path + file, output)
def download(video_id, max_workers, format='mkv', **kwargs):
print("Looking up video...")
video = twitch.get_video(video_id)
print("Fetching access token...")
access_token = twitch.get_access_token(video_id)
print("Fetching playlists...")
playlists = twitch.get_playlists(video_id, access_token)
playlist_url = _select_playlist_by_quality(playlists)
print("\nFetching playlist...")
base_url, filenames = twitch.get_playlist_urls(playlist_url)
target = _video_target_filename(video, format)
with tempfile.TemporaryDirectory() as directory:
print("Downloading...")
_download_files(base_url, directory, filenames, max_workers)
print("\n\nJoining files...")
_join_vods(directory, filenames, target)
print("\nDownloaded: {}".format(target))
def info(args):
video_id = utils.parse_video_identifier(args.identifier)
if video_id:
print_log("Fetching video...")
video = twitch.get_video(video_id)
if not video:
raise ConsoleError("Video {} not found".format(video_id))
print_log("Fetching access token...")
access_token = twitch.get_access_token(video_id)
print_log("Fetching playlists...")
playlists = twitch.get_playlists(video_id, access_token)
if video:
if args.json:
video_json(video, playlists)
else:
video_info(video, playlists)
return
clip_slug = utils.parse_clip_identifier(args.identifier)
if clip_slug:
print_log("Fetching clip...")
clip = twitch.get_clip(clip_slug)
if not clip:
raise ConsoleError("Clip {} not found".format(clip_slug))
if args.json:
print_json(clip)
else:
clip_info(clip)
return
raise ConsoleError("Invalid input: {}".format(args.video))
def _download_video(video_id, args):
if args.start and args.end and args.end <= args.start:
raise ConsoleError("End time must be greater than start time")
if os.path.isfile(str(Path.home()) + "/.twitchdownloads/" + video_id):
print("File already downloaded")
return
print_out("<dim>Looking up video...</dim>")
video = twitch.get_video(video_id)
save_json_video(video_id, video, _video_target_filename(video, "json"))
print_out("Found: <blue>{}</blue> by <yellow>{}</yellow>".format(
video['title'], video['channel']['display_name']))
print_out("<dim>Fetching access token...</dim>")
access_token = twitch.get_access_token(video_id)
print_out("<dim>Fetching playlists...</dim>")
playlists_m3u8 = twitch.get_playlists(video_id, access_token)
playlists = list(_parse_playlists(playlists_m3u8))
playlist_uri = (_get_playlist_by_name(playlists, args.quality) if
args.quality else _select_playlist_interactive(playlists))
print_out("<dim>Fetching playlist...</dim>")
response = requests.get(playlist_uri)
response.raise_for_status()
playlist = m3u8.loads(response.text)
base_uri = re.sub("/[^/]+$", "/", playlist_uri)
target_dir = _crete_temp_dir(base_uri)
vod_paths = _get_vod_paths(playlist, args.start, args.end)
# Save playlists for debugging purposes
with open(path.join(target_dir, "playlists.m3u8"), "w") as f:
f.write(playlists_m3u8)
with open(path.join(target_dir, "playlist.m3u8"), "w") as f:
f.write(response.text)
print_out("\nDownloading {} VODs using {} workers to {}".format(
len(vod_paths), args.max_workers, target_dir))
path_map = download_files(base_uri, target_dir, vod_paths,
args.max_workers)
# Make a modified playlist which references downloaded VODs
# Keep only the downloaded segments and skip the rest
org_segments = playlist.segments.copy()
playlist.segments.clear()
for segment in org_segments:
if segment.uri in path_map:
segment.uri = path_map[segment.uri]
playlist.segments.append(segment)
playlist_path = path.join(target_dir, "playlist_downloaded.m3u8")
playlist.dump(playlist_path)
print_out("\n\nJoining files...")
target = _video_target_filename(video, args.format)
_join_vods(playlist_path, target)
if args.keep:
print_out(
"\n<dim>Temporary files not deleted: {}</dim>".format(target_dir))
else:
print_out("\n<dim>Deleting temporary files...</dim>")
shutil.rmtree(target_dir)
print_out("\nDownloaded: <green>{}</green>".format(target))
def _download_video(video_id, args):
if args.start and args.end and args.end <= args.start:
raise ConsoleError("End time must be greater than start time")
print_out("<dim>Looking up video...</dim>")
video = twitch.get_video(video_id)
if not video:
raise ConsoleError("Video {} not found".format(video_id))
print_out("Found: <blue>{}</blue> by <yellow>{}</yellow>".format(
video['title'], video['creator']['displayName']))
target = _video_target_filename(video, args)
print_out("Output: <blue>{}</blue>".format(target))
if not args.overwrite and path.exists(target):
response = input("File exists. Overwrite? [Y/n]: ")
if response.lower().strip() not in ["", "y"]:
raise ConsoleError("Aborted")
args.overwrite = True
print_out("<dim>Fetching access token...</dim>")
access_token = twitch.get_access_token(video_id)
print_out("<dim>Fetching playlists...</dim>")
playlists_m3u8 = twitch.get_playlists(video_id, access_token)
playlists = list(_parse_playlists(playlists_m3u8))
playlist_uri = (_get_playlist_by_name(playlists, args.quality) if
args.quality else _select_playlist_interactive(playlists))
print_out("<dim>Fetching playlist...</dim>")
response = requests.get(playlist_uri)
response.raise_for_status()
playlist = m3u8.loads(response.text)
base_uri = re.sub("/[^/]+$", "/", playlist_uri)
target_dir = _crete_temp_dir(base_uri)
vod_paths = _get_vod_paths(playlist, args.start, args.end)
# Save playlists for debugging purposes
with open(path.join(target_dir, "playlists.m3u8"), "w") as f:
f.write(playlists_m3u8)
with open(path.join(target_dir, "playlist.m3u8"), "w") as f:
f.write(response.text)
print_out("\nDownloading {} VODs using {} workers to {}".format(
len(vod_paths), args.max_workers, target_dir))
path_map = download_files(base_uri, target_dir, vod_paths,
args.max_workers)
# Make a modified playlist which references downloaded VODs
# Keep only the downloaded segments and skip the rest
org_segments = playlist.segments.copy()
playlist.segments.clear()
for segment in org_segments:
if segment.uri in path_map:
segment.uri = path_map[segment.uri]
playlist.segments.append(segment)
playlist_path = path.join(target_dir, "playlist_downloaded.m3u8")
playlist.dump(playlist_path)
if args.no_join:
print_out("\n\n<dim>Skipping joining files...</dim>")
print_out("VODs downloaded to:\n<blue>{}</blue>".format(target_dir))
return
print_out("\n\nJoining files...")
_join_vods(playlist_path, target, args.overwrite, video)
if args.keep:
print_out(
"\n<dim>Temporary files not deleted: {}</dim>".format(target_dir))
else:
print_out("\n<dim>Deleting temporary files...</dim>")
shutil.rmtree(target_dir)
print_out("\nDownloaded: <green>{}</green>".format(target))