def dl_video(video: Vod, TEMP_DIR: Path, path: str, max_workers: int,
LOG_LEVEL: str):
video_id = video.id
# Grab access token
access_token = gql.get_access_token(video_id)
# Get M3U8 playlist, and parse them
# (first URI is always source quality!)
uris = get_playlist_uris(video_id, access_token)
source_uri = uris[0]
# Fetch playlist at proper quality
resp = requests.get(source_uri)
resp.raise_for_status()
playlist = m3u8.loads(resp.text)
# Create a temp dir in .vodbot/temp
tempdir = TEMP_DIR / video_id
make_dir(str(tempdir))
# Dump playlist to a file
playlist_path = tempdir / "playlist.m3u8"
playlist.dump(str(playlist_path))
# Get all the necessary vod paths for the uri
base_uri = "/".join(source_uri.split("/")[:-1]) + "/"
vod_paths = []
for segment in playlist.segments:
if segment.uri not in vod_paths:
vod_paths.append(segment.uri)
# Download VOD chunks to the temp folder
path_map = worker.download_files(video_id, base_uri, tempdir, vod_paths,
max_workers)
# TODO: rewrite this output to look nicer and remove ffmpeg output using COLOR_CODES["F"]
cprint("#dDone, now to ffmpeg join...#r")
# join the vods using ffmpeg at specified path
# TODO: change this to the concat function in video?
cwd = os.getcwd()
os.chdir(str(tempdir))
cmd = [
"ffmpeg", "-i",
str(playlist_path), "-c", "copy", path, "-y", "-stats", "-loglevel",
LOG_LEVEL
]
result = subprocess.run(cmd)
os.chdir(cwd)
if result.returncode != 0:
raise JoiningFailed()
# delete temp folder and contents
shutil.rmtree(str(tempdir))
def run(args):
# check if the config exists and ask if it should be overwritten
if exists(args.output):
input(
"It seems a file already exists here. Press enter if you wish to continue, otherwise press Ctrl+C to quit."
)
input(
"This operation will overwrite this file. Do you have a backup? Press enter again to continue, otherwise press Ctrl+C to quit."
)
input(
"Last time, are you absolutely certain you want to overwrite this file? Press enter again to continue, otherwise press Ctrl+C to quit."
)
print("Overwriting file...")
# test write the config.
try:
util.make_dir(args.output.parent)
with open(str(args.output), "w") as f:
pass
except FileNotFoundError:
util.exit_prog(67, f"Cannot create file at \"{args.output}\".")
# see which config we should make
if args.default:
# generate default config
cprint("#dCreating default config...#r")
else:
# generate config from inputs
create_config(args)
# create directories now
util.make_dir(DEFAULT_CONF['vod_dir'])
util.make_dir(DEFAULT_CONF['clip_dir'])
util.make_dir(DEFAULT_CONF['temp_dir'])
util.make_dir(DEFAULT_CONF['stage_dir'])
# now write the config
try:
#util.make_dir(args.output.parent) # redundant
with open(str(args.output), "w") as f:
json.dump(DEFAULT_CONF, f, indent=4)
except FileNotFoundError:
util.exit_prog(
68,
f"Cannot create file at \"{args.output}\" despite being able to before."
)
# list the location of the config and say what can be edited outside this command
cprint(
f"#fGFinished#r, the config can be edited at `#l{str(vodbotdir / 'conf.json')}#r`."
)
def dl_clip(clip: Clip, path: str):
clip_slug = clip.slug
clip_id = clip.id
# Get proper clip file URL
source_url = gql.get_clip_source(clip_slug)
# Download file to path
size, _existed = worker.download_file(source_url, path)
# Print progress
cprint(
f"#fM#lClip#r ({clip_id})`#fM{clip_slug}#r` #fB#l~{worker.format_size(size)}#r"
)
def check_title(default=None):
title = default
if not title:
title = ""
while title == "":
title = input(
colorize("#fW#lTitle of the Video#r #d(--title)#r: "))
# blank title
if title == "":
cprint("#fRTitle cannot be blank.#r")
# reserved names
if title in RESERVED_NAMES:
cprint("#fRTitle cannot be a reserved name.#r")
title = ""
return title
def check_streamers(default=None) -> List[str]:
streamers = ""
while not streamers:
streamers = input(
colorize(
f"#fW#lWho was in the VOD#r #d(default `{', '.join(default)}`, csv)#r: "
))
if streamers == "":
streamers = default
else:
streamers = streamers.replace(" ", "").split(",")
for streamer in streamers:
if len(streamer) == 0:
cprint("#l#fRMissing streamer name!#r")
streamers = ""
break
return streamers
def _print_progress(video_id, futures):
downloaded_count = 0
downloaded_size = 0
existing_size = 0
max_msg_size = 0
start_time = datetime.now()
total_count = len(futures)
try:
for future in as_completed(futures):
(size, existed) = future.result()
downloaded_count += 1
downloaded_size += size
if existed:
existing_size += size
percentage = 100 * downloaded_count // total_count
est_total_size = int(total_count * downloaded_size /
downloaded_count)
duration = (datetime.now() - start_time).seconds
speed = (downloaded_size -
existing_size) // duration if duration else 0
remaining = (total_count -
downloaded_count) * duration / downloaded_count
msg = " ".join([
f"#fM#lVOD#r `#fM{video_id}#r` pt#fC{downloaded_count}#r/#fB#l{total_count}#r,",
f"#fC{format_size(downloaded_size)}#r/#fB#l~{format_size(est_total_size)}#r #d({percentage}%)#r;",
f"at #fY~{format_size(speed)}/s#r;" if speed > 0 else "",
f"#fG~{format_duration(remaining)}#r left"
if speed > 0 else "",
])
max_msg_size = max(len(msg), max_msg_size)
cprint("\r" + msg.ljust(max_msg_size), end="")
except KeyboardInterrupt:
done, not_done = wait(futures, timeout=0)
for future in not_done:
future.cancel()
wait(not_done, timeout=None)
raise DownloadCancelled()
def check_description(formatdict, inputdefault=None):
desc = ""
if inputdefault:
try:
inputdefault = inputdefault.format(**formatdict).replace(
"\\n", "\n")
desc = inputdefault
except KeyError as err:
cprint(f"#fRDescription format error from default: {err}.#r")
desc = ""
while desc == "":
desc = input(colorize("#fW#lDescription of Video#r #d(--desc)#r: "))
if desc == "":
cprint("#fRDescription cannot be blank.#r")
continue
# Format the description
try:
desc = desc.format(**formatdict).replace("\\n", "\n")
except KeyError as err:
cprint(f"#fRDescription format error: {err}.#r")
desc = ""
return desc
def run(args):
conf = util.load_conf(args.config)
util.make_dir(vodbotdir)
stagedir = conf["stage_dir"]
path_stage = Path(stagedir)
util.make_dir(stagedir)
if args.action == "new":
_new(args, conf, stagedir)
elif args.action == "rm":
if not isfile(str(path_stage / (args.id + ".stage"))):
util.exit_prog(45, f'Could not find stage "{args.id}".')
try:
os_remove(str(path_stage / (args.id + ".stage")))
cprint(f'Stage "#fY#l{args.id}#r" has been #fRremoved#r.')
except OSError as err:
util.exit_prog(
88,
f'Stage "{args.id}" could not be removed due to an error. {err}'
)
elif args.action == "list":
_list(args, conf, stagedir)
def dl_video_chat(video: Vod, path: str):
video_id = video.id
# Download all chat from video
cprint(f"#fM#lVOD Chat#r `#fM{video_id}#r` (0%)", end="")
msgs = get_video_comments(video_id)
cprint(
f"\r#fM#lVOD Chat#r `#fM{video_id}#r` (100%); Done, now to write...",
end="")
chatlog.chat_to_logfile(msgs, path)
cprint(
f"\r#fM#lVOD Chat#r `#fM{video_id}#r` (100%); Done, now to write... Done"
)
def handle_stage(conf: dict, stage: StageData) -> Path:
tmpfile = None
try:
tmpfile = vbvid.process_stage(conf, stage)
except vbvid.FailedToSlice as e:
cprint(
f"#r#fRSkipping stage `{stage.id}`, failed to slice video with ID of `{e.vid}`.#r\n"
)
return None
except vbvid.FailedToConcat:
cprint(
f"#r#fRSkipping stage `{stage.id}`, failed to concatenate videos.#r\n"
)
return None
except vbvid.FailedToCleanUp as e:
cprint(
f"#r#fRSkipping stage `{stage.id}`, failed to clean up temp files.#r\n\n{e.vid}"
)
return None
return tmpfile
def run(args):
# load config
conf = util.load_conf(args.config)
# configure variables
global stagedir, tempdir
tempdir = Path(conf["temp_dir"])
stagedir = Path(conf["stage_dir"])
PICKLE_FILE = conf["youtube_pickle_path"]
CLIENT_SECRET_FILE = conf["youtube_client_path"]
API_NAME = 'youtube'
API_VERSION = 'v3'
SCOPES = [ # Only force-ssl is required, but both makes it explicit.
"https://www.googleapis.com/auth/youtube.upload", # Videos.insert
"https://www.googleapis.com/auth/youtube.force-ssl" # Captions.insert
]
# handle logout
if args.id == "logout":
try:
os_remove(PICKLE_FILE)
cprint("#dLogged out of Google API session#r")
except:
util.exit_prog(
11, "Failed to remove credentials for YouTube account.")
return
# load stages, but dont upload
# Handle id/all
stagedata = None
stagedatas = None
if args.id == "all":
cprint("#dLoading stages...", end=" ")
# create a list of all the hashes and sort by date streamed, upload chronologically
stagedatas = StageData.load_all_stages(stagedir)
stagedatas.sort(key=sort_stagedata)
else:
cprint("#dLoading stage...", end=" ")
# check if stage exists, and prep it for upload
stagedata = StageData.load_from_id(stagedir, args.id)
cprint(f"About to upload stage {stagedata.id}.#r")
# authenticate youtube service
if not os_exists(CLIENT_SECRET_FILE):
util.exit_prog(19, "Missing YouTube Client ID/Secret file.")
cprint("Authenticating with Google...", end=" ")
service = None
credentials = None
if os_exists(PICKLE_FILE):
with open(PICKLE_FILE, "rb") as f:
credentials = pickle.load(f)
if not credentials or credentials.expired:
try:
if credentials and credentials.expired and credentials.refresh_token:
credentials.refresh(Request())
else:
flow = InstalledAppFlow.from_client_secrets_file(
CLIENT_SECRET_FILE, SCOPES)
credentials = flow.run_console()
except RefreshError:
flow = InstalledAppFlow.from_client_secrets_file(
CLIENT_SECRET_FILE, SCOPES)
credentials = flow.run_console()
with open(PICKLE_FILE, "wb") as f:
pickle.dump(credentials, f)
try:
service = build(API_NAME, API_VERSION, credentials=credentials)
except Exception as err:
util.exit_prog(50, f"Failed to connect to YouTube API, \"{err}\"")
cprint("Authenticated.", end=" ")
# Handle id/all
if args.id == "all":
# begin to upload
cprint(f"About to upload {len(stagedatas)} stages.#r")
for stage in stagedatas:
video_id = upload_video(conf, service, stage)
if video_id is not None:
chat_success = True
if conf["chat_upload"]:
chat_success = upload_captions(conf, service, stage,
video_id)
if conf["stage_upload_delete"] and chat_success:
try:
os_remove(str(stagedir / f"{stage.id}.stage"))
except:
util.exit_prog(
90,
f"Failed to remove stage `{stage.id}` after upload."
)
print()
else:
# upload stage
cprint(f"About to upload stage {stagedata.id}.#r")
video_id = upload_video(conf, service, stagedata)
if video_id is not None:
chat_success = True
if conf["chat_upload"]:
chat_success = upload_captions(conf, service, stagedata,
video_id)
if conf["stage_upload_delete"] and chat_success:
try:
os_remove(str(stagedir / f"{stagedata.id}.stage"))
except:
util.exit_prog(
90,
f"Failed to remove stage `{stagedata.id}` after upload."
)
def upload_captions(conf: dict, service, stagedata: StageData,
vid_id: str) -> bool:
tmpfile = vbchat.process_stage(conf, stagedata, "upload")
if tmpfile == False:
return True
request_body = {
"snippet": {
"name": "Chat",
"videoId": vid_id,
"language": "en"
}
}
media_file = MediaFileUpload(str(tmpfile),
chunksize=1024 * 1024 * 100,
resumable=True)
response_upload = service.captions().insert(part="snippet",
body=request_body,
sync=False,
media_body=media_file)
resp = None
errn = 0
cprint(
f"#fCUploading stage chatlog #r`#fM{stagedata.id}#r`, progress: #fC0#fY%#r #d...#r",
end="\r")
while resp is None:
try:
status, resp = response_upload.next_chunk()
if status:
cprint(
f"#fCUploading stage chatlog #r`#fM{stagedata.id}#r`, progress: #fC{int(status.progress()*100)}#fY%#r #d...#r",
end="\r")
if resp is not None:
cprint(
f"#fCUploading stage chatlog #r`#fM{stagedata.id}#r`, progress: #fC100#fY%#r!"
)
cprint(f"#l#fGVideo captions were successfully uploaded!#r")
except ResumableUploadError as err:
if err.resp.status in [400, 401, 402, 403]:
try:
jsondata = json.loads(err.content)['error']['errors'][0]
util.exit_prog(
40,
f"API Error: `{jsondata['reason']}`. Message: `{jsondata['message']}`"
)
except (json.JSONDecodeError, KeyError):
util.exit_prog(
40,
f"Unknown API Error has occured, ({err.resp.status}, {err.content})"
)
print(
f"A Resumeable error has occured, retrying in 5 sec... ({err.resp.status}, {err.content})"
)
errn += 1
sleep(5)
except HttpError as err:
if err.resp.status in [500, 502, 503, 504]:
print(
f"An HTTP error has occured, retrying in 5 sec... ({err.resp.status}, {err.content})"
)
errn += 1
sleep(5)
else:
raise
except RETRIABLE_EXCEPTS as err:
print(f"An HTTP error has occured, retrying in 5 sec... ({err})")
errn += 1
sleep(5)
if errn >= 10:
print("Skipping chatlog upload, errored too many times.")
return False
# we're done, lets clean up
else:
try:
# delete vars to release the files
del media_file
del response_upload
sleep(1)
os_remove(str(tmpfile))
except Exception as e:
util.exit_prog(
90,
f"Failed to remove temp slice file of stage `{stagedata.id}` after upload. {e}"
)
return True
def create_config(args):
channels = []
timezone = "" # switch to UTC only
# these are for determining directories.
# we need to make sure any that are already provided are absolute right now.
# we do it now so that the user doesn't enter in anything before receiving
# an error that boots them out of the program.
parts = [[args.voddir, "VODs", DEFAULT_CONF['vod_dir']],
[args.clipdir, "Clips", DEFAULT_CONF['clip_dir']],
[args.tempdir, "temporary data", DEFAULT_CONF['temp_dir']],
[args.stagedir, "staged data", DEFAULT_CONF['stage_dir']]]
for part in parts:
if part[0]:
if not isabs(part[0]):
util.exit_prog(
150,
f"Path `{part[0]}` for {part[1]} is not an absolute path.")
part.append(part[0])
# Ask for channels
if not args.channels:
cprint(
"Enter the login names of the Twitch channels you'd like to have archived."
)
cprint("When you're done, just leave the input empty and press enter.")
cprint(
"Example: the url is `https://twitch.tv/notquiteapex`, enter `notquiteapex`"
)
while True:
channel = input("> ")
if channel == "" and len(channels) > 0:
break
elif channel == "" and len(channels) == 0:
cprint("#fRError#r, no channels given.")
elif not re.match(
r"^[a-zA-Z0-9][\w]{0,24}$",
channel): # https://discuss.dev.twitch.tv/t/3855/4
cprint(
"#fRError#r, channel names only contain characters A-Z, 0-9, and '_'. They also can't start with '_'."
)
else:
channels += [channel]
else:
channels = args.channels
# Ask for timezone as UTC string
# We only absolutely need the hours and minutes offset, and we regex it out of the string.
if not args.timezone:
cprint(
"Enter the UTC timezone for timedate referencing. Only use +, -, and numbers."
)
cprint(
"#dExample: `+0000` for UTC, `-0500` for America's EST. Default: `+0000`#r"
)
while True:
tz = input("> ")
if tz == "":
tz = "+0000"
break
elif not re.match(r"^[+-]\d\d\d\d$", tz):
cprint("Error, UTC string not recognized.")
else:
break
else:
timezone = args.timezone
# check that entered timezone is valid
# ask for directories (any provided by the terminal arguments are already handeled.)
cprint(
"Now let's get some directories to store data. The entered paths must be absolute, not relative."
)
cprint(
"If you'd like to use the default location listed, just leave the input blank and press enter."
)
for part in parts:
if not part[0]:
cprint(
f"Enter where {part[1]} should be stored.\n#dDefault: `#l{part[2]}`#r"
)
while True:
inpdir = input("> ")
if inpdir == "":
part.append(part[2])
break
elif isabs(inpdir):
# TODO: check if directory can be created? a file might already exist?
part.append(inpdir)
break
else:
cprint(
f"#fRError#r, directory `#l{inpdir}#r` is not an absolute path for a directory."
)
# ready to write it all, go!
cprint("#dWriting config...#r")
# Edit default config variable.
DEFAULT_CONF['twitch_channels'] = channels
DEFAULT_CONF['stage_timezone'] = timezone
DEFAULT_CONF['vod_dir'] = parts[0][3]
DEFAULT_CONF['clip_dir'] = parts[1][3]
DEFAULT_CONF['temp_dir'] = parts[2][3]
DEFAULT_CONF['stage_dir'] = parts[3][3]
def run(args):
global stagedir
conf = util.load_conf(args.config)
stagedir = Path(conf["stage_dir"])
util.make_dir(args.path)
args.path = Path(args.path)
# load stages, but dont slice
# Handle id/all
if args.id == "all":
cprint("#dLoading and slicing stages...#r")
# create a list of all the hashes and sort by date streamed, slice chronologically
stagedatas = StageData.load_all_stages(stagedir)
stagedatas.sort(key=sort_stagedata)
for stage in stagedatas:
tmpfile = None
tmpchat = None
# Export with ffmpeg
tmpfile = handle_stage(conf, stage)
# Export chat
tmpchat = vbchat.process_stage(conf, stage, "export")
title = stage.title.strip()
for x in DISALLOWED_CHARACTERS:
title = title.replace(x, "_")
# move appropriate files
if tmpfile is not None:
os_replace(tmpfile, args.path / (title + tmpfile.suffix))
if tmpchat is not None:
os_replace(tmpchat, args.path / (title + tmpchat.suffix))
# deal with old stage
if conf["stage_export_delete"]:
os_remove(str(stagedir / f"{stage.id}.stage"))
else:
cprint("#dLoading stage...", end=" ")
# check if stage exists, and prep it for slice
stagedata = StageData.load_from_id(stagedir, args.id)
tmpfile = None
tmpchat = None
# Export with ffmpeg
tmpfile = handle_stage(conf, stagedata)
# Export chat
tmpchat = vbchat.process_stage(conf, stagedata, "export")
title = stagedata.title.strip()
for x in DISALLOWED_CHARACTERS:
title = title.replace(x, "_")
# move appropriate files
if tmpfile is not None:
os_replace(tmpfile, args.path / (title + tmpfile.suffix))
if tmpchat is not None:
os_replace(tmpchat, args.path / (title + tmpchat.suffix))
# Deal with old stage
if conf["stage_export_delete"]:
os_remove(str(stagedir / f"{stagedata.id}.stage"))
# say "Done!"
cprint("#fG#lDone!#r")
def process_stage(conf: dict, stage: StageData, mode: str) -> Path:
tempdir = Path(conf["temp_dir"])
msg_duration = int(conf["chat_msg_time"])
cprint(f"#rLoading all chat messages for `#fM{stage.id}#r`.", end="")
total_offset = 0
chat_list = []
for slc in stage.slices:
# load up each stagedata's meta to see if chat exists
metapath = slc.filepath[:-4] + ".meta"
meta = None
with open(metapath, "r") as f:
meta = json.load(f)
has_chat = meta.get("has_chat", False)
duration = meta.get("length", 0)
chat_path = slc.filepath[:-4] + ".chat"
# start and end times as seconds
start_sec = timestring_as_seconds(slc.ss)
end_sec = timestring_as_seconds(slc.to, duration)
# keep each list of chat separate, compare timestamps to offsets to make
# sure theyre inbetween the slices
if has_chat:
msg_list = [
m for m in logfile_to_chat(chat_path)
if start_sec <= m.offset < end_sec
]
for m in msg_list:
m.offset = m.offset - start_sec + total_offset
chat_list += msg_list
# now take each stage's duration and apply it to the next chat list's
total_offset += end_sec - start_sec
# determine how to export, then return the resultant path
if mode != "upload" and mode != "export":
util.exit_prog(94, f"Cannot export chat with export mode {mode}")
export_type = conf["chat_" + mode]
if len(chat_list) == 0:
cprint(f" No chat found in `#fY{export_type}#r` stage. Skipping...")
return False
cprint(f" Exporting as format `#fY{export_type}#r`.")
returnpath = None
if export_type == "raw":
# chat to logfile time
returnpath = tempdir / (stage.id + ".chat")
chat_to_logfile(chat_list, str(returnpath))
elif export_type == "RealText":
# load from archive, parse and write to temp
returnpath = tempdir / (stage.id + ".rt")
chat_to_realtext(chat_list, str(returnpath), total_offset,
msg_duration)
elif export_type == "SAMI":
# load from archive, parse and write to temp
returnpath = tempdir / (stage.id + ".sami")
chat_to_sami(chat_list, str(returnpath), total_offset, msg_duration)
elif export_type == "YTT":
# load from archive, parse and write to temp
returnpath = tempdir / (stage.id + ".ytt")
chat_to_ytt(chat_list, str(returnpath), total_offset, msg_duration)
elif export_type == "TTML":
# load from archive, parse and write to temp
returnpath = tempdir / (stage.id + ".ttml")
chat_to_ttml(chat_list, str(returnpath), total_offset, msg_duration)
return returnpath
def check_time(prefix, resp, default=None):
output = resp
checkedonce = False
while True:
if checkedonce or not output:
a = 'ss' if prefix == 'Start' else 'to'
t = '0:0:0' if prefix == 'Start' else 'EOF'
f = f"#fW#l{prefix} time of the Video#r #d(--{a}, default {t})#r: "
output = input(colorize(f))
checkedonce = True
if output == "":
if prefix == "Start":
return default if default != None else "0:0:0"
elif prefix == "End":
return default if default != None else "EOF"
intime = output.split(":")
timelist = []
seconds = None
minutes = None
hours = None
if len(intime) > 3:
cprint(f"#fR{prefix} time: Time cannot have more than 3 units.#r")
continue
if len(intime) >= 1:
seconds = intime[-1]
try:
seconds = int(seconds)
except ValueError:
cprint(
f"#fR{prefix} time: Seconds does not appear to be a number.#r"
)
continue
if seconds > 59 or seconds < 0:
cprint(
f"#fR{prefix} time: Seconds must be in the range of 0 to 59.#r"
)
continue
timelist.insert(0, str(seconds))
if len(intime) >= 2:
minutes = intime[-2]
try:
minutes = int(minutes)
except ValueError:
cprint(
f"#fR{prefix} time: Minutes does not appear to be a number.#r"
)
continue
if minutes > 59 or minutes < 0:
cprint(
f"#fR{prefix} time: Minutes must be in the range of 0 to 59.#r"
)
continue
timelist.insert(0, str(minutes))
else:
timelist.insert(0, "0")
if len(intime) == 3:
hours = intime[-3]
try:
hours = int(hours)
except ValueError:
cprint(
f"#fR{prefix} time: Hours does not appear to be a number.#r"
)
continue
timelist.insert(0, str(hours))
else:
timelist.insert(0, "0")
output = ":".join(timelist)
break
return output
def _new(args, conf, stagedir):
VODS_DIR = conf["vod_dir"]
CLIPS_DIR = conf["clip_dir"]
STAGE_DIR = conf["stage_dir"]
stagedir = Path(STAGE_DIR)
# find the videos by their ids to confirm they exist
videos = []
for video in args.id:
try:
(filename, metadata,
videotype) = find_video_by_id(video, VODS_DIR, CLIPS_DIR)
videos += [{
"id": video,
"file": filename,
"meta": metadata,
"type": videotype
}]
except CouldntFindVideo:
util.exit_prog(13, f'Could not find video with ID "{args.id}"')
# Get what streamers were involved (usernames), always asked
default_streamers = []
for f in videos:
if f["meta"]["user_login"] not in default_streamers:
default_streamers.append(f["meta"]["user_login"])
args.streamers = check_streamers(default=default_streamers)
# get title
if not args.title:
args.title = check_title(default=None)
# get description
formatdict, datestring = create_format_dict(conf,
args.streamers,
utcdate=metadata["created_at"])
args.desc = check_description(formatdict, inputdefault=args.desc)
# get timestamps for each video through input
for x in range(len(videos)):
# skip times we dont need because we already have them
if x < len(args.ss):
continue
vid = videos[x]["meta"]
# grab times for this specific stream
cprint(f"#dTimestamps for `#r#fM{vid['title']}#r` #d({vid['id']})#r")
if "chapters" in vid and len(vid["chapters"]) > 0:
cprint(f"#dChapters: ", end="")
ch = []
for c in vid["chapters"]:
(pos, end) = posdur_to_timestamp(c['pos'], c['dur'])
ch.append(f"`{c['desc']}` ({pos}-{end})")
cprint(" | ".join(ch) + " | EOF#r")
else:
(pos, end) = posdur_to_timestamp(0, vid['length'])
cprint(f"#dChapter: `{vid['game_name']}` ({pos}-{end}) | EOF#r")
args.ss += [
check_time("Start", args.ss[x] if x < len(args.ss) else None)
]
args.to += [
check_time("End", args.to[x] if x < len(args.to) else None)
]
# make slice objects
slices = []
for x in range(len(videos)):
vid = videos[x]
vidslice = VideoSlice(video_id=vid["id"],
ss=args.ss[x],
to=args.to[x],
filepath=vid["file"])
slices += [vidslice]
# make stage object
stage = StageData(streamers=args.streamers,
title=args.title,
desc=args.desc,
datestring=datestring,
slices=slices)
# Check that new "id" does not collide
while check_stage_id(stage.id, STAGE_DIR):
stage.gen_new_id()
# shorter file name
#shortfile = stage.filename.replace(VODS_DIR, "$vods").replace(CLIPS_DIR, "$clips")
print()
cprint(
f"#r`#fC{stage.title}#r` #fM{' '.join(stage.streamers)}#r #d({stage.id})#r"
)
cprint(f"#d'''#fG{stage.desc}#r#d'''#r")
for vid in stage.slices:
cprint(f"#fM{vid.video_id}#r > #fY{vid.ss}#r - #fY{vid.to}#r")
# write stage
stagename = str(stagedir / (stage.id + ".stage"))
stage.write_stage(stagename)
def upload_video(conf: dict, service, stagedata: StageData) -> str:
tmpfile = None
try:
tmpfile = vbvid.process_stage(conf, stagedata)
except vbvid.FailedToSlice as e:
cprint(
f"#r#fRSkipping stage `{stagedata.id}`, failed to slice video with ID of `{e.vid}`.#r\n"
)
except vbvid.FailedToConcat:
cprint(
f"#r#fRSkipping stage `{stagedata.id}`, failed to concatenate videos.#r\n"
)
except vbvid.FailedToCleanUp as e:
cprint(
f"#r#fRSkipping stage `{stagedata.id}`, failed to clean up temp files.#r\n\n{e.vid}"
)
# send request to youtube to upload
request_body = {
"snippet": {
"categoryId": 20,
"title": stagedata.title,
"description": stagedata.desc
},
"status": {
"privacyStatus": "private",
"selfDeclaredMadeForKids": False
}
}
# create media file, 100 MiB chunks
media_file = MediaFileUpload(str(tmpfile),
chunksize=1024 * 1024 * 100,
resumable=True)
# create upload request and execute
response_upload = service.videos().insert(part="snippet,status",
body=request_body,
notifySubscribers=False,
media_body=media_file)
video_id = None # youtube video id
resp = None
errn = 0
cprint(
f"#fCUploading stage #r`#fM{stagedata.id}#r`, progress: #fC0#fY%#r #d...#r",
end="\r")
# Below is taken from Google's documentation, rewrite?
while resp is None:
try:
status, resp = response_upload.next_chunk()
if status:
cprint(
f"#fCUploading stage #r`#fM{stagedata.id}#r`, progress: #fC{int(status.progress()*100)}#fY%#r #d...#r",
end="\r")
if resp is not None:
if "id" in resp:
cprint(
f"#fCUploading stage #r`#fM{stagedata.id}#r`, progress: #fC100#fY%#r!"
)
cprint(
f"#l#fGVideo was successfully uploaded!#r #dhttps://youtu.be/{resp['id']}#r"
)
video_id = resp["id"]
else:
util.exit_prog(
99, f"Unexpected upload failure occurred, \"{resp}\"")
except ResumableUploadError as err:
if err.resp.status in [400, 401, 402, 403]:
try:
jsondata = json.loads(err.content)['error']['errors'][0]
util.exit_prog(
40,
f"API Error: `{jsondata['reason']}`. Message: `{jsondata['message']}`"
)
except (json.JSONDecodeError, KeyError):
util.exit_prog(
40,
f"Unknown API Error has occured, ({err.resp.status}, {err.content})"
)
print(
f"A Resumeable error has occured, retrying in 5 sec... ({err.resp.status}, {err.content})"
)
errn += 1
sleep(5)
except HttpError as err:
if err.resp.status in [500, 502, 503, 504]:
print(
f"An HTTP error has occured, retrying in 5 sec... ({err.resp.status}, {err.content})"
)
errn += 1
sleep(5)
else:
raise
except RETRIABLE_EXCEPTS as err:
print(f"An HTTP error has occured, retrying in 5 sec... ({err})")
errn += 1
sleep(5)
if errn >= 10:
print("Skipping video upload, errored too many times.")
break
# we're done, lets clean up
else:
try:
# delete vars to release the files
del media_file
del response_upload
sleep(1)
os_remove(str(tmpfile))
except Exception as e:
util.exit_prog(
90,
f"Failed to remove temp slice file of stage `{stagedata.id}` after upload. {e}"
)
return video_id
def _list(args, conf, stagedir):
VODS_DIR = conf["vod_dir"]
CLIPS_DIR = conf["clip_dir"]
STAGE_DIR = conf["stage_dir"]
stagedir = Path(STAGE_DIR)
if args.id == None:
stages = StageData.load_all_stages(stagedir)
for s in stages:
cprint(f'#r#fY#l{s.id}#r -- `#fC{s.title}#r` ', end="")
cprint(f'(#fM{" ".join([d.video_id for d in s.slices])}#r) -- ',
end="")
cprint(f'#l#fM{", ".join(s.streamers)}#r')
if len(stages) == 0:
cprint("#fBNothing staged right now.#r")
else:
stage = StageData.load_from_id(stagedir, args.id)
cprint(
f"#r`#fC{stage.title}#r` #fM{' '.join(stage.streamers)}#r #d({stage.id})#r"
)
cprint(f"#d'''#fG{stage.desc}#r#d'''#r")
for vid in stage.slices:
cprint(f"#fM{vid['id']}#r > #fY{vid['ss']}#r - #fY{vid['to']}#r")
def run(args):
# First determine what kind of data it is...
# We should allow people to paste full links or just the ID too.
tid = args.id
# Get the proper id and type of content we need to pull
cprint("#dDetermining type...#r", end=" ")
cid, ctype = get_type(tid)
# The call the appropriate info query for GQL
query = None
if ctype == "vod":
query = gql.GET_VIDEO_QUERY.format(video_id=cid)
elif ctype == "clip":
query = gql.GET_CLIP_QUERY.format(clip_slug=cid)
elif ctype == "channel":
query = gql.GET_CHANNEL_QUERY.format(channel_id=cid)
else:
util.exit_prog(92, "Could not determine content type from input.")
cword = KEYTRANS[ctype]
cprint(f"#dQuerying {cword} content for `{cid}`...#r")
# run the query
resp = gql.gql_query(query=query).json()
resp = resp["data"]
if not any([resp.get("video"), resp.get("clip"), resp.get("user")]):
util.exit_prog(93,
f"Could not query info on {cword} content from input.")
# Read the data back and out to the terminal
if ctype == "vod":
r = resp['video']
c = r['creator']
g = r['game']
cprint(f"#fGTitle: `{r['title']}`#r - ID: `{r['id']}`")
cprint(
f"#fMBroadcaster: {c['displayName']} - {c['login']} ({c['id']})#r")
cprint(f"#fCPlaying: {g['name']} ({g['id']})#r")
cprint(
f"#fYAt: {r['publishedAt']} - For: {r['lengthSeconds']} seconds#r")
elif ctype == "clip":
r = resp['clip']
c = r['curator']
b = r['broadcaster']
g = r['game']
cprint(f"#fGTitle: `{r['title']}`#r - ID: `{r['id']}`")
cprint(f"Slug: `{r['slug']}`")
cprint(f"#fBClipper: {c['displayName']} - {c['login']} ({c['id']})#r")
cprint(
f"#fMBroadcaster: {b['displayName']} - {b['login']} ({b['id']})#r")
cprint(f"#fCPlaying: {g['name']} ({g['id']})#r")
cprint(
f"#fYAt: {r['createdAt']} - For: {r['durationSeconds']} seconds - With: {r['viewCount']} views#r"
)
elif ctype == "channel":
r = resp['user']
cprint(
f"#fMBroadcaster: {r['displayName']} - {r['login']} (ID: `{r['id']}`)#r"
)
cprint(f"#fRDescription: {r['description']}#r")
cprint(f"#fYChannel Created At: {r['createdAt']}#r")
r = r['roles']
cprint(
f"#fBRoles: Affiliate={r['isAffiliate']} - Partner={r['isPartner']}#r"
)
def download_twitch_video(args):
# Load the config and set up the access token
cprint("#r#dLoading config...#r", end=" ", flush=True)
conf = util.load_conf(args.config)
CHANNEL_IDS = conf["twitch_channels"]
VODS_DIR = conf["vod_dir"]
CLIPS_DIR = conf["clip_dir"]
TEMP_DIR = conf["temp_dir"]
LOG_LEVEL = conf["ffmpeg_loglevel"]
PULL_CHAT = conf["pull_chat_logs"]
# If channel arguments are provided, override config
if args.channels:
CHANNEL_IDS = args.channels
cprint("#r#dLoading channel data...#r", flush=True)
channels = twitch.get_channels(CHANNEL_IDS)
contentnoun = "video" # temp contentnoun until the rest is reworked
# Setup directories for videos and temp
util.make_dir(TEMP_DIR)
voddir = Path(VODS_DIR)
util.make_dir(voddir)
clipdir = Path(CLIPS_DIR)
util.make_dir(clipdir)
# Get list of videos using channel object ID's from Twitch API
videos = []
totalvods = 0
totalclips = 0
channel_print = None
if args.type == "both":
channel_print = "Pulling #fM#lVOD#r & #fM#lClip#r list: #fY#l{}#r..."
elif args.type == "vods":
channel_print = "Pulling #fM#lVOD#r list: #fY#l{}#r..."
elif args.type == "clips":
channel_print = "Pulling #fM#lClip#r list: #fY#l{}#r..."
for channel in channels:
vods = None
clips = None
cprint(channel_print.format(channel.display_name), end=" ")
# Grab list of VODs and check against existing VODs
if args.type == "both" or args.type == "vods":
folder = voddir / channel.login
util.make_dir(folder)
allvods = twitch.get_channel_vods(channel)
vods = compare_existant_file(folder, allvods)
totalvods += len(vods)
# Grab list of Clips and check against existing Clips
if args.type == "both" or args.type == "clips":
folder = clipdir / channel.login
util.make_dir(folder)
allclips = twitch.get_channel_clips(channel)
clips = compare_existant_file(folder, allclips)
totalclips += len(clips)
# Print content found and save it
if args.type == "both":
cprint(
f"#fC#l{len(vods)} #fM#lVODSs#r & #fC#l{len(clips)} #fM#lClips#r"
)
videos += vods
videos += clips
elif args.type == "vods":
cprint(f"#fC#l{len(vods)} #fM#lVODSs#r")
videos += vods
elif args.type == "clips":
cprint(f"#fC#l{len(clips)} #fM#lClips#r")
videos += clips
if args.type == "both":
cprint(f"Total #fMVODs#r to download: #fC#l{totalvods}#r")
cprint(f"Total #fMClips#r to download: #fC#l{totalclips}#r")
cprint(f"Total #fM#lvideos#r to download: #fC#l{len(videos)}#r")
elif args.type == "vods":
cprint(f"Total #fMVODs#r to download: #fC#l{totalvods}#r")
elif args.type == "clips":
cprint(f"Total #fMClips#r to download: #fC#l{totalclips}#r")
# Download all the videos we need.
previouschannel = None
for vod in videos:
# Print if we're on to a new user.
if previouschannel != vod.user_id:
previouschannel = vod.user_id
cprint(
f"\nDownloading #fY#l{vod.user_name}#r's #fM#l{contentnoun}s#r..."
)
# Generate path for video
viddir = None
contentnoun = None
if isinstance(vod, Vod):
viddir = voddir / vod.user_name.lower()
contentnoun = "VOD"
elif isinstance(vod, Clip):
viddir = clipdir / vod.user_name.lower()
contentnoun = "Clip"
filepath = viddir / f"{vod.created_at}_{vod.id}".replace(":", ";")
filename = str(filepath) + ".mkv"
metaname = str(filepath) + ".meta"
chatname = str(filepath) + ".chat"
# Write video data and handle exceptions
try:
if isinstance(vod, Vod):
# download chat
if PULL_CHAT:
itd_dl.dl_video_chat(vod, chatname)
vod.has_chat = True
# download video
itd_dl.dl_video(vod, Path(TEMP_DIR), filename, 20, LOG_LEVEL)
# write meta file
vod.write_meta(metaname)
elif isinstance(vod, Clip):
# download clip
itd_dl.dl_clip(vod, filename)
# write meta file
vod.write_meta(metaname)
except itd_dl.JoiningFailed:
cprint(
f"#fR#lVOD `{vod.id}` joining failed! Preserving files...#r")
except itd_work.DownloadFailed:
cprint(f"Download failed! Skipping...#r")
except itd_work.DownloadCancelled:
cprint(f"\n#fR#l{contentnoun} download cancelled.#r")
raise KeyboardInterrupt()
cprint("\n#fM#l* All done, goodbye! *#r\n")
