def dl_video(video: Vod, TEMP_DIR: Path, path: str, max_workers: int, LOG_LEVEL: str): video_id = video.id # Grab access token access_token = gql.get_access_token(video_id) # Get M3U8 playlist, and parse them # (first URI is always source quality!) uris = get_playlist_uris(video_id, access_token) source_uri = uris[0] # Fetch playlist at proper quality resp = requests.get(source_uri) resp.raise_for_status() playlist = m3u8.loads(resp.text) # Create a temp dir in .vodbot/temp tempdir = TEMP_DIR / video_id make_dir(str(tempdir)) # Dump playlist to a file playlist_path = tempdir / "playlist.m3u8" playlist.dump(str(playlist_path)) # Get all the necessary vod paths for the uri base_uri = "/".join(source_uri.split("/")[:-1]) + "/" vod_paths = [] for segment in playlist.segments: if segment.uri not in vod_paths: vod_paths.append(segment.uri) # Download VOD chunks to the temp folder path_map = worker.download_files(video_id, base_uri, tempdir, vod_paths, max_workers) # TODO: rewrite this output to look nicer and remove ffmpeg output using COLOR_CODES["F"] cprint("#dDone, now to ffmpeg join...#r") # join the vods using ffmpeg at specified path # TODO: change this to the concat function in video? cwd = os.getcwd() os.chdir(str(tempdir)) cmd = [ "ffmpeg", "-i", str(playlist_path), "-c", "copy", path, "-y", "-stats", "-loglevel", LOG_LEVEL ] result = subprocess.run(cmd) os.chdir(cwd) if result.returncode != 0: raise JoiningFailed() # delete temp folder and contents shutil.rmtree(str(tempdir))
def run(args): # check if the config exists and ask if it should be overwritten if exists(args.output): input( "It seems a file already exists here. Press enter if you wish to continue, otherwise press Ctrl+C to quit." ) input( "This operation will overwrite this file. Do you have a backup? Press enter again to continue, otherwise press Ctrl+C to quit." ) input( "Last time, are you absolutely certain you want to overwrite this file? Press enter again to continue, otherwise press Ctrl+C to quit." ) print("Overwriting file...") # test write the config. try: util.make_dir(args.output.parent) with open(str(args.output), "w") as f: pass except FileNotFoundError: util.exit_prog(67, f"Cannot create file at \"{args.output}\".") # see which config we should make if args.default: # generate default config cprint("#dCreating default config...#r") else: # generate config from inputs create_config(args) # create directories now util.make_dir(DEFAULT_CONF['vod_dir']) util.make_dir(DEFAULT_CONF['clip_dir']) util.make_dir(DEFAULT_CONF['temp_dir']) util.make_dir(DEFAULT_CONF['stage_dir']) # now write the config try: #util.make_dir(args.output.parent) # redundant with open(str(args.output), "w") as f: json.dump(DEFAULT_CONF, f, indent=4) except FileNotFoundError: util.exit_prog( 68, f"Cannot create file at \"{args.output}\" despite being able to before." ) # list the location of the config and say what can be edited outside this command cprint( f"#fGFinished#r, the config can be edited at `#l{str(vodbotdir / 'conf.json')}#r`." )
def dl_clip(clip: Clip, path: str): clip_slug = clip.slug clip_id = clip.id # Get proper clip file URL source_url = gql.get_clip_source(clip_slug) # Download file to path size, _existed = worker.download_file(source_url, path) # Print progress cprint( f"#fM#lClip#r ({clip_id})`#fM{clip_slug}#r` #fB#l~{worker.format_size(size)}#r" )
def check_title(default=None): title = default if not title: title = "" while title == "": title = input( colorize("#fW#lTitle of the Video#r #d(--title)#r: ")) # blank title if title == "": cprint("#fRTitle cannot be blank.#r") # reserved names if title in RESERVED_NAMES: cprint("#fRTitle cannot be a reserved name.#r") title = "" return title
def check_streamers(default=None) -> List[str]: streamers = "" while not streamers: streamers = input( colorize( f"#fW#lWho was in the VOD#r #d(default `{', '.join(default)}`, csv)#r: " )) if streamers == "": streamers = default else: streamers = streamers.replace(" ", "").split(",") for streamer in streamers: if len(streamer) == 0: cprint("#l#fRMissing streamer name!#r") streamers = "" break return streamers
def _print_progress(video_id, futures): downloaded_count = 0 downloaded_size = 0 existing_size = 0 max_msg_size = 0 start_time = datetime.now() total_count = len(futures) try: for future in as_completed(futures): (size, existed) = future.result() downloaded_count += 1 downloaded_size += size if existed: existing_size += size percentage = 100 * downloaded_count // total_count est_total_size = int(total_count * downloaded_size / downloaded_count) duration = (datetime.now() - start_time).seconds speed = (downloaded_size - existing_size) // duration if duration else 0 remaining = (total_count - downloaded_count) * duration / downloaded_count msg = " ".join([ f"#fM#lVOD#r `#fM{video_id}#r` pt#fC{downloaded_count}#r/#fB#l{total_count}#r,", f"#fC{format_size(downloaded_size)}#r/#fB#l~{format_size(est_total_size)}#r #d({percentage}%)#r;", f"at #fY~{format_size(speed)}/s#r;" if speed > 0 else "", f"#fG~{format_duration(remaining)}#r left" if speed > 0 else "", ]) max_msg_size = max(len(msg), max_msg_size) cprint("\r" + msg.ljust(max_msg_size), end="") except KeyboardInterrupt: done, not_done = wait(futures, timeout=0) for future in not_done: future.cancel() wait(not_done, timeout=None) raise DownloadCancelled()
def check_description(formatdict, inputdefault=None): desc = "" if inputdefault: try: inputdefault = inputdefault.format(**formatdict).replace( "\\n", "\n") desc = inputdefault except KeyError as err: cprint(f"#fRDescription format error from default: {err}.#r") desc = "" while desc == "": desc = input(colorize("#fW#lDescription of Video#r #d(--desc)#r: ")) if desc == "": cprint("#fRDescription cannot be blank.#r") continue # Format the description try: desc = desc.format(**formatdict).replace("\\n", "\n") except KeyError as err: cprint(f"#fRDescription format error: {err}.#r") desc = "" return desc
def run(args): conf = util.load_conf(args.config) util.make_dir(vodbotdir) stagedir = conf["stage_dir"] path_stage = Path(stagedir) util.make_dir(stagedir) if args.action == "new": _new(args, conf, stagedir) elif args.action == "rm": if not isfile(str(path_stage / (args.id + ".stage"))): util.exit_prog(45, f'Could not find stage "{args.id}".') try: os_remove(str(path_stage / (args.id + ".stage"))) cprint(f'Stage "#fY#l{args.id}#r" has been #fRremoved#r.') except OSError as err: util.exit_prog( 88, f'Stage "{args.id}" could not be removed due to an error. {err}' ) elif args.action == "list": _list(args, conf, stagedir)
def dl_video_chat(video: Vod, path: str): video_id = video.id # Download all chat from video cprint(f"#fM#lVOD Chat#r `#fM{video_id}#r` (0%)", end="") msgs = get_video_comments(video_id) cprint( f"\r#fM#lVOD Chat#r `#fM{video_id}#r` (100%); Done, now to write...", end="") chatlog.chat_to_logfile(msgs, path) cprint( f"\r#fM#lVOD Chat#r `#fM{video_id}#r` (100%); Done, now to write... Done" )
def handle_stage(conf: dict, stage: StageData) -> Path: tmpfile = None try: tmpfile = vbvid.process_stage(conf, stage) except vbvid.FailedToSlice as e: cprint( f"#r#fRSkipping stage `{stage.id}`, failed to slice video with ID of `{e.vid}`.#r\n" ) return None except vbvid.FailedToConcat: cprint( f"#r#fRSkipping stage `{stage.id}`, failed to concatenate videos.#r\n" ) return None except vbvid.FailedToCleanUp as e: cprint( f"#r#fRSkipping stage `{stage.id}`, failed to clean up temp files.#r\n\n{e.vid}" ) return None return tmpfile
def run(args): # load config conf = util.load_conf(args.config) # configure variables global stagedir, tempdir tempdir = Path(conf["temp_dir"]) stagedir = Path(conf["stage_dir"]) PICKLE_FILE = conf["youtube_pickle_path"] CLIENT_SECRET_FILE = conf["youtube_client_path"] API_NAME = 'youtube' API_VERSION = 'v3' SCOPES = [ # Only force-ssl is required, but both makes it explicit. "https://www.googleapis.com/auth/youtube.upload", # Videos.insert "https://www.googleapis.com/auth/youtube.force-ssl" # Captions.insert ] # handle logout if args.id == "logout": try: os_remove(PICKLE_FILE) cprint("#dLogged out of Google API session#r") except: util.exit_prog( 11, "Failed to remove credentials for YouTube account.") return # load stages, but dont upload # Handle id/all stagedata = None stagedatas = None if args.id == "all": cprint("#dLoading stages...", end=" ") # create a list of all the hashes and sort by date streamed, upload chronologically stagedatas = StageData.load_all_stages(stagedir) stagedatas.sort(key=sort_stagedata) else: cprint("#dLoading stage...", end=" ") # check if stage exists, and prep it for upload stagedata = StageData.load_from_id(stagedir, args.id) cprint(f"About to upload stage {stagedata.id}.#r") # authenticate youtube service if not os_exists(CLIENT_SECRET_FILE): util.exit_prog(19, "Missing YouTube Client ID/Secret file.") cprint("Authenticating with Google...", end=" ") service = None credentials = None if os_exists(PICKLE_FILE): with open(PICKLE_FILE, "rb") as f: credentials = pickle.load(f) if not credentials or credentials.expired: try: if credentials and credentials.expired and credentials.refresh_token: credentials.refresh(Request()) else: flow = InstalledAppFlow.from_client_secrets_file( CLIENT_SECRET_FILE, SCOPES) credentials = flow.run_console() except RefreshError: flow = InstalledAppFlow.from_client_secrets_file( CLIENT_SECRET_FILE, SCOPES) credentials = flow.run_console() with open(PICKLE_FILE, "wb") as f: pickle.dump(credentials, f) try: service = build(API_NAME, API_VERSION, credentials=credentials) except Exception as err: util.exit_prog(50, f"Failed to connect to YouTube API, \"{err}\"") cprint("Authenticated.", end=" ") # Handle id/all if args.id == "all": # begin to upload cprint(f"About to upload {len(stagedatas)} stages.#r") for stage in stagedatas: video_id = upload_video(conf, service, stage) if video_id is not None: chat_success = True if conf["chat_upload"]: chat_success = upload_captions(conf, service, stage, video_id) if conf["stage_upload_delete"] and chat_success: try: os_remove(str(stagedir / f"{stage.id}.stage")) except: util.exit_prog( 90, f"Failed to remove stage `{stage.id}` after upload." ) print() else: # upload stage cprint(f"About to upload stage {stagedata.id}.#r") video_id = upload_video(conf, service, stagedata) if video_id is not None: chat_success = True if conf["chat_upload"]: chat_success = upload_captions(conf, service, stagedata, video_id) if conf["stage_upload_delete"] and chat_success: try: os_remove(str(stagedir / f"{stagedata.id}.stage")) except: util.exit_prog( 90, f"Failed to remove stage `{stagedata.id}` after upload." )
def upload_captions(conf: dict, service, stagedata: StageData, vid_id: str) -> bool: tmpfile = vbchat.process_stage(conf, stagedata, "upload") if tmpfile == False: return True request_body = { "snippet": { "name": "Chat", "videoId": vid_id, "language": "en" } } media_file = MediaFileUpload(str(tmpfile), chunksize=1024 * 1024 * 100, resumable=True) response_upload = service.captions().insert(part="snippet", body=request_body, sync=False, media_body=media_file) resp = None errn = 0 cprint( f"#fCUploading stage chatlog #r`#fM{stagedata.id}#r`, progress: #fC0#fY%#r #d...#r", end="\r") while resp is None: try: status, resp = response_upload.next_chunk() if status: cprint( f"#fCUploading stage chatlog #r`#fM{stagedata.id}#r`, progress: #fC{int(status.progress()*100)}#fY%#r #d...#r", end="\r") if resp is not None: cprint( f"#fCUploading stage chatlog #r`#fM{stagedata.id}#r`, progress: #fC100#fY%#r!" ) cprint(f"#l#fGVideo captions were successfully uploaded!#r") except ResumableUploadError as err: if err.resp.status in [400, 401, 402, 403]: try: jsondata = json.loads(err.content)['error']['errors'][0] util.exit_prog( 40, f"API Error: `{jsondata['reason']}`. Message: `{jsondata['message']}`" ) except (json.JSONDecodeError, KeyError): util.exit_prog( 40, f"Unknown API Error has occured, ({err.resp.status}, {err.content})" ) print( f"A Resumeable error has occured, retrying in 5 sec... ({err.resp.status}, {err.content})" ) errn += 1 sleep(5) except HttpError as err: if err.resp.status in [500, 502, 503, 504]: print( f"An HTTP error has occured, retrying in 5 sec... ({err.resp.status}, {err.content})" ) errn += 1 sleep(5) else: raise except RETRIABLE_EXCEPTS as err: print(f"An HTTP error has occured, retrying in 5 sec... ({err})") errn += 1 sleep(5) if errn >= 10: print("Skipping chatlog upload, errored too many times.") return False # we're done, lets clean up else: try: # delete vars to release the files del media_file del response_upload sleep(1) os_remove(str(tmpfile)) except Exception as e: util.exit_prog( 90, f"Failed to remove temp slice file of stage `{stagedata.id}` after upload. {e}" ) return True
def create_config(args): channels = [] timezone = "" # switch to UTC only # these are for determining directories. # we need to make sure any that are already provided are absolute right now. # we do it now so that the user doesn't enter in anything before receiving # an error that boots them out of the program. parts = [[args.voddir, "VODs", DEFAULT_CONF['vod_dir']], [args.clipdir, "Clips", DEFAULT_CONF['clip_dir']], [args.tempdir, "temporary data", DEFAULT_CONF['temp_dir']], [args.stagedir, "staged data", DEFAULT_CONF['stage_dir']]] for part in parts: if part[0]: if not isabs(part[0]): util.exit_prog( 150, f"Path `{part[0]}` for {part[1]} is not an absolute path.") part.append(part[0]) # Ask for channels if not args.channels: cprint( "Enter the login names of the Twitch channels you'd like to have archived." ) cprint("When you're done, just leave the input empty and press enter.") cprint( "Example: the url is `https://twitch.tv/notquiteapex`, enter `notquiteapex`" ) while True: channel = input("> ") if channel == "" and len(channels) > 0: break elif channel == "" and len(channels) == 0: cprint("#fRError#r, no channels given.") elif not re.match( r"^[a-zA-Z0-9][\w]{0,24}$", channel): # https://discuss.dev.twitch.tv/t/3855/4 cprint( "#fRError#r, channel names only contain characters A-Z, 0-9, and '_'. They also can't start with '_'." ) else: channels += [channel] else: channels = args.channels # Ask for timezone as UTC string # We only absolutely need the hours and minutes offset, and we regex it out of the string. if not args.timezone: cprint( "Enter the UTC timezone for timedate referencing. Only use +, -, and numbers." ) cprint( "#dExample: `+0000` for UTC, `-0500` for America's EST. Default: `+0000`#r" ) while True: tz = input("> ") if tz == "": tz = "+0000" break elif not re.match(r"^[+-]\d\d\d\d$", tz): cprint("Error, UTC string not recognized.") else: break else: timezone = args.timezone # check that entered timezone is valid # ask for directories (any provided by the terminal arguments are already handeled.) cprint( "Now let's get some directories to store data. The entered paths must be absolute, not relative." ) cprint( "If you'd like to use the default location listed, just leave the input blank and press enter." ) for part in parts: if not part[0]: cprint( f"Enter where {part[1]} should be stored.\n#dDefault: `#l{part[2]}`#r" ) while True: inpdir = input("> ") if inpdir == "": part.append(part[2]) break elif isabs(inpdir): # TODO: check if directory can be created? a file might already exist? part.append(inpdir) break else: cprint( f"#fRError#r, directory `#l{inpdir}#r` is not an absolute path for a directory." ) # ready to write it all, go! cprint("#dWriting config...#r") # Edit default config variable. DEFAULT_CONF['twitch_channels'] = channels DEFAULT_CONF['stage_timezone'] = timezone DEFAULT_CONF['vod_dir'] = parts[0][3] DEFAULT_CONF['clip_dir'] = parts[1][3] DEFAULT_CONF['temp_dir'] = parts[2][3] DEFAULT_CONF['stage_dir'] = parts[3][3]
def run(args): global stagedir conf = util.load_conf(args.config) stagedir = Path(conf["stage_dir"]) util.make_dir(args.path) args.path = Path(args.path) # load stages, but dont slice # Handle id/all if args.id == "all": cprint("#dLoading and slicing stages...#r") # create a list of all the hashes and sort by date streamed, slice chronologically stagedatas = StageData.load_all_stages(stagedir) stagedatas.sort(key=sort_stagedata) for stage in stagedatas: tmpfile = None tmpchat = None # Export with ffmpeg tmpfile = handle_stage(conf, stage) # Export chat tmpchat = vbchat.process_stage(conf, stage, "export") title = stage.title.strip() for x in DISALLOWED_CHARACTERS: title = title.replace(x, "_") # move appropriate files if tmpfile is not None: os_replace(tmpfile, args.path / (title + tmpfile.suffix)) if tmpchat is not None: os_replace(tmpchat, args.path / (title + tmpchat.suffix)) # deal with old stage if conf["stage_export_delete"]: os_remove(str(stagedir / f"{stage.id}.stage")) else: cprint("#dLoading stage...", end=" ") # check if stage exists, and prep it for slice stagedata = StageData.load_from_id(stagedir, args.id) tmpfile = None tmpchat = None # Export with ffmpeg tmpfile = handle_stage(conf, stagedata) # Export chat tmpchat = vbchat.process_stage(conf, stagedata, "export") title = stagedata.title.strip() for x in DISALLOWED_CHARACTERS: title = title.replace(x, "_") # move appropriate files if tmpfile is not None: os_replace(tmpfile, args.path / (title + tmpfile.suffix)) if tmpchat is not None: os_replace(tmpchat, args.path / (title + tmpchat.suffix)) # Deal with old stage if conf["stage_export_delete"]: os_remove(str(stagedir / f"{stagedata.id}.stage")) # say "Done!" cprint("#fG#lDone!#r")
def process_stage(conf: dict, stage: StageData, mode: str) -> Path: tempdir = Path(conf["temp_dir"]) msg_duration = int(conf["chat_msg_time"]) cprint(f"#rLoading all chat messages for `#fM{stage.id}#r`.", end="") total_offset = 0 chat_list = [] for slc in stage.slices: # load up each stagedata's meta to see if chat exists metapath = slc.filepath[:-4] + ".meta" meta = None with open(metapath, "r") as f: meta = json.load(f) has_chat = meta.get("has_chat", False) duration = meta.get("length", 0) chat_path = slc.filepath[:-4] + ".chat" # start and end times as seconds start_sec = timestring_as_seconds(slc.ss) end_sec = timestring_as_seconds(slc.to, duration) # keep each list of chat separate, compare timestamps to offsets to make # sure theyre inbetween the slices if has_chat: msg_list = [ m for m in logfile_to_chat(chat_path) if start_sec <= m.offset < end_sec ] for m in msg_list: m.offset = m.offset - start_sec + total_offset chat_list += msg_list # now take each stage's duration and apply it to the next chat list's total_offset += end_sec - start_sec # determine how to export, then return the resultant path if mode != "upload" and mode != "export": util.exit_prog(94, f"Cannot export chat with export mode {mode}") export_type = conf["chat_" + mode] if len(chat_list) == 0: cprint(f" No chat found in `#fY{export_type}#r` stage. Skipping...") return False cprint(f" Exporting as format `#fY{export_type}#r`.") returnpath = None if export_type == "raw": # chat to logfile time returnpath = tempdir / (stage.id + ".chat") chat_to_logfile(chat_list, str(returnpath)) elif export_type == "RealText": # load from archive, parse and write to temp returnpath = tempdir / (stage.id + ".rt") chat_to_realtext(chat_list, str(returnpath), total_offset, msg_duration) elif export_type == "SAMI": # load from archive, parse and write to temp returnpath = tempdir / (stage.id + ".sami") chat_to_sami(chat_list, str(returnpath), total_offset, msg_duration) elif export_type == "YTT": # load from archive, parse and write to temp returnpath = tempdir / (stage.id + ".ytt") chat_to_ytt(chat_list, str(returnpath), total_offset, msg_duration) elif export_type == "TTML": # load from archive, parse and write to temp returnpath = tempdir / (stage.id + ".ttml") chat_to_ttml(chat_list, str(returnpath), total_offset, msg_duration) return returnpath
def check_time(prefix, resp, default=None): output = resp checkedonce = False while True: if checkedonce or not output: a = 'ss' if prefix == 'Start' else 'to' t = '0:0:0' if prefix == 'Start' else 'EOF' f = f"#fW#l{prefix} time of the Video#r #d(--{a}, default {t})#r: " output = input(colorize(f)) checkedonce = True if output == "": if prefix == "Start": return default if default != None else "0:0:0" elif prefix == "End": return default if default != None else "EOF" intime = output.split(":") timelist = [] seconds = None minutes = None hours = None if len(intime) > 3: cprint(f"#fR{prefix} time: Time cannot have more than 3 units.#r") continue if len(intime) >= 1: seconds = intime[-1] try: seconds = int(seconds) except ValueError: cprint( f"#fR{prefix} time: Seconds does not appear to be a number.#r" ) continue if seconds > 59 or seconds < 0: cprint( f"#fR{prefix} time: Seconds must be in the range of 0 to 59.#r" ) continue timelist.insert(0, str(seconds)) if len(intime) >= 2: minutes = intime[-2] try: minutes = int(minutes) except ValueError: cprint( f"#fR{prefix} time: Minutes does not appear to be a number.#r" ) continue if minutes > 59 or minutes < 0: cprint( f"#fR{prefix} time: Minutes must be in the range of 0 to 59.#r" ) continue timelist.insert(0, str(minutes)) else: timelist.insert(0, "0") if len(intime) == 3: hours = intime[-3] try: hours = int(hours) except ValueError: cprint( f"#fR{prefix} time: Hours does not appear to be a number.#r" ) continue timelist.insert(0, str(hours)) else: timelist.insert(0, "0") output = ":".join(timelist) break return output
def _new(args, conf, stagedir): VODS_DIR = conf["vod_dir"] CLIPS_DIR = conf["clip_dir"] STAGE_DIR = conf["stage_dir"] stagedir = Path(STAGE_DIR) # find the videos by their ids to confirm they exist videos = [] for video in args.id: try: (filename, metadata, videotype) = find_video_by_id(video, VODS_DIR, CLIPS_DIR) videos += [{ "id": video, "file": filename, "meta": metadata, "type": videotype }] except CouldntFindVideo: util.exit_prog(13, f'Could not find video with ID "{args.id}"') # Get what streamers were involved (usernames), always asked default_streamers = [] for f in videos: if f["meta"]["user_login"] not in default_streamers: default_streamers.append(f["meta"]["user_login"]) args.streamers = check_streamers(default=default_streamers) # get title if not args.title: args.title = check_title(default=None) # get description formatdict, datestring = create_format_dict(conf, args.streamers, utcdate=metadata["created_at"]) args.desc = check_description(formatdict, inputdefault=args.desc) # get timestamps for each video through input for x in range(len(videos)): # skip times we dont need because we already have them if x < len(args.ss): continue vid = videos[x]["meta"] # grab times for this specific stream cprint(f"#dTimestamps for `#r#fM{vid['title']}#r` #d({vid['id']})#r") if "chapters" in vid and len(vid["chapters"]) > 0: cprint(f"#dChapters: ", end="") ch = [] for c in vid["chapters"]: (pos, end) = posdur_to_timestamp(c['pos'], c['dur']) ch.append(f"`{c['desc']}` ({pos}-{end})") cprint(" | ".join(ch) + " | EOF#r") else: (pos, end) = posdur_to_timestamp(0, vid['length']) cprint(f"#dChapter: `{vid['game_name']}` ({pos}-{end}) | EOF#r") args.ss += [ check_time("Start", args.ss[x] if x < len(args.ss) else None) ] args.to += [ check_time("End", args.to[x] if x < len(args.to) else None) ] # make slice objects slices = [] for x in range(len(videos)): vid = videos[x] vidslice = VideoSlice(video_id=vid["id"], ss=args.ss[x], to=args.to[x], filepath=vid["file"]) slices += [vidslice] # make stage object stage = StageData(streamers=args.streamers, title=args.title, desc=args.desc, datestring=datestring, slices=slices) # Check that new "id" does not collide while check_stage_id(stage.id, STAGE_DIR): stage.gen_new_id() # shorter file name #shortfile = stage.filename.replace(VODS_DIR, "$vods").replace(CLIPS_DIR, "$clips") print() cprint( f"#r`#fC{stage.title}#r` #fM{' '.join(stage.streamers)}#r #d({stage.id})#r" ) cprint(f"#d'''#fG{stage.desc}#r#d'''#r") for vid in stage.slices: cprint(f"#fM{vid.video_id}#r > #fY{vid.ss}#r - #fY{vid.to}#r") # write stage stagename = str(stagedir / (stage.id + ".stage")) stage.write_stage(stagename)
def upload_video(conf: dict, service, stagedata: StageData) -> str: tmpfile = None try: tmpfile = vbvid.process_stage(conf, stagedata) except vbvid.FailedToSlice as e: cprint( f"#r#fRSkipping stage `{stagedata.id}`, failed to slice video with ID of `{e.vid}`.#r\n" ) except vbvid.FailedToConcat: cprint( f"#r#fRSkipping stage `{stagedata.id}`, failed to concatenate videos.#r\n" ) except vbvid.FailedToCleanUp as e: cprint( f"#r#fRSkipping stage `{stagedata.id}`, failed to clean up temp files.#r\n\n{e.vid}" ) # send request to youtube to upload request_body = { "snippet": { "categoryId": 20, "title": stagedata.title, "description": stagedata.desc }, "status": { "privacyStatus": "private", "selfDeclaredMadeForKids": False } } # create media file, 100 MiB chunks media_file = MediaFileUpload(str(tmpfile), chunksize=1024 * 1024 * 100, resumable=True) # create upload request and execute response_upload = service.videos().insert(part="snippet,status", body=request_body, notifySubscribers=False, media_body=media_file) video_id = None # youtube video id resp = None errn = 0 cprint( f"#fCUploading stage #r`#fM{stagedata.id}#r`, progress: #fC0#fY%#r #d...#r", end="\r") # Below is taken from Google's documentation, rewrite? while resp is None: try: status, resp = response_upload.next_chunk() if status: cprint( f"#fCUploading stage #r`#fM{stagedata.id}#r`, progress: #fC{int(status.progress()*100)}#fY%#r #d...#r", end="\r") if resp is not None: if "id" in resp: cprint( f"#fCUploading stage #r`#fM{stagedata.id}#r`, progress: #fC100#fY%#r!" ) cprint( f"#l#fGVideo was successfully uploaded!#r #dhttps://youtu.be/{resp['id']}#r" ) video_id = resp["id"] else: util.exit_prog( 99, f"Unexpected upload failure occurred, \"{resp}\"") except ResumableUploadError as err: if err.resp.status in [400, 401, 402, 403]: try: jsondata = json.loads(err.content)['error']['errors'][0] util.exit_prog( 40, f"API Error: `{jsondata['reason']}`. Message: `{jsondata['message']}`" ) except (json.JSONDecodeError, KeyError): util.exit_prog( 40, f"Unknown API Error has occured, ({err.resp.status}, {err.content})" ) print( f"A Resumeable error has occured, retrying in 5 sec... ({err.resp.status}, {err.content})" ) errn += 1 sleep(5) except HttpError as err: if err.resp.status in [500, 502, 503, 504]: print( f"An HTTP error has occured, retrying in 5 sec... ({err.resp.status}, {err.content})" ) errn += 1 sleep(5) else: raise except RETRIABLE_EXCEPTS as err: print(f"An HTTP error has occured, retrying in 5 sec... ({err})") errn += 1 sleep(5) if errn >= 10: print("Skipping video upload, errored too many times.") break # we're done, lets clean up else: try: # delete vars to release the files del media_file del response_upload sleep(1) os_remove(str(tmpfile)) except Exception as e: util.exit_prog( 90, f"Failed to remove temp slice file of stage `{stagedata.id}` after upload. {e}" ) return video_id
def _list(args, conf, stagedir): VODS_DIR = conf["vod_dir"] CLIPS_DIR = conf["clip_dir"] STAGE_DIR = conf["stage_dir"] stagedir = Path(STAGE_DIR) if args.id == None: stages = StageData.load_all_stages(stagedir) for s in stages: cprint(f'#r#fY#l{s.id}#r -- `#fC{s.title}#r` ', end="") cprint(f'(#fM{" ".join([d.video_id for d in s.slices])}#r) -- ', end="") cprint(f'#l#fM{", ".join(s.streamers)}#r') if len(stages) == 0: cprint("#fBNothing staged right now.#r") else: stage = StageData.load_from_id(stagedir, args.id) cprint( f"#r`#fC{stage.title}#r` #fM{' '.join(stage.streamers)}#r #d({stage.id})#r" ) cprint(f"#d'''#fG{stage.desc}#r#d'''#r") for vid in stage.slices: cprint(f"#fM{vid['id']}#r > #fY{vid['ss']}#r - #fY{vid['to']}#r")
def run(args): # First determine what kind of data it is... # We should allow people to paste full links or just the ID too. tid = args.id # Get the proper id and type of content we need to pull cprint("#dDetermining type...#r", end=" ") cid, ctype = get_type(tid) # The call the appropriate info query for GQL query = None if ctype == "vod": query = gql.GET_VIDEO_QUERY.format(video_id=cid) elif ctype == "clip": query = gql.GET_CLIP_QUERY.format(clip_slug=cid) elif ctype == "channel": query = gql.GET_CHANNEL_QUERY.format(channel_id=cid) else: util.exit_prog(92, "Could not determine content type from input.") cword = KEYTRANS[ctype] cprint(f"#dQuerying {cword} content for `{cid}`...#r") # run the query resp = gql.gql_query(query=query).json() resp = resp["data"] if not any([resp.get("video"), resp.get("clip"), resp.get("user")]): util.exit_prog(93, f"Could not query info on {cword} content from input.") # Read the data back and out to the terminal if ctype == "vod": r = resp['video'] c = r['creator'] g = r['game'] cprint(f"#fGTitle: `{r['title']}`#r - ID: `{r['id']}`") cprint( f"#fMBroadcaster: {c['displayName']} - {c['login']} ({c['id']})#r") cprint(f"#fCPlaying: {g['name']} ({g['id']})#r") cprint( f"#fYAt: {r['publishedAt']} - For: {r['lengthSeconds']} seconds#r") elif ctype == "clip": r = resp['clip'] c = r['curator'] b = r['broadcaster'] g = r['game'] cprint(f"#fGTitle: `{r['title']}`#r - ID: `{r['id']}`") cprint(f"Slug: `{r['slug']}`") cprint(f"#fBClipper: {c['displayName']} - {c['login']} ({c['id']})#r") cprint( f"#fMBroadcaster: {b['displayName']} - {b['login']} ({b['id']})#r") cprint(f"#fCPlaying: {g['name']} ({g['id']})#r") cprint( f"#fYAt: {r['createdAt']} - For: {r['durationSeconds']} seconds - With: {r['viewCount']} views#r" ) elif ctype == "channel": r = resp['user'] cprint( f"#fMBroadcaster: {r['displayName']} - {r['login']} (ID: `{r['id']}`)#r" ) cprint(f"#fRDescription: {r['description']}#r") cprint(f"#fYChannel Created At: {r['createdAt']}#r") r = r['roles'] cprint( f"#fBRoles: Affiliate={r['isAffiliate']} - Partner={r['isPartner']}#r" )
def download_twitch_video(args): # Load the config and set up the access token cprint("#r#dLoading config...#r", end=" ", flush=True) conf = util.load_conf(args.config) CHANNEL_IDS = conf["twitch_channels"] VODS_DIR = conf["vod_dir"] CLIPS_DIR = conf["clip_dir"] TEMP_DIR = conf["temp_dir"] LOG_LEVEL = conf["ffmpeg_loglevel"] PULL_CHAT = conf["pull_chat_logs"] # If channel arguments are provided, override config if args.channels: CHANNEL_IDS = args.channels cprint("#r#dLoading channel data...#r", flush=True) channels = twitch.get_channels(CHANNEL_IDS) contentnoun = "video" # temp contentnoun until the rest is reworked # Setup directories for videos and temp util.make_dir(TEMP_DIR) voddir = Path(VODS_DIR) util.make_dir(voddir) clipdir = Path(CLIPS_DIR) util.make_dir(clipdir) # Get list of videos using channel object ID's from Twitch API videos = [] totalvods = 0 totalclips = 0 channel_print = None if args.type == "both": channel_print = "Pulling #fM#lVOD#r & #fM#lClip#r list: #fY#l{}#r..." elif args.type == "vods": channel_print = "Pulling #fM#lVOD#r list: #fY#l{}#r..." elif args.type == "clips": channel_print = "Pulling #fM#lClip#r list: #fY#l{}#r..." for channel in channels: vods = None clips = None cprint(channel_print.format(channel.display_name), end=" ") # Grab list of VODs and check against existing VODs if args.type == "both" or args.type == "vods": folder = voddir / channel.login util.make_dir(folder) allvods = twitch.get_channel_vods(channel) vods = compare_existant_file(folder, allvods) totalvods += len(vods) # Grab list of Clips and check against existing Clips if args.type == "both" or args.type == "clips": folder = clipdir / channel.login util.make_dir(folder) allclips = twitch.get_channel_clips(channel) clips = compare_existant_file(folder, allclips) totalclips += len(clips) # Print content found and save it if args.type == "both": cprint( f"#fC#l{len(vods)} #fM#lVODSs#r & #fC#l{len(clips)} #fM#lClips#r" ) videos += vods videos += clips elif args.type == "vods": cprint(f"#fC#l{len(vods)} #fM#lVODSs#r") videos += vods elif args.type == "clips": cprint(f"#fC#l{len(clips)} #fM#lClips#r") videos += clips if args.type == "both": cprint(f"Total #fMVODs#r to download: #fC#l{totalvods}#r") cprint(f"Total #fMClips#r to download: #fC#l{totalclips}#r") cprint(f"Total #fM#lvideos#r to download: #fC#l{len(videos)}#r") elif args.type == "vods": cprint(f"Total #fMVODs#r to download: #fC#l{totalvods}#r") elif args.type == "clips": cprint(f"Total #fMClips#r to download: #fC#l{totalclips}#r") # Download all the videos we need. previouschannel = None for vod in videos: # Print if we're on to a new user. if previouschannel != vod.user_id: previouschannel = vod.user_id cprint( f"\nDownloading #fY#l{vod.user_name}#r's #fM#l{contentnoun}s#r..." ) # Generate path for video viddir = None contentnoun = None if isinstance(vod, Vod): viddir = voddir / vod.user_name.lower() contentnoun = "VOD" elif isinstance(vod, Clip): viddir = clipdir / vod.user_name.lower() contentnoun = "Clip" filepath = viddir / f"{vod.created_at}_{vod.id}".replace(":", ";") filename = str(filepath) + ".mkv" metaname = str(filepath) + ".meta" chatname = str(filepath) + ".chat" # Write video data and handle exceptions try: if isinstance(vod, Vod): # download chat if PULL_CHAT: itd_dl.dl_video_chat(vod, chatname) vod.has_chat = True # download video itd_dl.dl_video(vod, Path(TEMP_DIR), filename, 20, LOG_LEVEL) # write meta file vod.write_meta(metaname) elif isinstance(vod, Clip): # download clip itd_dl.dl_clip(vod, filename) # write meta file vod.write_meta(metaname) except itd_dl.JoiningFailed: cprint( f"#fR#lVOD `{vod.id}` joining failed! Preserving files...#r") except itd_work.DownloadFailed: cprint(f"Download failed! Skipping...#r") except itd_work.DownloadCancelled: cprint(f"\n#fR#l{contentnoun} download cancelled.#r") raise KeyboardInterrupt() cprint("\n#fM#l* All done, goodbye! *#r\n")