def run(args): # check if the config exists and ask if it should be overwritten if exists(args.output): input( "It seems a file already exists here. Press enter if you wish to continue, otherwise press Ctrl+C to quit." ) input( "This operation will overwrite this file. Do you have a backup? Press enter again to continue, otherwise press Ctrl+C to quit." ) input( "Last time, are you absolutely certain you want to overwrite this file? Press enter again to continue, otherwise press Ctrl+C to quit." ) print("Overwriting file...") # test write the config. try: util.make_dir(args.output.parent) with open(str(args.output), "w") as f: pass except FileNotFoundError: util.exit_prog(67, f"Cannot create file at \"{args.output}\".") # see which config we should make if args.default: # generate default config cprint("#dCreating default config...#r") else: # generate config from inputs create_config(args) # create directories now util.make_dir(DEFAULT_CONF['vod_dir']) util.make_dir(DEFAULT_CONF['clip_dir']) util.make_dir(DEFAULT_CONF['temp_dir']) util.make_dir(DEFAULT_CONF['stage_dir']) # now write the config try: #util.make_dir(args.output.parent) # redundant with open(str(args.output), "w") as f: json.dump(DEFAULT_CONF, f, indent=4) except FileNotFoundError: util.exit_prog( 68, f"Cannot create file at \"{args.output}\" despite being able to before." ) # list the location of the config and say what can be edited outside this command cprint( f"#fGFinished#r, the config can be edited at `#l{str(vodbotdir / 'conf.json')}#r`." )
def load_from_id(stagedir: Path, sid: str) -> 'StageData': jsonread = None try: with open(str(stagedir / (sid + ".stage"))) as f: jsonread = json.load(f) except FileNotFoundError: util.exit_prog(46, f'Could not find stage "{sid}". (FileNotFound)') except KeyError: util.exit_prog( 46, f'Could not parse stage "{sid}" as JSON. Is this file corrupted?' ) return StageData.load_from_json(jsonread)
def create_format_dict(conf, streamers, utcdate=None, truedate=None): thistz = None datestring = None if truedate == None: try: # https://stackoverflow.com/a/37097784/13977827 sign, hours, minutes = re.match('([+\-]?)(\d{2})(\d{2})', conf['stage_timezone']).groups() sign = -1 if sign == '-' else 1 hours, minutes = int(hours), int(minutes) thistz = timezone(sign * timedelta(hours=hours, minutes=minutes)) except: util.exit_prog(73, f"Unknown timezone {conf['stage_timezone']}") date = datetime.strptime( utcdate, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc) datestring = date.astimezone(thistz).strftime("%Y/%m/%d") else: datestring = truedate formatdict = { "date": datestring, "link": f"https://twitch.tv/{streamers[0]}", "streamer": streamers[0], "links": " ".join([f"https://twitch.tv/{s}" for s in streamers]), "streamers": streamers, } # first pass format for x in range(2): for item, string in conf["stage_format"].items(): try: string = string.format(**formatdict) formatdict[item] = string except KeyError as err: # ignore errors on first pass if x == 1: util.exit_prog(81, f"Format failed: {err}") return formatdict, datestring
def run(args): conf = util.load_conf(args.config) util.make_dir(vodbotdir) stagedir = conf["stage_dir"] path_stage = Path(stagedir) util.make_dir(stagedir) if args.action == "new": _new(args, conf, stagedir) elif args.action == "rm": if not isfile(str(path_stage / (args.id + ".stage"))): util.exit_prog(45, f'Could not find stage "{args.id}".') try: os_remove(str(path_stage / (args.id + ".stage"))) cprint(f'Stage "#fY#l{args.id}#r" has been #fRremoved#r.') except OSError as err: util.exit_prog( 88, f'Stage "{args.id}" could not be removed due to an error. {err}' ) elif args.action == "list": _list(args, conf, stagedir)
def upload_video(conf: dict, service, stagedata: StageData) -> str: tmpfile = None try: tmpfile = vbvid.process_stage(conf, stagedata) except vbvid.FailedToSlice as e: cprint( f"#r#fRSkipping stage `{stagedata.id}`, failed to slice video with ID of `{e.vid}`.#r\n" ) except vbvid.FailedToConcat: cprint( f"#r#fRSkipping stage `{stagedata.id}`, failed to concatenate videos.#r\n" ) except vbvid.FailedToCleanUp as e: cprint( f"#r#fRSkipping stage `{stagedata.id}`, failed to clean up temp files.#r\n\n{e.vid}" ) # send request to youtube to upload request_body = { "snippet": { "categoryId": 20, "title": stagedata.title, "description": stagedata.desc }, "status": { "privacyStatus": "private", "selfDeclaredMadeForKids": False } } # create media file, 100 MiB chunks media_file = MediaFileUpload(str(tmpfile), chunksize=1024 * 1024 * 100, resumable=True) # create upload request and execute response_upload = service.videos().insert(part="snippet,status", body=request_body, notifySubscribers=False, media_body=media_file) video_id = None # youtube video id resp = None errn = 0 cprint( f"#fCUploading stage #r`#fM{stagedata.id}#r`, progress: #fC0#fY%#r #d...#r", end="\r") # Below is taken from Google's documentation, rewrite? while resp is None: try: status, resp = response_upload.next_chunk() if status: cprint( f"#fCUploading stage #r`#fM{stagedata.id}#r`, progress: #fC{int(status.progress()*100)}#fY%#r #d...#r", end="\r") if resp is not None: if "id" in resp: cprint( f"#fCUploading stage #r`#fM{stagedata.id}#r`, progress: #fC100#fY%#r!" ) cprint( f"#l#fGVideo was successfully uploaded!#r #dhttps://youtu.be/{resp['id']}#r" ) video_id = resp["id"] else: util.exit_prog( 99, f"Unexpected upload failure occurred, \"{resp}\"") except ResumableUploadError as err: if err.resp.status in [400, 401, 402, 403]: try: jsondata = json.loads(err.content)['error']['errors'][0] util.exit_prog( 40, f"API Error: `{jsondata['reason']}`. Message: `{jsondata['message']}`" ) except (json.JSONDecodeError, KeyError): util.exit_prog( 40, f"Unknown API Error has occured, ({err.resp.status}, {err.content})" ) print( f"A Resumeable error has occured, retrying in 5 sec... ({err.resp.status}, {err.content})" ) errn += 1 sleep(5) except HttpError as err: if err.resp.status in [500, 502, 503, 504]: print( f"An HTTP error has occured, retrying in 5 sec... ({err.resp.status}, {err.content})" ) errn += 1 sleep(5) else: raise except RETRIABLE_EXCEPTS as err: print(f"An HTTP error has occured, retrying in 5 sec... ({err})") errn += 1 sleep(5) if errn >= 10: print("Skipping video upload, errored too many times.") break # we're done, lets clean up else: try: # delete vars to release the files del media_file del response_upload sleep(1) os_remove(str(tmpfile)) except Exception as e: util.exit_prog( 90, f"Failed to remove temp slice file of stage `{stagedata.id}` after upload. {e}" ) return video_id
def run(args): # load config conf = util.load_conf(args.config) # configure variables global stagedir, tempdir tempdir = Path(conf["temp_dir"]) stagedir = Path(conf["stage_dir"]) PICKLE_FILE = conf["youtube_pickle_path"] CLIENT_SECRET_FILE = conf["youtube_client_path"] API_NAME = 'youtube' API_VERSION = 'v3' SCOPES = [ # Only force-ssl is required, but both makes it explicit. "https://www.googleapis.com/auth/youtube.upload", # Videos.insert "https://www.googleapis.com/auth/youtube.force-ssl" # Captions.insert ] # handle logout if args.id == "logout": try: os_remove(PICKLE_FILE) cprint("#dLogged out of Google API session#r") except: util.exit_prog( 11, "Failed to remove credentials for YouTube account.") return # load stages, but dont upload # Handle id/all stagedata = None stagedatas = None if args.id == "all": cprint("#dLoading stages...", end=" ") # create a list of all the hashes and sort by date streamed, upload chronologically stagedatas = StageData.load_all_stages(stagedir) stagedatas.sort(key=sort_stagedata) else: cprint("#dLoading stage...", end=" ") # check if stage exists, and prep it for upload stagedata = StageData.load_from_id(stagedir, args.id) cprint(f"About to upload stage {stagedata.id}.#r") # authenticate youtube service if not os_exists(CLIENT_SECRET_FILE): util.exit_prog(19, "Missing YouTube Client ID/Secret file.") cprint("Authenticating with Google...", end=" ") service = None credentials = None if os_exists(PICKLE_FILE): with open(PICKLE_FILE, "rb") as f: credentials = pickle.load(f) if not credentials or credentials.expired: try: if credentials and credentials.expired and credentials.refresh_token: credentials.refresh(Request()) else: flow = InstalledAppFlow.from_client_secrets_file( CLIENT_SECRET_FILE, SCOPES) credentials = flow.run_console() except RefreshError: flow = InstalledAppFlow.from_client_secrets_file( CLIENT_SECRET_FILE, SCOPES) credentials = flow.run_console() with open(PICKLE_FILE, "wb") as f: pickle.dump(credentials, f) try: service = build(API_NAME, API_VERSION, credentials=credentials) except Exception as err: util.exit_prog(50, f"Failed to connect to YouTube API, \"{err}\"") cprint("Authenticated.", end=" ") # Handle id/all if args.id == "all": # begin to upload cprint(f"About to upload {len(stagedatas)} stages.#r") for stage in stagedatas: video_id = upload_video(conf, service, stage) if video_id is not None: chat_success = True if conf["chat_upload"]: chat_success = upload_captions(conf, service, stage, video_id) if conf["stage_upload_delete"] and chat_success: try: os_remove(str(stagedir / f"{stage.id}.stage")) except: util.exit_prog( 90, f"Failed to remove stage `{stage.id}` after upload." ) print() else: # upload stage cprint(f"About to upload stage {stagedata.id}.#r") video_id = upload_video(conf, service, stagedata) if video_id is not None: chat_success = True if conf["chat_upload"]: chat_success = upload_captions(conf, service, stagedata, video_id) if conf["stage_upload_delete"] and chat_success: try: os_remove(str(stagedir / f"{stagedata.id}.stage")) except: util.exit_prog( 90, f"Failed to remove stage `{stagedata.id}` after upload." )
def upload_captions(conf: dict, service, stagedata: StageData, vid_id: str) -> bool: tmpfile = vbchat.process_stage(conf, stagedata, "upload") if tmpfile == False: return True request_body = { "snippet": { "name": "Chat", "videoId": vid_id, "language": "en" } } media_file = MediaFileUpload(str(tmpfile), chunksize=1024 * 1024 * 100, resumable=True) response_upload = service.captions().insert(part="snippet", body=request_body, sync=False, media_body=media_file) resp = None errn = 0 cprint( f"#fCUploading stage chatlog #r`#fM{stagedata.id}#r`, progress: #fC0#fY%#r #d...#r", end="\r") while resp is None: try: status, resp = response_upload.next_chunk() if status: cprint( f"#fCUploading stage chatlog #r`#fM{stagedata.id}#r`, progress: #fC{int(status.progress()*100)}#fY%#r #d...#r", end="\r") if resp is not None: cprint( f"#fCUploading stage chatlog #r`#fM{stagedata.id}#r`, progress: #fC100#fY%#r!" ) cprint(f"#l#fGVideo captions were successfully uploaded!#r") except ResumableUploadError as err: if err.resp.status in [400, 401, 402, 403]: try: jsondata = json.loads(err.content)['error']['errors'][0] util.exit_prog( 40, f"API Error: `{jsondata['reason']}`. Message: `{jsondata['message']}`" ) except (json.JSONDecodeError, KeyError): util.exit_prog( 40, f"Unknown API Error has occured, ({err.resp.status}, {err.content})" ) print( f"A Resumeable error has occured, retrying in 5 sec... ({err.resp.status}, {err.content})" ) errn += 1 sleep(5) except HttpError as err: if err.resp.status in [500, 502, 503, 504]: print( f"An HTTP error has occured, retrying in 5 sec... ({err.resp.status}, {err.content})" ) errn += 1 sleep(5) else: raise except RETRIABLE_EXCEPTS as err: print(f"An HTTP error has occured, retrying in 5 sec... ({err})") errn += 1 sleep(5) if errn >= 10: print("Skipping chatlog upload, errored too many times.") return False # we're done, lets clean up else: try: # delete vars to release the files del media_file del response_upload sleep(1) os_remove(str(tmpfile)) except Exception as e: util.exit_prog( 90, f"Failed to remove temp slice file of stage `{stagedata.id}` after upload. {e}" ) return True
def create_config(args): channels = [] timezone = "" # switch to UTC only # these are for determining directories. # we need to make sure any that are already provided are absolute right now. # we do it now so that the user doesn't enter in anything before receiving # an error that boots them out of the program. parts = [[args.voddir, "VODs", DEFAULT_CONF['vod_dir']], [args.clipdir, "Clips", DEFAULT_CONF['clip_dir']], [args.tempdir, "temporary data", DEFAULT_CONF['temp_dir']], [args.stagedir, "staged data", DEFAULT_CONF['stage_dir']]] for part in parts: if part[0]: if not isabs(part[0]): util.exit_prog( 150, f"Path `{part[0]}` for {part[1]} is not an absolute path.") part.append(part[0]) # Ask for channels if not args.channels: cprint( "Enter the login names of the Twitch channels you'd like to have archived." ) cprint("When you're done, just leave the input empty and press enter.") cprint( "Example: the url is `https://twitch.tv/notquiteapex`, enter `notquiteapex`" ) while True: channel = input("> ") if channel == "" and len(channels) > 0: break elif channel == "" and len(channels) == 0: cprint("#fRError#r, no channels given.") elif not re.match( r"^[a-zA-Z0-9][\w]{0,24}$", channel): # https://discuss.dev.twitch.tv/t/3855/4 cprint( "#fRError#r, channel names only contain characters A-Z, 0-9, and '_'. They also can't start with '_'." ) else: channels += [channel] else: channels = args.channels # Ask for timezone as UTC string # We only absolutely need the hours and minutes offset, and we regex it out of the string. if not args.timezone: cprint( "Enter the UTC timezone for timedate referencing. Only use +, -, and numbers." ) cprint( "#dExample: `+0000` for UTC, `-0500` for America's EST. Default: `+0000`#r" ) while True: tz = input("> ") if tz == "": tz = "+0000" break elif not re.match(r"^[+-]\d\d\d\d$", tz): cprint("Error, UTC string not recognized.") else: break else: timezone = args.timezone # check that entered timezone is valid # ask for directories (any provided by the terminal arguments are already handeled.) cprint( "Now let's get some directories to store data. The entered paths must be absolute, not relative." ) cprint( "If you'd like to use the default location listed, just leave the input blank and press enter." ) for part in parts: if not part[0]: cprint( f"Enter where {part[1]} should be stored.\n#dDefault: `#l{part[2]}`#r" ) while True: inpdir = input("> ") if inpdir == "": part.append(part[2]) break elif isabs(inpdir): # TODO: check if directory can be created? a file might already exist? part.append(inpdir) break else: cprint( f"#fRError#r, directory `#l{inpdir}#r` is not an absolute path for a directory." ) # ready to write it all, go! cprint("#dWriting config...#r") # Edit default config variable. DEFAULT_CONF['twitch_channels'] = channels DEFAULT_CONF['stage_timezone'] = timezone DEFAULT_CONF['vod_dir'] = parts[0][3] DEFAULT_CONF['clip_dir'] = parts[1][3] DEFAULT_CONF['temp_dir'] = parts[2][3] DEFAULT_CONF['stage_dir'] = parts[3][3]
def process_stage(conf: dict, stage: StageData, mode: str) -> Path: tempdir = Path(conf["temp_dir"]) msg_duration = int(conf["chat_msg_time"]) cprint(f"#rLoading all chat messages for `#fM{stage.id}#r`.", end="") total_offset = 0 chat_list = [] for slc in stage.slices: # load up each stagedata's meta to see if chat exists metapath = slc.filepath[:-4] + ".meta" meta = None with open(metapath, "r") as f: meta = json.load(f) has_chat = meta.get("has_chat", False) duration = meta.get("length", 0) chat_path = slc.filepath[:-4] + ".chat" # start and end times as seconds start_sec = timestring_as_seconds(slc.ss) end_sec = timestring_as_seconds(slc.to, duration) # keep each list of chat separate, compare timestamps to offsets to make # sure theyre inbetween the slices if has_chat: msg_list = [ m for m in logfile_to_chat(chat_path) if start_sec <= m.offset < end_sec ] for m in msg_list: m.offset = m.offset - start_sec + total_offset chat_list += msg_list # now take each stage's duration and apply it to the next chat list's total_offset += end_sec - start_sec # determine how to export, then return the resultant path if mode != "upload" and mode != "export": util.exit_prog(94, f"Cannot export chat with export mode {mode}") export_type = conf["chat_" + mode] if len(chat_list) == 0: cprint(f" No chat found in `#fY{export_type}#r` stage. Skipping...") return False cprint(f" Exporting as format `#fY{export_type}#r`.") returnpath = None if export_type == "raw": # chat to logfile time returnpath = tempdir / (stage.id + ".chat") chat_to_logfile(chat_list, str(returnpath)) elif export_type == "RealText": # load from archive, parse and write to temp returnpath = tempdir / (stage.id + ".rt") chat_to_realtext(chat_list, str(returnpath), total_offset, msg_duration) elif export_type == "SAMI": # load from archive, parse and write to temp returnpath = tempdir / (stage.id + ".sami") chat_to_sami(chat_list, str(returnpath), total_offset, msg_duration) elif export_type == "YTT": # load from archive, parse and write to temp returnpath = tempdir / (stage.id + ".ytt") chat_to_ytt(chat_list, str(returnpath), total_offset, msg_duration) elif export_type == "TTML": # load from archive, parse and write to temp returnpath = tempdir / (stage.id + ".ttml") chat_to_ttml(chat_list, str(returnpath), total_offset, msg_duration) return returnpath
def run(args): # First determine what kind of data it is... # We should allow people to paste full links or just the ID too. tid = args.id # Get the proper id and type of content we need to pull cprint("#dDetermining type...#r", end=" ") cid, ctype = get_type(tid) # The call the appropriate info query for GQL query = None if ctype == "vod": query = gql.GET_VIDEO_QUERY.format(video_id=cid) elif ctype == "clip": query = gql.GET_CLIP_QUERY.format(clip_slug=cid) elif ctype == "channel": query = gql.GET_CHANNEL_QUERY.format(channel_id=cid) else: util.exit_prog(92, "Could not determine content type from input.") cword = KEYTRANS[ctype] cprint(f"#dQuerying {cword} content for `{cid}`...#r") # run the query resp = gql.gql_query(query=query).json() resp = resp["data"] if not any([resp.get("video"), resp.get("clip"), resp.get("user")]): util.exit_prog(93, f"Could not query info on {cword} content from input.") # Read the data back and out to the terminal if ctype == "vod": r = resp['video'] c = r['creator'] g = r['game'] cprint(f"#fGTitle: `{r['title']}`#r - ID: `{r['id']}`") cprint( f"#fMBroadcaster: {c['displayName']} - {c['login']} ({c['id']})#r") cprint(f"#fCPlaying: {g['name']} ({g['id']})#r") cprint( f"#fYAt: {r['publishedAt']} - For: {r['lengthSeconds']} seconds#r") elif ctype == "clip": r = resp['clip'] c = r['curator'] b = r['broadcaster'] g = r['game'] cprint(f"#fGTitle: `{r['title']}`#r - ID: `{r['id']}`") cprint(f"Slug: `{r['slug']}`") cprint(f"#fBClipper: {c['displayName']} - {c['login']} ({c['id']})#r") cprint( f"#fMBroadcaster: {b['displayName']} - {b['login']} ({b['id']})#r") cprint(f"#fCPlaying: {g['name']} ({g['id']})#r") cprint( f"#fYAt: {r['createdAt']} - For: {r['durationSeconds']} seconds - With: {r['viewCount']} views#r" ) elif ctype == "channel": r = resp['user'] cprint( f"#fMBroadcaster: {r['displayName']} - {r['login']} (ID: `{r['id']}`)#r" ) cprint(f"#fRDescription: {r['description']}#r") cprint(f"#fYChannel Created At: {r['createdAt']}#r") r = r['roles'] cprint( f"#fBRoles: Affiliate={r['isAffiliate']} - Partner={r['isPartner']}#r" )
def _new(args, conf, stagedir): VODS_DIR = conf["vod_dir"] CLIPS_DIR = conf["clip_dir"] STAGE_DIR = conf["stage_dir"] stagedir = Path(STAGE_DIR) # find the videos by their ids to confirm they exist videos = [] for video in args.id: try: (filename, metadata, videotype) = find_video_by_id(video, VODS_DIR, CLIPS_DIR) videos += [{ "id": video, "file": filename, "meta": metadata, "type": videotype }] except CouldntFindVideo: util.exit_prog(13, f'Could not find video with ID "{args.id}"') # Get what streamers were involved (usernames), always asked default_streamers = [] for f in videos: if f["meta"]["user_login"] not in default_streamers: default_streamers.append(f["meta"]["user_login"]) args.streamers = check_streamers(default=default_streamers) # get title if not args.title: args.title = check_title(default=None) # get description formatdict, datestring = create_format_dict(conf, args.streamers, utcdate=metadata["created_at"]) args.desc = check_description(formatdict, inputdefault=args.desc) # get timestamps for each video through input for x in range(len(videos)): # skip times we dont need because we already have them if x < len(args.ss): continue vid = videos[x]["meta"] # grab times for this specific stream cprint(f"#dTimestamps for `#r#fM{vid['title']}#r` #d({vid['id']})#r") if "chapters" in vid and len(vid["chapters"]) > 0: cprint(f"#dChapters: ", end="") ch = [] for c in vid["chapters"]: (pos, end) = posdur_to_timestamp(c['pos'], c['dur']) ch.append(f"`{c['desc']}` ({pos}-{end})") cprint(" | ".join(ch) + " | EOF#r") else: (pos, end) = posdur_to_timestamp(0, vid['length']) cprint(f"#dChapter: `{vid['game_name']}` ({pos}-{end}) | EOF#r") args.ss += [ check_time("Start", args.ss[x] if x < len(args.ss) else None) ] args.to += [ check_time("End", args.to[x] if x < len(args.to) else None) ] # make slice objects slices = [] for x in range(len(videos)): vid = videos[x] vidslice = VideoSlice(video_id=vid["id"], ss=args.ss[x], to=args.to[x], filepath=vid["file"]) slices += [vidslice] # make stage object stage = StageData(streamers=args.streamers, title=args.title, desc=args.desc, datestring=datestring, slices=slices) # Check that new "id" does not collide while check_stage_id(stage.id, STAGE_DIR): stage.gen_new_id() # shorter file name #shortfile = stage.filename.replace(VODS_DIR, "$vods").replace(CLIPS_DIR, "$clips") print() cprint( f"#r`#fC{stage.title}#r` #fM{' '.join(stage.streamers)}#r #d({stage.id})#r" ) cprint(f"#d'''#fG{stage.desc}#r#d'''#r") for vid in stage.slices: cprint(f"#fM{vid.video_id}#r > #fY{vid.ss}#r - #fY{vid.to}#r") # write stage stagename = str(stagedir / (stage.id + ".stage")) stage.write_stage(stagename)