def tile_marked_livestream_upcoming(self, video: Video): """ Mark the video as live broadcast content (upcoming) :return: """ video.kind = VIDEO_KIND_LIVE_SCHEDULED self.update_and_repaint_tile(video)
def compare_db_filtered(videos, limit, discarded=False, downloaded=False): logger.info("Comparing filtered videos with DB") return_list = [] counter = 0 filter_days = read_config('Requests', 'filter_videos_days_old') DatabaseListener.static_instance.startRead.emit(threading.get_ident()) for video in videos: if filter_days >= 0: date = datetime.datetime.utcnow() - datetime.timedelta( days=filter_days) if video.date_published < date: break db_vid = get_vid_by_id(video.video_id) if db_vid: if db_vid.downloaded: if downloaded: continue if db_vid.discarded: if discarded: continue return_list.append(Video.to_video_d(video)) counter += 1 else: return_list.append(video) counter += 1 if counter >= limit: break DatabaseListener.static_instance.finishRead.emit(threading.get_ident()) db_session.remove() return return_list
def tile_unmarked_livestream(self, video: Video): """ Unmark the video as live broadcast content (live) :return: """ video.kind = VIDEO_KIND_VOD self.update_and_repaint_tile(video)
def run(self): self.db_id = threading.get_ident() DatabaseListener.static_instance.startWrite.emit(self.db_id) """ Override threading.Thread.run() with its own code :return: """ # self.logger.debug("Run") # start = default_timer() lock.acquire() stmt = get_video_by_vidd_stmt(Video.video_d_to_video(self.video_d)) db_video = engine.execute(stmt).first() if db_video: if self.update_existing: engine.execute(update_video_statement_full(self.video_d)) else: pass else: engine.execute(Video.__table__.insert(), insert_item(self.video_d)) # print('Updated: {}'.format(self.video_d.title)) lock.release() if self.finished_listeners: for listener in self.finished_listeners: listener.emit() DatabaseListener.static_instance.finishWrite.emit(self.db_id)
def get_videos_by_ids(video_ids): DatabaseListener.static_instance.startRead.emit(threading.get_ident()) db_videos = engine.execute( Video.__table__.select(Video.video_id.in_(video_ids))) DatabaseListener.static_instance.finishRead.emit(threading.get_ident()) return_videos = Video.to_video_ds(db_videos) return return_videos
def tile_unmarked_premiere(self, video: Video): """ Unmark the video as live broadcast content (premiere) A premiere is: upcoming stream --> live stream --> vod :return: """ video.kind = VIDEO_KIND_VOD self.update_and_repaint_tile(video)
def run(self): """ Override threading.Thread.run() with its own code :return: """ self.db_id = threading.get_ident() if self.uniques_check: self.video_list = check_for_unique(self.video_list) items_to_add = [] items_to_update = [] # bulk_items_to_add = [] lock.acquire() self.logger.debug("Thread {} - Acquired lock".format( self.db_id, len(items_to_update))) DatabaseListener.static_instance.startRead.emit(self.db_id) select_step = 500 for i in range(0, len(self.video_list), select_step): videos_bulk = self.video_list[i:i + select_step] video_ids_bulk = set(video.video_id for video in videos_bulk) stmt = get_video_ids_by_video_ids_stmt(video_ids_bulk) db_videos = Video.to_video_ds(engine.execute(stmt)) db_videos_ids = set(video.video_id for video in db_videos) items_to_add.extend( insert_item(video) for video in videos_bulk if video.video_id not in db_videos_ids) if self.update_existing: items_to_update.extend(video for video in videos_bulk if video.video_id in db_videos_ids) DatabaseListener.static_instance.finishRead.emit(self.db_id) DatabaseListener.static_instance.startWrite.emit(self.db_id) step = 1000 if len(items_to_add) > 0: self.logger.debug("Thread {} - inserting {} new videos".format( self.db_id, len(items_to_add))) for i in range(0, len(items_to_add), step): engine.execute(Video.__table__.insert(), items_to_add[i:i + step]) if len(items_to_update) > 0: self.logger.debug("Thread {} - updating {} items".format( self.db_id, len(items_to_update))) update_list = [] # FIXME: add step to update for item in items_to_update: item_dict = item.__dict__ item_dict["_video_id"] = item.video_id update_list.append(item_dict) engine.execute(update_video_stmt(), update_list) DatabaseListener.static_instance.finishWrite.emit(self.db_id) lock.release() if self.finished_listeners: for listener in self.finished_listeners: listener.emit()
def tile_unwatched(self, video: Video): """ Action to take if tile has been un-flagged as watched. Called by Views: Playback :param video: :return: """ video.watched = False if not read_config('GridView', 'show_watched'): self.logger.info("Mark unwatched: {} - {}".format(video.title, video.__dict__)) self.model.unhide_video_item(video, self.widget_id) self.update_and_repaint_tile(video)
def tile_undiscarded(self, video: Video): """ Action to take if tile has been un-flagged as dismissed. Called by Views: Subfeed and Playback :param video: :return: """ video.discarded = False if not read_config('GridView', 'show_dismissed'): self.logger.info("Un-hide video (Un-discarded): {}".format(video)) self.model.unhide_video_item(video, self.widget_id) self.update_and_repaint_tile(video)
def __init__(self, download_tile, db_download_tile=False): self.finished = download_tile.finished self.started_date = download_tile.started_date self.finished_date = download_tile.finished_date if db_download_tile: self.video = Video.to_video_d(download_tile.video) else: self.video = download_tile.video self.video_downloaded = download_tile.video_downloaded self.total_bytes = download_tile.total_bytes self.last_event = download_tile.last_event self.cleared = download_tile.cleared # Other self.progress_listener = None
def tile_delete_downloaded_data(self, video: Video): """ Action to take if tile has been told to delete its downloaded data. This will delete the video file. :param video: :return: """ # Delete downloaded video file os.remove(video.vid_path) self.logger.info("Deleted: {}".format(video.vid_path)) # Unset path to signal that the data isn't on disk. video.vid_path = None # Repaint the tile to reflect the action. self.update_and_repaint_tile(video)
def get_new_and_updated_videos(vid_paths): """ :param vid_paths: dict(video_id, vid_path) :return: """ logger = create_logger(__name__ + ".new_and_updated") db_videos = get_videos_by_ids(vid_paths.keys()) return_videos = [] for video in db_videos: if not video.vid_path or not video.downloaded: video_d = Video.to_video_d(video) video_d.vid_path = vid_paths[video.video_id] video_d.downloaded = True video_d.date_downloaded = datetime.datetime.utcnow() logger.info("Found video needing update: {} - {}".format( video.video_id, video.title)) return_videos.append(video_d) vid_paths.pop(video.video_id, None) new_videos = [] if len(vid_paths) > 0: youtube_keys = load_keys(1) logger.info( "Grabbing new video(s) information from youtube for: {}".format( vid_paths.keys())) response_videos = list_uploaded_videos_videos(youtube_keys[0], vid_paths.keys(), 30) for video in response_videos: video.vid_path = vid_paths[video.video_id] video.downloaded = True video.watched = False video.date_downloaded = datetime.datetime.utcnow() logger.info("Found new video: {} - {}".format( video.video_id, video.title)) new_videos.append(video) return_videos.extend(new_videos) logger.info("Downloading thumbnails for: {}".format(return_videos)) download_thumbnails_threaded(return_videos) return return_videos
def get_videos_by_ids(video_ids): """ Get videos by IDs using a __table__ SELECT WHERE engine statement. :param video_ids: Video ID strings :return: """ # Signal DB listener about started read operation (Used in DB status indicator and logs) DatabaseListener.static_instance.startRead.emit(threading.get_ident()) # Execute SELECT engine statement for each video. db_videos = engine.execute( Video.__table__.select(Video.video_id.in_(video_ids))) # Signal DB listener about finished read operation (Used in DB status indicator and logs) DatabaseListener.static_instance.finishRead.emit(threading.get_ident()) # Convert Video objects to VideoD (Detached model) objects. videos = Video.to_video_ds(db_videos) return videos
def get_db_videos_playback(limit, filters=(~Video.watched, Video.downloaded, ~Video.discarded), sort_method=(asc(Video.watch_prio), desc(Video.date_downloaded), desc(Video.date_published))): """ Get a list of videos from the database, limited by limit and filters. Sorted by sort_method. :param filters: Tuple of SQLAlchemy Column(Boolean) objects. :param sort_method: Tuple of SQLAlchemy sort expressions. :param limit: Integer value determining how many videos to grab. :return: A list of VideoD objects. """ # Initiate DB Query session. db_query = db_session.query(Video) # Apply filters to query. db_query = db_query.filter(*filters) # Signal DB listener about started read operation (Used in DB status indicator and logs) DatabaseListener.static_instance.startRead.emit(threading.get_ident()) # Execute query, ordered by sort_method and limited by limit. db_videos = db_query.order_by(*sort_method).limit(limit).all() # Signal DB listener about finished read operation (Used in DB status indicator and logs) DatabaseListener.static_instance.finishRead.emit(threading.get_ident()) # Convert Video objects to VideoD (Detached model) objects. videos = Video.to_video_ds(db_videos) # Close DB Query session. db_session.remove() logger.debug5("sane_yt_subfeed.debug.date_formats: {}".format( sane_yt_subfeed.debug.date_formats)) sane_yt_subfeed.debug.clear_date_formats() return videos
def get_best_downloaded_videos(limit, filters=(~Video.watched, Video.downloaded, ~Video.discarded), sort_method=(asc(Video.watch_prio), desc(Video.date_downloaded), desc(Video.date_published))): """ :param filters: Tuple of filters :param sort_method: :param limit: :return: list(VideoD) """ db_query = db_session.query(Video) db_query = db_query.filter(*filters) DatabaseListener.static_instance.startRead.emit(threading.get_ident()) db_videos = db_query.order_by(*sort_method).limit(limit).all() DatabaseListener.static_instance.finishRead.emit(threading.get_ident()) videos = Video.to_video_ds(db_videos) db_session.remove() return videos
def get_db_videos_subfeed(limit, filters=(~Video.downloaded, ~Video.discarded)): """ Get a list of videos from the database, limited by limit and filters. :param filters: Tuple of SQLAlchemy Column(Boolean) objects. :param limit: Integer value determining how many videos to grab. :return: A list of VideoD objects. """ logger.info("Getting newest stored videos (filters={})".format(filters)) # Check whether "filter by video age" is enabled, if so apply the filter to filters. filter_days = read_config('Requests', 'filter_videos_days_old') if filter_days >= 0: date = datetime.datetime.utcnow() - datetime.timedelta( days=filter_days) filters = filters + (Video.date_published > date, ) # Signal DB listener about started read operation (Used in DB status indicator and logs) DatabaseListener.static_instance.startRead.emit(threading.get_ident()) # Execute query, ordered by publish date (descending), filtered by filters and limited by limit. db_videos = db_session.query(Video).order_by(desc( Video.date_published)).filter(*filters).limit(limit).all() # Signal DB listener about finished read operation (Used in DB status indicator and logs) DatabaseListener.static_instance.finishRead.emit(threading.get_ident()) # Convert Video objects to VideoD (Detached model) objects. videos = Video.to_video_ds(db_videos) # Close DB Query session. db_session.remove() logger.debug5("sane_yt_subfeed.debug.date_formats: {}".format( sane_yt_subfeed.debug.date_formats)) sane_yt_subfeed.debug.clear_date_formats() return videos
def get_newest_stored_videos(limit, filters=(~Video.downloaded, ~Video.discarded)): """ :param filters: :param limit: :return: list(VideoD) """ logger.info("Getting newest stored videos (filters={})".format(filters)) filter_days = read_config('Requests', 'filter_videos_days_old') if filter_days >= 0: date = datetime.datetime.utcnow() - datetime.timedelta( days=filter_days) filters = filters + (Video.date_published > date, ) DatabaseListener.static_instance.startRead.emit(threading.get_ident()) db_videos = db_session.query(Video).order_by(desc( Video.date_published)).filter(*filters).limit(limit).all() DatabaseListener.static_instance.finishRead.emit(threading.get_ident()) videos = Video.to_video_ds(db_videos) db_session.remove() return videos
def run(): video_item = { 'kind': 'youtube#searchResult', 'etag': '"XI7nbFXulYBIpL0ayR_gDh3eu1k/rrGI6j0VT4vX91p6eQyuqFQjf8M"', 'id': { 'kind': 'youtube#video', 'videoId': '8NG1PEFceG0' }, 'snippet': { 'publishedAt': '2017-06-22T19:26:41.000Z', 'channelId': 'UCzORJV8l3FWY4cFO8ot-F2w', 'title': '[Vinesauce] Vinny - The Legend of Zelda: Clips of the Wild', 'description': 'A compilation of user-created Twitch clips from Breath of the Wild! Streamed live on Vinesauce. Check out my other youtube channel for full streams.', 'thumbnails': { 'default': { 'url': 'https://i.ytimg.com/vi/8NG1PEFceG0/default.jpg', 'width': 120, 'height': 90 }, 'medium': { 'url': 'https://i.ytimg.com/vi/8NG1PEFceG0/mqdefault.jpg', 'width': 320, 'height': 180 }, 'high': { 'url': 'https://i.ytimg.com/vi/8NG1PEFceG0/hqdefault.jpg', 'width': 480, 'height': 360 } }, 'channelTitle': 'vinesauce', 'liveBroadcastContent': 'none' } } channel_item = { "publishedAt": "2015-01-09T12:46:03.000Z", "title": "OfficialNerdCubed", "description": "Recommending good games every Monday and Wednesday. Going strange every Saturday.", "resourceId": { "kind": "youtube#channel", "channelId": "UCKab3hYnOoTZZbEUQBMx-ww" }, "channelId": "UCv5TbAM7zHQOoDr0fw2Cy8w", "thumbnails": { "default": { "url": "https://yt3.ggpht.com/-2AuBRBi8yE8/AAAAAAAAAAI/AAAAAAAAAAA/PsaY8p1aJ2U/s88-c-k-no-mo-rj-c0xffffff/photo.jpg" }, "medium": { "url": "https://yt3.ggpht.com/-2AuBRBi8yE8/AAAAAAAAAAI/AAAAAAAAAAA/PsaY8p1aJ2U/s240-c-k-no-mo-rj-c0xffffff/photo.jpg" }, "high": { "url": "https://yt3.ggpht.com/-2AuBRBi8yE8/AAAAAAAAAAI/AAAAAAAAAAA/PsaY8p1aJ2U/s800-c-k-no-mo-rj-c0xffffff/photo.jpg" } } } channel = Channel(channel_item, 'UUv5TbAM7zHQOoDr0fw2Cy8w') video = Video(video_item) insrt_txt = { "video_id": video.video_id, "channel_title": video.channel_title, 'title': video.title, "date_published": video.date_published, "description": video.description, "thumbnail_path": video.thumbnail_path, "playlist_pos": video.playlist_pos, "url_video": video.url_video, "url_playlist_video": video.url_playlist_video, "thumbnails": video.thumbnails, "downloaded": video.downloaded, "search_item": video.search_item } # stmt_1 = Video.__table__.insert(), [insrt_txt] # engine.execute(Video.__table__.insert(), [insrt_txt]) # stmt = update_statement_full(video) # stmt = get_video_by_id_stmt(video) # # stmt = Video.__table__.select('video.video_id').where(text("video_id = '{}'".format(video.video_id))) # # print(db_session.execute(stmt).first()) # print(engine.execute(stmt).first()) # video2 = engine.execute(Video.__table__.select(Video.title).where('video_id' == video.video_id).limit(1)) stmt = get_channel_by_id_stmt(channel) print(engine.execute(stmt).first())
def filter_videos(videos, limit, filter_discarded=False, filter_downloaded=False): """ Takes a list of videos and excludes items that match any enabled exclusion filter. :param videos: A list of Video objects to be compared. :param limit: Integer value determining how many videos to return. :param filter_discarded: Boolean determining whether or not to exclude discarded videos. :param filter_downloaded: Boolean determining whether or not to exclude filter_downloaded videos. :return: A list of VideoD objects. """ logger.info("Comparing filtered videos with DB") return_list = [] counter = 0 # Check whether "filter by video age" is enabled, if so apply the filter to filters. filter_days = read_config('Requests', 'filter_videos_days_old') # Signal DB listener about started read operation (Used in DB status indicator and logs) DatabaseListener.static_instance.startRead.emit(threading.get_ident()) # Iterate through videos: for video in videos: # If "filter by video age" is enabled, skip videos older than filter_days days. if filter_days >= 0: date = datetime.datetime.utcnow() - datetime.timedelta( days=filter_days) if video.date_published < date: break # Check if video is in database, if so, convert it and add it to list, unless it matches an exclusion filter. db_vid = get_vid_by_id(video.video_id) if db_vid: # Skip video if it matches an enabled exclusion filter. if (db_vid.downloaded and filter_downloaded) or (db_vid.discarded and filter_discarded): continue # Convert Video object to VideoD (Detached model) object and append it to the list, then increment counter. return_list.append(Video.to_video_d(video)) counter += 1 # If video isn't in database, no filters apply and it can simply be appended to the list. else: # Append VideoD object to list, then increment counter. return_list.append(video) counter += 1 # Break once the limit has been reached. if counter >= limit: break # Signal DB listener about finished read operation (Used in DB status indicator and logs) DatabaseListener.static_instance.finishRead.emit(threading.get_ident()) # Close DB Query session. db_session.remove() return return_list