def cli(no_gui, test_channels, update_watch_prio, set_watched_day): logger = create_logger(__name__) if no_gui: run_print() if update_watch_prio: videos = db_session.query(Video).all() watch_prio = read_config('Play', 'default_watch_prio') logger.debug("Setting watch_prio {}, for: {} videos".format(watch_prio, len(videos))) for video in videos: video.watch_prio = watch_prio db_session.commit() return if set_watched_day: videos = db_session.query(Video).filter(or_(Video.downloaded == True, (Video.vid_path.is_(None)))).all() for video in videos: vid_age = datetime.datetime.utcnow() - video.date_published if vid_age > datetime.timedelta(days=int(set_watched_day)): logger.debug("Setting watched, {} - {} - {}".format(vid_age, video.title, video.__dict__)) video.watched = True db_session.commit() return if test_channels: run_channels_test() else: """ PyQT raises and catches exceptions, but doesn't pass them along. Instead it just exits with a status of 1 to show an exception was caught. """ # Back up the reference to the exceptionhook sys._excepthook = sys.excepthook def my_exception_hook(exctype, value, traceback): # Ignore KeyboardInterrupt so a console python program can exit with Ctrl + C. if issubclass(exctype, KeyboardInterrupt): sys.__excepthook__(exctype, value, traceback) return # Log the exception with the logger logger.critical("Intercepted Exception", exc_info=(exctype, value, traceback)) # Call the normal Exception hook after sys._excepthook(exctype, value, traceback) # sys.exit(1) # Alternatively, exit # Set the exception hook to our wrapping function sys.excepthook = my_exception_hook run_with_gui()
def get_stored_subscriptions(): logger.info("Getting subscriptions from DB.") channels = db_session.query(Channel).filter( or_(Channel.subscribed, Channel.subscribed_override)).all() if len(channels) < 1: return get_remote_subscriptions_cached_oauth() return channels
def __init__(self, download_tile): self.finished = download_tile.finished self.started_date = download_tile.started_date self.finished_date = download_tile.finished_date self.video = db_session.query(Video).get(download_tile.video.video_id) self.video_downloaded = download_tile.video_downloaded self.total_bytes = download_tile.total_bytes self.last_event = download_tile.last_event self.cleared = download_tile.cleared
def run(self): """ Override threading.Thread.run() with its own code :return: """ try: # youtube = youtube_auth_keys() # self.videos = get_channel_uploads(self.youtube, channel_id) use_tests = read_config('Requests', 'use_tests') if self.deep_search: temp_videos = [] list_uploaded_videos_search(self.youtube, self.channel_id, temp_videos, self.search_pages) list_uploaded_videos(self.youtube, temp_videos, self.playlist_id, self.list_pages) self.merge_same_videos_in_list(temp_videos) self.videos.extend(temp_videos) elif use_tests: channel = db_session.query(Channel).get(self.channel_id) miss = read_config('Requests', 'miss_limit') pages = read_config('Requests', 'test_pages') extra_pages = read_config('Requests', 'extra_list_pages') list_pages = 0 list_videos = [] search_videos = [] for test in channel.tests: if test.test_pages > list_pages: list_pages = test.test_pages if test.test_miss < miss or test.test_pages > pages: db_session.remove() list_uploaded_videos_search(self.youtube, self.channel_id, search_videos, self.search_pages) break db_session.remove() list_uploaded_videos(self.youtube, list_videos, self.playlist_id, min(pages + extra_pages, list_pages + extra_pages)) if len(search_videos) > 0: return_videos = self.merge_two_videos_list_grab_info(list_videos, search_videos) else: return_videos = list_videos self.videos.extend(return_videos) else: use_playlist_items = read_config('Debug', 'use_playlistItems') if use_playlist_items: list_uploaded_videos(self.youtube, self.videos, self.playlist_id, self.list_pages) else: list_uploaded_videos_search(self.youtube, self.channel_id, self.videos, self.search_pages) except Exception as e: # Save the exception details, but don't rethrow. self.exc = e pass self.job_done = True
def delete_sub_not_in_list(subs): delete_channels = db_session.query(Channel).filter( ~Channel.id.in_(subs)).all() for channel in delete_channels: if channel.subscribed or channel.subscribed is None: channel.subscribed = False create_logger(__name__).warning( "Setting unsubscribed for channel: {} - {}".format( channel.title, channel.__dict__)) stmt = update_channel_from_remote(channel) engine.execute(stmt)
def update_download_tile(self, download_tile): result = db_session.query(DBDownloadTile).filter( DBDownloadTile.video_id == format( download_tile.video.video_id)).first() # stmt = DBDownloadTile.__table__.select().where( # text("video_id = '{}'".format(download_tile.video.video_id))) # result = engine.execute(stmt).first() if result: result.update_tile(download_tile) db_session.commit() db_session.remove() else: self.logger.warning( "Download tile not found in db, so no update was performed: {}" .format(download_tile.__dict__))
def new_download_tile(self, new_tile): lock.acquire() result = db_session.query(DBDownloadTile).filter( DBDownloadTile.video_id == format( new_tile.video.video_id)).first() if not result: download_tile = DBDownloadTile(new_tile) if not download_tile.video: self.logger.error("No video in new tile: {}".format( download_tile.__dict__), exc_info=True) return db_session.add(download_tile) db_session.commit() db_session.remove() lock.release()
def load_db_download_tiles(self): db_result = db_session.query(DBDownloadTile).filter( DBDownloadTile.cleared == false()).all() detached_db_result = DDBDownloadTile.list_detach(db_result) use_youtube_dl = read_config('Youtube-dl', 'use_youtube_dl') download_finished_signals = [ static_listeners.STATIC_GRID_VIEW_LISTENER.downloadFinished ] for tile in detached_db_result: if use_youtube_dl and not tile.finished: self.logger.info( "Starting paused in progress download for: {}".format( tile.video.__dict__)) tile.progress_listener = \ DownloadHandler.download_using_youtube_dl(tile.video, youtube_dl_finished_listener=download_finished_signals, wait=True) self.dbDownloadTiles.emit(detached_db_result)
def get_db_videos_playback(limit, filters=(~Video.watched, Video.downloaded, ~Video.discarded), sort_method=(asc(Video.watch_prio), desc(Video.date_downloaded), desc(Video.date_published))): """ Get a list of videos from the database, limited by limit and filters. Sorted by sort_method. :param filters: Tuple of SQLAlchemy Column(Boolean) objects. :param sort_method: Tuple of SQLAlchemy sort expressions. :param limit: Integer value determining how many videos to grab. :return: A list of VideoD objects. """ # Initiate DB Query session. db_query = db_session.query(Video) # Apply filters to query. db_query = db_query.filter(*filters) # Signal DB listener about started read operation (Used in DB status indicator and logs) DatabaseListener.static_instance.startRead.emit(threading.get_ident()) # Execute query, ordered by sort_method and limited by limit. db_videos = db_query.order_by(*sort_method).limit(limit).all() # Signal DB listener about finished read operation (Used in DB status indicator and logs) DatabaseListener.static_instance.finishRead.emit(threading.get_ident()) # Convert Video objects to VideoD (Detached model) objects. videos = Video.to_video_ds(db_videos) # Close DB Query session. db_session.remove() logger.debug5("sane_yt_subfeed.debug.date_formats: {}".format( sane_yt_subfeed.debug.date_formats)) sane_yt_subfeed.debug.clear_date_formats() return videos
def get_db_videos_subfeed(limit, filters=(~Video.downloaded, ~Video.discarded)): """ Get a list of videos from the database, limited by limit and filters. :param filters: Tuple of SQLAlchemy Column(Boolean) objects. :param limit: Integer value determining how many videos to grab. :return: A list of VideoD objects. """ logger.info("Getting newest stored videos (filters={})".format(filters)) # Check whether "filter by video age" is enabled, if so apply the filter to filters. filter_days = read_config('Requests', 'filter_videos_days_old') if filter_days >= 0: date = datetime.datetime.utcnow() - datetime.timedelta( days=filter_days) filters = filters + (Video.date_published > date, ) # Signal DB listener about started read operation (Used in DB status indicator and logs) DatabaseListener.static_instance.startRead.emit(threading.get_ident()) # Execute query, ordered by publish date (descending), filtered by filters and limited by limit. db_videos = db_session.query(Video).order_by(desc( Video.date_published)).filter(*filters).limit(limit).all() # Signal DB listener about finished read operation (Used in DB status indicator and logs) DatabaseListener.static_instance.finishRead.emit(threading.get_ident()) # Convert Video objects to VideoD (Detached model) objects. videos = Video.to_video_ds(db_videos) # Close DB Query session. db_session.remove() logger.debug5("sane_yt_subfeed.debug.date_formats: {}".format( sane_yt_subfeed.debug.date_formats)) sane_yt_subfeed.debug.clear_date_formats() return videos
def get_best_downloaded_videos(limit, filters=(~Video.watched, Video.downloaded, ~Video.discarded), sort_method=(asc(Video.watch_prio), desc(Video.date_downloaded), desc(Video.date_published))): """ :param filters: Tuple of filters :param sort_method: :param limit: :return: list(VideoD) """ db_query = db_session.query(Video) db_query = db_query.filter(*filters) DatabaseListener.static_instance.startRead.emit(threading.get_ident()) db_videos = db_query.order_by(*sort_method).limit(limit).all() DatabaseListener.static_instance.finishRead.emit(threading.get_ident()) videos = Video.to_video_ds(db_videos) db_session.remove() return videos
def get_newest_stored_videos(limit, filters=(~Video.downloaded, ~Video.discarded)): """ :param filters: :param limit: :return: list(VideoD) """ logger.info("Getting newest stored videos (filters={})".format(filters)) filter_days = read_config('Requests', 'filter_videos_days_old') if filter_days >= 0: date = datetime.datetime.utcnow() - datetime.timedelta( days=filter_days) filters = filters + (Video.date_published > date, ) DatabaseListener.static_instance.startRead.emit(threading.get_ident()) db_videos = db_session.query(Video).order_by(desc( Video.date_published)).filter(*filters).limit(limit).all() DatabaseListener.static_instance.finishRead.emit(threading.get_ident()) videos = Video.to_video_ds(db_videos) db_session.remove() return videos
def new_file(self, vid_id, vid_path): vid = db_session.query(Video).get(vid_id) if vid: if not vid.downloaded: vid.vid_path = vid_path vid.date_downloaded = datetime.datetime.utcnow() vid.downloaded = True thumb_path = os.path.join(THUMBNAILS_PATH, '{}.jpg'.format(vid.video_id)) downloaded_thumbnail = os.path.isfile(thumb_path) if downloaded_thumbnail and (not vid.thumbnail_path): vid.thumbnail_path = thumb_path self.logger.warning( "Thumbnail downloaded, but path didn't exist in db, for video: {}" .format(vid.__dict__)) elif (not vid.thumbnail_path) or (not downloaded_thumbnail): if not downloaded_thumbnail: self.logger.warning( "Thumbnail path in db, but not on disk, for video: {}" .format(vid.__dict__)) self.logger.info("Downloading thumbnail for: {}".format( vid.__dict__)) download_thumbnails_threaded([vid]) self.logger.info( "Updating existing record in db: {} - {}".format( vid.title, vid.__dict__)) db_session.commit() self.model.update_subfeed_videos_from_db() self.model.update_playback_videos_from_db() else: self.logger.info( "File already downloaded by this system: {} - {}".format( vid.title, vid.__dict__)) db_session.remove() else: db_session.remove() youtube_keys = load_keys(1) self.logger.info( "Grabbing new video information from youtube: {}".format( vid_id)) response_videos = list_uploaded_videos_videos( youtube_keys[0], [vid_id], 1) if len(response_videos) > 0: video = response_videos[0] video.vid_path = vid_path video.downloaded = True video.watched = False video.date_downloaded = datetime.datetime.utcnow() self.logger.info("Downloading thumbnail: {} - {}".format( video.title, video.__dict__)) download_thumbnails_threaded([video]) self.logger.info("Adding new file to db: {} - {}".format( video.title, video.__dict__)) UpdateVideo(video, finished_listeners=[ self.model.playback_grid_view_listener. downloadedVideosChangedinDB ]).start() else: self.logger.warning( "Video with id {}, not found on youtube servers".format( vid_id))
def cli(no_gui, test_channels, update_watch_prio, set_watched_day, refresh_and_print_subfeed, print_subscriptions, print_watched_videos, print_discarded_videos, print_downloaded_videos, print_playlist_items, print_playlist_items_url_only, auth_oauth2): logger = create_logger(__name__) if update_watch_prio: videos = db_session.query(Video).all() watch_prio = read_config('Play', 'default_watch_prio') logger.debug("Setting watch_prio {}, for: {} videos".format( watch_prio, len(videos))) for video in videos: video.watch_prio = watch_prio db_session.commit() return if set_watched_day: videos = db_session.query(Video).filter( or_(Video.downloaded == True, (Video.vid_path.is_(None)))).all() for video in videos: vid_age = datetime.datetime.utcnow() - video.date_published if vid_age > datetime.timedelta(days=int(set_watched_day)): logger.debug("Setting watched, {} - {} - {}".format( vid_age, video.title, video.__dict__)) video.watched = True db_session.commit() return if test_channels: run_channels_test() if refresh_and_print_subfeed: cli_refresh_and_print_subfeed() if print_subscriptions: cached_subs = True subs = get_subscriptions(cached_subs) for channel in subs: if channel.subscribed_override: print(("[{}] {} [Subscription override]".format( channel.id, channel.title))) else: print(("[{}] {}".format(channel.id, channel.title))) if print_watched_videos: videos = db_session.query(Video).filter( and_(Video.watched is True, (Video.vid_path.isnot(None)))).all() print_functions.print_videos(videos, path_only=True) if print_discarded_videos: videos = db_session.query(Video).filter( and_(Video.discarded is True, (Video.vid_path.isnot(None)))).all() print_functions.print_videos(videos, path_only=True) if print_downloaded_videos: videos = db_session.query(Video).filter( and_(Video.downloaded is True, (Video.vid_path.isnot(None)))).all() print_functions.print_videos(videos, path_only=True) if print_playlist_items: youtube_auth_resource = load_keys(1)[0] playlist_video_items = [] youtube.youtube_requests.list_uploaded_videos(youtube_auth_resource, playlist_video_items, print_playlist_items, 50) for vid in playlist_video_items: if print_playlist_items_url_only: print(vid.url_video) else: print(vid) if auth_oauth2: youtube_oauth = youtube_auth_oauth() if youtube_oauth is None: logger.critical("Failed to authenticate YouTube API OAuth2!") return None save_youtube_resource_oauth(youtube_oauth) if no_gui: run_with_cli() else: if LEGACY_EXCEPTION_HANDLER: """ PyQT raises and catches exceptions, but doesn't pass them along. Instead it just exits with a status of 1 to show an exception was caught. """ # Back up the reference to the exceptionhook sys._excepthook = sys.excepthook def my_exception_hook(exctype, value, traceback): global exc_id, exceptions # Ignore KeyboardInterrupt so a console python program can exit with Ctrl + C. if issubclass(exctype, KeyboardInterrupt): sys.__excepthook__(exctype, value, traceback) return # Log the exception with the logger logger.exception("Intercepted Exception #{}".format(exc_id), exc_info=(exctype, value, traceback)) # Store intercepted exceptions in a reference list of lists exceptions.append([exctype, value, traceback, exc_id]) # Increment Exception Identifier exc_id += 1 # Call the normal Exception hook after # noinspection PyProtectedMember sys._excepthook(exctype, value, traceback) # sys.exit(1) # Alternatively, exit # Set the exception hook to our wrapping function sys.excepthook = my_exception_hook run_with_gui()