def compare_db_filtered(videos, limit, discarded=False, downloaded=False):
logger.info("Comparing filtered videos with DB")
return_list = []
counter = 0
filter_days = read_config('Requests', 'filter_videos_days_old')
DatabaseListener.static_instance.startRead.emit(threading.get_ident())
for video in videos:
if filter_days >= 0:
date = datetime.datetime.utcnow() - datetime.timedelta(
days=filter_days)
if video.date_published < date:
break
db_vid = get_vid_by_id(video.video_id)
if db_vid:
if db_vid.downloaded:
if downloaded:
continue
if db_vid.discarded:
if discarded:
continue
return_list.append(Video.to_video_d(video))
counter += 1
else:
return_list.append(video)
counter += 1
if counter >= limit:
break
DatabaseListener.static_instance.finishRead.emit(threading.get_ident())
db_session.remove()
return return_list
def __init__(self, download_tile, db_download_tile=False):
self.finished = download_tile.finished
self.started_date = download_tile.started_date
self.finished_date = download_tile.finished_date
if db_download_tile:
self.video = Video.to_video_d(download_tile.video)
else:
self.video = download_tile.video
self.video_downloaded = download_tile.video_downloaded
self.total_bytes = download_tile.total_bytes
self.last_event = download_tile.last_event
self.cleared = download_tile.cleared
# Other
self.progress_listener = None
def get_new_and_updated_videos(vid_paths):
"""
:param vid_paths: dict(video_id, vid_path)
:return:
"""
logger = create_logger(__name__ + ".new_and_updated")
db_videos = get_videos_by_ids(vid_paths.keys())
return_videos = []
for video in db_videos:
if not video.vid_path or not video.downloaded:
video_d = Video.to_video_d(video)
video_d.vid_path = vid_paths[video.video_id]
video_d.downloaded = True
video_d.date_downloaded = datetime.datetime.utcnow()
logger.info("Found video needing update: {} - {}".format(
video.video_id, video.title))
return_videos.append(video_d)
vid_paths.pop(video.video_id, None)
new_videos = []
if len(vid_paths) > 0:
youtube_keys = load_keys(1)
logger.info(
"Grabbing new video(s) information from youtube for: {}".format(
vid_paths.keys()))
response_videos = list_uploaded_videos_videos(youtube_keys[0],
vid_paths.keys(), 30)
for video in response_videos:
video.vid_path = vid_paths[video.video_id]
video.downloaded = True
video.watched = False
video.date_downloaded = datetime.datetime.utcnow()
logger.info("Found new video: {} - {}".format(
video.video_id, video.title))
new_videos.append(video)
return_videos.extend(new_videos)
logger.info("Downloading thumbnails for: {}".format(return_videos))
download_thumbnails_threaded(return_videos)
return return_videos
def filter_videos(videos,
limit,
filter_discarded=False,
filter_downloaded=False):
"""
Takes a list of videos and excludes items that match any enabled exclusion filter.
:param videos: A list of Video objects to be compared.
:param limit: Integer value determining how many videos to return.
:param filter_discarded: Boolean determining whether or not to exclude discarded videos.
:param filter_downloaded: Boolean determining whether or not to exclude filter_downloaded videos.
:return: A list of VideoD objects.
"""
logger.info("Comparing filtered videos with DB")
return_list = []
counter = 0
# Check whether "filter by video age" is enabled, if so apply the filter to filters.
filter_days = read_config('Requests', 'filter_videos_days_old')
# Signal DB listener about started read operation (Used in DB status indicator and logs)
DatabaseListener.static_instance.startRead.emit(threading.get_ident())
# Iterate through videos:
for video in videos:
# If "filter by video age" is enabled, skip videos older than filter_days days.
if filter_days >= 0:
date = datetime.datetime.utcnow() - datetime.timedelta(
days=filter_days)
if video.date_published < date:
break
# Check if video is in database, if so, convert it and add it to list, unless it matches an exclusion filter.
db_vid = get_vid_by_id(video.video_id)
if db_vid:
# Skip video if it matches an enabled exclusion filter.
if (db_vid.downloaded
and filter_downloaded) or (db_vid.discarded
and filter_discarded):
continue
# Convert Video object to VideoD (Detached model) object and append it to the list, then increment counter.
return_list.append(Video.to_video_d(video))
counter += 1
# If video isn't in database, no filters apply and it can simply be appended to the list.
else:
# Append VideoD object to list, then increment counter.
return_list.append(video)
counter += 1
# Break once the limit has been reached.
if counter >= limit:
break
# Signal DB listener about finished read operation (Used in DB status indicator and logs)
DatabaseListener.static_instance.finishRead.emit(threading.get_ident())
# Close DB Query session.
db_session.remove()
return return_list