def get_videos_by_ids(video_ids):
DatabaseListener.static_instance.startRead.emit(threading.get_ident())
db_videos = engine.execute(
Video.__table__.select(Video.video_id.in_(video_ids)))
DatabaseListener.static_instance.finishRead.emit(threading.get_ident())
return_videos = Video.to_video_ds(db_videos)
return return_videos
def run(self):
"""
Override threading.Thread.run() with its own code
:return:
"""
self.db_id = threading.get_ident()
if self.uniques_check:
self.video_list = check_for_unique(self.video_list)
items_to_add = []
items_to_update = []
# bulk_items_to_add = []
lock.acquire()
self.logger.debug("Thread {} - Acquired lock".format(
self.db_id, len(items_to_update)))
DatabaseListener.static_instance.startRead.emit(self.db_id)
select_step = 500
for i in range(0, len(self.video_list), select_step):
videos_bulk = self.video_list[i:i + select_step]
video_ids_bulk = set(video.video_id for video in videos_bulk)
stmt = get_video_ids_by_video_ids_stmt(video_ids_bulk)
db_videos = Video.to_video_ds(engine.execute(stmt))
db_videos_ids = set(video.video_id for video in db_videos)
items_to_add.extend(
insert_item(video) for video in videos_bulk
if video.video_id not in db_videos_ids)
if self.update_existing:
items_to_update.extend(video for video in videos_bulk
if video.video_id in db_videos_ids)
DatabaseListener.static_instance.finishRead.emit(self.db_id)
DatabaseListener.static_instance.startWrite.emit(self.db_id)
step = 1000
if len(items_to_add) > 0:
self.logger.debug("Thread {} - inserting {} new videos".format(
self.db_id, len(items_to_add)))
for i in range(0, len(items_to_add), step):
engine.execute(Video.__table__.insert(),
items_to_add[i:i + step])
if len(items_to_update) > 0:
self.logger.debug("Thread {} - updating {} items".format(
self.db_id, len(items_to_update)))
update_list = []
# FIXME: add step to update
for item in items_to_update:
item_dict = item.__dict__
item_dict["_video_id"] = item.video_id
update_list.append(item_dict)
engine.execute(update_video_stmt(), update_list)
DatabaseListener.static_instance.finishWrite.emit(self.db_id)
lock.release()
if self.finished_listeners:
for listener in self.finished_listeners:
listener.emit()
def get_videos_by_ids(video_ids):
"""
Get videos by IDs using a __table__ SELECT WHERE engine statement.
:param video_ids: Video ID strings
:return:
"""
# Signal DB listener about started read operation (Used in DB status indicator and logs)
DatabaseListener.static_instance.startRead.emit(threading.get_ident())
# Execute SELECT engine statement for each video.
db_videos = engine.execute(
Video.__table__.select(Video.video_id.in_(video_ids)))
# Signal DB listener about finished read operation (Used in DB status indicator and logs)
DatabaseListener.static_instance.finishRead.emit(threading.get_ident())
# Convert Video objects to VideoD (Detached model) objects.
videos = Video.to_video_ds(db_videos)
return videos
def get_db_videos_playback(limit,
filters=(~Video.watched, Video.downloaded,
~Video.discarded),
sort_method=(asc(Video.watch_prio),
desc(Video.date_downloaded),
desc(Video.date_published))):
"""
Get a list of videos from the database, limited by limit and filters. Sorted by sort_method.
:param filters: Tuple of SQLAlchemy Column(Boolean) objects.
:param sort_method: Tuple of SQLAlchemy sort expressions.
:param limit: Integer value determining how many videos to grab.
:return: A list of VideoD objects.
"""
# Initiate DB Query session.
db_query = db_session.query(Video)
# Apply filters to query.
db_query = db_query.filter(*filters)
# Signal DB listener about started read operation (Used in DB status indicator and logs)
DatabaseListener.static_instance.startRead.emit(threading.get_ident())
# Execute query, ordered by sort_method and limited by limit.
db_videos = db_query.order_by(*sort_method).limit(limit).all()
# Signal DB listener about finished read operation (Used in DB status indicator and logs)
DatabaseListener.static_instance.finishRead.emit(threading.get_ident())
# Convert Video objects to VideoD (Detached model) objects.
videos = Video.to_video_ds(db_videos)
# Close DB Query session.
db_session.remove()
logger.debug5("sane_yt_subfeed.debug.date_formats: {}".format(
sane_yt_subfeed.debug.date_formats))
sane_yt_subfeed.debug.clear_date_formats()
return videos
def get_db_videos_subfeed(limit,
filters=(~Video.downloaded, ~Video.discarded)):
"""
Get a list of videos from the database, limited by limit and filters.
:param filters: Tuple of SQLAlchemy Column(Boolean) objects.
:param limit: Integer value determining how many videos to grab.
:return: A list of VideoD objects.
"""
logger.info("Getting newest stored videos (filters={})".format(filters))
# Check whether "filter by video age" is enabled, if so apply the filter to filters.
filter_days = read_config('Requests', 'filter_videos_days_old')
if filter_days >= 0:
date = datetime.datetime.utcnow() - datetime.timedelta(
days=filter_days)
filters = filters + (Video.date_published > date, )
# Signal DB listener about started read operation (Used in DB status indicator and logs)
DatabaseListener.static_instance.startRead.emit(threading.get_ident())
# Execute query, ordered by publish date (descending), filtered by filters and limited by limit.
db_videos = db_session.query(Video).order_by(desc(
Video.date_published)).filter(*filters).limit(limit).all()
# Signal DB listener about finished read operation (Used in DB status indicator and logs)
DatabaseListener.static_instance.finishRead.emit(threading.get_ident())
# Convert Video objects to VideoD (Detached model) objects.
videos = Video.to_video_ds(db_videos)
# Close DB Query session.
db_session.remove()
logger.debug5("sane_yt_subfeed.debug.date_formats: {}".format(
sane_yt_subfeed.debug.date_formats))
sane_yt_subfeed.debug.clear_date_formats()
return videos
def get_best_downloaded_videos(limit,
filters=(~Video.watched, Video.downloaded,
~Video.discarded),
sort_method=(asc(Video.watch_prio),
desc(Video.date_downloaded),
desc(Video.date_published))):
"""
:param filters: Tuple of filters
:param sort_method:
:param limit:
:return: list(VideoD)
"""
db_query = db_session.query(Video)
db_query = db_query.filter(*filters)
DatabaseListener.static_instance.startRead.emit(threading.get_ident())
db_videos = db_query.order_by(*sort_method).limit(limit).all()
DatabaseListener.static_instance.finishRead.emit(threading.get_ident())
videos = Video.to_video_ds(db_videos)
db_session.remove()
return videos
def get_newest_stored_videos(limit,
filters=(~Video.downloaded, ~Video.discarded)):
"""
:param filters:
:param limit:
:return: list(VideoD)
"""
logger.info("Getting newest stored videos (filters={})".format(filters))
filter_days = read_config('Requests', 'filter_videos_days_old')
if filter_days >= 0:
date = datetime.datetime.utcnow() - datetime.timedelta(
days=filter_days)
filters = filters + (Video.date_published > date, )
DatabaseListener.static_instance.startRead.emit(threading.get_ident())
db_videos = db_session.query(Video).order_by(desc(
Video.date_published)).filter(*filters).limit(limit).all()
DatabaseListener.static_instance.finishRead.emit(threading.get_ident())
videos = Video.to_video_ds(db_videos)
db_session.remove()
return videos