def tile_marked_livestream_upcoming(self, video: Video):
"""
Mark the video as live broadcast content (upcoming)
:return:
"""
video.kind = VIDEO_KIND_LIVE_SCHEDULED
self.update_and_repaint_tile(video)
def compare_db_filtered(videos, limit, discarded=False, downloaded=False):
logger.info("Comparing filtered videos with DB")
return_list = []
counter = 0
filter_days = read_config('Requests', 'filter_videos_days_old')
DatabaseListener.static_instance.startRead.emit(threading.get_ident())
for video in videos:
if filter_days >= 0:
date = datetime.datetime.utcnow() - datetime.timedelta(
days=filter_days)
if video.date_published < date:
break
db_vid = get_vid_by_id(video.video_id)
if db_vid:
if db_vid.downloaded:
if downloaded:
continue
if db_vid.discarded:
if discarded:
continue
return_list.append(Video.to_video_d(video))
counter += 1
else:
return_list.append(video)
counter += 1
if counter >= limit:
break
DatabaseListener.static_instance.finishRead.emit(threading.get_ident())
db_session.remove()
return return_list
def tile_unmarked_livestream(self, video: Video):
"""
Unmark the video as live broadcast content (live)
:return:
"""
video.kind = VIDEO_KIND_VOD
self.update_and_repaint_tile(video)
def run(self):
self.db_id = threading.get_ident()
DatabaseListener.static_instance.startWrite.emit(self.db_id)
"""
Override threading.Thread.run() with its own code
:return:
"""
# self.logger.debug("Run")
# start = default_timer()
lock.acquire()
stmt = get_video_by_vidd_stmt(Video.video_d_to_video(self.video_d))
db_video = engine.execute(stmt).first()
if db_video:
if self.update_existing:
engine.execute(update_video_statement_full(self.video_d))
else:
pass
else:
engine.execute(Video.__table__.insert(), insert_item(self.video_d))
# print('Updated: {}'.format(self.video_d.title))
lock.release()
if self.finished_listeners:
for listener in self.finished_listeners:
listener.emit()
DatabaseListener.static_instance.finishWrite.emit(self.db_id)
def get_videos_by_ids(video_ids):
DatabaseListener.static_instance.startRead.emit(threading.get_ident())
db_videos = engine.execute(
Video.__table__.select(Video.video_id.in_(video_ids)))
DatabaseListener.static_instance.finishRead.emit(threading.get_ident())
return_videos = Video.to_video_ds(db_videos)
return return_videos
def tile_unmarked_premiere(self, video: Video):
"""
Unmark the video as live broadcast content (premiere)
A premiere is: upcoming stream --> live stream --> vod
:return:
"""
video.kind = VIDEO_KIND_VOD
self.update_and_repaint_tile(video)
def run(self):
"""
Override threading.Thread.run() with its own code
:return:
"""
self.db_id = threading.get_ident()
if self.uniques_check:
self.video_list = check_for_unique(self.video_list)
items_to_add = []
items_to_update = []
# bulk_items_to_add = []
lock.acquire()
self.logger.debug("Thread {} - Acquired lock".format(
self.db_id, len(items_to_update)))
DatabaseListener.static_instance.startRead.emit(self.db_id)
select_step = 500
for i in range(0, len(self.video_list), select_step):
videos_bulk = self.video_list[i:i + select_step]
video_ids_bulk = set(video.video_id for video in videos_bulk)
stmt = get_video_ids_by_video_ids_stmt(video_ids_bulk)
db_videos = Video.to_video_ds(engine.execute(stmt))
db_videos_ids = set(video.video_id for video in db_videos)
items_to_add.extend(
insert_item(video) for video in videos_bulk
if video.video_id not in db_videos_ids)
if self.update_existing:
items_to_update.extend(video for video in videos_bulk
if video.video_id in db_videos_ids)
DatabaseListener.static_instance.finishRead.emit(self.db_id)
DatabaseListener.static_instance.startWrite.emit(self.db_id)
step = 1000
if len(items_to_add) > 0:
self.logger.debug("Thread {} - inserting {} new videos".format(
self.db_id, len(items_to_add)))
for i in range(0, len(items_to_add), step):
engine.execute(Video.__table__.insert(),
items_to_add[i:i + step])
if len(items_to_update) > 0:
self.logger.debug("Thread {} - updating {} items".format(
self.db_id, len(items_to_update)))
update_list = []
# FIXME: add step to update
for item in items_to_update:
item_dict = item.__dict__
item_dict["_video_id"] = item.video_id
update_list.append(item_dict)
engine.execute(update_video_stmt(), update_list)
DatabaseListener.static_instance.finishWrite.emit(self.db_id)
lock.release()
if self.finished_listeners:
for listener in self.finished_listeners:
listener.emit()
def tile_unwatched(self, video: Video):
"""
Action to take if tile has been un-flagged as watched.
Called by Views: Playback
:param video:
:return:
"""
video.watched = False
if not read_config('GridView', 'show_watched'):
self.logger.info("Mark unwatched: {} - {}".format(video.title, video.__dict__))
self.model.unhide_video_item(video, self.widget_id)
self.update_and_repaint_tile(video)
def tile_undiscarded(self, video: Video):
"""
Action to take if tile has been un-flagged as dismissed.
Called by Views: Subfeed and Playback
:param video:
:return:
"""
video.discarded = False
if not read_config('GridView', 'show_dismissed'):
self.logger.info("Un-hide video (Un-discarded): {}".format(video))
self.model.unhide_video_item(video, self.widget_id)
self.update_and_repaint_tile(video)
def __init__(self, download_tile, db_download_tile=False):
self.finished = download_tile.finished
self.started_date = download_tile.started_date
self.finished_date = download_tile.finished_date
if db_download_tile:
self.video = Video.to_video_d(download_tile.video)
else:
self.video = download_tile.video
self.video_downloaded = download_tile.video_downloaded
self.total_bytes = download_tile.total_bytes
self.last_event = download_tile.last_event
self.cleared = download_tile.cleared
# Other
self.progress_listener = None
def tile_delete_downloaded_data(self, video: Video):
"""
Action to take if tile has been told to delete its downloaded data.
This will delete the video file.
:param video:
:return:
"""
# Delete downloaded video file
os.remove(video.vid_path)
self.logger.info("Deleted: {}".format(video.vid_path))
# Unset path to signal that the data isn't on disk.
video.vid_path = None
# Repaint the tile to reflect the action.
self.update_and_repaint_tile(video)
def get_new_and_updated_videos(vid_paths):
"""
:param vid_paths: dict(video_id, vid_path)
:return:
"""
logger = create_logger(__name__ + ".new_and_updated")
db_videos = get_videos_by_ids(vid_paths.keys())
return_videos = []
for video in db_videos:
if not video.vid_path or not video.downloaded:
video_d = Video.to_video_d(video)
video_d.vid_path = vid_paths[video.video_id]
video_d.downloaded = True
video_d.date_downloaded = datetime.datetime.utcnow()
logger.info("Found video needing update: {} - {}".format(
video.video_id, video.title))
return_videos.append(video_d)
vid_paths.pop(video.video_id, None)
new_videos = []
if len(vid_paths) > 0:
youtube_keys = load_keys(1)
logger.info(
"Grabbing new video(s) information from youtube for: {}".format(
vid_paths.keys()))
response_videos = list_uploaded_videos_videos(youtube_keys[0],
vid_paths.keys(), 30)
for video in response_videos:
video.vid_path = vid_paths[video.video_id]
video.downloaded = True
video.watched = False
video.date_downloaded = datetime.datetime.utcnow()
logger.info("Found new video: {} - {}".format(
video.video_id, video.title))
new_videos.append(video)
return_videos.extend(new_videos)
logger.info("Downloading thumbnails for: {}".format(return_videos))
download_thumbnails_threaded(return_videos)
return return_videos
def get_videos_by_ids(video_ids):
"""
Get videos by IDs using a __table__ SELECT WHERE engine statement.
:param video_ids: Video ID strings
:return:
"""
# Signal DB listener about started read operation (Used in DB status indicator and logs)
DatabaseListener.static_instance.startRead.emit(threading.get_ident())
# Execute SELECT engine statement for each video.
db_videos = engine.execute(
Video.__table__.select(Video.video_id.in_(video_ids)))
# Signal DB listener about finished read operation (Used in DB status indicator and logs)
DatabaseListener.static_instance.finishRead.emit(threading.get_ident())
# Convert Video objects to VideoD (Detached model) objects.
videos = Video.to_video_ds(db_videos)
return videos
def get_db_videos_playback(limit,
filters=(~Video.watched, Video.downloaded,
~Video.discarded),
sort_method=(asc(Video.watch_prio),
desc(Video.date_downloaded),
desc(Video.date_published))):
"""
Get a list of videos from the database, limited by limit and filters. Sorted by sort_method.
:param filters: Tuple of SQLAlchemy Column(Boolean) objects.
:param sort_method: Tuple of SQLAlchemy sort expressions.
:param limit: Integer value determining how many videos to grab.
:return: A list of VideoD objects.
"""
# Initiate DB Query session.
db_query = db_session.query(Video)
# Apply filters to query.
db_query = db_query.filter(*filters)
# Signal DB listener about started read operation (Used in DB status indicator and logs)
DatabaseListener.static_instance.startRead.emit(threading.get_ident())
# Execute query, ordered by sort_method and limited by limit.
db_videos = db_query.order_by(*sort_method).limit(limit).all()
# Signal DB listener about finished read operation (Used in DB status indicator and logs)
DatabaseListener.static_instance.finishRead.emit(threading.get_ident())
# Convert Video objects to VideoD (Detached model) objects.
videos = Video.to_video_ds(db_videos)
# Close DB Query session.
db_session.remove()
logger.debug5("sane_yt_subfeed.debug.date_formats: {}".format(
sane_yt_subfeed.debug.date_formats))
sane_yt_subfeed.debug.clear_date_formats()
return videos
def get_best_downloaded_videos(limit,
filters=(~Video.watched, Video.downloaded,
~Video.discarded),
sort_method=(asc(Video.watch_prio),
desc(Video.date_downloaded),
desc(Video.date_published))):
"""
:param filters: Tuple of filters
:param sort_method:
:param limit:
:return: list(VideoD)
"""
db_query = db_session.query(Video)
db_query = db_query.filter(*filters)
DatabaseListener.static_instance.startRead.emit(threading.get_ident())
db_videos = db_query.order_by(*sort_method).limit(limit).all()
DatabaseListener.static_instance.finishRead.emit(threading.get_ident())
videos = Video.to_video_ds(db_videos)
db_session.remove()
return videos
def get_db_videos_subfeed(limit,
filters=(~Video.downloaded, ~Video.discarded)):
"""
Get a list of videos from the database, limited by limit and filters.
:param filters: Tuple of SQLAlchemy Column(Boolean) objects.
:param limit: Integer value determining how many videos to grab.
:return: A list of VideoD objects.
"""
logger.info("Getting newest stored videos (filters={})".format(filters))
# Check whether "filter by video age" is enabled, if so apply the filter to filters.
filter_days = read_config('Requests', 'filter_videos_days_old')
if filter_days >= 0:
date = datetime.datetime.utcnow() - datetime.timedelta(
days=filter_days)
filters = filters + (Video.date_published > date, )
# Signal DB listener about started read operation (Used in DB status indicator and logs)
DatabaseListener.static_instance.startRead.emit(threading.get_ident())
# Execute query, ordered by publish date (descending), filtered by filters and limited by limit.
db_videos = db_session.query(Video).order_by(desc(
Video.date_published)).filter(*filters).limit(limit).all()
# Signal DB listener about finished read operation (Used in DB status indicator and logs)
DatabaseListener.static_instance.finishRead.emit(threading.get_ident())
# Convert Video objects to VideoD (Detached model) objects.
videos = Video.to_video_ds(db_videos)
# Close DB Query session.
db_session.remove()
logger.debug5("sane_yt_subfeed.debug.date_formats: {}".format(
sane_yt_subfeed.debug.date_formats))
sane_yt_subfeed.debug.clear_date_formats()
return videos
def get_newest_stored_videos(limit,
filters=(~Video.downloaded, ~Video.discarded)):
"""
:param filters:
:param limit:
:return: list(VideoD)
"""
logger.info("Getting newest stored videos (filters={})".format(filters))
filter_days = read_config('Requests', 'filter_videos_days_old')
if filter_days >= 0:
date = datetime.datetime.utcnow() - datetime.timedelta(
days=filter_days)
filters = filters + (Video.date_published > date, )
DatabaseListener.static_instance.startRead.emit(threading.get_ident())
db_videos = db_session.query(Video).order_by(desc(
Video.date_published)).filter(*filters).limit(limit).all()
DatabaseListener.static_instance.finishRead.emit(threading.get_ident())
videos = Video.to_video_ds(db_videos)
db_session.remove()
return videos
def run():
video_item = {
'kind': 'youtube#searchResult',
'etag': '"XI7nbFXulYBIpL0ayR_gDh3eu1k/rrGI6j0VT4vX91p6eQyuqFQjf8M"',
'id': {
'kind': 'youtube#video',
'videoId': '8NG1PEFceG0'
},
'snippet': {
'publishedAt': '2017-06-22T19:26:41.000Z',
'channelId': 'UCzORJV8l3FWY4cFO8ot-F2w',
'title':
'[Vinesauce] Vinny - The Legend of Zelda: Clips of the Wild',
'description':
'A compilation of user-created Twitch clips from Breath of the Wild! Streamed live on Vinesauce. Check out my other youtube channel for full streams.',
'thumbnails': {
'default': {
'url': 'https://i.ytimg.com/vi/8NG1PEFceG0/default.jpg',
'width': 120,
'height': 90
},
'medium': {
'url': 'https://i.ytimg.com/vi/8NG1PEFceG0/mqdefault.jpg',
'width': 320,
'height': 180
},
'high': {
'url': 'https://i.ytimg.com/vi/8NG1PEFceG0/hqdefault.jpg',
'width': 480,
'height': 360
}
},
'channelTitle': 'vinesauce',
'liveBroadcastContent': 'none'
}
}
channel_item = {
"publishedAt": "2015-01-09T12:46:03.000Z",
"title": "OfficialNerdCubed",
"description":
"Recommending good games every Monday and Wednesday. Going strange every Saturday.",
"resourceId": {
"kind": "youtube#channel",
"channelId": "UCKab3hYnOoTZZbEUQBMx-ww"
},
"channelId": "UCv5TbAM7zHQOoDr0fw2Cy8w",
"thumbnails": {
"default": {
"url":
"https://yt3.ggpht.com/-2AuBRBi8yE8/AAAAAAAAAAI/AAAAAAAAAAA/PsaY8p1aJ2U/s88-c-k-no-mo-rj-c0xffffff/photo.jpg"
},
"medium": {
"url":
"https://yt3.ggpht.com/-2AuBRBi8yE8/AAAAAAAAAAI/AAAAAAAAAAA/PsaY8p1aJ2U/s240-c-k-no-mo-rj-c0xffffff/photo.jpg"
},
"high": {
"url":
"https://yt3.ggpht.com/-2AuBRBi8yE8/AAAAAAAAAAI/AAAAAAAAAAA/PsaY8p1aJ2U/s800-c-k-no-mo-rj-c0xffffff/photo.jpg"
}
}
}
channel = Channel(channel_item, 'UUv5TbAM7zHQOoDr0fw2Cy8w')
video = Video(video_item)
insrt_txt = {
"video_id": video.video_id,
"channel_title": video.channel_title,
'title': video.title,
"date_published": video.date_published,
"description": video.description,
"thumbnail_path": video.thumbnail_path,
"playlist_pos": video.playlist_pos,
"url_video": video.url_video,
"url_playlist_video": video.url_playlist_video,
"thumbnails": video.thumbnails,
"downloaded": video.downloaded,
"search_item": video.search_item
}
# stmt_1 = Video.__table__.insert(), [insrt_txt]
# engine.execute(Video.__table__.insert(), [insrt_txt])
# stmt = update_statement_full(video)
# stmt = get_video_by_id_stmt(video)
# # stmt = Video.__table__.select('video.video_id').where(text("video_id = '{}'".format(video.video_id)))
# # print(db_session.execute(stmt).first())
# print(engine.execute(stmt).first())
# video2 = engine.execute(Video.__table__.select(Video.title).where('video_id' == video.video_id).limit(1))
stmt = get_channel_by_id_stmt(channel)
print(engine.execute(stmt).first())
def filter_videos(videos,
limit,
filter_discarded=False,
filter_downloaded=False):
"""
Takes a list of videos and excludes items that match any enabled exclusion filter.
:param videos: A list of Video objects to be compared.
:param limit: Integer value determining how many videos to return.
:param filter_discarded: Boolean determining whether or not to exclude discarded videos.
:param filter_downloaded: Boolean determining whether or not to exclude filter_downloaded videos.
:return: A list of VideoD objects.
"""
logger.info("Comparing filtered videos with DB")
return_list = []
counter = 0
# Check whether "filter by video age" is enabled, if so apply the filter to filters.
filter_days = read_config('Requests', 'filter_videos_days_old')
# Signal DB listener about started read operation (Used in DB status indicator and logs)
DatabaseListener.static_instance.startRead.emit(threading.get_ident())
# Iterate through videos:
for video in videos:
# If "filter by video age" is enabled, skip videos older than filter_days days.
if filter_days >= 0:
date = datetime.datetime.utcnow() - datetime.timedelta(
days=filter_days)
if video.date_published < date:
break
# Check if video is in database, if so, convert it and add it to list, unless it matches an exclusion filter.
db_vid = get_vid_by_id(video.video_id)
if db_vid:
# Skip video if it matches an enabled exclusion filter.
if (db_vid.downloaded
and filter_downloaded) or (db_vid.discarded
and filter_discarded):
continue
# Convert Video object to VideoD (Detached model) object and append it to the list, then increment counter.
return_list.append(Video.to_video_d(video))
counter += 1
# If video isn't in database, no filters apply and it can simply be appended to the list.
else:
# Append VideoD object to list, then increment counter.
return_list.append(video)
counter += 1
# Break once the limit has been reached.
if counter >= limit:
break
# Signal DB listener about finished read operation (Used in DB status indicator and logs)
DatabaseListener.static_instance.finishRead.emit(threading.get_ident())
# Close DB Query session.
db_session.remove()
return return_list