def run(self):
self.db_id = threading.get_ident()
DatabaseListener.static_instance.startWrite.emit(self.db_id)
"""
Override threading.Thread.run() with its own code
:return:
"""
# self.logger.debug("Run")
# start = default_timer()
lock.acquire()
stmt = get_video_by_vidd_stmt(Video.video_d_to_video(self.video_d))
db_video = engine.execute(stmt).first()
if db_video:
if self.update_existing:
engine.execute(update_video_statement_full(self.video_d))
else:
pass
else:
engine.execute(Video.__table__.insert(), insert_item(self.video_d))
# print('Updated: {}'.format(self.video_d.title))
lock.release()
if self.finished_listeners:
for listener in self.finished_listeners:
listener.emit()
DatabaseListener.static_instance.finishWrite.emit(self.db_id)
def update_event_download_tile(download_tile):
lock.acquire()
stmt = DBDownloadTile.__table__.update().where(
DBDownloadTile.video_id == download_tile.video.video_id).values(
{'last_event': download_tile.last_event})
engine.execute(stmt)
lock.release()
def run(self):
"""
Override threading.Thread.run() with its own code
:return:
"""
self.db_id = threading.get_ident()
if self.uniques_check:
self.video_list = check_for_unique(self.video_list)
items_to_add = []
items_to_update = []
# bulk_items_to_add = []
lock.acquire()
self.logger.debug("Thread {} - Acquired lock".format(
self.db_id, len(items_to_update)))
DatabaseListener.static_instance.startRead.emit(self.db_id)
select_step = 500
for i in range(0, len(self.video_list), select_step):
videos_bulk = self.video_list[i:i + select_step]
video_ids_bulk = set(video.video_id for video in videos_bulk)
stmt = get_video_ids_by_video_ids_stmt(video_ids_bulk)
db_videos = Video.to_video_ds(engine.execute(stmt))
db_videos_ids = set(video.video_id for video in db_videos)
items_to_add.extend(
insert_item(video) for video in videos_bulk
if video.video_id not in db_videos_ids)
if self.update_existing:
items_to_update.extend(video for video in videos_bulk
if video.video_id in db_videos_ids)
DatabaseListener.static_instance.finishRead.emit(self.db_id)
DatabaseListener.static_instance.startWrite.emit(self.db_id)
step = 1000
if len(items_to_add) > 0:
self.logger.debug("Thread {} - inserting {} new videos".format(
self.db_id, len(items_to_add)))
for i in range(0, len(items_to_add), step):
engine.execute(Video.__table__.insert(),
items_to_add[i:i + step])
if len(items_to_update) > 0:
self.logger.debug("Thread {} - updating {} items".format(
self.db_id, len(items_to_update)))
update_list = []
# FIXME: add step to update
for item in items_to_update:
item_dict = item.__dict__
item_dict["_video_id"] = item.video_id
update_list.append(item_dict)
engine.execute(update_video_stmt(), update_list)
DatabaseListener.static_instance.finishWrite.emit(self.db_id)
lock.release()
if self.finished_listeners:
for listener in self.finished_listeners:
listener.emit()
def delete_sub_not_in_list(subs):
delete_channels = db_session.query(Channel).filter(
~Channel.id.in_(subs)).all()
for channel in delete_channels:
if channel.subscribed or channel.subscribed is None:
channel.subscribed = False
create_logger(__name__).warning(
"Setting unsubscribed for channel: {} - {}".format(
channel.title, channel.__dict__))
stmt = update_channel_from_remote(channel)
engine.execute(stmt)
def get_videos_by_ids(video_ids):
DatabaseListener.static_instance.startRead.emit(threading.get_ident())
db_videos = engine.execute(
Video.__table__.select(Video.video_id.in_(video_ids)))
DatabaseListener.static_instance.finishRead.emit(threading.get_ident())
return_videos = Video.to_video_ds(db_videos)
return return_videos
def check_for_new(videos, deep_refresh=False):
logger.info("Checking for new videos{}".format(
(" (deep refresh)" if deep_refresh else "")))
# FIXME: add to progress bar
# start_time = timeit.default_timer()
DatabaseListener.static_instance.startRead.emit(threading.get_ident())
for vid in videos:
stmt = get_video_by_vidd_stmt(vid)
db_video = engine.execute(stmt).first()
if not db_video:
vid_age = datetime.datetime.utcnow() - vid.date_published
if deep_refresh:
if vid_age > datetime.timedelta(hours=1):
vid.missed = True
logger.info("Missed video: {} - {} [{}]".format(
vid.channel_title, vid.title, vid.url_video))
else:
vid.new = True
logger.info("New video: {} - {} [{}]".format(
vid.channel_title, vid.title, vid.url_video))
else:
if vid_age > datetime.timedelta(hours=12):
vid.missed = True
logger.info("Missed video: {} - {} [{}]".format(
vid.channel_title, vid.title, vid.url_video))
else:
vid.new = True
logger.info("New video: {} - {} [{}]".format(
vid.channel_title, vid.title, vid.url_video))
else:
pass
# print(timeit.default_timer() - start_time)
DatabaseListener.static_instance.finishRead.emit(threading.get_ident())
return videos
def run(self):
"""
Override threading.Thread.run() with its own code
:return:
"""
self.db_id = threading.get_ident()
lock.acquire()
DatabaseListener.static_instance.startWrite.emit(self.db_id)
update_list = []
if len(self.video_list):
for item in self.video_list:
if not item.thumbnail_path:
self.logger.warning(
"Video missing thumbnail for update: {}".format(item))
elif not item.duration:
self.logger.warning(
"Video missing duration for update: {}".format(item))
else:
update_list.append({
"thumbnail_path": item.thumbnail_path,
"_video_id": item.video_id,
"duration": item.duration
})
try:
engine.execute(update_extra_information_stmt(), update_list)
except Exception as e:
self.logger.critical(
"Failed to update extra information: {} - {}".format(
e, update_list),
exc_info=1)
else:
self.logger.info("Skipping update as self.video_list is empty")
DatabaseListener.static_instance.finishWrite.emit(self.db_id)
lock.release()
if self.finished_listeners:
for listener in self.finished_listeners:
listener.emit()
def get_vid_by_id(video_id):
"""
Get video by id using a __table__ SELECT WHERE engine statement.
:param video_id: Video ID string
:return:
"""
# Create a __table__ SELECT WHERE video_id=this_video_id engine statement.
stmt = get_video_by_id_stmt(video_id)
# Signal DB listener about started read operation (Used in DB status indicator and logs)
DatabaseListener.static_instance.startRead.emit(threading.get_ident())
# Execute engine statement, and only return the first result.
db_video = engine.execute(stmt).first()
# Signal DB listener about finished read operation (Used in DB status indicator and logs)
DatabaseListener.static_instance.finishRead.emit(threading.get_ident())
return db_video
def get_videos_by_ids(video_ids):
"""
Get videos by IDs using a __table__ SELECT WHERE engine statement.
:param video_ids: Video ID strings
:return:
"""
# Signal DB listener about started read operation (Used in DB status indicator and logs)
DatabaseListener.static_instance.startRead.emit(threading.get_ident())
# Execute SELECT engine statement for each video.
db_videos = engine.execute(
Video.__table__.select(Video.video_id.in_(video_ids)))
# Signal DB listener about finished read operation (Used in DB status indicator and logs)
DatabaseListener.static_instance.finishRead.emit(threading.get_ident())
# Convert Video objects to VideoD (Detached model) objects.
videos = Video.to_video_ds(db_videos)
return videos
def check_for_new(videos, refresh_type):
"""
Check for new video uploads.
:param videos:
:param refresh_type:
:return:
"""
logger.info("Checking for new videos{}".format(
(" (deep refresh)"
if refresh_type == LISTENER_SIGNAL_DEEP_REFRESH else "")))
# FIXME: add to progress bar
# Signal DB listener about started read operation (Used in DB status indicator and logs)
DatabaseListener.static_instance.startRead.emit(threading.get_ident())
# Iterate through videos.
for vid in videos:
# Get the Video id using an engine statement and check if the video is in the database.
stmt = get_video_by_vidd_stmt(vid)
db_video = engine.execute(stmt).first()
# If video is not in database, apply new/missing conditions to it.
if not db_video:
# Determine the
vid_age = datetime.datetime.utcnow() - vid.date_published
# FIXME: Make missed video age condition user configurable.
if vid_age > datetime.timedelta(hours=12):
vid.missed = True
logger.info("Missed video: {}".format(vid))
else:
vid.new = True
logger.info("New video: {}".format(vid))
# If video is in database, do nothing.
else:
pass
# Signal DB listener about finished read operation (Used in DB status indicator and logs)
DatabaseListener.static_instance.finishRead.emit(threading.get_ident())
return videos
def engine_execute(stmt):
engine.execute(stmt)
def engine_execute_first(stmt):
return engine.execute(stmt).first()
def get_vid_by_id(video_id):
stmt = get_video_by_id_stmt(video_id)
DatabaseListener.static_instance.startRead.emit(threading.get_ident())
db_video = engine.execute(stmt).first()
DatabaseListener.static_instance.finishRead.emit(threading.get_ident())
return db_video
def run(self):
"""
Override threading.Thread.run() with its own code
:return:
"""
self.db_id = threading.get_ident()
lock.acquire()
DatabaseListener.static_instance.startWrite.emit(self.db_id)
update_list = []
if len(self.video_list):
for item in self.video_list:
vid_info = "{}".format(item)
if item.thumbnail_path is None:
self.logger.warning(
"Video missing thumbnail for update: {}".format(
vid_info))
self.logger.error(self.obj_attrs_to_dict(item))
elif item.duration is None:
self.logger.warning(
"Video missing duration for update: {}".format(
vid_info))
self.logger.error(self.obj_attrs_to_dict(item))
elif item.has_caption is None:
self.logger.warning(
"Video missing has_caption for update: {}".format(
vid_info))
self.logger.error(self.obj_attrs_to_dict(item))
elif item.dimension is None:
self.logger.warning(
"Video missing dimension for update: {}".format(
vid_info))
self.logger.error(self.obj_attrs_to_dict(item))
elif item.definition is None:
self.logger.warning(
"Video missing definition for update: {}".format(
vid_info))
self.logger.error(self.obj_attrs_to_dict(item))
elif item.projection is None:
self.logger.warning(
"Video missing projection for update: {}".format(
vid_info))
self.logger.error(self.obj_attrs_to_dict(item))
elif item.region_restriction_allowed is None:
self.logger.warning(
"Video missing region_restriction_allowed for update: {}"
.format(vid_info))
self.logger.error(self.obj_attrs_to_dict(item))
elif item.region_restriction_blocked is None:
self.logger.warning(
"Video missing region_restriction_blocked for update: {}"
.format(vid_info))
self.logger.error(self.obj_attrs_to_dict(item))
elif item.kind is None:
self.logger.warning(
"Video missing kind for update: {}".format(vid_info))
self.logger.error(self.obj_attrs_to_dict(item))
else:
update_list.append({
"thumbnail_path": item.thumbnail_path,
"_video_id": item.video_id,
"duration": item.duration,
"has_caption": item.has_caption,
"dimension": item.dimension,
"definition": item.definition,
"projection": item.projection,
"region_restriction_allowed":
item.region_restriction_allowed,
"region_restriction_blocked":
item.region_restriction_blocked,
"kind": item.kind
})
try:
engine.execute(update_extra_information_stmt(), update_list)
except Exception as e:
self.logger.critical(
"Failed to update extra information: {} - {}".format(
e, update_list),
exc_info=1)
else:
self.logger.info("Skipping update as self.video_list is empty")
DatabaseListener.static_instance.finishWrite.emit(self.db_id)
lock.release()
if self.finished_listeners:
for listener in self.finished_listeners:
listener.emit()