def add_subscription_local(youtube_auth, channel_id, by_username=False):
"""
Add a YouTube subscription (Local/DB).
:param by_username:
:param youtube_auth:
:param channel_id:
:return:
"""
# FIXME: Somewhat duplicate code of get_remote_subscriptions, move to own function -- START
# Get ID of uploads playlist
# channel_uploads_playlist_id = get_channel_uploads_playlist_id(youtube_auth, channel_id)
if by_username:
channel_response = get_channel_by_username(youtube_auth, channel_id)
else:
channel_response = get_channel_by_id(youtube_auth, channel_id)
channel_uploads_playlist_id = channel_response['contentDetails'][
'relatedPlaylists']['uploads']
channel = Channel(channel_response,
channel_uploads_playlist_id,
channel_list_response=True)
db_channel = engine_execute_first(get_channel_by_id_stmt(channel))
if db_channel:
engine_execute(update_channel_from_remote(channel))
else:
# TODO: change to sqlalchemy core stmt
create_logger(__name__ + ".subscriptions").info(
"Added channel {} - {}".format(channel.title, channel.id))
db_session.add(channel)
db_session.commit()
# FIXME: Somewhat duplicate code of get_remote_subscriptions, move to own function -- END
logger.info("Added subscription (Local/DB): {} / {}".format(
channel_id, channel.title))
def delete_sub_not_in_list(subs):
delete_channels = db_session.query(Channel).filter(
~Channel.id.in_(subs)).all()
for channel in delete_channels:
if channel.subscribed or channel.subscribed is None:
channel.subscribed = False
create_logger(__name__).warning(
"Setting unsubscribed for channel: {} - {}".format(
channel.title, channel.__dict__))
stmt = update_channel_from_remote(channel)
engine.execute(stmt)
def add_subscription_remote(channel_id):
"""
Add a YouTube subscription (On YouTube).
DEPRECATED: Google doesn't let you, see supported op table https://developers.google.com/youtube/v3/getting-started
:param youtube_oauth:
:param channel_id:
:return: returns response or raises exception
"""
youtube_oauth = load_youtube()
response = youtube_oauth.subscriptions().insert(
part='snippet',
body=dict(snippet=dict(resourceId=dict(channelId=channel_id))))
try:
response.execute()
except HttpError as exc_http:
_msg = "Failed adding subscription to '{}', HTTP Error {}".format(
channel_id, exc_http.resp.status)
logger.error("{}: {}".format(_msg, exc_http.content),
exc_info=exc_http)
raise exc_http
except Exception as exc:
_msg = "Unhandled exception occurred when adding subscription to '{}'".format(
channel_id)
logger.critical("{} | response={}".format(_msg, response.__dict__),
exc_info=exc)
raise exc
# FIXME: Somewhat duplicate code of get_remote_subscriptions, move to own function -- START
# Get ID of uploads playlist
channel_uploads_playlist_id = response['items'][0]['contentDetails'][
'relatedPlaylists']['uploads']
channel = Channel(channel_id, channel_uploads_playlist_id)
db_channel = engine_execute_first(get_channel_by_id_stmt(channel))
if db_channel:
engine_execute(update_channel_from_remote(channel))
# subs.append(channel)
else:
# TODO: change to sqlalchemy core stmt
create_logger(__name__ + ".subscriptions").info(
"Added channel {} - {}".format(channel.title, channel.id))
db_session.add(channel)
# subs.append(channel)
db_session.commit()
# FIXME: Somewhat duplicate code of get_remote_subscriptions, move to own function -- END
logger.info("Added subscription: {} / {}".format(
channel_id, response['snippet']['title']))
return response
def get_remote_subscriptions(youtube_oauth):
"""
Get a list of the authenticated user's subscriptions.
:param youtube_oauth:
:param stats:
:param info: debug lite
:param debug:
:param traverse_pages:
:return: [total, subs, statistics]
"""
subscription_list_request = youtube_oauth.subscriptions().list(
part='snippet', mine=True, maxResults=50)
subs = []
# Retrieve the list of subscribed channels for authenticated user's channel.
update_stmts = []
channel_ids = []
while subscription_list_request:
subscription_list_response = subscription_list_request.execute()
# Grab information about each subscription page
for page in tqdm(subscription_list_response['items'],
desc="Adding and updating channels by page",
disable=read_config('Debug', 'disable_tqdm')):
# Get channel
channel_response = channels_list(
youtube_oauth,
part='contentDetails',
id=page['snippet']['resourceId']['channelId'])
# Get ID of uploads playlist
channel_uploads_playlist_id = channel_response['items'][0][
'contentDetails']['relatedPlaylists']['uploads']
channel = Channel(page['snippet'], channel_uploads_playlist_id)
db_channel = engine_execute_first(get_channel_by_id_stmt(channel))
if db_channel:
engine_execute(update_channel_from_remote(channel))
subs.append(channel)
else:
# TODO: change to sqlalchemy core stmt
create_logger(__name__ + ".subscriptions").info(
"Added channel {} - {}".format(channel.title, channel.id))
db_session.add(channel)
subs.append(channel)
channel_ids.append(channel.id)
subscription_list_request = youtube_oauth.playlistItems().list_next(
subscription_list_request, subscription_list_response)
delete_sub_not_in_list(channel_ids)
db_session.commit()
return subs
def __init__(self, parent, root, main_model: MainModel):
super(GridView, self).__init__(parent=parent)
self.logger = create_logger(__name__)
self.setMinimumSize(0, 0)
self.parent = parent
self.root = root # MainWindow
self.clipboard = self.root.clipboard
self.status_bar = self.root.status_bar
self.buffer = 10
self.bar_correction = 0
self.main_model = main_model
self.pref_tile_height = read_config('Gui', 'tile_pref_height')
self.pref_tile_width = read_config('Gui', 'tile_pref_width')
self.q_labels = {}
self.grid = QGridLayout()
self.items_x = read_config('Gui', 'grid_view_x')
self.items_y = read_config('Gui', 'grid_view_y')
self.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Preferred)
self.grid.setContentsMargins(5, 5, 0, 0)
self.grid.setSpacing(2)
self.grid.setAlignment(Qt.AlignTop)
self.setLayout(self.grid)
self.setAutoFillBackground(True)
self.set_bgcolor()
self.main_model.grid_view_listener.redrawVideos.connect(self.redraw_videos)
def __init__(self, sane_parent, main_model):
super(DbStateIcon, self).__init__(parent=sane_parent)
self.logger = create_logger(__name__)
self.sane_parent = sane_parent
self.main_model = main_model
self.base_icon = QPixmap(os.path.join(ICONS_PATH, 'database.png'))
self.base_icon = self.base_icon.scaled(
QSize(self.height(), self.height()), Qt.KeepAspectRatio,
Qt.SmoothTransformation)
self.full_icon = self.base_icon.copy()
self.sane_painter = QPainter(self.full_icon)
self.default_dot = QPixmap(os.path.join(
ICONS_PATH, 'default_dot.png')).scaled(
QSize(self.height() * 0.3,
self.height() * 0.3), Qt.KeepAspectRatio,
Qt.SmoothTransformation)
self.read_dot = QPixmap(os.path.join(ICONS_PATH,
'blue_dot.png')).scaled(
QSize(self.height() * 0.3,
self.height() * 0.3),
Qt.KeepAspectRatio,
Qt.SmoothTransformation)
self.write_dot = QPixmap(os.path.join(
ICONS_PATH, 'green_dot.png')).scaled(
QSize(self.height() * 0.3,
self.height() * 0.3), Qt.KeepAspectRatio,
Qt.SmoothTransformation)
self.setPixmap(self.full_icon)
DatabaseListener.static_instance.dbStateChanged.connect(
self.change_state)
def download_thumbnails_threaded(input_vid_list, progress_listener=None):
logger = create_logger(__name__ + ".download_thumbnails_threaded")
thread_list = []
thread_limit = int(read_config('Threading', 'img_threads'))
force_dl_best = read_config('Thumbnails', 'force_download_best')
if progress_listener:
progress_listener.setText.emit('Downloading thumbnails')
vid_list = []
chunk_size = math.ceil(len(input_vid_list) / thread_limit)
for i in range(0, len(input_vid_list), max(chunk_size, 1)):
vid_list.append(input_vid_list[i:i + chunk_size])
counter = 0
logger.info(
"Starting thumbnail download threads for {} videos in {} threads".
format(len(input_vid_list), len(vid_list)))
for vid_list_chunk in tqdm(vid_list,
desc="Starting thumbnail threads",
disable=read_config('Debug', 'disable_tqdm')):
t = DownloadThumbnail(vid_list_chunk,
force_dl_best=force_dl_best,
progress_listener=progress_listener)
thread_list.append(t)
t.start()
counter += 1
if progress_listener:
progress_listener.updateProgress.emit()
for t in tqdm(thread_list,
desc="Waiting on thumbnail threads",
disable=read_config('Debug', 'disable_tqdm')):
t.join()
def __init__(self, parent):
super(AboutView, self).__init__(parent)
self.logger = create_logger(__name__)
self.root = parent # MainWindow
self.clipboard = self.root.clipboard
self.status_bar = self.root.status_bar
self.init_ui()
def __init__(self, parent, root):
"""
A GUI Widget that reads and sets config.ini settings
:param parent:
:param clipboard:
:param status_bar:
"""
super(InputSuper, self).__init__(parent)
self.parent = parent
self.root = root # MainWindow
self.logger = create_logger(__name__)
self.clipboard = self.root.clipboard
self.status_bar = self.root.status_bar
self.offset = 0
self.section_count = 0
self.section_fontsize = 10
self.section_fontfamily = "Helvetica"
self.section_fontstyle = QFont.Black
self.layout = QGridLayout()
self.layout.setAlignment(Qt.AlignTop)
# self.layout = QVBoxLayout()
# Create tabs - Part 2: Set self layout and add tabs to it
self.setLayout(self.layout)
def __init__(self, parent):
QLabel.__init__(self, parent)
self.parent = parent
self.logger = create_logger(__name__ + ".ThumbnailTitle")
margins = self.parent.layout.getContentsMargins()
self.setFixedSize(self.parent.width() - margins[0] - margins[2],
(self.parent.height() - 4 * margins[3]) * 0.6)
def __init__(self, model):
super().__init__()
self.model = model
self.refreshVideos.connect(self.refresh_videos)
self.refreshSubs.connect(self.refresh_subs)
self.testChannels.connect(self.test_channels)
self.getSingleVideo.connect(self.get_single_video)
self.addYouTubeChannelSubscriptionById.connect(
self.add_youtube_channel_subscription_by_id)
self.addYouTubeChannelSubscriptionByUsername.connect(
self.add_youtube_channel_subscription_by_username)
self.logger = create_logger(__name__ + '.MainWindowListener')
def __init__(self, parent, root, main_model: MainModel):
super().__init__(parent, root, main_model)
self.logger = create_logger(__name__ + ".PlayView")
self.main_model.grid_view_listener.downloadedVideosChanged.connect(
self.videos_changed)
self.main_model.grid_view_listener.downloadedVideosUpdated.connect(
self.update_videos)
self.logger.debug("Init grid")
self.update_grid()
self.logger.info("Initialized PlayView")
def __init__(self, parent, vid_limit=40):
super(ListTiledView, self).__init__(parent)
self.logger = create_logger(__name__)
self.root = parent # MainWindow
self.eqlabels = []
self.vid_limit = vid_limit
self.clipboard = self.root.clipboard
self.status_bar = self.root.status_bar
# The layouts
main_layout = QVBoxLayout()
main_layout.setSpacing(10)
self.setLayout(main_layout)
header_layout = QHBoxLayout()
body_layout = QHBoxLayout()
main_layout.addLayout(header_layout)
main_layout.addLayout(body_layout)
description_layout = QVBoxLayout()
image_layout = QVBoxLayout()
body_layout.addLayout(image_layout)
body_layout.addLayout(description_layout)
video_title_layout = QHBoxLayout()
more_layout = QHBoxLayout()
published_layout = QHBoxLayout()
description_layout.addLayout(video_title_layout)
description_layout.addLayout(more_layout)
description_layout.addLayout(published_layout)
video_title = QLabel('video title')
view_count = QLabel('View count')
date = QLabel('date')
published = QLabel('more')
view_description = QLabel('view description')
image_label = QLabel('image')
icon_label = QLabel('icon')
title_label = QLabel('title')
header_layout.addWidget(icon_label)
header_layout.addWidget(title_label)
image_layout.addWidget(image_label)
video_title_layout.addWidget(video_title)
more_layout.addWidget(view_count)
more_layout.addWidget(date)
more_layout.addWidget(published)
description_layout.addWidget(view_description)
def __init__(self, video_list, finished_listeners=None):
"""
Init GetUploadsThread
:param thread_id:
:param channel:
:param info:
:param debug:
"""
self.logger = create_logger(__name__ + ".UpdateVideosThread")
threading.Thread.__init__(self)
self.video_list = video_list
self.finished_listeners = finished_listeners
self.db_id = 0
def __init__(self, model, progress_bar):
super().__init__()
self.model = model
self.progress_bar = progress_bar
self.progress_bar.setTextVisible(True)
self.progress_bar.setAlignment(Qt.AlignCenter)
self.setMaximum.connect(self.set_maximum)
self.setProgress.connect(self.set_progress)
self.updateProgress.connect(self.update_progress)
self.setText.connect(self.set_text)
self.resetBar.connect(self.reset_bar)
self.logger = create_logger(__name__ + '.ProgressBar')
def cli(no_gui, test_channels, update_watch_prio, set_watched_day):
logger = create_logger(__name__)
if no_gui:
run_print()
if update_watch_prio:
videos = db_session.query(Video).all()
watch_prio = read_config('Play', 'default_watch_prio')
logger.debug("Setting watch_prio {}, for: {} videos".format(watch_prio, len(videos)))
for video in videos:
video.watch_prio = watch_prio
db_session.commit()
return
if set_watched_day:
videos = db_session.query(Video).filter(or_(Video.downloaded == True, (Video.vid_path.is_(None)))).all()
for video in videos:
vid_age = datetime.datetime.utcnow() - video.date_published
if vid_age > datetime.timedelta(days=int(set_watched_day)):
logger.debug("Setting watched, {} - {} - {}".format(vid_age, video.title, video.__dict__))
video.watched = True
db_session.commit()
return
if test_channels:
run_channels_test()
else:
"""
PyQT raises and catches exceptions, but doesn't pass them along.
Instead it just exits with a status of 1 to show an exception was caught.
"""
# Back up the reference to the exceptionhook
sys._excepthook = sys.excepthook
def my_exception_hook(exctype, value, traceback):
# Ignore KeyboardInterrupt so a console python program can exit with Ctrl + C.
if issubclass(exctype, KeyboardInterrupt):
sys.__excepthook__(exctype, value, traceback)
return
# Log the exception with the logger
logger.critical("Intercepted Exception", exc_info=(exctype, value, traceback))
# Call the normal Exception hook after
sys._excepthook(exctype, value, traceback)
# sys.exit(1) # Alternatively, exit
# Set the exception hook to our wrapping function
sys.excepthook = my_exception_hook
run_with_gui()
def __init__(self,
app: QApplication,
main_model: MainModel,
dimensions=None,
position=None):
super().__init__()
self.logger = create_logger(__name__)
self.app = app
self.main_model = main_model
self.themes_list = THEMES_LIST
self.current_theme = None
self.current_theme_idx = 0
self.clipboard = QApplication.clipboard()
self.status_bar = self.statusBar()
self.menus = {}
if dimensions:
self.dimensions = dimensions
else:
self.dimensions = [770, 700]
self.position = position
self.central_widget = QStackedWidget()
self.setCentralWidget(self.central_widget)
self.grid_view = GridScrollArea(self, main_model)
self.play_view = GridScrollArea(self, main_model)
self.grid_view.set_view(SubFeedView(self.grid_view, self, main_model),
GRID_VIEW_ID)
self.play_view.set_view(PlayView(self.play_view, self, main_model),
PLAY_VIEW_ID)
self.download_view = DownloadScrollArea(self, main_model)
self.hotkeys_view = ConfigWindow(self)
self.hotkeys_view.setWidget(
HotkeysViewWidget(self.hotkeys_view,
self,
icon=QIcon(
os.path.join(ICONS_PATH, 'hotkeys.png'))))
self.config_view = ConfigViewTabs(
self, icon=QIcon(os.path.join(ICONS_PATH, 'preferences.png')))
self.list_detailed_view = ListDetailedView(self)
self.list_tiled_view = ListTiledView(self)
self.subs_view = SubscriptionsView(self)
self.about_view = AboutView(self)
self.init_ui()
def __init__(self, model):
super().__init__()
self.logger = create_logger(__name__ + '.YtDirListener')
self.model = model
self.newFile.connect(self.new_file)
self.manualCheck.connect(self.manual_check)
disable_dir_observer = read_config('Play', 'disable_dir_listener')
if not disable_dir_observer:
path = read_config('Play', 'yt_file_path', literal_eval=False)
event_handler = VidEventHandler(self)
self.observer = Observer()
self.observer.schedule(event_handler, path)
self.observer.start()
def __init__(self, videos, videos_limit):
super().__init__()
self.logger = create_logger(__name__)
self.videos_limit = videos_limit
self.downloaded_videos_limit = videos_limit
self.videos = videos
self.filtered_videos = []
self.downloaded_videos = []
self.download_progress_signals = []
self.logger.info("Creating listeners and threads")
self.grid_view_listener = GridViewListener(self)
self.grid_thread = QThread()
self.grid_thread.setObjectName('grid_thread')
self.grid_view_listener.moveToThread(self.grid_thread)
self.grid_thread.start()
self.database_listener = DatabaseListener(self)
self.db_thread = QThread()
self.db_thread.setObjectName('db_thread')
self.database_listener.moveToThread(self.db_thread)
self.db_thread.start()
self.main_window_listener = MainWindowListener(self)
self.main_w_thread = QThread()
self.main_w_thread.setObjectName('main_w_thread')
self.main_window_listener.moveToThread(self.main_w_thread)
self.main_w_thread.start()
self.download_handler = DownloadHandler(self)
self.download_thread = QThread()
self.download_thread.setObjectName('download_thread')
self.download_handler.moveToThread(self.download_thread)
self.download_thread.start()
if read_config("Play", "yt_file_path", literal_eval=False):
self.yt_dir_listener = YtDirListener(self)
self.yt_dir_thread = QThread()
self.yt_dir_thread.setObjectName('yt_dir_thread')
self.yt_dir_listener.moveToThread(self.yt_dir_thread)
self.yt_dir_thread.start()
else:
self.logger.warning(
"No youtube file path provided, directory listener is disabled"
)
def __init__(self, model):
super().__init__()
self.model = model
self.logger = create_logger(__name__ + '.GridViewListener')
GridViewListener.static_self = self
static_listeners.STATIC_GRID_VIEW_LISTENER = self
self.tileDownloaded.connect(self.tile_downloaded)
self.tileWatched.connect(self.tile_watched)
self.tileDiscarded.connect(self.tile_discarded)
self.downloadFinished.connect(self.download_finished)
self.downloadedVideosChangedinDB.connect(self.download_finished_in_db)
self.updateGridViewFromDb.connect(self.update_grid_view_from_db)
self.updateFromDb.connect(self.update_from_db)
self.scrollReachedEndGrid.connect(self.scroll_reached_end_grid)
self.scrollReachedEndPlay.connect(self.scroll_reached_end_play)
self.thumbnailDownload.connect(self.thumbnail_download)
self.decreaseWatchPrio.connect(self.decrease_watch_prio)
class UpdateVideo(threading.Thread):
logger = create_logger(__name__ + ".UpdateVideo")
def __init__(self,
video_d,
update_existing=False,
finished_listeners=None):
"""
Init GetUploadsThread
:param video_d:
"""
threading.Thread.__init__(self)
self.finished_listeners = finished_listeners
self.video_d = video_d
self.update_existing = update_existing
self.db_id = 0
# TODO: Handle failed requests
def run(self):
self.db_id = threading.get_ident()
DatabaseListener.static_instance.startWrite.emit(self.db_id)
"""
Override threading.Thread.run() with its own code
:return:
"""
# self.logger.debug("Run")
# start = default_timer()
lock.acquire()
stmt = get_video_by_vidd_stmt(Video.video_d_to_video(self.video_d))
db_video = engine.execute(stmt).first()
if db_video:
if self.update_existing:
engine.execute(update_video_statement_full(self.video_d))
else:
pass
else:
engine.execute(Video.__table__.insert(), insert_item(self.video_d))
# print('Updated: {}'.format(self.video_d.title))
lock.release()
if self.finished_listeners:
for listener in self.finished_listeners:
listener.emit()
DatabaseListener.static_instance.finishWrite.emit(self.db_id)
def __init__(self, parent, *args, **kwargs):
super(DownloadView, self).__init__(parent, *args, **kwargs)
self.sane_paret = parent
self.logger = create_logger(__name__)
self.sane_layout = QVBoxLayout()
self.sane_layout.setAlignment(Qt.AlignTop)
self.buttons_tile = ButtonsTile(self)
self.sane_layout.addWidget(self.buttons_tile)
self.setLayout(self.sane_layout)
self.widgets = []
DownloadHandler.static_self.dbDownloadTiles.connect(
self.update_widgets)
DownloadHandler.static_self.newYTDLDownlaod.connect(self.new_download)
DownloadHandler.static_self.loadDBDownloadTiles.emit()
def __init__(self, thread_id, youtube, channel_id, playlist_id, videos, list_pages, search_pages,
deep_search=False):
"""
Init GetUploadsThread
:param thread_id:
:param channel:
:param info:
:param debug:
"""
threading.Thread.__init__(self)
self.logger = create_logger(__name__)
self.videos = videos
self.thread_id = thread_id
self.job_done = False
self.youtube = youtube
self.search_pages = search_pages
self.list_pages = list_pages
self.channel_id = channel_id
self.playlist_id = playlist_id
self.deep_search = deep_search
self.exc = None
def __init__(self, parent, vid_limit=40, headers=None):
"""
ListDetailedView, a table representation of relevant parts of the Subscription Feed
:param clipboard:
:param status_bar:
:param vid_limit:
"""
super(ListDetailedView, self).__init__(parent)
self.logger = create_logger(__name__)
self.config_file = None
self.root = parent # MainWindow
self.clipboard = self.root.clipboard
self.status_bar = self.root.status_bar
self.vid_limit = vid_limit
if headers:
self.headers = headers
else:
self.headers = [
'Channel', 'Title', 'URL', 'Published', 'Description',
'Missed?', 'Downloaded?', 'Discarded?'
]
self.init_ui()
def __init__(self, parent, headers=None):
"""
SubscriptionsView, a table representation of relevant parts of the Subscriptions list
:param parent:
:param clipboard:
:param status_bar:
:param headers:
"""
super(SubscriptionsView, self).__init__(parent)
self.logger = create_logger(__name__)
self.root = parent # MainWindow
self.clipboard = self.root.clipboard
self.status_bar = self.root.status_bar
if headers:
self.headers = headers
else:
self.headers = [
'ID', 'Title', 'Uploaded Videos Playlist ID', 'Description',
'Subscribed on YouTube?', 'Force subscription?'
]
self.init_ui()
import os
import json
from google_auth_oauthlib.flow import InstalledAppFlow
from googleapiclient.discovery import build
from sane_yt_subfeed.log_handler import create_logger
# FIXME: module level logger not suggested: https://fangpenlin.com/posts/2012/08/26/good-logging-practice-in-python/
logger = create_logger(__name__)
OS_PATH = os.path.dirname(__file__)
CLIENT_SECRETS_FILE = os.path.join(OS_PATH, 'resources', 'client_secret.json')
KEYS_FILE = os.path.join(OS_PATH, 'resources', 'keys.json')
SCOPES = ['https://www.googleapis.com/auth/youtube.readonly']
API_SERVICE_NAME = 'youtube'
API_VERSION = 'v3'
def get_api_key():
with open(KEYS_FILE) as keys_json_data_file:
keys = json.load(keys_json_data_file)
return keys['api_key']
def youtube_auth_oauth():
"""
Authorize the request and store authorization credentials.
:return:
def __init__(self, parent, video, vid_id, clipboard, status_bar):
super().__init__(parent, video, vid_id, clipboard, status_bar)
self.logger = create_logger(__name__)
class UpdateVideosThread(threading.Thread):
logger = create_logger(__name__ + ".UpdateVideosThread")
def __init__(self,
video_list,
update_existing=False,
uniques_check=True,
finished_listeners=None,
only_thumbnails=False):
"""
Init GetUploadsThread
:param thread_id:
:param channel:
:param info:
:param debug:
"""
threading.Thread.__init__(self)
self.video_list = video_list
self.update_existing = update_existing
self.uniques_check = uniques_check
self.finished_listeners = finished_listeners
self.db_id = 0
self.only_thumbnails = only_thumbnails
# TODO: Handle failed requests
def run(self):
"""
Override threading.Thread.run() with its own code
:return:
"""
self.db_id = threading.get_ident()
if self.uniques_check:
self.video_list = check_for_unique(self.video_list)
items_to_add = []
items_to_update = []
# bulk_items_to_add = []
lock.acquire()
self.logger.debug("Thread {} - Acquired lock".format(
self.db_id, len(items_to_update)))
DatabaseListener.static_instance.startRead.emit(self.db_id)
select_step = 500
for i in range(0, len(self.video_list), select_step):
videos_bulk = self.video_list[i:i + select_step]
video_ids_bulk = set(video.video_id for video in videos_bulk)
stmt = get_video_ids_by_video_ids_stmt(video_ids_bulk)
db_videos = Video.to_video_ds(engine.execute(stmt))
db_videos_ids = set(video.video_id for video in db_videos)
items_to_add.extend(
insert_item(video) for video in videos_bulk
if video.video_id not in db_videos_ids)
if self.update_existing:
items_to_update.extend(video for video in videos_bulk
if video.video_id in db_videos_ids)
DatabaseListener.static_instance.finishRead.emit(self.db_id)
DatabaseListener.static_instance.startWrite.emit(self.db_id)
step = 1000
if len(items_to_add) > 0:
self.logger.debug("Thread {} - inserting {} new videos".format(
self.db_id, len(items_to_add)))
for i in range(0, len(items_to_add), step):
engine.execute(Video.__table__.insert(),
items_to_add[i:i + step])
if len(items_to_update) > 0:
self.logger.debug("Thread {} - updating {} items".format(
self.db_id, len(items_to_update)))
update_list = []
# FIXME: add step to update
for item in items_to_update:
item_dict = item.__dict__
item_dict["_video_id"] = item.video_id
update_list.append(item_dict)
engine.execute(update_video_stmt(), update_list)
DatabaseListener.static_instance.finishWrite.emit(self.db_id)
lock.release()
if self.finished_listeners:
for listener in self.finished_listeners:
listener.emit()
def __init__(self):
super().__init__()
self.logger = create_logger(__name__)
from sane_yt_subfeed.database.models import Channel
from sane_yt_subfeed.database.orm import db_session, engine
from sane_yt_subfeed.database.write_operations import engine_execute_first, engine_execute, delete_sub_not_in_list
from sane_yt_subfeed.database.engine_statements import update_channel_from_remote, get_channel_by_id_stmt
from sane_yt_subfeed.log_handler import create_logger
from sane_yt_subfeed.pickle_handler import load_sub_list, load_youtube, dump_youtube, dump_sub_list
from sane_yt_subfeed.print_functions import remove_empty_kwargs
import datetime
YOUTUBE_URL = "https://www.youtube.com/"
YOUTUBE_PARM_VIDEO = "watch?v="
YOUTUBE_PARM_PLIST = "playlist?list ="
YT_VIDEO_URL = YOUTUBE_URL + YOUTUBE_PARM_VIDEO
# FIXME: module level logger not suggested: https://fangpenlin.com/posts/2012/08/26/good-logging-practice-in-python/
logger = create_logger(__name__)
# FIXME: integrate with socket system
logger_list_search = create_logger("youtube_requests: list()/search()",
logfile="debug_list_search.log")
def get_channel_uploads_playlist_id(youtube_key, channel_id):
"""
Get a channel's "Uploaded videos" playlist ID, given channel ID.
:param youtube_key:
:param channel_id:
:return: list_uploaded_videos(channel_uploads_playlist_id, debug=debug, limit=limit)
"""
# Get channel
channel = channels_list(youtube_key, part='contentDetails',
