def check_videos_exist(self, media_data_obj):
"""Called by self.tidy_directory().
Checks all child videos of the specified media data object. If the
video should exist, but doesn't (or vice-versa), modify the media.Video
object's IVs accordingly.
Args:
media_data_obj (media.Channel, media.Playlist or media.Folder):
The media data object whose directory must be tidied up
"""
if DEBUG_FUNC_FLAG:
utils.debug_time('top 642 check_videos_exist')
for video_obj in media_data_obj.compile_all_videos( [] ):
if video_obj.file_name is not None:
video_path = video_obj.get_actual_path(self.app_obj)
if not video_obj.dl_flag \
and os.path.isfile(video_path):
# File exists, but is marked as not downloaded
self.app_obj.mark_video_downloaded(
video_obj,
True, # Video is downloaded
True, # ...but don't mark it as new
)
self.video_exist_count += 1
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _(
'Video file exists:',
) + ' \'' + video_obj.name + '\'',
)
elif video_obj.dl_flag \
and not os.path.isfile(video_path):
# File doesn't exist, but is marked as downloaded
self.app_obj.mark_video_downloaded(
video_obj,
False, # Video is not downloaded
)
self.video_no_exist_count += 1
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _(
'Video file doesn\'t exist:',
) + ' \'' + video_obj.name + '\'',
)
def setup_finish_page_default(self):
"""Called by self.setup_page().
Sets up the widget layout for a page, for all operating systems except
MS Windows.
"""
self.add_image(
self.app_obj.main_win_obj.icon_dict['ready_icon'],
0,
0,
1,
1,
)
self.add_label(
'<span font_size="large" font_weight="bold">' \
+ _('All done!') + '</span>',
0, 1, 1, 1,
)
# (Empty label for spacing)
self.add_empty_label(0, 2, 1, 1)
self.add_label(
'<span font_size="large" style="italic">' \
+ 'It is stronly recommended that you install FFmpeg.</span>',
0, 3, 1, 1,
)
self.add_label(
'<span font_size="large" style="italic">' \
+ utils.tidy_up_long_string(
_(
'Without FFmpeg, Tartube cannot download high-resolution' \
+ ' videos, and cannot display video thumbnails from' \
+ ' YouTube.',
),
60,
) + '</span>',
0, 4, 1, 1,
)
# (Empty label for spacing)
self.add_empty_label(0, 5, 1, 1)
self.add_label(
'<span font_size="large" style="italic">' \
+ _('Click the <b>OK</b> button to start Tartube!') \
+ '</span>',
0, 6, 1, 1,
)
def call_moviepy(self, video_obj, video_path):
"""Called by thread inside self.check_video_corrupt().
When we call moviepy.editor.VideoFileClip() on a corrupted video file,
moviepy freezes indefinitely.
This function is called inside a thread, so a timeout of (by default)
ten seconds can be applied.
Args:
video_obj (media.Video): The video object being updated
video_path (str): The path to the video file itself
"""
try:
clip = moviepy.editor.VideoFileClip(video_path)
except:
self.video_corrupt_count += 1
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Video file might be corrupt:') + ' \'' \
+ video_obj.name + '\'',
)
def tidy_directory(self, media_data_obj):
"""Called by self.run().
Tidy up the directory of a single channel, playlist or folder.
Args:
media_data_obj (media.Channel, media.Playlist or media.Folder):
The media data object whose directory must be tidied up
"""
if DEBUG_FUNC_FLAG:
utils.debug_time('top 502 tidy_directory')
# Update the main window's progress bar
self.job_count += 1
GObject.timeout_add(
0,
self.app_obj.main_win_obj.update_progress_bar,
media_data_obj.name,
self.job_count,
self.job_total,
)
media_type = media_data_obj.get_type()
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
_('Checking:') + ' \'' + media_data_obj.name + '\'',
)
if self.corrupt_flag:
self.check_video_corrupt(media_data_obj)
if self.exist_flag:
self.check_videos_exist(media_data_obj)
if self.del_video_flag:
self.delete_video(media_data_obj)
if self.del_descrip_flag:
self.delete_descrip(media_data_obj)
if self.del_json_flag:
self.delete_json(media_data_obj)
if self.del_xml_flag:
self.delete_xml(media_data_obj)
if self.del_thumb_flag:
self.delete_thumb(media_data_obj)
if self.del_webp_flag:
self.delete_webp(media_data_obj)
if self.del_archive_flag:
self.delete_archive(media_data_obj)
def setup_page(self):
"""Called initially by self.setup(), then by .on_button_next_clicked()
or .on_button_prev_clicked().
Sets up the page specified by self.current_page.
"""
index = self.current_page
page_func = self.page_list[self.current_page]
if page_func is None:
# Emergency fallback
index = 0
page_func = self.page_list[0]
if len(self.page_list) <= 1:
self.next_button.set_sensitive(False)
self.prev_button.set_sensitive(False)
elif index == 0:
self.next_button.set_sensitive(True)
self.prev_button.set_sensitive(False)
else:
self.next_button.set_sensitive(True)
self.prev_button.set_sensitive(True)
if index >= len(self.page_list) - 1:
self.next_button.set_label(_('OK'))
else:
self.next_button.set_label(_('Next'))
self.next_button.get_child().set_width_chars(10)
# Replace the inner grid...
self.vbox.remove(self.inner_grid)
self.inner_grid = Gtk.Grid()
self.vbox.pack_start(self.inner_grid, True, False, 0)
self.inner_grid.set_row_spacing(self.spacing_size)
self.inner_grid.set_column_spacing(self.spacing_size)
# ...and then refill it, with the widget layout for the new page
method = getattr(self, page_func)
method()
self.show_all()
def setup_finish_page_mswin(self):
"""Called by self.setup_page().
Sets up the widget layout for a page, shown only on MS Windows.
"""
self.add_image(
self.app_obj.main_win_obj.icon_dict['ready_icon'],
0,
0,
1,
1,
)
self.add_label(
'<span font_size="large" font_weight="bold">' \
+ _('All done!') + '</span>',
0, 1, 1, 1,
)
# (Empty label for spacing)
self.add_empty_label(0, 2, 1, 1)
self.add_label(
'<span font_size="large" style="italic">' \
+ utils.tidy_up_long_string(
_(
'If you need to re-install or update the downloader or' \
+ ' FFmpeg, you can do it from the main window\'s menu.',
),
60,
) + '</span>',
0, 3, 1, 1,
)
# (Empty label for spacing)
self.add_empty_label(0, 4, 1, 1)
self.add_label(
'<span font_size="large" style="italic">' \
+ _('Click the <b>OK</b> button to start Tartube!') \
+ '</span>',
0, 5, 1, 1,
)
def run(self):
"""Called as a result of self.__init__().
Calls FFmpegManager.run_ffmpeg for every media.Video object in the
list.
Then informs the main application that the process operation is
complete.
"""
# Show information about the process operation in the Output Tab
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
_('Starting process operation'),
)
# Process each video in turn
while self.running_flag and self.video_list:
self.process_video(self.video_list.pop(0))
# Pause a moment, before the next iteration of the loop (don't want
# to hog resources)
time.sleep(self.sleep_time)
# Operation complete. Set the stop time
self.stop_time = int(time.time())
# Show a confirmation in the Output Tab
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
_('Process operation finished'),
)
# Let the timer run for a few more seconds to prevent Gtk errors (for
# systems with Gtk < 3.24)
GObject.timeout_add(
0,
self.app_obj.process_manager_halt_timer,
)
def convert_next_button(self):
"""Can be called by anything.
Converts the 'Next' to an 'OK' button, and sensitises it.
Should usually be called from the last page, when the code is ready to
let the window finish the wizard.
"""
self.next_button.set_label(_('Finish'))
self.next_button.get_child().set_width_chars(10)
self.next_button.set_sensitive(True)
def on_button_choose_folder_clicked(self, button, label):
"""Called from a callback in self.setup_db_page().
Opens a file chooser dialogue, so the user can set the location of
Tartube's data directory.
Args:
button (Gtk.Button): The widget clicked
label (Gtk.Label): Once set, the path to the directory is displayed
in this label
"""
if not self.mswin_flag:
title = _('Select Tartube\'s data directory')
else:
title = _('Select Tartube\'s data folder')
dialogue_win = self.app_obj.dialogue_manager_obj.show_file_chooser(
title,
self,
Gtk.FileChooserAction.SELECT_FOLDER,
)
# Get the user's response
response = dialogue_win.run()
if response == Gtk.ResponseType.OK:
self.data_dir = dialogue_win.get_filename()
label.set_markup(
'<span font_size="large" font_weight="bold">' + self.data_dir \
+ '</span>',
)
# Data directory set, so re-enable the Next button
self.next_button.set_sensitive(True)
dialogue_win.destroy()
def setup_button_strip(self):
"""Called by self.setup().
Creates a strip of buttons at the bottom of the window: a 'cancel'
button on the left, and 'next'/'previous' buttons on the right.
The window is closed by using the 'cancel' button, or by clicking the
'next' button on the last page.
"""
hbox = Gtk.HBox()
self.grid.attach(hbox, 0, 1, 1, 1)
# 'Cancel' button
self.cancel_button = Gtk.Button(_('Cancel'))
hbox.pack_start(self.cancel_button, False, False, 0)
self.cancel_button.get_child().set_width_chars(10)
self.cancel_button.set_tooltip_text(
_('Close this window without completing it'), )
self.cancel_button.connect('clicked', self.on_button_cancel_clicked)
# 'Next' button
self.next_button = Gtk.Button(_('Next'))
hbox.pack_end(self.next_button, False, False, 0)
self.next_button.get_child().set_width_chars(10)
self.next_button.set_tooltip_text(_('Go to the next page'))
self.next_button.connect('clicked', self.on_button_next_clicked)
# 'Previous' button
self.prev_button = Gtk.Button(_('Previous'))
hbox.pack_end(self.prev_button, False, False, self.spacing_size)
self.prev_button.get_child().set_width_chars(10)
self.prev_button.set_tooltip_text(_('Go to the previous page'))
self.prev_button.connect('clicked', self.on_button_prev_clicked)
def create_child_process(self, cmd_list):
"""Called by self.install_ffmpeg() or .install_ytdl().
Based on code from downloads.VideoDownloader.create_child_process().
Executes the system command, creating a new child process which
executes youtube-dl.
Args:
cmd_list (list): Python list that contains the command to execute.
"""
if DEBUG_FUNC_FLAG:
utils.debug_time('uop 167 create_child_process')
info = preexec = None
if os.name == 'nt':
# Hide the child process window that MS Windows helpfully creates
# for us
info = subprocess.STARTUPINFO()
info.dwFlags |= subprocess.STARTF_USESHOWWINDOW
else:
# Make this child process the process group leader, so that we can
# later kill the whole process group with os.killpg
preexec = os.setsid
try:
self.child_process = subprocess.Popen(
cmd_list,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
preexec_fn=preexec,
startupinfo=info,
)
except (ValueError, OSError) as error:
# (The code in self.run() will spot that the child process did not
# start)
self.stderr_list.append(_('Child process did not start'))
def run(self):
"""Called as a result of self.__init__().
Compiles a list of media data objects (channels, playlists and folders)
to tidy up. If self.init_obj is not set, only that channel/playlist/
folder (and its child channels/playlists/folders) are tidied up;
otherwise the whole data directory is tidied up.
Then calls self.tidy_directory() for each item in the list.
Finally informs the main application that the tidy operation is
complete.
"""
# Show information about the tidy operation in the Output Tab
if not self.init_obj:
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
_('Starting tidy operation, tidying up whole data directory'),
)
else:
media_type = self.init_obj.get_type()
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
_('Starting tidy operation, tidying up \'{0}\'').format(
self.init_obj.name, ))
if self.corrupt_flag:
text = _('YES')
else:
text = _('NO')
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Check videos are not corrupted:') + ' ' + text,
)
if self.corrupt_flag:
if self.del_corrupt_flag:
text = _('YES')
else:
text = _('NO')
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Delete corrupted videos:') + ' ' + text,
)
if self.exist_flag:
text = _('YES')
else:
text = _('NO')
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Check videos do/don\'t exist:') + ' ' + text,
)
if self.del_video_flag:
text = _('YES')
else:
text = _('NO')
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Delete all video files:') + ' ' + text,
)
if self.del_video_flag:
if self.del_others_flag:
text = _('YES')
else:
text = _('NO')
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Delete other video/audio files:') + ' ' + text,
)
if self.del_archive_flag:
text = _('YES')
else:
text = _('NO')
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Delete downloader archive files:') + ' ' + text,
)
if self.move_thumb_flag:
text = _('YES')
else:
text = _('NO')
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Move thumbnails into own folder:') + ' ' + text,
)
if self.del_thumb_flag:
text = _('YES')
else:
text = _('NO')
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Delete all thumbnail files:') + ' ' + text,
)
if self.convert_webp_flag:
text = _('YES')
else:
text = _('NO')
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Convert .webp thumbnails to .jpg:') + ' ' + text,
)
if self.move_data_flag:
text = _('YES')
else:
text = _('NO')
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Move other metadata files into own folder:') \
+ ' ' + text,
)
if self.del_descrip_flag:
text = _('YES')
else:
text = _('NO')
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Delete all description files:') + ' ' + text,
)
if self.del_json_flag:
text = _('YES')
else:
text = _('NO')
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Delete all metadata (JSON) files:') + ' ' + text,
)
if self.del_xml_flag:
text = _('YES')
else:
text = _('NO')
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Delete all annotation files:') + ' ' + text,
)
# Compile a list of channels, playlists and folders to tidy up (each
# one has their own sub-directory inside Tartube's data directory)
obj_list = []
if self.init_obj:
# Add this channel/playlist/folder, and any child channels/
# playlists/folders (but not videos, obviously)
obj_list = self.init_obj.compile_all_containers(obj_list)
else:
# Add all channels/playlists/folders in the database
for dbid in list(self.app_obj.media_name_dict.values()):
obj = self.app_obj.media_reg_dict[dbid]
# Don't add private folders
if not isinstance(obj, media.Folder) or not obj.priv_flag:
obj_list.append(obj)
self.job_total = len(obj_list)
# Check each sub-directory in turn, updating the media data registry
# as we go
while self.running_flag and obj_list:
self.tidy_directory(obj_list.pop(0))
# Pause a moment, before the next iteration of the loop (don't want
# to hog resources)
time.sleep(self.sleep_time)
# Operation complete. Set the stop time
self.stop_time = int(time.time())
# Show a confirmation in the Output Tab
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
_('Tidy operation finished'),
)
if self.corrupt_flag:
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Corrupted videos found:') + ' ' \
+ str(self.video_corrupt_count),
)
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Corrupted videos deleted:') + ' ' \
+ str(self.video_corrupt_deleted_count),
)
if self.exist_flag:
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('New video files detected:') + ' ' \
+ str(self.video_exist_count),
)
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Missing video files detected:') + ' ' \
+ str(self.video_no_exist_count),
)
if self.del_video_flag:
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Non-corrupted video files deleted:') + ' ' \
+ str(self.video_deleted_count),
)
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Other video/audio files deleted:') + ' ' \
+ str(self.other_deleted_count),
)
if self.del_archive_flag:
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Downloader archive files deleted:') + ' ' \
+ str(self.archive_deleted_count),
)
if self.move_thumb_flag:
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Thumbnail files moved:') + ' ' \
+ str(self.thumb_moved_count),
)
if self.del_thumb_flag:
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Thumbnail files deleted:') + ' ' \
+ str(self.thumb_deleted_count),
)
if self.convert_webp_flag:
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('.webp thumbnails converted to .jpg:') + ' ' \
+ str(self.webp_converted_count),
)
if self.move_data_flag:
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Other metadata files moved:') + ' ' \
+ str(self.data_moved_count),
)
if self.del_descrip_flag:
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Description files deleted:') + ' ' \
+ str(self.descrip_deleted_count),
)
if self.del_json_flag:
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Metadata (JSON) files deleted:') + ' ' \
+ str(self.json_deleted_count),
)
if self.del_xml_flag:
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Annotation files deleted:') + ' ' \
+ str(self.xml_deleted_count),
)
# Let the timer run for a few more seconds to prevent Gtk errors (for
# systems with Gtk < 3.24)
GObject.timeout_add(
0,
self.app_obj.tidy_manager_halt_timer,
)
def refresh_from_actual_destination(self, media_data_obj):
"""Called by self.run().
A modified version of self.refresh_from_default_destination().
Refreshes a single channel, playlist or folder, for which an
alternative download destination has been set.
If a file is missing in the alternative download destination, mark the
video object as not downloaded.
Don't check for unexpected video files in the alternative download
destination - we expect that they exist.
Args:
media_data_obj (media.Channel, media.Playlist or media.Folder):
The media data object to refresh
"""
if DEBUG_FUNC_FLAG:
utils.debug_time('rop 519 refresh_from_actual_destination')
# Update the main window's progress bar
self.job_count += 1
GObject.timeout_add(
0,
self.app_obj.main_win_obj.update_progress_bar,
media_data_obj.name,
self.job_count,
self.job_total,
)
# Keep a running total of matched videos for this channel, playlist or
# folder
local_total_count = 0
local_match_count = 0
# (No new media.Video objects are created)
local_missing_count = 0
# Update our progress in the Output Tab
if isinstance(media_data_obj, media.Channel):
string = _('Channel:') + ' '
elif isinstance(media_data_obj, media.Playlist):
string = _('Playlist:') + ' '
else:
string = _('Folder:') + ' '
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
string + media_data_obj.name,
)
# Get the alternative download destination
dir_path = media_data_obj.get_actual_dir(self.app_obj)
# Get a list of video files in that sub-directory
try:
init_list = os.listdir(dir_path)
except:
# Can't read the directory
return
# Now check each media.Video object, to see if the video file still
# exists (or not)
for child_obj in media_data_obj.child_list:
if isinstance(child_obj, media.Video) and child_obj.file_name:
this_file = child_obj.file_name + child_obj.file_ext
if child_obj.dl_flag and not this_file in init_list:
local_missing_count += 1
# Video doesn't exist, so mark it as not downloaded
self.app_obj.mark_video_downloaded(child_obj, False)
# Update our progress in the Output Tab (if required)
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Missing:') + ' ' + child_obj.name,
)
elif not child_obj.dl_flag and this_file in init_list:
self.video_total_count += 1
local_total_count += 1
self.video_match_count += 1
local_match_count += 1
# Video exists, so mark it as downloaded (but don't mark it
# as new)
self.app_obj.mark_video_downloaded(child_obj, True, True)
# Update our progress in the Output Tab (if required)
if self.app_obj.refresh_output_videos_flag:
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Match:') + ' ' + child_obj.name,
)
# Check complete, display totals
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Total videos:') + ' ' + str(local_total_count) \
+ ', ' + _('matched:') + ' ' + str(local_match_count) \
+ ', ' + _('missing:') + ' ' + str(local_missing_count),
)
def setup_db_page(self):
"""Called by self.setup_page().
Sets up the widget layout for a page.
"""
grid_width = 3
self.add_label(
'<span font_size="large" style="italic">' \
+ _('Tartube stores all of its downloads in one place.') \
+ '</span>',
0, 0, grid_width, 1,
)
# (Empty label for spacing)
self.add_empty_label(0, 1, grid_width, 1)
if not self.mswin_flag:
msg = utils.tidy_up_long_string(
_(
'If you don\'t want to use the default location, then' \
+ ' click <b>Choose</b> to select a different one.',
),
60,
)
msg2 = utils.tidy_up_long_string(
_(
'If you have used Tartube before, you can select an' \
+ ' existing directory, instead of creating a new one.',
),
60,
)
else:
msg = _('Click <b>Choose</b> to create a new folder.')
msg2 = utils.tidy_up_long_string(
_(
'If you have used Tartube before, you can select an' \
+ ' existing folder, instead of creating a new one.',
),
60,
)
self.add_label(
'<span font_size="large" style="italic">' + msg + '</span>',
0,
2,
grid_width,
1,
)
# (Empty label for spacing)
self.add_empty_label(0, 3, grid_width, 1)
self.add_label(
'<span font_size="large" style="italic">' + msg2 + '</span>',
0,
4,
grid_width,
1,
)
# (Empty label for spacing)
self.add_empty_label(0, 5, grid_width, 1)
button = Gtk.Button(_('Choose'))
self.inner_grid.attach(button, 1, 6, 1, 1)
# (Signal connect appears below)
if not self.mswin_flag:
button2 = Gtk.Button(_('Use default location'))
self.inner_grid.attach(button2, 1, 7, 1, 1)
# (Signal connect appears below)
# (Empty label for spacing)
self.add_empty_label(0, 8, grid_width, 1)
# The specified path appears here, after it has been selected
if self.data_dir is None:
label = self.add_label(
'',
0,
9,
grid_width,
1,
)
else:
label = self.add_label(
'<span font_size="large" font_weight="bold">' \
+ self.data_dir + '</span>',
0, 9, grid_width, 1,
)
# (Signal connects from above)
button.connect(
'clicked',
self.on_button_choose_folder_clicked,
label,
)
if not self.mswin_flag:
button2.connect(
'clicked',
self.on_button_default_folder_clicked,
label,
)
# Disable the Next button until a folder has been created/selected
if self.data_dir is None:
self.next_button.set_sensitive(False)
def install_ffmpeg(self):
"""Called by self.run().
A modified version of self.install_ytdl, that installs FFmpeg on an
MS Windows system.
Creates a child process to run the installation process.
Reads from the child process STDOUT and STDERR, and calls the main
application with the result of the update (success or failure).
"""
# Show information about the update operation in the Output Tab
self.install_ffmpeg_write_output(
_('Starting update operation, installing FFmpeg'), )
# Create a new child process to install either the 64-bit or 32-bit
# version of FFmpeg, as appropriate
if sys.maxsize <= 2147483647:
binary = 'mingw-w64-i686-ffmpeg'
else:
binary = 'mingw-w64-x86_64-ffmpeg'
self.create_child_process(['pacman', '-S', binary, '--noconfirm'], )
# Show the system command in the Output Tab
self.install_ffmpeg_write_output(
' '.join(['pacman', '-S', binary, '--noconfirm']),
True, # A system command, not a message
)
# So that we can read from the child process STDOUT and STDERR, attach
# a file descriptor to the PipeReader objects
if self.child_process is not None:
self.stdout_reader.attach_file_descriptor(
self.child_process.stdout, )
self.stderr_reader.attach_file_descriptor(
self.child_process.stderr, )
while self.is_child_process_alive():
# Read from the child process STDOUT, and convert into unicode for
# Python's convenience
while not self.stdout_queue.empty():
stdout = self.stdout_queue.get_nowait().rstrip()
stdout = stdout.decode('cp1252')
if stdout:
# Show command line output in the Output Tab (or wizard
# window textview)
self.install_ffmpeg_write_output(stdout)
# The child process has finished
while not self.stderr_queue.empty():
# Read from the child process STDERR queue (we don't need to read
# it in real time), and convert into unicode for python's
# convenience
stderr = self.stderr_queue.get_nowait().rstrip()
stderr = stderr.decode('cp1252')
# Ignore pacman warning messages, e.g. 'warning: dependency cycle
# detected:'
if stderr and not re.match('warning\:', stderr):
self.stderr_list.append(stderr)
# Show command line output in the Output Tab (or wizard window
# textview)
self.install_ffmpeg_write_output(stderr)
# (Generate our own error messages for debugging purposes, in certain
# situations)
if self.child_process is None:
self.stderr_list.append(_('FFmpeg installation did not start'))
elif self.child_process.returncode > 0:
self.stderr_list.append(
_('Child process exited with non-zero code: {}').format(
self.child_process.returncode, ))
# Operation complete. self.success_flag is checked by
# mainapp.TartubeApp.update_manager_finished
if not self.stderr_list:
self.success_flag = True
# Show a confirmation in the the Output Tab (or wizard window textview)
self.install_ffmpeg_write_output(_('Update operation finished'))
# Let the timer run for a few more seconds to prevent Gtk errors (for
# systems with Gtk < 3.24)
GObject.timeout_add(
0,
self.app_obj.update_manager_halt_timer,
)
def run(self):
"""Called as a result of self.__init__().
Creates a child process to run the youtube-dl system command.
Reads from the child process STDOUT and STDERR, and calls the main
application with the result of the process (success or failure).
"""
# Checking for a new release of Tartube doesn't involve any system
# commands or child processes, so it is handled by a separate
# function
if self.info_type == 'version':
return self.run_check_version()
# Show information about the info operation in the Output Tab
if self.info_type == 'test_ytdl':
msg = _(
'Starting info operation, testing downloader with specified' \
+ ' options',
)
else:
if self.info_type == 'formats':
msg = _(
'Starting info operation, fetching list of video/audio'\
+ ' formats for \'{0}\'',
).format(self.video_obj.name)
else:
msg = _(
'Starting info operation, fetching list of subtitles'\
+ ' for \'{0}\'',
).format(self.video_obj.name)
self.app_obj.main_win_obj.output_tab_write_stdout(1, msg)
# Convert a path beginning with ~ (not on MS Windows)
ytdl_path = self.app_obj.check_downloader(self.app_obj.ytdl_path)
if os.name != 'nt':
ytdl_path = re.sub('^\~', os.path.expanduser('~'), ytdl_path)
# Prepare the system command
if self.info_type == 'formats':
cmd_list = [
ytdl_path,
'--list-formats',
self.video_obj.source,
]
elif self.info_type == 'subs':
cmd_list = [
ytdl_path,
'--list-subs',
self.video_obj.source,
]
else:
if app_obj.ytdl_path_custom_flag:
cmd_list = ['python3'] + [ytdl_path]
else:
cmd_list = [ytdl_path]
if self.options_string is not None \
and self.options_string != '':
# Parse the string into a list. It was obtained from a
# Gtk.TextView, so it can contain newline and/or multiple
# whitepsace characters. Whitespace characters within
# double quotes "..." must be preserved
option_list = utils.parse_options(self.options_string)
for item in option_list:
cmd_list.append(item)
if self.url_string is not None \
and self.url_string != '':
cmd_list.append('-o')
cmd_list.append(
os.path.join(
self.app_obj.temp_test_dir,
'%(title)s.%(ext)s',
),
)
cmd_list.append(self.url_string)
# Create the new child process
self.create_child_process(cmd_list)
# Show the system command in the Output Tab
space = ' '
self.app_obj.main_win_obj.output_tab_write_system_cmd(
1,
space.join(cmd_list),
)
# So that we can read from the child process STDOUT and STDERR, attach
# a file descriptor to the PipeReader objects
if self.child_process is not None:
self.stdout_reader.attach_file_descriptor(
self.child_process.stdout,
)
self.stderr_reader.attach_file_descriptor(
self.child_process.stderr,
)
while self.is_child_process_alive():
# Read from the child process STDOUT, and convert into unicode for
# Python's convenience
while not self.stdout_queue.empty():
stdout = self.stdout_queue.get_nowait().rstrip()
if stdout:
if os.name == 'nt':
stdout = stdout.decode('cp1252')
else:
stdout = stdout.decode('utf-8')
self.output_list.append(stdout)
self.stdout_list.append(stdout)
# Show command line output in the Output Tab
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
stdout,
)
# The child process has finished
while not self.stderr_queue.empty():
# Read from the child process STDERR queue (we don't need to read
# it in real time), and convert into unicode for python's
# convenience
stderr = self.stderr_queue.get_nowait().rstrip()
if os.name == 'nt':
stderr = stderr.decode('cp1252')
else:
stderr = stderr.decode('utf-8')
if stderr:
# While testing youtube-dl, don't treat anything as an error
if self.info_type == 'test_ytdl':
self.stdout_list.append(stderr)
# When fetching subtitles from a video that has none, don't
# treat youtube-dl WARNING: messages as something that
# makes the info operation fail
elif self.info_type == 'subs':
if not re.match('WARNING\:', stderr):
self.stderr_list.append(stderr)
# When fetching formats, recognise all warnings as errors
else:
self.stderr_list.append(stderr)
# Show command line output in the Output Tab
self.app_obj.main_win_obj.output_tab_write_stderr(
1,
stderr,
)
# (Generate our own error messages for debugging purposes, in certain
# situations)
if self.child_process is None:
msg = _('System process did not start')
self.stderr_list.append(msg)
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
msg,
)
elif self.child_process.returncode > 0:
msg = _('Child process exited with non-zero code: {}').format(
self.child_process.returncode,
)
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
msg,
)
# Operation complete. self.success_flag is checked by
# mainapp.TartubeApp.info_manager_finished()
if not self.stderr_list:
self.success_flag = True
# Show a confirmation in the the Output Tab
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
_('Info operation finished'),
)
# Let the timer run for a few more seconds to prevent Gtk errors (for
# systems with Gtk < 3.24)
GObject.timeout_add(
0,
self.app_obj.info_manager_halt_timer,
)
def do_translate(config_flag=False):
"""Function called for the first time below, setting various values.
If mainapp.TartubeApp.load_config() changes the locale to something else,
called for a second time to update those values.
Args:
config_flag (bool): False for the initial call, True for the second
call from mainapp.TartubeApp.load_config()
"""
global FOLDER_ALL_VIDEOS, FOLDER_BOOKMARKS, FOLDER_FAVOURITE_VIDEOS, \
FOLDER_LIVESTREAMS, FOLDER_MISSING_VIDEOS, FOLDER_NEW_VIDEOS, \
FOLDER_WAITING_VIDEOS, FOLDER_TEMPORARY_VIDEOS, FOLDER_UNSORTED_VIDEOS
global YTDL_UPDATE_DICT
global MAIN_STAGE_QUEUED, MAIN_STAGE_ACTIVE, MAIN_STAGE_PAUSED, \
MAIN_STAGE_COMPLETED, MAIN_STAGE_ERROR, ACTIVE_STAGE_PRE_PROCESS, \
ACTIVE_STAGE_DOWNLOAD, ACTIVE_STAGE_POST_PROCESS, ACTIVE_STAGE_CHECKING, \
COMPLETED_STAGE_FINISHED, COMPLETED_STAGE_WARNING, \
COMPLETED_STAGE_ALREADY, ERROR_STAGE_ERROR, ERROR_STAGE_STOPPED, \
ERROR_STAGE_ABORT
global TIME_METRIC_TRANS_DICT
global FILE_OUTPUT_NAME_DICT, FILE_OUTPUT_CONVERT_DICT
global VIDEO_OPTION_LIST, VIDEO_OPTION_DICT
# System folder names
FOLDER_ALL_VIDEOS = _('All Videos')
FOLDER_BOOKMARKS = _('Bookmarks')
FOLDER_FAVOURITE_VIDEOS = _('Favourite Videos')
FOLDER_LIVESTREAMS = _('Livestreams')
FOLDER_MISSING_VIDEOS = _('Missing Videos')
FOLDER_NEW_VIDEOS = _('New Videos')
FOLDER_WAITING_VIDEOS = _('Waiting Videos')
FOLDER_TEMPORARY_VIDEOS = _('Temporary Videos')
FOLDER_UNSORTED_VIDEOS = _('Unsorted Videos')
# youtube-dl update shell commands
YTDL_UPDATE_DICT = {
'ytdl_update_default_path': _('Update using default youtube-dl path'),
'ytdl_update_local_path': _('Update using local youtube-dl path'),
'ytdl_update_pip': _('Update using pip'),
'ytdl_update_pip_omit_user':
_('Update using pip (omit --user option)'),
'ytdl_update_pip3': _('Update using pip3'),
'ytdl_update_pip3_omit_user':
_('Update using pip3 (omit --user option)'),
'ytdl_update_pip3_recommend': _('Update using pip3 (recommended)'),
'ytdl_update_pypi_path': _('Update using PyPI youtube-dl path'),
'ytdl_update_win_32': _('Windows 32-bit update (recommended)'),
'ytdl_update_win_64': _('Windows 64-bit update (recommended)'),
'ytdl_update_disabled': _('youtube-dl updates are disabled'),
}
# Download operation stages
MAIN_STAGE_QUEUED = _('Queued')
MAIN_STAGE_ACTIVE = _('Active')
MAIN_STAGE_PAUSED = _('Paused') # (not actually used)
MAIN_STAGE_COMPLETED = _('Completed') # (not actually used)
MAIN_STAGE_ERROR = _('Error')
# Sub-stages of the 'Active' stage
ACTIVE_STAGE_PRE_PROCESS = _('Pre-processing')
ACTIVE_STAGE_DOWNLOAD = _('Downloading')
ACTIVE_STAGE_POST_PROCESS = _('Post-processing')
ACTIVE_STAGE_CHECKING = _('Checking')
# Sub-stages of the 'Completed' stage
COMPLETED_STAGE_FINISHED = _('Finished')
COMPLETED_STAGE_WARNING = _('Warning')
COMPLETED_STAGE_ALREADY = _('Already downloaded')
# Sub-stages of the 'Error' stage
ERROR_STAGE_ERROR = _('Error') # (not actually used)
ERROR_STAGE_STOPPED = _('Stopped')
ERROR_STAGE_ABORT = _('Filesize abort')
if config_flag:
for key in TIME_METRIC_TRANS_DICT:
TIME_METRIC_TRANS_DICT[key] = _(key)
# File output templates use a combination of English words, each of
# which must be translated
translate_note = _(
'TRANSLATOR\'S NOTE: ID refers to a video\'s unique ID on the' \
+ ' website, e.g. on YouTube "CS9OO0S5w2k"',
)
new_name_dict = {}
for key in FILE_OUTPUT_NAME_DICT.keys():
mod_value \
= re.sub('Custom', _('Custom'), FILE_OUTPUT_NAME_DICT[key])
mod_value = re.sub('ID', _('ID'), mod_value)
mod_value = re.sub('Title', _('Title'), mod_value)
mod_value = re.sub('Quality', _('Quality'), mod_value)
mod_value = re.sub('Autonumber', _('Autonumber'), mod_value)
new_name_dict[key] = mod_value
FILE_OUTPUT_NAME_DICT = new_name_dict
# Video/audio formats. A number of them contain 'Any format', which
# must be translated
new_list = []
new_dict = {}
for item in VIDEO_OPTION_LIST:
mod_item = re.sub('Any format', _('Any format'), item)
new_list.append(mod_item)
new_dict[mod_item] = VIDEO_OPTION_DICT[item]
VIDEO_OPTION_LIST = new_list
VIDEO_OPTION_DICT = new_dict
# End of this function
return
LOCALE_LIST.append(key)
LOCALE_DICT[key] = value
# Some icons are different at Christmas
today = datetime.date.today()
day = today.strftime("%d")
month = today.strftime("%m")
if (int(month) == 12 and int(day) >= 24) \
or (int(month) == 1 and int(day) <= 5):
xmas_flag = True
else:
xmas_flag = False
# Standard list and dictionaries
time_metric_setup_list = [
'seconds', _('seconds'), 1,
'minutes', _('minutes'), 60,
'hours', _('hours'), int(60 * 60),
'days', _('days'), int(60 * 60 * 24),
'weeks', _('weeks'), int(60 * 60 * 24 * 7),
'years', _('years'), int(60 * 60 * 24 * 365),
]
TIME_METRIC_LIST = []
TIME_METRIC_DICT = {}
TIME_METRIC_TRANS_DICT = {}
while time_metric_setup_list:
key = time_metric_setup_list.pop(0)
trans_key = time_metric_setup_list.pop(0)
value = time_metric_setup_list.pop(0)
LOCALE_DICT[key] = value
# Some icons are different at Christmas
today = datetime.date.today()
day = today.strftime("%d")
month = today.strftime("%m")
if (int(month) == 12 and int(day) >= 24) \
or (int(month) == 1 and int(day) <= 5):
xmas_flag = True
else:
xmas_flag = False
# Standard list and dictionaries
time_metric_setup_list = [
'seconds',
_('seconds'),
1,
'minutes',
_('minutes'),
60,
'hours',
_('hours'),
int(60 * 60),
'days',
_('days'),
int(60 * 60 * 24),
'weeks',
_('weeks'),
int(60 * 60 * 24 * 7),
'years',
_('years'),
def install_ytdl(self):
"""Called by self.run().
Based on code from downloads.VideoDownloader.do_download().
Creates a child process to run the youtube-dl update.
Reads from the child process STDOUT and STDERR, and calls the main
application with the result of the update (success or failure).
"""
# Show information about the update operation in the Output tab (or in
# the setup wizard window, if called from there)
downloader = self.app_obj.get_downloader(self.wiz_win_obj)
self.install_ytdl_write_output(
_('Starting update operation, installing/updating ' +
downloader), )
# Prepare the system command
# The user can change the system command for updating youtube-dl,
# depending on how it was installed
# (For example, if youtube-dl was installed via pip, then it must be
# updated via pip)
if self.wiz_win_obj \
and self.wiz_win_obj.ytdl_update_current is not None:
ytdl_update_current = self.wiz_win_obj.ytdl_update_current
else:
ytdl_update_current = self.app_obj.ytdl_update_current
# Special case: install yt-dlp with no dependencies, if required
if (
(
not self.wiz_win_obj \
and self.app_obj.ytdl_fork == 'yt-dlp' \
and self.app_obj.ytdl_fork_no_dependency_flag
) or (
self.wiz_win_obj \
and self.wiz_win_obj.ytdl_fork == 'yt-dlp' \
and self.wiz_win_obj.ytdl_fork_no_dependency_flag
)
):
if ytdl_update_current == 'ytdl_update_pip':
ytdl_update_current = 'ytdl_update_pip_no_dependencies'
elif ytdl_update_current == 'ytdl_update_pip3' \
or ytdl_update_current == 'ytdl_update_pip3_recommend':
ytdl_update_current = 'ytdl_update_pip3_no_dependencies'
elif ytdl_update_current == 'ytdl_update_win_64':
ytdl_update_current = 'ytdl_update_win_64_no_dependencies'
elif ytdl_update_current == 'ytdl_update_win_32':
ytdl_update_current = 'ytdl_update_win_32_no_dependencies'
# Prepare a system command...
if os.name == 'nt' \
and ytdl_update_current == 'ytdl_update_custom_path' \
and re.search('\.exe$', self.app_obj.ytdl_path):
# Special case: on MS Windows, a custom path may point at an .exe,
# therefore 'python3' must be removed from the system command
# (we can't run 'python3.exe youtube-dl.exe' or anything like
# that)
cmd_list = [self.app_obj.ytdl_path, '-U']
else:
cmd_list = self.app_obj.ytdl_update_dict[ytdl_update_current]
mod_list = []
for arg in cmd_list:
# Substitute in the fork, if one is specified
arg = self.app_obj.check_downloader(arg, self.wiz_win_obj)
# Convert a path beginning with ~ (not on MS Windows)
if os.name != 'nt':
arg = re.sub('^\~', os.path.expanduser('~'), arg)
mod_list.append(arg)
# ...and display it in the Output tab (if required)
self.install_ytdl_write_output(
' '.join(mod_list),
True, # A system command, not a message
)
# Create a new child process using that command...
self.create_child_process(mod_list)
# ...and set up the PipeReader objects to read from the child process
# STDOUT and STDERR
if self.child_process is not None:
self.stdout_reader.attach_fh(self.child_process.stdout)
self.stderr_reader.attach_fh(self.child_process.stderr)
while self.is_child_process_alive():
# Pause a moment between each iteration of the loop (we don't want
# to hog system resources)
time.sleep(self.sleep_time)
# Read from the child process STDOUT and STDERR, in the correct
# order, until there is nothing left to read
while self.read_ytdl_child_process(downloader):
pass
# (Generate our own error messages for debugging purposes, in certain
# situations)
if self.child_process is None:
msg = _('Update did not start')
self.stderr_list.append(msg)
self.install_ytdl_write_output(msg)
elif self.child_process.returncode > 0:
msg = _('Child process exited with non-zero code: {}').format(
self.child_process.returncode, )
self.stderr_list.append(msg)
self.install_ytdl_write_output(msg)
# Operation complete. self.success_flag is checked by
# mainapp.TartubeApp.update_manager_finished
if not self.stderr_list:
self.success_flag = True
# Show a confirmation in the the Output tab (or wizard window textview)
self.install_ytdl_write_output(_('Update operation finished'))
# Let the timer run for a few more seconds to prevent Gtk errors (for
# systems with Gtk < 3.24)
GObject.timeout_add(
0,
self.app_obj.update_manager_halt_timer,
)
def install_streamlink(self):
"""Called by self.run().
A modified version of self.install_ytdl, that installs streamlink on an
MS Windows system.
Creates a child process to run the installation process.
Reads from the child process STDOUT and STDERR, and calls the main
application with the result of the update (success or failure).
"""
# Show information about the update operation in the Output tab
self.install_streamlink_write_output(
_('Starting update operation, installing streamlink'), )
# Create a new child process to install either the 64-bit or 32-bit
# version of streamlink, as appropriate
if sys.maxsize <= 2147483647:
binary = 'mingw-w64-i686-streamlink'
else:
binary = 'mingw-w64-x86_64-streamlink'
# Prepare a system command...
cmd_list = ['pacman', '-S', binary, '--noconfirm']
# ...and display it in the Output tab (if required)
self.install_streamlink_write_output(
' '.join(cmd_list),
True, # A system command, not a message
)
# Create a new child process using that command...
self.create_child_process(cmd_list)
# ...and set up the PipeReader objects to read from the child process
# STDOUT and STDERR
if self.child_process is not None:
self.stdout_reader.attach_fh(self.child_process.stdout)
self.stderr_reader.attach_fh(self.child_process.stderr)
while self.is_child_process_alive():
# Pause a moment between each iteration of the loop (we don't want
# to hog system resources)
time.sleep(self.sleep_time)
# Read from the child process STDOUT and STDERR, in the correct
# order, until there is nothing left to read
while self.read_streamlink_child_process():
pass
# (Generate our own error messages for debugging purposes, in certain
# situations)
if self.child_process is None:
self.stderr_list.append(_('streamlink installation did not start'))
elif self.child_process.returncode > 0:
self.stderr_list.append(
_('Child process exited with non-zero code: {}').format(
self.child_process.returncode, ))
# Operation complete. self.success_flag is checked by
# mainapp.TartubeApp.update_manager_finished()
if not self.stderr_list:
self.success_flag = True
# Show a confirmation in the the Output tab (or wizard window textview)
self.install_streamlink_write_output(_('Update operation finished'))
# Let the timer run for a few more seconds to prevent Gtk errors
GObject.timeout_add(
0,
self.app_obj.update_manager_halt_timer,
)
def check_video_corrupt(self, media_data_obj):
"""Called by self.tidy_directory().
Checks all child videos of the specified media data object. If the
video are corrupted, don't delete them (let the user do that manually).
Args:
media_data_obj (media.Channel, media.Playlist or media.Folder):
The media data object whose directory must be tidied up
"""
for video_obj in media_data_obj.compile_all_videos([]):
if video_obj.file_name is not None \
and video_obj.dl_flag:
video_path = video_obj.get_actual_path(self.app_obj)
if os.path.isfile(video_path):
# Code copied from
# mainapp.TartubeApp.update_video_from_filesystem()
# When the video file is corrupted, moviepy freezes
# indefinitely
# Instead, let's try placing the procedure inside a thread
# (unlike the original function, this one is never called
# if .refresh_moviepy_timeout is 0)
this_thread = threading.Thread(
target=self.call_moviepy,
args=(
video_obj,
video_path,
),
)
this_thread.daemon = True
this_thread.start()
this_thread.join(self.app_obj.refresh_moviepy_timeout)
if this_thread.is_alive():
# moviepy timed out, so assume the video is corrupted
self.video_corrupt_count += 1
if self.del_corrupt_flag \
and os.path.isfile(video_path):
# Delete the corrupted file
os.remove(video_path)
self.video_corrupt_deleted_count += 1
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' +
_('Deleted (possibly) corrupted video file:', )
+ ' \'' + video_obj.name + '\'',
)
self.app_obj.mark_video_downloaded(
video_obj,
False,
)
else:
# Don't delete it
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Video file might be corrupt:', ) +
' \'' + video_obj.name + '\'',
)
LOCALE_DICT[key] = value
# Some icons are different at Christmas
today = datetime.date.today()
day = today.strftime("%d")
month = today.strftime("%m")
if (int(month) == 12 and int(day) >= 24) \
or (int(month) == 1 and int(day) <= 5):
xmas_flag = True
else:
xmas_flag = False
# Standard list and dictionaries
time_metric_setup_list = [
'seconds',
_('seconds'),
1,
'minutes',
_('minutes'),
60,
'hours',
_('hours'),
int(60 * 60),
'days',
_('days'),
int(60 * 60 * 24),
'weeks',
_('weeks'),
int(60 * 60 * 24 * 7),
'years',
_('years'),
def process_video(self, video_obj):
"""Called by self.run().
Sends a single video to FFmpeg for post-processing.
Args:
video_obj (media.Video): The video to be sent to FFmpeg
"""
# Get the path to the video file, which might be in the directory of
# its parent channel/playlist/folder, or in a different directory
# altogether
input_path = video_obj.get_actual_path(self.app_obj)
# Set the output path; the same as the input path, unless the user has
# requested changes
output_file, output_ext = os.path.splitext(input_path)
if self.add_string != '':
output_file += self.add_string
if self.substitute_string != '':
output_file = re.sub(
self.regex_string,
self.substitute_string,
output_file,
)
if self.ext_string != '':
output_ext = self.ext_string
output_path = output_file + output_ext
# Update the main window's progress bar
self.job_count += 1
GObject.timeout_add(
0,
self.app_obj.main_win_obj.update_progress_bar,
video_obj.name,
self.job_count,
self.job_total,
)
# Update our progress in the Output Tab
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Video') + ' ' + str(self.job_count) + '/' \
+ str(self.job_total) + ': ' + video_obj.name,
)
# Show the system command we're about to execute...
test_list = self.app_obj.ffmpeg_manager_obj.run_ffmpeg(
input_path,
output_path,
self.option_list,
True,
)
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Input:') + ' ' + ' '.join(test_list[1]),
)
# ...and then send the command to FFmpeg for processing, which returns
# a list in the form (success_flag, optional_message)
result_list = self.app_obj.ffmpeg_manager_obj.run_ffmpeg(
input_path,
output_path,
self.option_list,
)
if not result_list or not result_list[0]:
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Output: FAILED:') + ' ' + result_list[1],
)
else:
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Output:') + ' ' + output_path,
)
# Delete the original video file, and update media.Video IVs, if
# required
if self.delete_flag \
and os.path.isfile(input_path) \
and os.path.isfile(output_path) \
and input_path != output_path:
os.remove(input_path)
video_obj.set_file(output_file, output_ext)
def process_video(self, video_obj):
"""Called by self.run().
Sends a single video to FFmpeg for post-processing.
Args:
video_obj (media.Video): The video to be sent to FFmpeg
"""
self.job_count += 1
# Update our progress in the Output Tab
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
_('Video') + ' ' + str(self.job_count) + '/' \
+ str(self.job_total) + ': ' + video_obj.name,
)
# mainwin.MainWin.on_video_catalogue_process_ffmpeg_multi() should have
# filtered any media.Video objects whose .file_name is unknown, but
# just in case, check again
# (Special case: 'dummy' video objects (those downloaded in the Classic
# Mode tab) use different IVs)
if video_obj.file_name is None \
and (not video_obj.dummy_flag or video_obj.dummy_path is None):
self.app_obj.main_win_obj.output_tab_write_stderr(
1,
_('FAILED: File name is not known'),
)
self.fail_count += 1
return
# Get the source/output files, ahd the FFmpeg system command (as a
# list)
source_path, output_path, cmd_list = self.options_obj.get_system_cmd(
self.app_obj,
video_obj,
)
if source_path is None:
self.app_obj.main_win_obj.output_tab_write_stderr(
1,
_('FAILED: File not found'),
)
self.fail_count += 1
return
# Update the main window's progress bar
GObject.timeout_add(
0,
self.app_obj.main_win_obj.update_progress_bar,
video_obj.name,
self.job_count,
self.job_total,
)
# Update the Output Tab again
self.app_obj.main_win_obj.output_tab_write_system_cmd(
1,
' '.join(cmd_list),
)
# Process the video
success_flag, msg \
= self.app_obj.ffmpeg_manager_obj.run_ffmpeg_with_options(
video_obj,
source_path,
cmd_list,
)
if not success_flag:
self.fail_count += 1
self.app_obj.main_win_obj.output_tab_write_stderr(
1,
_('FAILED:') + ' ' + msg,
)
else:
self.success_count += 1
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
_('Output file:') + ' ' + output_path,
)
# Delete the original video file, if required
if self.options_obj.options_dict['delete_original_flag'] \
and os.path.isfile(source_path) \
and os.path.isfile(output_path) \
and source_path != output_path:
try:
os.remove(source_path)
except:
self.fail_count += 1
self.app_obj.main_win_obj.output_tab_write_stderr(
1,
_('Could not delete the original file:') + ' ' \
+ source_path,
)
# Ignoring changes to the extension, has the video/audio filename
# changed?
new_dir, new_file = os.path.split(output_path)
new_name, new_ext = os.path.splitext(new_file)
old_name = video_obj.name
rename_flag = False
if (
self.options_obj.options_dict['add_end_filename'] != '' \
or self.options_obj.options_dict['regex_match_filename'] \
!= '' \
) and old_name != new_name:
rename_flag = True
# If the flag is set, rename a thumbnail file to match the
# video file
if rename_flag \
and self.options_obj.options_dict['rename_both_flag']:
thumb_path = utils.find_thumbnail(
self.app_obj,
video_obj,
True, # Rename a temporary thumbnail too
)
if thumb_path:
thumb_name, thumb_ext = os.path.splitext(thumb_path)
new_thumb_path = os.path.abspath(
os.path.join(
new_dir,
new_name + thumb_ext,
), )
# (Don't call utils.rename_file(), as we need our own
# try/except)
try:
# (On MSWin, can't do os.rename if the destination file
# already exists)
if os.path.isfile(new_thumb_path):
os.remove(new_thumb_path)
# (os.rename sometimes fails on external hard drives;
# this is safer)
shutil.move(thumb_path, new_thumb_path)
except:
self.fail_count += 1
self.app_obj.main_win_obj.output_tab_write_stderr(
1,
_('Could not rename the thumbnail:') + ' ' \
+ thumb_path,
)
# If a video/audio file was processed, update its filename
if self.options_obj.options_dict['input_mode'] != 'thumb':
if not video_obj.dummy_flag:
video_obj.set_file_from_path(output_path)
else:
video_obj.set_dummy_path(output_path)
# Also update its .name IV (but its .nickname)
if rename_flag:
video_obj.set_name(new_name)
def run(self):
"""Called as a result of self.__init__().
Compiles a list of media data objects (channels, playlists and folders)
to refresh. If self.init_obj is not set, only that channel/playlist/
folder (and its child channels/playlists/folders) are refreshed;
otherwise the whole media registry is refreshed.
Then calls self.refresh_from_default_destination() for each item in the
list.
Finally informs the main application that the refresh operation is
complete.
"""
if DEBUG_FUNC_FLAG:
utils.debug_time('rop 143 run')
# Show information about the refresh operation in the Output Tab
if not self.init_obj:
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
_('Starting refresh operation, analysing whole database'),
)
else:
media_type = self.init_obj.get_type()
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
_('Starting refresh operation, analysing \'{}\'').format(
self.init_obj.name,
),
)
# Compile a list of channels, playlists and folders to refresh (each
# one has their own sub-directory inside Tartube's data directory)
obj_list = []
if self.init_obj:
# Add this channel/playlist/folder, and any child channels/
# playlists/folders (but not videos, obviously)
obj_list = self.init_obj.compile_all_containers(obj_list)
else:
# Add all channels/playlists/folders in the database
for dbid in list(self.app_obj.media_name_dict.values()):
obj = self.app_obj.media_reg_dict[dbid]
# Don't add private folders
if not isinstance(obj, media.Folder) or not obj.priv_flag:
obj_list.append(obj)
self.job_total = len(obj_list)
# Check each sub-directory in turn, updating the media data registry
# as we go
while self.running_flag and obj_list:
obj = obj_list.pop(0)
if obj.dbid == obj.master_dbid:
self.refresh_from_default_destination(obj)
else:
self.refresh_from_actual_destination(obj)
# Pause a moment, before the next iteration of the loop (don't want
# to hog resources)
time.sleep(self.sleep_time)
# Operation complete. Set the stop time
self.stop_time = int(time.time())
# Show a confirmation in the Output Tab
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
_('Refresh operation finished'),
)
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Number of video files analysed:') + ' ' \
+ str(self.video_total_count),
)
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Video files already in the database:') + ' ' \
+ str(self.video_match_count),
)
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('New videos found and added to the database:') + ' ' \
+ str(self.video_new_count),
)
# Let the timer run for a few more seconds to prevent Gtk errors (for
# systems with Gtk < 3.24)
GObject.timeout_add(
0,
self.app_obj.refresh_manager_halt_timer,
)
def run_check_version(self):
"""Called by self.run().
Checking for a new release of Tartube doesn't involve any system
commands or child processes, so it is handled separately by this
function.
There is a stable release at Sourceforge, and a development release at
Github. Fetch the VERSION file from each, and store the stable/
development versions, so that mainapp.TartubeApp.info_manager_finished
can display them.
"""
# Show information about the info operation in the Output Tab
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
_('Starting info operation, checking for new releases of Tartube'),
)
# Check the stable version, http://tartube.sourceforge.io/VERSION
stable_path = __main__.__website__ + '/VERSION'
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
_('Checking stable release...'),
)
self.app_obj.main_win_obj.output_tab_write_system_cmd(1, stable_path)
try:
request_obj = requests.get(
stable_path,
timeout = self.app_obj.request_get_timeout,
)
response = utils.strip_whitespace(request_obj.text)
if not re.search('^\d+\.\d+\.\d+\s*$', response):
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
_('Ignoring invalid version'),
)
else:
self.stable_version = response
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
_('Retrieved version:') + ' ' + str(response),
)
except:
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
_('Connection failed'),
)
# Check the development version,
# http://raw.githubusercontent.com/axcore/tartube/master/VERSION
dev_path = __main__.__website_dev__ + '/VERSION'
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
_('Checking development release...'),
)
self.app_obj.main_win_obj.output_tab_write_system_cmd(1, dev_path)
try:
request_obj = requests.get(
dev_path,
timeout = self.app_obj.request_get_timeout,
)
response = utils.strip_whitespace(request_obj.text)
if not re.search('^\d+\.\d+\.\d+\s*$', response):
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
_('Ignoring invalid version'),
)
else:
self.dev_version = response
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
_('Retrieved version:') + ' ' + str(response),
)
except:
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
_('Connection failed'),
)
# Operation complete. self.success_flag is checked by
# mainapp.TartubeApp.info_manager_finished()
self.success_flag = True
# Show a confirmation in the the Output Tab
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
_('Info operation finished'),
)
# Let the timer run for a few more seconds to prevent Gtk errors (for
# systems with Gtk < 3.24)
GObject.timeout_add(
0,
self.app_obj.info_manager_halt_timer,
)
def refresh_from_default_destination(self, media_data_obj):
"""Called by self.run().
Refreshes a single channel, playlist or folder, for which an
alternative download destination has not been set.
If a file is missing in the channel/playlist/folder's sub-directory,
mark the video object as not downloaded.
If unexpected video files exist in the sub-directory, create a new
media.Video object for them.
Args:
media_data_obj (media.Channel, media.Playlist or media.Folder):
The media data object to refresh
"""
if DEBUG_FUNC_FLAG:
utils.debug_time('rop 252 refresh_from_default_destination')
# Update the main window's progress bar
self.job_count += 1
GObject.timeout_add(
0,
self.app_obj.main_win_obj.update_progress_bar,
media_data_obj.name,
self.job_count,
self.job_total,
)
# Keep a running total of matched/new videos for this channel, playlist
# or folder
local_total_count = 0
local_match_count = 0
local_new_count = 0
# Update our progress in the Output Tab
if isinstance(media_data_obj, media.Channel):
string = _('Channel:') + ' '
elif isinstance(media_data_obj, media.Playlist):
string = _('Playlist:') + ' '
else:
string = _('Folder:') + ' '
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
string + media_data_obj.name,
)
# Get the sub-directory for this media data object
dir_path = media_data_obj.get_default_dir(self.app_obj)
# Get a list of video files in the sub-directory
try:
init_list = os.listdir(dir_path)
except:
# Can't read the directory
return
# From this list, filter out files without a recognised file extension
# (.mp4, .webm, etc)
mod_list = []
for relative_path in init_list:
# (If self.stop_refresh_operation() has been called, give up
# immediately)
if not self.running_flag:
return
filename, ext = os.path.splitext(relative_path)
# (Remove the initial .)
ext = ext[1:]
if ext in formats.VIDEO_FORMAT_DICT:
mod_list.append(relative_path)
# From the new list, filter out duplicate filenames (e.g. if the list
# contains both 'my_video.mp4' and 'my_video.webm', filter out the
# second one, adding to a list of alternative files)
filter_list = []
filter_dict = {}
alt_list = []
for relative_path in mod_list:
# (If self.stop_refresh_operation() has been called, give up
# immediately)
if not self.running_flag:
return
filename, ext = os.path.splitext(relative_path)
if not filename in filter_dict:
filter_list.append(relative_path)
filter_dict[filename] = relative_path
else:
alt_list.append(relative_path)
# Now compile a dictionary of media.Video objects in this channel/
# playlist/folder, so we can eliminate them one by one
check_dict = {}
for child_obj in media_data_obj.child_list:
# (If self.stop_refresh_operation() has been called, give up
# immediately)
if not self.running_flag:
return
if isinstance(child_obj, media.Video) and child_obj.file_name:
# Does the video file still exist?
this_file = child_obj.file_name + child_obj.file_ext
if child_obj.dl_flag and not this_file in init_list:
self.app_obj.mark_video_downloaded(child_obj, False)
else:
check_dict[child_obj.file_name] = child_obj
# If this channel/playlist/folder is the alternative download
# destination for other channels/playlists/folders, compile a
# dicationary of their media.Video objects
# (If we find a video we weren't expecting, before creating a new
# media.Video object, we must first check it isn't one of them)
slave_dict = {}
for slave_dbid in media_data_obj.slave_dbid_list:
# (If self.stop_refresh_operation() has been called, give up
# immediately)
if not self.running_flag:
return
slave_obj = self.app_obj.media_reg_dict[slave_dbid]
for child_obj in slave_obj.child_list:
if isinstance(child_obj, media.Video) and child_obj.file_name:
slave_dict[child_obj.file_name] = child_obj
# Now try to match each video file (in filter_list) with an existing
# media.Video object (in check_dict)
# If there is no match, and if the video file doesn't match a video
# in another channel/playlist/folder (for which this is the
# alternative download destination), then we can create a new
# media.Video object
for relative_path in filter_list:
# (If self.stop_refresh_operation() has been called, give up
# immediately)
if not self.running_flag:
return
filename, ext = os.path.splitext(relative_path)
if self.app_obj.refresh_output_videos_flag:
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Checking:') + ' ' + filename,
)
if filename in check_dict:
# File matched
self.video_total_count += 1
local_total_count += 1
self.video_match_count += 1
local_match_count += 1
# If it is not marked as downloaded, we can mark it so now
child_obj = check_dict[filename]
if not child_obj.dl_flag:
self.app_obj.mark_video_downloaded(child_obj, True)
# Make sure the stored extension is correct (e.g. if we've
# matched an existing .webm video file, with an expected
# .mp4 video file)
if child_obj.file_ext != ext:
child_relative_path \
= child_obj.file_name + child_obj.file_ext
if not child_relative_path in alt_list:
child_obj.set_file(filename, ext)
# Eliminate this media.Video object; no other video file should
# match it
del check_dict[filename]
# Update our progress in the Output Tab (if required)
if self.app_obj.refresh_output_videos_flag:
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Match:') + ' ' + filename,
)
elif filename not in slave_dict:
# File didn't match a media.Video object
self.video_total_count += 1
local_total_count += 1
self.video_new_count += 1
local_new_count += 1
# Display the list of non-matching videos, if required
if self.app_obj.refresh_output_videos_flag \
and self.app_obj.refresh_output_verbose_flag:
for failed_path in check_dict.keys():
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Non-match:') + ' ' + filename,
)
# Create a new media.Video object
video_obj = self.app_obj.add_video(media_data_obj, None)
video_path = os.path.abspath(
os.path.join(
dir_path,
filter_dict[filename],
)
)
# Set the new video object's IVs
filename, ext = os.path.splitext(filter_dict[filename])
video_obj.set_name(filename)
video_obj.set_nickname(filename)
video_obj.set_file(filename, ext)
if ext == '.mkv':
video_obj.set_mkv()
video_obj.set_file_size(
os.path.getsize(
os.path.abspath(
os.path.join(dir_path, filter_dict[filename]),
),
),
)
# If the video's JSON file exists downloaded, we can extract
# video statistics from it
self.app_obj.update_video_from_json(video_obj)
# For any of those statistics that haven't been set (because
# the JSON file was missing or didn't contain the right
# statistics), set them directly
self.app_obj.update_video_from_filesystem(
video_obj,
video_path,
)
# This call marks the video as downloaded, and also updates the
# Video Index and Video Catalogue (if required)
self.app_obj.mark_video_downloaded(video_obj, True)
if self.app_obj.refresh_output_videos_flag:
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('New video:') + ' ' + filename,
)
# Check complete, display totals
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Total videos:') + ' ' + str(local_total_count) \
+ ', ' + _('matched:') + ' ' + str(local_match_count) \
+ ', ' + _('new:') + ' ' + str(local_new_count),
)
def install_ytdl(self):
"""Called by self.run().
Based on code from downloads.VideoDownloader.do_download().
Creates a child process to run the youtube-dl update.
Reads from the child process STDOUT and STDERR, and calls the main
application with the result of the update (success or failure).
"""
# Show information about the update operation in the Output Tab (or in
# the setup wizard window, if called from there)
downloader = self.app_obj.get_downloader(self.wiz_win_obj)
self.install_ytdl_write_output(
_('Starting update operation, installing/updating ' +
downloader), )
# Prepare the system command
# The user can change the system command for updating youtube-dl,
# depending on how it was installed
# (For example, if youtube-dl was installed via pip, then it must be
# updated via pip)
if self.wiz_win_obj \
and self.wiz_win_obj.ytdl_update_current is not None:
ytdl_update_current = self.wiz_win_obj.ytdl_update_current
else:
ytdl_update_current = self.app_obj.ytdl_update_current
cmd_list = self.app_obj.ytdl_update_dict[ytdl_update_current]
mod_list = []
for arg in cmd_list:
# Substitute in the fork, if one is specified
arg = self.app_obj.check_downloader(arg, self.wiz_win_obj)
# Convert a path beginning with ~ (not on MS Windows)
if os.name != 'nt':
arg = re.sub('^\~', os.path.expanduser('~'), arg)
mod_list.append(arg)
# Create a new child process using that command
self.create_child_process(mod_list)
# Show the system command in the Output Tab
self.install_ytdl_write_output(
' '.join(mod_list),
True, # A system command, not a message
)
# So that we can read from the child process STDOUT and STDERR, attach
# a file descriptor to the PipeReader objects
if self.child_process is not None:
self.stdout_reader.attach_file_descriptor(
self.child_process.stdout, )
self.stderr_reader.attach_file_descriptor(
self.child_process.stderr, )
while self.is_child_process_alive():
# Read from the child process STDOUT, and convert into unicode for
# Python's convenience
while not self.stdout_queue.empty():
stdout = self.stdout_queue.get_nowait().rstrip()
if stdout:
if os.name == 'nt':
stdout = stdout.decode('cp1252')
else:
stdout = stdout.decode('utf-8')
# "It looks like you installed youtube-dl with a package
# manager, pip, setup.py or a tarball. Please use that to
# update."
# "The script youtube-dl is installed in '...' which is not
# on PATH. Consider adding this directory to PATH..."
if re.search('It looks like you installed', stdout) \
or re.search(
'The script ' + downloader + ' is installed',
stdout,
):
self.stderr_list.append(stdout)
else:
# Try to intercept the new version number for
# youtube-dl
self.intercept_version_from_stdout(stdout)
self.stdout_list.append(stdout)
# Show command line output in the Output Tab (or wizard
# window textview)
self.install_ytdl_write_output(stdout)
# The child process has finished
while not self.stderr_queue.empty():
# Read from the child process STDERR queue (we don't need to read
# it in real time), and convert into unicode for python's
# convenience
stderr = self.stderr_queue.get_nowait().rstrip()
if os.name == 'nt':
stderr = stderr.decode('cp1252')
else:
stderr = stderr.decode('utf-8')
if stderr:
# If the user has pip installed, rather than pip3, they will by
# now (mid-2019) be seeing a Python 2.7 deprecation warning.
# Ignore that message, if received
# If a newer version of pip is available, the user will see a
# 'You should consider upgrading' warning. Ignore that too,
# if received
if not re.search('DEPRECATION', stderr) \
and not re.search('You are using pip version', stderr) \
and not re.search('You should consider upgrading', stderr):
self.stderr_list.append(stderr)
# Show command line output in the Output Tab (or wizard window
# textview)
self.install_ytdl_write_output(stderr)
# (Generate our own error messages for debugging purposes, in certain
# situations)
if self.child_process is None:
msg = _('Update did not start')
self.stderr_list.append(msg)
self.install_ytdl_write_output(msg)
elif self.child_process.returncode > 0:
msg = _('Child process exited with non-zero code: {}').format(
self.child_process.returncode, )
self.stderr_list.append(msg)
self.install_ytdl_write_output(msg)
# Operation complete. self.success_flag is checked by
# mainapp.TartubeApp.update_manager_finished
if not self.stderr_list:
self.success_flag = True
# Show a confirmation in the the Output Tab (or wizard window textview)
self.install_ytdl_write_output(_('Update operation finished'))
# Let the timer run for a few more seconds to prevent Gtk errors (for
# systems with Gtk < 3.24)
GObject.timeout_add(
0,
self.app_obj.update_manager_halt_timer,
)
def setup_set_downloader_page(self):
"""Called by self.setup_page().
Sets up the widget layout for a page.
"""
grid_width = 3
self.add_label(
'<span font_size="large" style="italic">' \
+ _('Choose which downloader to use.') \
+ '</span>',
0, 0, grid_width, 1,
)
# youtube-dlc
radiobutton = self.setup_set_downloader_page_add_button(
1, # Row number
'<b>youtube-dlc</b>: <i>' \
+ self.app_obj.ytdl_fork_descrip_dict['youtube-dlc'] \
+ '</i>',
_('Use youtube-dlc'),
)
# youtube-dl
radiobutton2 = self.setup_set_downloader_page_add_button(
2, # Row number
'<b>youtube-dl</b>: <i>' \
+ self.app_obj.ytdl_fork_descrip_dict['youtube-dl'] \
+ '</i>',
_('Use youtube-dl'),
radiobutton,
)
# Any other fork
radiobutton3, entry = self.setup_set_downloader_page_add_button(
3, # Row number
self.app_obj.ytdl_fork_descrip_dict['custom'],
_('Use a different fork of youtube-dl'),
radiobutton2,
True, # Show an entry
)
# Set widgets to their initial state
if self.ytdl_fork is None or self.ytdl_fork == 'youtube-dl':
radiobutton2.set_active(True)
entry.set_sensitive(False)
elif self.ytdl_fork == 'youtube-dlc':
radiobutton.set_active(True)
entry.set_sensitive(False)
else:
radiobutton3.set_active(True)
if self.ytdl_fork is not None:
entry.set_text(self.ytdl_fork)
else:
entry.set_text('')
entry.set_sensitive(True)
# (Signal connects from the call to
# self.setup_set_downloader_page_add_button() )
radiobutton.connect(
'toggled',
self.on_button_ytdl_fork_toggled,
entry,
'youtube-dlc',
)
radiobutton2.connect(
'toggled',
self.on_button_ytdl_fork_toggled,
entry,
'youtube-dl',
)
radiobutton3.connect(
'toggled',
self.on_button_ytdl_fork_toggled,
entry,
)
entry.connect(
'changed',
self.on_entry_ytdl_fork_changed,
radiobutton3,
)