def call_moviepy(self, video_obj, video_path):
"""Called by thread inside self.check_video_corrupt().
When we call moviepy.editor.VideoFileClip() on a corrupted video file,
moviepy freezes indefinitely.
This function is called inside a thread, so a timeout of (by default)
ten seconds can be applied.
Args:
video_obj (media.Video): The video object being updated
video_path (str): The path to the video file itself
"""
if DEBUG_FUNC_FLAG:
utils.debug_time('top 1063 call_moviepy')
try:
clip = moviepy.editor.VideoFileClip(video_path)
except:
self.video_corrupt_count += 1
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Video file might be corrupt:') + ' \'' \
+ video_obj.name + '\'',
)
def check_videos_exist(self, media_data_obj):
"""Called by self.tidy_directory().
Checks all child videos of the specified media data object. If the
video should exist, but doesn't (or vice-versa), modify the media.Video
object's IVs accordingly.
Args:
media_data_obj (media.Channel, media.Playlist or media.Folder):
The media data object whose directory must be tidied up
"""
if DEBUG_FUNC_FLAG:
utils.debug_time('top 642 check_videos_exist')
for video_obj in media_data_obj.compile_all_videos( [] ):
if video_obj.file_name is not None:
video_path = video_obj.get_actual_path(self.app_obj)
if not video_obj.dl_flag \
and os.path.isfile(video_path):
# File exists, but is marked as not downloaded
self.app_obj.mark_video_downloaded(
video_obj,
True, # Video is downloaded
True, # ...but don't mark it as new
)
self.video_exist_count += 1
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _(
'Video file exists:',
) + ' \'' + video_obj.name + '\'',
)
elif video_obj.dl_flag \
and not os.path.isfile(video_path):
# File doesn't exist, but is marked as downloaded
self.app_obj.mark_video_downloaded(
video_obj,
False, # Video is not downloaded
)
self.video_no_exist_count += 1
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _(
'Video file doesn\'t exist:',
) + ' \'' + video_obj.name + '\'',
)
def delete_archive(self, media_data_obj):
"""Called by self.tidy_directory().
Checks the specified media data object's directory. If a youtube-dl
archive file is found there, delete it.
Args:
media_data_obj (media.Channel, media.Playlist or media.Folder):
The media data object whose directory must be tidied up
"""
if DEBUG_FUNC_FLAG:
utils.debug_time('top 1028 delete_archive')
archive_path = os.path.abspath(
os.path.join(
media_data_obj.get_default_dir(self.app_obj),
'ytdl-archive.txt',
),
)
if os.path.isfile(archive_path):
# Delete the archive file
os.remove(archive_path)
self.archive_deleted_count += 1
def intercept_version_from_stdout(self, stdout):
"""Called by self.install_yt_dl() only.
Check a STDOUT message, hoping to intercept the new youtube-dl version
number.
Args:
stdout (str): The STDOUT message
"""
if DEBUG_FUNC_FLAG:
utils.debug_time('uop 489 intercept_version_from_stdout')
substring = re.search(
'Requirement already up\-to\-date.*\(([\d\.]+)\)\s*$',
stdout,
)
if substring:
self.ytdl_version = substring.group(1)
else:
substring = re.search(
'Successfully installed youtube\-dl\-([\d\.]+)\s*$',
stdout,
)
if substring:
self.ytdl_version = substring.group(1)
def stop_update_operation(self):
"""Called by mainapp.TartubeApp.do_shutdown(), .stop_continue(),
.on_button_stop_operation() and mainwin.MainWin.on_stop_menu_item().
Based on code from downloads.VideoDownloader.stop().
Terminates the child process.
"""
if DEBUG_FUNC_FLAG:
utils.debug_time('uop 545 stop_update_operation')
if self.is_child_process_alive():
if os.name == 'nt':
# os.killpg is not available on MS Windows (see
# https://bugs.python.org/issue5115 )
self.child_process.kill()
# When we kill the child process on MS Windows the return code
# gets set to 1, so we want to reset the return code back to
# 0
self.child_process.returncode = 0
else:
os.killpg(self.child_process.pid, signal.SIGKILL)
def tidy_directory(self, media_data_obj):
"""Called by self.run().
Tidy up the directory of a single channel, playlist or folder.
Args:
media_data_obj (media.Channel, media.Playlist or media.Folder):
The media data object whose directory must be tidied up
"""
if DEBUG_FUNC_FLAG:
utils.debug_time('top 502 tidy_directory')
# Update the main window's progress bar
self.job_count += 1
GObject.timeout_add(
0,
self.app_obj.main_win_obj.update_progress_bar,
media_data_obj.name,
self.job_count,
self.job_total,
)
media_type = media_data_obj.get_type()
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
_('Checking:') + ' \'' + media_data_obj.name + '\'',
)
if self.corrupt_flag:
self.check_video_corrupt(media_data_obj)
if self.exist_flag:
self.check_videos_exist(media_data_obj)
if self.del_video_flag:
self.delete_video(media_data_obj)
if self.del_descrip_flag:
self.delete_descrip(media_data_obj)
if self.del_json_flag:
self.delete_json(media_data_obj)
if self.del_xml_flag:
self.delete_xml(media_data_obj)
if self.del_thumb_flag:
self.delete_thumb(media_data_obj)
if self.del_webp_flag:
self.delete_webp(media_data_obj)
if self.del_archive_flag:
self.delete_archive(media_data_obj)
def check_video_in_actual_dir(self, container_obj, video_obj, file_path):
"""Called by self.delete_video(), .delete_descrip(), .delete_json(),
.delete_xml() and .delete_thumb().
If the video's parent container has an alternative download destination
set, we must check the corresponding media data object. If the latter
also has a media.Video object matching this video, then this function
returns None and nothing is deleted. Otherwise, the specified file_path
is returned, so it can be deleted.
Args:
container_obj (media.Channel, media.Playlist, media.Folder): A
channel, playlist or folder
video_obj (media.Video): A video contained in that channel,
playlist or folder
file_path (str): The path to a file which the calling function
wants to delete
Returns:
The specified file_path if it can be deleted, or None if it should
not be deleted
"""
if DEBUG_FUNC_FLAG:
utils.debug_time('top 1108 check_video_in_actual_dir')
if container_obj.dbid == container_obj.master_dbid:
# No alternative download destination to check
return file_path
else:
# Get the channel/playlist/folder acting as container_obj's
# alternative download destination
master_obj = self.app_obj.media_reg_dict[container_obj.master_dbid]
# Check its videos. Are there any videos with the same name?
for child_obj in master_obj.child_list:
if child_obj.file_name is not None \
and child_obj.file_name == video_obj.file_name:
# Don't delete the file associated with this video
return None
# There are no videos with the same name, so the file can be
# deleted
return file_path
def stop_refresh_operation(self):
"""Called by mainapp.TartubeApp.do_shutdown(), .stop_continue(),
.on_button_stop_operation() and mainwin.MainWin.on_stop_menu_item().
Stops the refresh operation.
"""
if DEBUG_FUNC_FLAG:
utils.debug_time('rop 613 stop_refresh_operation')
self.running_flag = False
def run(self):
"""Called as a result of self.__init__().
Initiates the download.
"""
if DEBUG_FUNC_FLAG:
utils.debug_time('uop 143 run')
if self.update_type == 'ffmpeg':
self.install_ffmpeg()
else:
self.install_ytdl()
def __init__(self, app_obj, init_obj=None):
if DEBUG_FUNC_FLAG:
utils.debug_time('rop 73 __init__')
super(RefreshManager, self).__init__()
# IV list - class objects
# -----------------------
# The mainapp.TartubeApp object
self.app_obj = app_obj
# The media data object (channel, playlist or folder) to refresh, or
# None if the whole media data registry is to be refreshed
self.init_obj = init_obj
# IV list - other
# ---------------
# Flag set to False if self.stop_refresh_operation() is called, which
# halts the operation immediately
self.running_flag = True
# The time at which the refresh operation began (in seconds since
# epoch)
self.start_time = int(time.time())
# The time at which the refresh operation completed (in seconds since
# epoch)
self.stop_time = None
# The time (in seconds) between iterations of the loop in self.run()
self.sleep_time = 0.25
# The number of media data objects refreshed so far...
self.job_count = 0
# ...and the total number to refresh (these numbers are displayed in
# the progress bar in the Videos tab)
self.job_total = 0
# Total number of videos analysed
self.video_total_count = 0
# Number of videos matched with a media.Video object in the database
self.video_match_count = 0
# Number of videos not matched, and therefore given a new media.Video
# object
self.video_new_count = 0
# Code
# ----
# Let's get this party started!
self.start()
def __init__(self, app_obj, update_type):
if DEBUG_FUNC_FLAG:
utils.debug_time('uop 83 __init__')
super(UpdateManager, self).__init__()
# IV list - class objects
# -----------------------
# The mainapp.TartubeApp object
self.app_obj = app_obj
# This object reads from the child process STDOUT and STDERR in an
# asynchronous way
# Standard Python synchronised queue classes
self.stdout_queue = queue.Queue()
self.stderr_queue = queue.Queue()
# The downloads.PipeReader objects created to handle reading from the
# pipes
self.stdout_reader = downloads.PipeReader(self.stdout_queue)
self.stderr_reader = downloads.PipeReader(self.stderr_queue)
# The child process created by self.create_child_process()
self.child_process = None
# IV list - other
# ---------------
# 'ffmpeg' to install FFmpeg (on MS Windows only), or 'ytdl' to
# install/update youtube-dl
self.update_type = update_type
# Flag set to True if the update operation succeeds, False if it fails
self.success_flag = False
# The youtube-dl version number as a string, if captured from the child
# process (e.g. '2019.07.02')
self.ytdl_version = None
# (For debugging purposes, store any STDOUT/STDERR messages received;
# otherwise we would just set a flag if a STDERR message was
# received)
self.stdout_list = []
self.stderr_list = []
# Code
# ----
# Let's get this party started!
self.start()
def delete_webp(self, media_data_obj):
"""Called by self.tidy_directory().
Checks all child videos of the specified media data object. If the
associated thumbnail file in a .webp or malformed .jpg format exists,
delete it
Args:
media_data_obj (media.Channel, media.Playlist or media.Folder):
The media data object whose directory must be tidied up
"""
if DEBUG_FUNC_FLAG:
utils.debug_time('top 983 delete_webp')
for video_obj in media_data_obj.compile_all_videos( [] ):
if video_obj.file_name is not None:
# Thumbnails might be in one of two locations
thumb_path = utils.find_thumbnail_webp(self.app_obj, video_obj)
# If the video's parent container has an alternative download
# destination set, we must check the corresponding media
# data object. If the latter also has a media.Video object
# matching this video, then this function returns None and
# nothing is deleted
if thumb_path is not None:
thumb_path = self.check_video_in_actual_dir(
media_data_obj,
video_obj,
thumb_path,
)
if thumb_path is not None \
and os.path.isfile(thumb_path):
# Delete the thumbnail file
os.remove(thumb_path)
self.webp_deleted_count += 1
def create_child_process(self, cmd_list):
"""Called by self.install_ffmpeg() or .install_ytdl().
Based on code from downloads.VideoDownloader.create_child_process().
Executes the system command, creating a new child process which
executes youtube-dl.
Args:
cmd_list (list): Python list that contains the command to execute.
"""
if DEBUG_FUNC_FLAG:
utils.debug_time('uop 167 create_child_process')
info = preexec = None
if os.name == 'nt':
# Hide the child process window that MS Windows helpfully creates
# for us
info = subprocess.STARTUPINFO()
info.dwFlags |= subprocess.STARTF_USESHOWWINDOW
else:
# Make this child process the process group leader, so that we can
# later kill the whole process group with os.killpg
preexec = os.setsid
try:
self.child_process = subprocess.Popen(
cmd_list,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
preexec_fn=preexec,
startupinfo=info,
)
except (ValueError, OSError) as error:
# (The code in self.run() will spot that the child process did not
# start)
self.stderr_list.append(_('Child process did not start'))
def delete_xml(self, media_data_obj):
"""Called by self.tidy_directory().
Checks all child videos of the specified media data object. If the
associated annotation file exists, delete it.
Args:
media_data_obj (media.Channel, media.Playlist or media.Folder):
The media data object whose directory must be tidied up
"""
if DEBUG_FUNC_FLAG:
utils.debug_time('top 892 delete_xml')
for video_obj in media_data_obj.compile_all_videos( [] ):
if video_obj.file_name is not None:
xml_path = video_obj.get_actual_path_by_ext(
self.app_obj,
'.annotations.xml',
)
# If the video's parent container has an alternative download
# destination set, we must check the corresponding media
# data object. If the latter also has a media.Video object
# matching this video, then this function returns None and
# nothing is deleted
xml_path = self.check_video_in_actual_dir(
media_data_obj,
video_obj,
xml_path,
)
if xml_path is not None \
and os.path.isfile(xml_path):
# Delete the annotation file
os.remove(xml_path)
self.xml_deleted_count += 1
def is_child_process_alive(self):
"""Called by self.run() and .stop_info_operation().
Based on code from downloads.VideoDownloader.is_child_process_alive().
Called continuously during the self.run() loop to check whether the
child process has finished or not.
Returns:
True if the child process is alive, otherwise returns False.
"""
if DEBUG_FUNC_FLAG:
utils.debug_time('iop 440 is_child_process_alive')
if self.child_process is None:
return False
return self.child_process.poll() is None
def run(self):
"""Called as a result of self.__init__().
Creates a child process to run the youtube-dl system command.
Reads from the child process STDOUT and STDERR, and calls the main
application with the result of the process (success or failure).
"""
if DEBUG_FUNC_FLAG:
utils.debug_time('iop 180 run')
# Show information about the info operation in the Output Tab
if self.info_type == 'test_ytdl':
msg = _(
'Starting info operation, testing youtube-dl with specified' \
+ ' options',
)
else:
if self.info_type == 'formats':
msg = _(
'Starting info operation, fetching list of video/audio'\
+ ' formats for \'{0}\'',
).format(self.video_obj.name)
else:
msg = _(
'Starting info operation, fetching list of subtitles'\
+ ' for \'{0}\'',
).format(self.video_obj.name)
self.app_obj.main_win_obj.output_tab_write_stdout(1, msg)
# Convert a path beginning with ~ (not on MS Windows)
ytdl_path = self.app_obj.ytdl_path
if os.name != 'nt':
ytdl_path = re.sub('^\~', os.path.expanduser('~'), ytdl_path)
# Prepare the system command
if self.info_type == 'formats':
cmd_list = [
ytdl_path,
'--list-formats',
self.video_obj.source,
]
elif self.info_type == 'subs':
cmd_list = [
ytdl_path,
'--list-subs',
self.video_obj.source,
]
else:
cmd_list = [ytdl_path]
if self.options_string is not None \
and self.options_string != '':
# Parse the string into a list. It was obtained from a
# Gtk.TextView, so it can contain newline and/or multiple
# whitepsace characters. Whitespace characters within
# double quotes "..." must be preserved
option_list = utils.parse_ytdl_options(self.options_string)
for item in option_list:
cmd_list.append(item)
if self.url_string is not None \
and self.url_string != '':
cmd_list.append('-o')
cmd_list.append(
os.path.join(
self.app_obj.temp_test_dir,
'%(title)s.%(ext)s',
), )
cmd_list.append(self.url_string)
# Create the new child process
self.create_child_process(cmd_list)
# Show the system command in the Output Tab
space = ' '
self.app_obj.main_win_obj.output_tab_write_system_cmd(
1,
space.join(cmd_list),
)
# So that we can read from the child process STDOUT and STDERR, attach
# a file descriptor to the PipeReader objects
if self.child_process is not None:
self.stdout_reader.attach_file_descriptor(
self.child_process.stdout, )
self.stderr_reader.attach_file_descriptor(
self.child_process.stderr, )
while self.is_child_process_alive():
# Read from the child process STDOUT, and convert into unicode for
# Python's convenience
while not self.stdout_queue.empty():
stdout = self.stdout_queue.get_nowait().rstrip()
if stdout:
if os.name == 'nt':
stdout = stdout.decode('cp1252')
else:
stdout = stdout.decode('utf-8')
self.output_list.append(stdout)
self.stdout_list.append(stdout)
# Show command line output in the Output Tab
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
stdout,
)
# The child process has finished
while not self.stderr_queue.empty():
# Read from the child process STDERR queue (we don't need to read
# it in real time), and convert into unicode for python's
# convenience
stderr = self.stderr_queue.get_nowait().rstrip()
if os.name == 'nt':
stderr = stderr.decode('cp1252')
else:
stderr = stderr.decode('utf-8')
if stderr:
# While testing youtube-dl, don't treat anything as an error
if self.info_type == 'test_ytdl':
self.stdout_list.append(stderr)
# When fetching subtitles from a video that has none, don't
# treat youtube-dl WARNING: messages as something that
# makes the info operation fail
elif self.info_type == 'subs':
if not re.match('WARNING\:', stderr):
self.stderr_list.append(stderr)
# When fetching formats, recognise all warnings as errors
else:
self.stderr_list.append(stderr)
# Show command line output in the Output Tab
self.app_obj.main_win_obj.output_tab_write_stderr(
1,
stderr,
)
# (Generate our own error messages for debugging purposes, in certain
# situations)
if self.child_process is None:
msg = _('youtube-dl process did not start')
self.stderr_list.append(msg)
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
msg,
)
elif self.child_process.returncode > 0:
msg = _('Child process exited with non-zero code: {}').format(
self.child_process.returncode, )
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
msg,
)
# Operation complete. self.success_flag is checked by
# mainapp.TartubeApp.info_manager_finished
if not self.stderr_list:
self.success_flag = True
# Show a confirmation in the the Output Tab
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
_('Info operation finished'),
)
# Let the timer run for a few more seconds to prevent Gtk errors (for
# systems with Gtk < 3.24)
GObject.timeout_add(
0,
self.app_obj.info_manager_halt_timer,
)
def __init__(self, app_obj, info_type, media_data_obj, url_string,
options_string):
if DEBUG_FUNC_FLAG:
utils.debug_time('iop 102 __init__')
super(InfoManager, self).__init__()
# IV list - class objects
# -----------------------
# The mainapp.TartubeApp object
self.app_obj = app_obj
# The video for which information will be fetched (None if
# self.info_type is 'test_ytdl')
self.video_obj = media_data_obj
# This object reads from the child process STDOUT and STDERR in an
# asynchronous way
# Standard Python synchronised queue classes
self.stdout_queue = queue.Queue()
self.stderr_queue = queue.Queue()
# The downloads.PipeReader objects created to handle reading from the
# pipes
self.stdout_reader = downloads.PipeReader(self.stdout_queue)
self.stderr_reader = downloads.PipeReader(self.stderr_queue)
# The child process created by self.create_child_process()
self.child_process = None
# IV list - other
# ---------------
# The type of information to fetch: 'formats' for a list of video
# formats, 'subs' for a list of subtitles, or 'test_ytdl' to test
# youtube-dl with specified options
self.info_type = info_type
# For 'test_ytdl', the video URL to download (can be None or an empty
# string, if no download is required, for example
# 'youtube-dl --version'. For 'formats' and 'subs', set to None
self.url_string = url_string
# For 'test_ytdl', a string containing one or more youtube-dl download
# options. The string, generated by a Gtk.TextView, typically
# contains newline and/or multiple whitespace characters; the
# info.InfoManager code deals with that. Can be None or an empty
# string, if no download options are required. For 'formats' and
# 'subs', set to None
self.options_string = options_string
# Flag set to True if the info operation succeeds, False if it fails
self.success_flag = False
# The list of formats/subtitles extracted from STDOUT
self.output_list = []
# (For debugging purposes, store any STDOUT/STDERR messages received;
# otherwise we would just set a flag if a STDERR message was
# received)
self.stdout_list = []
self.stderr_list = []
# Code
# ----
# Let's get this party started!
self.start()
def install_ytdl(self):
"""Called by self.run().
Based on code from downloads.VideoDownloader.do_download().
Creates a child process to run the youtube-dl update.
Reads from the child process STDOUT and STDERR, and calls the main
application with the result of the update (success or failure).
"""
if DEBUG_FUNC_FLAG:
utils.debug_time('uop 330 install_ytdl')
# Show information about the update operation in the Output Tab
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
_('Starting update operation, installing/updating youtube-dl'),
)
# Prepare the system command
# The user can change the system command for updating youtube-dl,
# depending on how it was installed
# (For example, if youtube-dl was installed via pip, then it must be
# updated via pip)
cmd_list \
= self.app_obj.ytdl_update_dict[self.app_obj.ytdl_update_current]
# Convert a path beginning with ~ (not on MS Windows)
if os.name != 'nt':
cmd_list[0] = re.sub('^\~', os.path.expanduser('~'), cmd_list[0])
# Create a new child process using that command
self.create_child_process(cmd_list)
# Show the system command in the Output Tab
space = ' '
self.app_obj.main_win_obj.output_tab_write_system_cmd(
1,
space.join(cmd_list),
)
# So that we can read from the child process STDOUT and STDERR, attach
# a file descriptor to the PipeReader objects
if self.child_process is not None:
self.stdout_reader.attach_file_descriptor(
self.child_process.stdout, )
self.stderr_reader.attach_file_descriptor(
self.child_process.stderr, )
while self.is_child_process_alive():
# Read from the child process STDOUT, and convert into unicode for
# Python's convenience
while not self.stdout_queue.empty():
stdout = self.stdout_queue.get_nowait().rstrip()
if stdout:
if os.name == 'nt':
stdout = stdout.decode('cp1252')
else:
stdout = stdout.decode('utf-8')
# "It looks like you installed youtube-dl with a package
# manager, pip, setup.py or a tarball. Please use that to
# update."
# "The script youtube-dl is installed in '...' which is not
# on PATH. Consider adding this directory to PATH..."
if re.search('It looks like you installed', stdout) \
or re.search('The script youtube-dl is installed', stdout):
self.stderr_list.append(stdout)
else:
# Try to intercept the new version number for
# youtube-dl
self.intercept_version_from_stdout(stdout)
self.stdout_list.append(stdout)
# Show command line output in the Output Tab
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
stdout,
)
# The child process has finished
while not self.stderr_queue.empty():
# Read from the child process STDERR queue (we don't need to read
# it in real time), and convert into unicode for python's
# convenience
stderr = self.stderr_queue.get_nowait().rstrip()
if os.name == 'nt':
stderr = stderr.decode('cp1252')
else:
stderr = stderr.decode('utf-8')
if stderr:
# If the user has pip installed, rather than pip3, they will by
# now (mid-2019) be seeing a Python 2.7 deprecation warning.
# Ignore that message, if received
# If a newer version of pip is available, the user will see a
# 'You should consider upgrading' warning. Ignore that too,
# if received
if not re.search('DEPRECATION', stderr) \
and not re.search('You are using pip version', stderr) \
and not re.search('You should consider upgrading', stderr):
self.stderr_list.append(stderr)
# Show command line output in the Output Tab
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
stderr,
)
# (Generate our own error messages for debugging purposes, in certain
# situations)
if self.child_process is None:
msg = _('youtube-dl update did not start')
self.stderr_list.append(msg)
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
msg,
)
elif self.child_process.returncode > 0:
msg = _('Child process exited with non-zero code: {}').format(
self.child_process.returncode, )
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
msg,
)
# Operation complete. self.success_flag is checked by
# mainapp.TartubeApp.update_manager_finished
if not self.stderr_list:
self.success_flag = True
# Show a confirmation in the the Output Tab
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
_('Update operation finished'),
)
# Let the timer run for a few more seconds to prevent Gtk errors (for
# systems with Gtk < 3.24)
GObject.timeout_add(
0,
self.app_obj.update_manager_halt_timer,
)
def install_ffmpeg(self):
"""Called by self.run().
A modified version of self.install_ytdl, that installs FFmpeg on an
MS Windows system.
Creates a child process to run the installation process.
Reads from the child process STDOUT and STDERR, and calls the main
application with the result of the update (success or failure).
"""
if DEBUG_FUNC_FLAG:
utils.debug_time('uop 210 install_ffmpeg')
# Show information about the update operation in the Output Tab
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
_('Starting update operation, installing FFmpeg'),
)
# Create a new child process to install either the 64-bit or 32-bit
# version of FFmpeg, as appropriate
if sys.maxsize <= 2147483647:
binary = 'mingw-w64-i686-ffmpeg'
else:
binary = 'mingw-w64-x86_64-ffmpeg'
self.create_child_process(['pacman', '-S', binary, '--noconfirm'], )
# Show the system command in the Output Tab
space = ' '
self.app_obj.main_win_obj.output_tab_write_system_cmd(
1,
space.join(['pacman', '-S', binary, '--noconfirm']),
)
# So that we can read from the child process STDOUT and STDERR, attach
# a file descriptor to the PipeReader objects
if self.child_process is not None:
self.stdout_reader.attach_file_descriptor(
self.child_process.stdout, )
self.stderr_reader.attach_file_descriptor(
self.child_process.stderr, )
while self.is_child_process_alive():
# Read from the child process STDOUT, and convert into unicode for
# Python's convenience
while not self.stdout_queue.empty():
stdout = self.stdout_queue.get_nowait().rstrip()
stdout = stdout.decode('cp1252')
if stdout:
# Show command line output in the Output Tab
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
stdout,
)
# The child process has finished
while not self.stderr_queue.empty():
# Read from the child process STDERR queue (we don't need to read
# it in real time), and convert into unicode for python's
# convenience
stderr = self.stderr_queue.get_nowait().rstrip()
stderr = stderr.decode('cp1252')
# Ignore pacman warning messages, e.g. 'warning: dependency cycle
# detected:'
if stderr and not re.match('warning\:', stderr):
self.stderr_list.append(stderr)
# Show command line output in the Output Tab
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
stderr,
)
# (Generate our own error messages for debugging purposes, in certain
# situations)
if self.child_process is None:
self.stderr_list.append(_('FFmpeg installation did not start'))
elif self.child_process.returncode > 0:
self.stderr_list.append(
_('Child process exited with non-zero code: {}').format(
self.child_process.returncode, ))
# Operation complete. self.success_flag is checked by
# mainapp.TartubeApp.update_manager_finished
if not self.stderr_list:
self.success_flag = True
# Show a confirmation in the the Output Tab
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
_('Update operation finished'),
)
# Let the timer run for a few more seconds to prevent Gtk errors (for
# systems with Gtk < 3.24)
GObject.timeout_add(
0,
self.app_obj.update_manager_halt_timer,
)
def __init__(self, app_obj, choices_dict):
if DEBUG_FUNC_FLAG:
utils.debug_time('top 112 __init__')
super(TidyManager, self).__init__()
# IV list - class objects
# -----------------------
# The mainapp.TartubeApp object
self.app_obj = app_obj
# The media data object (channel, playlist or folder) to be tidied up,
# or None if the whole data directory is to be tidied up
# If specified, the channel/playlist/folder and all of its descendants
# are checked
self.init_obj = choices_dict['media_data_obj']
# IV list - other
# ---------------
# Flag set to False if self.stop_tidy_operation() is called, which
# halts the operation immediately
self.running_flag = True
# The time at which the tidy operation began (in seconds since epoch)
self.start_time = int(time.time())
# The time at which the tidy operation completed (in seconds since
# epoch)
self.stop_time = None
# The time (in seconds) between iterations of the loop in self.run()
self.sleep_time = 0.25
# Flags specifying which actions should be applied
# True if video files should be checked for corruption
self.corrupt_flag = choices_dict['corrupt_flag']
# True if corrupted video files should be deleted
self.del_corrupt_flag = choices_dict['del_corrupt_flag']
# True if video files that should exist should be checked, in case they
# don't (and vice-versa)
self.exist_flag = choices_dict['exist_flag']
# True if downloaded video files should be deleted
self.del_video_flag = choices_dict['del_video_flag']
# True if all video/audio files with the same name should be deleted
# (as artefacts of post-processing with FFmpeg or AVConv)
self.del_others_flag = choices_dict['del_others_flag']
# True if all description files should be deleted
self.del_descrip_flag = choices_dict['del_descrip_flag']
# True if all metadata (JSON) files should be deleted
self.del_json_flag = choices_dict['del_json_flag']
# True if all annotation files should be deleted
self.del_xml_flag = choices_dict['del_xml_flag']
# True if all thumbnail files should be deleted
self.del_thumb_flag = choices_dict['del_thumb_flag']
# v2.1.027. In June 2020, YouTube started serving .webp thumbnails.
# At the time of writing, Gtk can't display them. A youtube-dl fix is
# expected, which will convert .webp thumbnails to .jpg; in
# anticipation of that, we add an option to remove .webp files
# True if all thumbnail files in .webp or malformed .jpg format should
# be deleted
self.del_webp_flag = choices_dict['del_webp_flag']
# True if all youtube-dl archive files should be deleted
self.del_archive_flag = choices_dict['del_archive_flag']
# The number of media data objects whose directories have been tidied
# so far...
self.job_count = 0
# ...and the total number to tidy (these numbers are displayed in the
# progress bar in the Videos tab)
self.job_total = 0
# Individual counts, updated as we go
self.video_corrupt_count = 0
self.video_corrupt_deleted_count = 0
self.video_exist_count = 0
self.video_no_exist_count = 0
self.video_deleted_count = 0
self.other_deleted_count = 0
self.descrip_deleted_count = 0
self.json_deleted_count = 0
self.xml_deleted_count = 0
self.thumb_deleted_count = 0
self.webp_deleted_count = 0
self.archive_deleted_count = 0
# Code
# ----
# Let's get this party started!
self.start()
def run(self):
"""Called as a result of self.__init__().
Compiles a list of media data objects (channels, playlists and folders)
to tidy up. If self.init_obj is not set, only that channel/playlist/
folder (and its child channels/playlists/folders) are tidied up;
otherwise the whole data directory is tidied up.
Then calls self.tidy_directory() for each item in the list.
Finally informs the main application that the tidy operation is
complete.
"""
if DEBUG_FUNC_FLAG:
utils.debug_time('top 220 run')
# Show information about the tidy operation in the Output Tab
if not self.init_obj:
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
_('Starting tidy operation, tidying up whole data directory'),
)
else:
media_type = self.init_obj.get_type()
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
_('Starting tidy operation, tidying up \'{0}\'').format(
self.init_obj.name,
)
)
if self.corrupt_flag:
text = _('YES')
else:
text = _('NO')
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Check videos are not corrupted:') + ' ' + text,
)
if self.corrupt_flag:
if self.del_corrupt_flag:
text = _('YES')
else:
text = _('NO')
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Delete corrupted videos:') + ' ' + text,
)
if self.exist_flag:
text = _('YES')
else:
text = _('NO')
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Check videos do/don\'t exist:') + ' ' + text,
)
if self.del_video_flag:
text = _('YES')
else:
text = _('NO')
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Delete all video files:') + ' ' + text,
)
if self.del_video_flag:
if self.del_others_flag:
text = _('YES')
else:
text = _('NO')
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Delete other video/audio files:') + ' ' + text,
)
if self.del_descrip_flag:
text = _('YES')
else:
text = _('NO')
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Delete all description files:') + ' ' + text,
)
if self.del_json_flag:
text = _('YES')
else:
text = _('NO')
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Delete all metadata (JSON) files:') + ' ' + text,
)
if self.del_xml_flag:
text = _('YES')
else:
text = _('NO')
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Delete all annotation files:') + ' ' + text,
)
if self.del_thumb_flag:
text = _('YES')
else:
text = _('NO')
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Delete all thumbnail files:') + ' ' + text,
)
if self.del_webp_flag:
text = _('YES')
else:
text = _('NO')
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Delete .webp/malformed .jpg files:') + ' ' + text,
)
if self.del_archive_flag:
text = _('YES')
else:
text = _('NO')
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Delete youtube-dl archive files:') + ' ' + text,
)
# Compile a list of channels, playlists and folders to tidy up (each
# one has their own sub-directory inside Tartube's data directory)
obj_list = []
if self.init_obj:
# Add this channel/playlist/folder, and any child channels/
# playlists/folders (but not videos, obviously)
obj_list = self.init_obj.compile_all_containers(obj_list)
else:
# Add all channels/playlists/folders in the database
for dbid in list(self.app_obj.media_name_dict.values()):
obj = self.app_obj.media_reg_dict[dbid]
# Don't add private folders
if not isinstance(obj, media.Folder) or not obj.priv_flag:
obj_list.append(obj)
self.job_total = len(obj_list)
# Check each sub-directory in turn, updating the media data registry
# as we go
while self.running_flag and obj_list:
self.tidy_directory(obj_list.pop(0))
# Pause a moment, before the next iteration of the loop (don't want
# to hog resources)
time.sleep(self.sleep_time)
# Operation complete. Set the stop time
self.stop_time = int(time.time())
# Show a confirmation in the Output Tab
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
_('Tidy operation finished'),
)
if self.corrupt_flag:
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Corrupted videos found:') + ' ' \
+ str(self.video_corrupt_count),
)
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Corrupted videos deleted:') + ' ' \
+ str(self.video_corrupt_deleted_count),
)
if self.exist_flag:
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('New video files detected:') + ' ' \
+ str(self.video_exist_count),
)
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Missing video files detected:') + ' ' \
+ str(self.video_no_exist_count),
)
if self.del_video_flag:
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Non-corrupted video files deleted:') + ' ' \
+ str(self.video_deleted_count),
)
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Other video/audio files deleted:') + ' ' \
+ str(self.other_deleted_count),
)
if self.del_descrip_flag:
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Description files deleted:') + ' ' \
+ str(self.descrip_deleted_count),
)
if self.del_json_flag:
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Metadata (JSON) files deleted:') + ' ' \
+ str(self.json_deleted_count),
)
if self.del_xml_flag:
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Annotation files deleted:') + ' ' \
+ str(self.xml_deleted_count),
)
if self.del_thumb_flag:
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Thumbnail files deleted:') + ' ' \
+ str(self.thumb_deleted_count),
)
if self.del_webp_flag:
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('.webp/malformed .jpg files deleted:') + ' ' \
+ str(self.webp_deleted_count),
)
if self.del_archive_flag:
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('youtube-dl archive files deleted:') + ' ' \
+ str(self.archive_deleted_count),
)
# Let the timer run for a few more seconds to prevent Gtk errors (for
# systems with Gtk < 3.24)
GObject.timeout_add(
0,
self.app_obj.tidy_manager_halt_timer,
)
def refresh_from_actual_destination(self, media_data_obj):
"""Called by self.run().
A modified version of self.refresh_from_default_destination().
Refreshes a single channel, playlist or folder, for which an
alternative download destination has been set.
If a file is missing in the alternative download destination, mark the
video object as not downloaded.
Don't check for unexpected video files in the alternative download
destination - we expect that they exist.
Args:
media_data_obj (media.Channel, media.Playlist or media.Folder):
The media data object to refresh
"""
if DEBUG_FUNC_FLAG:
utils.debug_time('rop 519 refresh_from_actual_destination')
# Update the main window's progress bar
self.job_count += 1
GObject.timeout_add(
0,
self.app_obj.main_win_obj.update_progress_bar,
media_data_obj.name,
self.job_count,
self.job_total,
)
# Keep a running total of matched videos for this channel, playlist or
# folder
local_total_count = 0
local_match_count = 0
# (No new media.Video objects are created)
local_missing_count = 0
# Update our progress in the Output Tab
if isinstance(media_data_obj, media.Channel):
string = _('Channel:') + ' '
elif isinstance(media_data_obj, media.Playlist):
string = _('Playlist:') + ' '
else:
string = _('Folder:') + ' '
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
string + media_data_obj.name,
)
# Get the alternative download destination
dir_path = media_data_obj.get_actual_dir(self.app_obj)
# Get a list of video files in that sub-directory
try:
init_list = os.listdir(dir_path)
except:
# Can't read the directory
return
# Now check each media.Video object, to see if the video file still
# exists (or not)
for child_obj in media_data_obj.child_list:
if isinstance(child_obj, media.Video) and child_obj.file_name:
this_file = child_obj.file_name + child_obj.file_ext
if child_obj.dl_flag and not this_file in init_list:
local_missing_count += 1
# Video doesn't exist, so mark it as not downloaded
self.app_obj.mark_video_downloaded(child_obj, False)
# Update our progress in the Output Tab (if required)
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Missing:') + ' ' + child_obj.name,
)
elif not child_obj.dl_flag and this_file in init_list:
self.video_total_count += 1
local_total_count += 1
self.video_match_count += 1
local_match_count += 1
# Video exists, so mark it as downloaded (but don't mark it
# as new)
self.app_obj.mark_video_downloaded(child_obj, True, True)
# Update our progress in the Output Tab (if required)
if self.app_obj.refresh_output_videos_flag:
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Match:') + ' ' + child_obj.name,
)
# Check complete, display totals
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Total videos:') + ' ' + str(local_total_count) \
+ ', ' + _('matched:') + ' ' + str(local_match_count) \
+ ', ' + _('missing:') + ' ' + str(local_missing_count),
)
def refresh_from_default_destination(self, media_data_obj):
"""Called by self.run().
Refreshes a single channel, playlist or folder, for which an
alternative download destination has not been set.
If a file is missing in the channel/playlist/folder's sub-directory,
mark the video object as not downloaded.
If unexpected video files exist in the sub-directory, create a new
media.Video object for them.
Args:
media_data_obj (media.Channel, media.Playlist or media.Folder):
The media data object to refresh
"""
if DEBUG_FUNC_FLAG:
utils.debug_time('rop 252 refresh_from_default_destination')
# Update the main window's progress bar
self.job_count += 1
GObject.timeout_add(
0,
self.app_obj.main_win_obj.update_progress_bar,
media_data_obj.name,
self.job_count,
self.job_total,
)
# Keep a running total of matched/new videos for this channel, playlist
# or folder
local_total_count = 0
local_match_count = 0
local_new_count = 0
# Update our progress in the Output Tab
if isinstance(media_data_obj, media.Channel):
string = _('Channel:') + ' '
elif isinstance(media_data_obj, media.Playlist):
string = _('Playlist:') + ' '
else:
string = _('Folder:') + ' '
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
string + media_data_obj.name,
)
# Get the sub-directory for this media data object
dir_path = media_data_obj.get_default_dir(self.app_obj)
# Get a list of video files in the sub-directory
try:
init_list = os.listdir(dir_path)
except:
# Can't read the directory
return
# From this list, filter out files without a recognised file extension
# (.mp4, .webm, etc)
mod_list = []
for relative_path in init_list:
# (If self.stop_refresh_operation() has been called, give up
# immediately)
if not self.running_flag:
return
filename, ext = os.path.splitext(relative_path)
# (Remove the initial .)
ext = ext[1:]
if ext in formats.VIDEO_FORMAT_DICT:
mod_list.append(relative_path)
# From the new list, filter out duplicate filenames (e.g. if the list
# contains both 'my_video.mp4' and 'my_video.webm', filter out the
# second one, adding to a list of alternative files)
filter_list = []
filter_dict = {}
alt_list = []
for relative_path in mod_list:
# (If self.stop_refresh_operation() has been called, give up
# immediately)
if not self.running_flag:
return
filename, ext = os.path.splitext(relative_path)
if not filename in filter_dict:
filter_list.append(relative_path)
filter_dict[filename] = relative_path
else:
alt_list.append(relative_path)
# Now compile a dictionary of media.Video objects in this channel/
# playlist/folder, so we can eliminate them one by one
check_dict = {}
for child_obj in media_data_obj.child_list:
# (If self.stop_refresh_operation() has been called, give up
# immediately)
if not self.running_flag:
return
if isinstance(child_obj, media.Video) and child_obj.file_name:
# Does the video file still exist?
this_file = child_obj.file_name + child_obj.file_ext
if child_obj.dl_flag and not this_file in init_list:
self.app_obj.mark_video_downloaded(child_obj, False)
else:
check_dict[child_obj.file_name] = child_obj
# If this channel/playlist/folder is the alternative download
# destination for other channels/playlists/folders, compile a
# dicationary of their media.Video objects
# (If we find a video we weren't expecting, before creating a new
# media.Video object, we must first check it isn't one of them)
slave_dict = {}
for slave_dbid in media_data_obj.slave_dbid_list:
# (If self.stop_refresh_operation() has been called, give up
# immediately)
if not self.running_flag:
return
slave_obj = self.app_obj.media_reg_dict[slave_dbid]
for child_obj in slave_obj.child_list:
if isinstance(child_obj, media.Video) and child_obj.file_name:
slave_dict[child_obj.file_name] = child_obj
# Now try to match each video file (in filter_list) with an existing
# media.Video object (in check_dict)
# If there is no match, and if the video file doesn't match a video
# in another channel/playlist/folder (for which this is the
# alternative download destination), then we can create a new
# media.Video object
for relative_path in filter_list:
# (If self.stop_refresh_operation() has been called, give up
# immediately)
if not self.running_flag:
return
filename, ext = os.path.splitext(relative_path)
if self.app_obj.refresh_output_videos_flag:
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Checking:') + ' ' + filename,
)
if filename in check_dict:
# File matched
self.video_total_count += 1
local_total_count += 1
self.video_match_count += 1
local_match_count += 1
# If it is not marked as downloaded, we can mark it so now
child_obj = check_dict[filename]
if not child_obj.dl_flag:
self.app_obj.mark_video_downloaded(child_obj, True)
# Make sure the stored extension is correct (e.g. if we've
# matched an existing .webm video file, with an expected
# .mp4 video file)
if child_obj.file_ext != ext:
child_relative_path \
= child_obj.file_name + child_obj.file_ext
if not child_relative_path in alt_list:
child_obj.set_file(filename, ext)
# Eliminate this media.Video object; no other video file should
# match it
del check_dict[filename]
# Update our progress in the Output Tab (if required)
if self.app_obj.refresh_output_videos_flag:
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Match:') + ' ' + filename,
)
elif filename not in slave_dict:
# File didn't match a media.Video object
self.video_total_count += 1
local_total_count += 1
self.video_new_count += 1
local_new_count += 1
# Display the list of non-matching videos, if required
if self.app_obj.refresh_output_videos_flag \
and self.app_obj.refresh_output_verbose_flag:
for failed_path in check_dict.keys():
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Non-match:') + ' ' + filename,
)
# Create a new media.Video object
video_obj = self.app_obj.add_video(media_data_obj, None)
video_path = os.path.abspath(
os.path.join(
dir_path,
filter_dict[filename],
)
)
# Set the new video object's IVs
filename, ext = os.path.splitext(filter_dict[filename])
video_obj.set_name(filename)
video_obj.set_nickname(filename)
video_obj.set_file(filename, ext)
if ext == '.mkv':
video_obj.set_mkv()
video_obj.set_file_size(
os.path.getsize(
os.path.abspath(
os.path.join(dir_path, filter_dict[filename]),
),
),
)
# If the video's JSON file exists downloaded, we can extract
# video statistics from it
self.app_obj.update_video_from_json(video_obj)
# For any of those statistics that haven't been set (because
# the JSON file was missing or didn't contain the right
# statistics), set them directly
self.app_obj.update_video_from_filesystem(
video_obj,
video_path,
)
# This call marks the video as downloaded, and also updates the
# Video Index and Video Catalogue (if required)
self.app_obj.mark_video_downloaded(video_obj, True)
if self.app_obj.refresh_output_videos_flag:
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('New video:') + ' ' + filename,
)
# Check complete, display totals
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Total videos:') + ' ' + str(local_total_count) \
+ ', ' + _('matched:') + ' ' + str(local_match_count) \
+ ', ' + _('new:') + ' ' + str(local_new_count),
)
def run(self):
"""Called as a result of self.__init__().
Compiles a list of media data objects (channels, playlists and folders)
to refresh. If self.init_obj is not set, only that channel/playlist/
folder (and its child channels/playlists/folders) are refreshed;
otherwise the whole media registry is refreshed.
Then calls self.refresh_from_default_destination() for each item in the
list.
Finally informs the main application that the refresh operation is
complete.
"""
if DEBUG_FUNC_FLAG:
utils.debug_time('rop 143 run')
# Show information about the refresh operation in the Output Tab
if not self.init_obj:
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
_('Starting refresh operation, analysing whole database'),
)
else:
media_type = self.init_obj.get_type()
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
_('Starting refresh operation, analysing \'{}\'').format(
self.init_obj.name,
),
)
# Compile a list of channels, playlists and folders to refresh (each
# one has their own sub-directory inside Tartube's data directory)
obj_list = []
if self.init_obj:
# Add this channel/playlist/folder, and any child channels/
# playlists/folders (but not videos, obviously)
obj_list = self.init_obj.compile_all_containers(obj_list)
else:
# Add all channels/playlists/folders in the database
for dbid in list(self.app_obj.media_name_dict.values()):
obj = self.app_obj.media_reg_dict[dbid]
# Don't add private folders
if not isinstance(obj, media.Folder) or not obj.priv_flag:
obj_list.append(obj)
self.job_total = len(obj_list)
# Check each sub-directory in turn, updating the media data registry
# as we go
while self.running_flag and obj_list:
obj = obj_list.pop(0)
if obj.dbid == obj.master_dbid:
self.refresh_from_default_destination(obj)
else:
self.refresh_from_actual_destination(obj)
# Pause a moment, before the next iteration of the loop (don't want
# to hog resources)
time.sleep(self.sleep_time)
# Operation complete. Set the stop time
self.stop_time = int(time.time())
# Show a confirmation in the Output Tab
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
_('Refresh operation finished'),
)
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Number of video files analysed:') + ' ' \
+ str(self.video_total_count),
)
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('Video files already in the database:') + ' ' \
+ str(self.video_match_count),
)
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _('New videos found and added to the database:') + ' ' \
+ str(self.video_new_count),
)
# Let the timer run for a few more seconds to prevent Gtk errors (for
# systems with Gtk < 3.24)
GObject.timeout_add(
0,
self.app_obj.refresh_manager_halt_timer,
)
def check_video_corrupt(self, media_data_obj):
"""Called by self.tidy_directory().
Checks all child videos of the specified media data object. If the
video are corrupted, don't delete them (let the user do that manually).
Args:
media_data_obj (media.Channel, media.Playlist or media.Folder):
The media data object whose directory must be tidied up
"""
if DEBUG_FUNC_FLAG:
utils.debug_time('top 564 check_video_corrupt')
for video_obj in media_data_obj.compile_all_videos( [] ):
if video_obj.file_name is not None \
and video_obj.dl_flag:
video_path = video_obj.get_actual_path(self.app_obj)
if os.path.isfile(video_path):
# Code copied from
# mainapp.TartubeApp.update_video_from_filesystem()
# When the video file is corrupted, moviepy freezes
# indefinitely
# Instead, let's try placing the procedure inside a thread
# (unlike the original function, this one is never called
# if .refresh_moviepy_timeout is 0)
this_thread = threading.Thread(
target=self.call_moviepy,
args=(video_obj, video_path,),
)
this_thread.daemon = True
this_thread.start()
this_thread.join(self.app_obj.refresh_moviepy_timeout)
if this_thread.is_alive():
# moviepy timed out, so assume the video is corrupted
self.video_corrupt_count += 1
if self.del_corrupt_flag \
and os.path.isfile(video_path):
# Delete the corrupted file
os.remove(video_path)
self.video_corrupt_deleted_count += 1
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _(
'Deleted (possibly) corrupted video file:',
) + ' \'' + video_obj.name + '\'',
)
self.app_obj.mark_video_downloaded(
video_obj,
False,
)
else:
# Don't delete it
self.app_obj.main_win_obj.output_tab_write_stdout(
1,
' ' + _(
'Video file might be corrupt:',
) + ' \'' + video_obj.name + '\'',
)
def delete_video(self, media_data_obj):
"""Called by self.tidy_directory().
Checks all child videos of the specified media data object. If the
video exists, delete it.
Args:
media_data_obj (media.Channel, media.Playlist or media.Folder):
The media data object whose directory must be tidied up
"""
if DEBUG_FUNC_FLAG:
utils.debug_time('top 703 delete_video')
ext_list = formats.VIDEO_FORMAT_LIST.copy()
ext_list.extend(formats.AUDIO_FORMAT_LIST)
for video_obj in media_data_obj.compile_all_videos( [] ):
video_path = None
if video_obj.file_name is not None:
video_path = video_obj.get_actual_path(self.app_obj)
# If the video's parent container has an alternative download
# destination set, we must check the corresponding media
# data object. If the latter also has a media.Video object
# matching this video, then this function returns None and
# nothing is deleted
video_path = self.check_video_in_actual_dir(
media_data_obj,
video_obj,
video_path,
)
if video_path is not None:
if video_obj.dl_flag \
and os.path.isfile(video_path):
# Delete the downloaded video file
os.remove(video_path)
# Mark the video as not downloaded
self.app_obj.mark_video_downloaded(video_obj, False)
self.video_deleted_count += 1
if self.del_others_flag:
# Also delete all video/audio files with the same name
# There might be thousands of files in the directory, so
# using os.walk() or something like that might be too
# expensive
# Also, post-processing might create various artefacts, all
# of which must be deleted
for ext in ext_list:
other_path = video_obj.get_actual_path_by_ext(
self.app_obj,
ext,
)
if os.path.isfile(other_path):
os.remove(other_path)
self.other_deleted_count += 1
# For an encore, delete all post-processing artefacts in the form
# VIDEO_NAME.fNNN.ext, where NNN is an integer and .ext is one of
# the video extensions specified by formats.VIDEO_FORMAT_LIST
# (.mkv, etc)
# (The previous code won't pick them up, but we can delete them all
# now.)
# (The alternative download destination, if set, is not affected.)
check_list = []
search_path = media_data_obj.get_default_dir(self.app_obj)
for (dir_path, dir_name_list, file_name_list) in os.walk(search_path):
check_list.extend(file_name_list)
char = '|'
regex = '\.f\d+\.(' + char.join(formats.VIDEO_FORMAT_LIST) + ')$'
for check_path in check_list:
if re.search(regex, check_path):
full_path = os.path.abspath(
os.path.join(search_path, check_path),
)
if os.path.isfile(full_path):
os.remove(full_path)
self.other_deleted_count += 1
