def get_value(self, section, key, key_type=None): """\ Retrieve a value for a given section and key. @param section: the ini file section @type section: C{str} @param key: the ini file key in the given section @type key: C{str} @return: the value for the given section and key @rtype: class """ if key_type is None: key_type = self.get_type(section, key) if self.iniConfig.has_option(section, key): if key_type is types.BooleanType: return self.iniConfig.getboolean(section, key) elif key_type is types.IntType: return self.iniConfig.getint(section, key) elif key_type is types.ListType: _val = self.iniConfig.get(section, key) _val = _val.strip() if _val.startswith('[') and _val.endswith(']'): return eval(_val) elif ',' in _val: _val = [ v.strip() for v in _val.split(',') ] else: _val = [ _val ] return _val else: _val = self.iniConfig.get(section, key) return _val.decode(utils.get_encoding())
def load_text(self, full_path): """Can be called by anything. Given the full path to a text file, loads it. Args: full_path (str): The full path to the text file Returns: The contents of the text file as a string, or or None if the file is missing or can't be loaded """ if not os.path.isfile(full_path): return None with open( full_path, 'r', encoding=utils.get_encoding(), errors='ignore', ) as text_file: try: text = text_file.read() return text except: return None
def _storeValue(self, section, key, value): """\ Stores a value for a given section and key. This methods affects a SafeConfigParser object held in RAM. No configuration file is affected by this method. To write the configuration to disk use the L{write()} method. @param section: the ini file section @type section: C{str} @param key: the ini file key in the given section @type key: C{str} @param value: the value for the given section and key @type value: C{str}, C{list}, C{booAl}, ... """ if type(value) == type(u''): value = value.encode(utils.get_encoding()) if type(value) is types.BooleanType: self.iniConfig.set(section, key, str(int(value))) elif type(value) in (types.ListType, types.TupleType): self.iniConfig.set(section, key, ", ".join(value)) else: self.iniConfig.set(section, key, str(value))
def __init__(self, config_path, add_time=False): self.config_path = config_path self.add_time = add_time self.log_file = os.path.join(config_path, self.LOG_FILENAME) self._encoding = get_encoding() self._init_log() self._auto_clear_log()
def load_json(self, full_path): """Can be called by anything. Given the full path to a JSON file, loads the file into a Python dictionary and returns the dictionary. Args: full_path (str): The full path to the JSON file Returns: The JSON data, converted to a Python dictionary (an empty dictionary if the file is missing or can't be loaded) """ empty_dict = {} if not os.path.isfile(full_path): return empty_dict with open( full_path, 'r', encoding=utils.get_encoding(), errors='ignore', ) as json_file: try: json_dict = json.load(json_file) return json_dict except: return empty_dict
def run_ffmpeg_with_options(self, video_obj, source_path, cmd_list): """Modified version of self.run_ffmpeg(), called by process.ProcessManager.process_video(). Adapted from youtube-dl/youtube-dl/postprocessor/ffmpeg.py. Prepares the FFmpeg system command, and then executes it. Args: video_obj (media.Video): The video object to be processed source_path (str): The full path to the source file cmd_list (list): The FFmpeg system command to use, as a list Return values: Returns a list of two items, in the form (success_flag, optional_message) """ # Get the file's modification time mod_time = os.stat(source_path).st_mtime # Execute the system command in a subprocess try: p = subprocess.Popen( cmd_list, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, ) except: # If FFmpeg is not installed on the user's system, this is the # result return [False, 'Could not find FFmpeg'] stdout, stderr = p.communicate() if p.returncode != 0: stderr = stderr.decode(utils.get_encoding(), 'replace') return [False, stderr.strip().split('\n')[-1]] else: return [self.try_utime(source_path, mod_time, mod_time), '']
def message(self, msg, loglevel=loglevel_NONE, tag=None): """\ Log a message. @param msg: log message text @type msg: C{str} @param loglevel: log level of this message @type loglevel: C{int} @param tag: additional tag for this log entry @type tag: C{str} """ if tag is None: tag = self.tag if loglevel & self.loglevel: msg = msg.replace('\n', ' ') msg = msg.encode(utils.get_encoding()) from datetime import datetime minsec = datetime.now().strftime("%H:%M:%S.%f") if self.tag is not None: self.destination.write('%s: %s[%s] (%s) %s: %s\n' % (minsec, self.name, self.progpid, tag, self._loglevel_NAMES[loglevel].upper(), msg)) else: self.destination.write('%s: %s[%s] %s: %s\n' % (minsec, self.name, self.progpid, self._loglevel_NAMES[loglevel].upper(), msg))
def run_ffmpeg_multiple_files(self, input_path_list, out_path, opt_list, \ test_flag=False): """Can be called by anything (currently called only by self.run_ffmpeg() ). Adapted from youtube-dl/youtube-dl/postprocessor/ffmpeg.py. Prepares the FFmpeg system command, and then executes it. Args: input_path_list (list): List of full paths to files to be processed by FFmpeg. At the moment, Tartube only processes one file at a time out_path (str): Full path to FFmpeg's output file opt_list (list): List of FFmpeg command line options (may be an empty list) test_flag (bool): If True, just returns the FFmpeg system command, rather than executing it Return values: Returns a list of two items, in the form (success_flag, optional_message) """ # Get the modification time for the oldest file oldest_mtime = min(os.stat(path).st_mtime for path in input_path_list) # Prepare the system command files_cmd_list = [] for path in input_path_list: files_cmd_list.extend(['-i', self._ffmpeg_filename_argument(path)]) cmd_list = [self.get_executable(), '-y'] cmd_list += ['-loglevel', 'repeat+info'] cmd_list += ( files_cmd_list + opt_list + [self._ffmpeg_filename_argument(out_path)] ) # Return the system command only, if required if test_flag: return [ True, cmd_list ] # Execute the system command in a subprocess try: p = subprocess.Popen( cmd_list, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, ) except Exception as e: return [ False, str(e) ] stdout, stderr = p.communicate() if p.returncode != 0: stderr = stderr.decode(utils.get_encoding(), 'replace') return [ False, stderr.strip().split('\n')[-1] ] else: return [ self.try_utime(out_path, oldest_mtime, oldest_mtime), '' ]
def read_ytdl_child_process(self, downloader): """Called by self.install_ytdl(). Reads from the child process STDOUT and STDERR, in the correct order. Args: downloader (str): e.g. 'youtube-dl' Return values: True if either STDOUT or STDERR were read, None if both queues were empty """ # mini_list is in the form [time, pipe_type, data] try: mini_list = self.queue.get_nowait() except: # Nothing left to read return None # Failsafe check if not mini_list \ or (mini_list[1] != 'stdout' and mini_list[1] != 'stderr'): # Just in case... GObject.timeout_add( 0, self.app_obj.system_error, 704, 'Malformed STDOUT or STDERR data', ) # STDOUT or STDERR has been read data = mini_list[2].rstrip() # On MS Windows we use cp1252, so that Tartube can communicate with the # Windows console data = data.decode(utils.get_encoding(), 'replace') # STDOUT if mini_list[1] == 'stdout': # "It looks like you installed youtube-dl with a package manager, # pip, setup.py or a tarball. Please use that to update." # "The script youtube-dl is installed in '...' which is not on # PATH. Consider adding this directory to PATH..." if re.search('It looks like you installed', data) \ or re.search( 'The script ' + downloader + ' is installed', data, ): self.stderr_list.append(data) else: # Try to intercept the new version number for youtube-dl self.intercept_version_from_stdout(data, downloader) self.stdout_list.append(data) # Show command line output in the Output tab (or wizard window # textview) self.install_ytdl_write_output(data) # STDERR else: # If the user has pip installed, rather than pip3, they will by now # (mid-2019) be seeing a Python 2.7 deprecation warning. Ignore # that message, if received # If a newer version of pip is available, the user will see a # 'You should consider upgrading' warning. Ignore that too, if # received if not re.search('DEPRECATION', data) \ and not re.search('You are using pip version', data) \ and not re.search('You should consider upgrading', data): self.stderr_list.append(data) # Show command line output in the Output tab (or wizard window # textview) self.install_ytdl_write_output(data) # Either (or both) of STDOUT and STDERR were non-empty self.queue.task_done() return True
def read_streamlink_child_process(self): """Called by self.install_matplotlib(). Reads from the child process STDOUT and STDERR, in the correct order. Return values: True if either STDOUT or STDERR were read, None if both queues were empty """ # mini_list is in the form [time, pipe_type, data] try: mini_list = self.queue.get_nowait() except: # Nothing left to read return None # Failsafe check if not mini_list \ or (mini_list[1] != 'stdout' and mini_list[1] != 'stderr'): # Just in case... GObject.timeout_add( 0, self.app_obj.system_error, 703, 'Malformed STDOUT or STDERR data', ) # STDOUT or STDERR has been read data = mini_list[2].rstrip() # On MS Windows we use cp1252, so that Tartube can communicate with the # Windows console data = data.decode(utils.get_encoding(), 'replace') # STDOUT if mini_list[1] == 'stdout': # Show command line output in the Output tab (or wizard window # textview) self.install_streamlink_write_output(data) # STDERR else: # Ignore pacman warning messages, e.g. 'warning: dependency cycle # detected:' if data and not re.search('^warning\:', data): self.stderr_list.append(data) # Show command line output in the Output tab (or wizard window # textview) self.install_streamlink_write_output(data) # Either (or both) of STDOUT and STDERR were non-empty self.queue.task_done() return True
def install_ytdl(self): """Called by self.run(). Based on code from downloads.VideoDownloader.do_download(). Creates a child process to run the youtube-dl update. Reads from the child process STDOUT and STDERR, and calls the main application with the result of the update (success or failure). """ # Show information about the update operation in the Output tab (or in # the setup wizard window, if called from there) downloader = self.app_obj.get_downloader(self.wiz_win_obj) self.install_ytdl_write_output( _('Starting update operation, installing/updating ' + downloader), ) # Prepare the system command # The user can change the system command for updating youtube-dl, # depending on how it was installed # (For example, if youtube-dl was installed via pip, then it must be # updated via pip) if self.wiz_win_obj \ and self.wiz_win_obj.ytdl_update_current is not None: ytdl_update_current = self.wiz_win_obj.ytdl_update_current else: ytdl_update_current = self.app_obj.ytdl_update_current # Special case: install yt-dlp with no dependencies, if required if ( ( not self.wiz_win_obj \ and self.app_obj.ytdl_fork == 'yt-dlp' \ and self.app_obj.ytdl_fork_no_dependency_flag ) or ( self.wiz_win_obj \ and self.wiz_win_obj.ytdl_fork == 'yt-dlp' \ and self.wiz_win_obj.ytdl_fork_no_dependency_flag ) ): if ytdl_update_current == 'ytdl_update_pip': ytdl_update_current = 'ytdl_update_pip_no_dependencies' elif ytdl_update_current == 'ytdl_update_pip3' \ or ytdl_update_current == 'ytdl_update_pip3_recommend': ytdl_update_current = 'ytdl_update_pip3_no_dependencies' elif ytdl_update_current == 'ytdl_update_win_64': ytdl_update_current = 'ytdl_update_win_64_no_dependencies' elif ytdl_update_current == 'ytdl_update_win_32': ytdl_update_current = 'ytdl_update_win_32_no_dependencies' # Set the system command cmd_list = self.app_obj.ytdl_update_dict[ytdl_update_current] mod_list = [] for arg in cmd_list: # Substitute in the fork, if one is specified arg = self.app_obj.check_downloader(arg, self.wiz_win_obj) # Convert a path beginning with ~ (not on MS Windows) if os.name != 'nt': arg = re.sub('^\~', os.path.expanduser('~'), arg) mod_list.append(arg) # Create a new child process using that command self.create_child_process(mod_list) # Show the system command in the Output tab self.install_ytdl_write_output( ' '.join(mod_list), True, # A system command, not a message ) # So that we can read from the child process STDOUT and STDERR, attach # a file descriptor to the PipeReader objects if self.child_process is not None: self.stdout_reader.attach_file_descriptor( self.child_process.stdout, ) self.stderr_reader.attach_file_descriptor( self.child_process.stderr, ) while self.is_child_process_alive(): # Read from the child process STDOUT, and convert into unicode for # Python's convenience while not self.stdout_queue.empty(): stdout = self.stdout_queue.get_nowait().rstrip() if stdout: stdout = stdout.decode(utils.get_encoding(), 'replace') # "It looks like you installed youtube-dl with a package # manager, pip, setup.py or a tarball. Please use that to # update." # "The script youtube-dl is installed in '...' which is not # on PATH. Consider adding this directory to PATH..." if re.search('It looks like you installed', stdout) \ or re.search( 'The script ' + downloader + ' is installed', stdout, ): self.stderr_list.append(stdout) else: # Try to intercept the new version number for # youtube-dl self.intercept_version_from_stdout(stdout, downloader) self.stdout_list.append(stdout) # Show command line output in the Output tab (or wizard # window textview) self.install_ytdl_write_output(stdout) # The child process has finished while not self.stderr_queue.empty(): # Read from the child process STDERR queue (we don't need to read # it in real time), and convert into unicode for python's # convenience stderr = self.stderr_queue.get_nowait().rstrip() if stderr: stderr = stderr.decode(utils.get_encoding(), 'replace') # If the user has pip installed, rather than pip3, they will by # now (mid-2019) be seeing a Python 2.7 deprecation warning. # Ignore that message, if received # If a newer version of pip is available, the user will see a # 'You should consider upgrading' warning. Ignore that too, # if received if not re.search('DEPRECATION', stderr) \ and not re.search('You are using pip version', stderr) \ and not re.search('You should consider upgrading', stderr): self.stderr_list.append(stderr) # Show command line output in the Output tab (or wizard window # textview) self.install_ytdl_write_output(stderr) # (Generate our own error messages for debugging purposes, in certain # situations) if self.child_process is None: msg = _('Update did not start') self.stderr_list.append(msg) self.install_ytdl_write_output(msg) elif self.child_process.returncode > 0: msg = _('Child process exited with non-zero code: {}').format( self.child_process.returncode, ) self.stderr_list.append(msg) self.install_ytdl_write_output(msg) # Operation complete. self.success_flag is checked by # mainapp.TartubeApp.update_manager_finished if not self.stderr_list: self.success_flag = True # Show a confirmation in the the Output tab (or wizard window textview) self.install_ytdl_write_output(_('Update operation finished')) # Let the timer run for a few more seconds to prevent Gtk errors (for # systems with Gtk < 3.24) GObject.timeout_add( 0, self.app_obj.update_manager_halt_timer, )
def install_ffmpeg(self): """Called by self.run(). A modified version of self.install_ytdl, that installs FFmpeg on an MS Windows system. Creates a child process to run the installation process. Reads from the child process STDOUT and STDERR, and calls the main application with the result of the update (success or failure). """ # Show information about the update operation in the Output tab self.install_ffmpeg_write_output( _('Starting update operation, installing FFmpeg'), ) # Create a new child process to install either the 64-bit or 32-bit # version of FFmpeg, as appropriate if sys.maxsize <= 2147483647: binary = 'mingw-w64-i686-ffmpeg' else: binary = 'mingw-w64-x86_64-ffmpeg' self.create_child_process(['pacman', '-S', binary, '--noconfirm'], ) # Show the system command in the Output tab self.install_ffmpeg_write_output( ' '.join(['pacman', '-S', binary, '--noconfirm']), True, # A system command, not a message ) # So that we can read from the child process STDOUT and STDERR, attach # a file descriptor to the PipeReader objects if self.child_process is not None: self.stdout_reader.attach_file_descriptor( self.child_process.stdout, ) self.stderr_reader.attach_file_descriptor( self.child_process.stderr, ) while self.is_child_process_alive(): # Read from the child process STDOUT, and convert into unicode for # Python's convenience while not self.stdout_queue.empty(): stdout = self.stdout_queue.get_nowait().rstrip() stdout = stdout.decode(utils.get_encoding(), 'replace') if stdout: # Show command line output in the Output tab (or wizard # window textview) self.install_ffmpeg_write_output(stdout) # The child process has finished while not self.stderr_queue.empty(): # Read from the child process STDERR queue (we don't need to read # it in real time), and convert into unicode for python's # convenience stderr = self.stderr_queue.get_nowait().rstrip() stderr = stderr.decode(utils.get_encoding(), 'replace') # Ignore pacman warning messages, e.g. 'warning: dependency cycle # detected:' if stderr and not re.match('warning\:', stderr): self.stderr_list.append(stderr) # Show command line output in the Output tab (or wizard window # textview) self.install_ffmpeg_write_output(stderr) # (Generate our own error messages for debugging purposes, in certain # situations) if self.child_process is None: self.stderr_list.append(_('FFmpeg installation did not start')) elif self.child_process.returncode > 0: self.stderr_list.append( _('Child process exited with non-zero code: {}').format( self.child_process.returncode, )) # Operation complete. self.success_flag is checked by # mainapp.TartubeApp.update_manager_finished if not self.stderr_list: self.success_flag = True # Show a confirmation in the the Output tab (or wizard window textview) self.install_ffmpeg_write_output(_('Update operation finished')) # Let the timer run for a few more seconds to prevent Gtk errors (for # systems with Gtk < 3.24) GObject.timeout_add( 0, self.app_obj.update_manager_halt_timer, )
def run(self): """Called as a result of self.__init__(). Creates a child process to run the youtube-dl system command. Reads from the child process STDOUT and STDERR, and calls the main application with the result of the process (success or failure). """ # Checking for a new release of Tartube doesn't involve any system # commands or child processes, so it is handled by a separate # function if self.info_type == 'version': return self.run_check_version() # Show information about the info operation in the Output tab if self.info_type == 'test_ytdl': msg = _( 'Starting info operation, testing downloader with specified' \ + ' options', ) else: if self.info_type == 'formats': msg = _( 'Starting info operation, fetching list of video/audio'\ + ' formats for \'{0}\'', ).format(self.video_obj.name) else: msg = _( 'Starting info operation, fetching list of subtitles'\ + ' for \'{0}\'', ).format(self.video_obj.name) self.app_obj.main_win_obj.output_tab_write_stdout(1, msg) # Convert a path beginning with ~ (not on MS Windows) ytdl_path = self.app_obj.check_downloader(self.app_obj.ytdl_path) if os.name != 'nt': ytdl_path = re.sub('^\~', os.path.expanduser('~'), ytdl_path) # Prepare the system command if self.info_type == 'formats': cmd_list = [ ytdl_path, '--list-formats', self.video_obj.source, ] elif self.info_type == 'subs': cmd_list = [ ytdl_path, '--list-subs', self.video_obj.source, ] else: if self.app_obj.ytdl_path_custom_flag: cmd_list = ['python3'] + [ytdl_path] else: cmd_list = [ytdl_path] if self.options_string is not None \ and self.options_string != '': # Parse the string into a list. It was obtained from a # Gtk.TextView, so it can contain newline and/or multiple # whitepsace characters. Whitespace characters within # double quotes "..." must be preserved option_list = utils.parse_options(self.options_string) for item in option_list: cmd_list.append(item) if self.url_string is not None \ and self.url_string != '': cmd_list.append('-o') cmd_list.append( os.path.join( self.app_obj.temp_test_dir, '%(title)s.%(ext)s', ), ) cmd_list.append(self.url_string) # Create the new child process self.create_child_process(cmd_list) # Show the system command in the Output tab space = ' ' self.app_obj.main_win_obj.output_tab_write_system_cmd( 1, space.join(cmd_list), ) # So that we can read from the child process STDOUT and STDERR, attach # a file descriptor to the PipeReader objects if self.child_process is not None: self.stdout_reader.attach_file_descriptor( self.child_process.stdout, ) self.stderr_reader.attach_file_descriptor( self.child_process.stderr, ) while self.is_child_process_alive(): # Read from the child process STDOUT, and convert into unicode for # Python's convenience while not self.stdout_queue.empty(): stdout = self.stdout_queue.get_nowait().rstrip() if stdout: stdout = stdout.decode(utils.get_encoding(), 'replace') self.output_list.append(stdout) self.stdout_list.append(stdout) # Show command line output in the Output tab self.app_obj.main_win_obj.output_tab_write_stdout( 1, stdout, ) # The child process has finished while not self.stderr_queue.empty(): # Read from the child process STDERR queue (we don't need to read # it in real time), and convert into unicode for python's # convenience stderr = self.stderr_queue.get_nowait().rstrip() if stderr: stderr = stderr.decode(utils.get_encoding(), 'replace') # While testing youtube-dl, don't treat anything as an error if self.info_type == 'test_ytdl': self.stdout_list.append(stderr) # When fetching subtitles from a video that has none, don't # treat youtube-dl WARNING: messages as something that # makes the info operation fail elif self.info_type == 'subs': if not re.match('WARNING\:', stderr): self.stderr_list.append(stderr) # When fetching formats, recognise all warnings as errors else: self.stderr_list.append(stderr) # Show command line output in the Output tab self.app_obj.main_win_obj.output_tab_write_stderr( 1, stderr, ) # (Generate our own error messages for debugging purposes, in certain # situations) if self.child_process is None: msg = _('System process did not start') self.stderr_list.append(msg) self.app_obj.main_win_obj.output_tab_write_stdout( 1, msg, ) elif self.child_process.returncode > 0: msg = _('Child process exited with non-zero code: {}').format( self.child_process.returncode, ) self.app_obj.main_win_obj.output_tab_write_stdout( 1, msg, ) # Operation complete. self.success_flag is checked by # mainapp.TartubeApp.info_manager_finished() if not self.stderr_list: self.success_flag = True # Show a confirmation in the the Output tab self.app_obj.main_win_obj.output_tab_write_stdout( 1, _('Info operation finished'), ) # Let the timer run for a few more seconds to prevent Gtk errors (for # systems with Gtk < 3.24) GObject.timeout_add( 0, self.app_obj.info_manager_halt_timer, )
def read_child_process(self): """Called by self.run(). Reads from the child process STDOUT and STDERR, in the correct order. Return values: True if either STDOUT or STDERR were read, None if both queues were empty """ # mini_list is in the form [time, pipe_type, data] try: mini_list = self.queue.get_nowait() except: # Nothing left to read return None # Failsafe check if not mini_list \ or (mini_list[1] != 'stdout' and mini_list[1] != 'stderr'): # Just in case... GObject.timeout_add( 0, self.app_obj.system_error, 601, 'Malformed STDOUT or STDERR data', ) # STDOUT or STDERR has been read data = mini_list[2].rstrip() # On MS Windows we use cp1252, so that Tartube can communicate with the # Windows console data = data.decode(utils.get_encoding(), 'replace') # STDOUT if mini_list[1] == 'stdout': self.output_list.append(data) self.stdout_list.append(data) # Show command line output in the Output tab self.app_obj.main_win_obj.output_tab_write_stdout( 1, data, ) # STDERR else: # While testing youtube-dl, don't treat anything as an error if self.info_type == 'test_ytdl': self.stdout_list.append(data) # When fetching subtitles from a video that has none, don't treat # youtube-dl WARNING: messages as something that makes the info # operation fail elif self.info_type == 'subs': if not re.search('^WARNING\:', data): self.stderr_list.append(data) # When fetching formats, recognise all warnings as errors else: self.stderr_list.append(data) # Show command line output in the Output tab self.app_obj.main_win_obj.output_tab_write_stderr( 1, data, ) # Either (or both) of STDOUT and STDERR were non-empty self.queue.task_done() return True