def __init__(self, configfile: ConfigFile): self.queues = dict() self.configfile = configfile self.ffmpeg = FFmpeg(self.configfile.ffmpeg_path) # # initialize the queues # self.queues['_default_'] = Queue() for qname in configfile.queues.keys(): self.queues[qname] = Queue()
def __init__(self, queuename, queue: Queue, configfile: ConfigFile, manager): """ :param queuename: Name of the queue, for thread naming purposes only :param queue: Thread-safe queue containing files to be encoded :param configfile: Instance of the parsed configuration (transcode.yml) :param manager: Reference to object that manages this thread """ super().__init__(name=queuename, group=None, daemon=True) self.queue = queue self.config = configfile self._manager = manager self.ffmpeg = FFmpeg(self.config.ffmpeg_path)
def get_processor_by_name(self, name: str) -> Processor: if name == 'ffmpeg': return FFmpeg(self.ffmpeg_path) if self.hbcli_path: return Handbrake(self.hbcli_path) print(f'Unknown processor type "{name}" for host "{self.name}"') sys.exit(1)
def get_processor_by_name(self, name: str) -> Processor: if name == 'ffmpeg': return FFmpeg(self.ffmpeg_path) if self.hbcli_path: return Handbrake(self.hbcli_path) print('Missing "ffmpeg" or "hbcli" path') sys.exit(1)
def __init__(self, hostname, props, queue, cluster): """ :param hostname: name of host from config/clusters :param props: dictionary of properties from config/clusters :param queue: Work queue assigned to this thread, could be many-to-one in the future. :param cluster: Reference to parent Cluster object """ super().__init__(name=hostname, group=None, daemon=True) self.hostname = hostname self.props = props self.queue = queue self._complete = list() self._manager = cluster self.ffmpeg = FFmpeg(props.ffmpeg_path)
def __init__(self, name, configs: Dict, config: ConfigFile, ssh: str): """ :param name: Cluster name, used only for thread naming :param configs: The "clusters" section of the global config :param config: The full configuration object :param ssh: Path to local ssh """ super().__init__(name=name, group=None, daemon=True) self.queues: Dict[str, Queue] = dict() self.ssh = ssh self.hosts: List[ManagedHost] = list() self.config = config self.verbose = verbose self.ffmpeg = FFmpeg(config.ffmpeg_path) self.lock = Cluster.terminal_lock self.completed: List = list() for host, props in configs.items(): hostprops = RemoteHostProperties(host, props) if not hostprops.is_enabled: continue hosttype = hostprops.host_type # # make sure Queue exists for name # host_queues: Dict = hostprops.queues if len(host_queues) > 0: for host_queue in host_queues: if host_queue not in self.queues: self.queues[host_queue] = Queue() _h = None if hosttype == 'local': # special case - using pytranscoder host also as cluster host for host_queue, slots in host_queues.items(): # # for each queue configured for this host create a dedicated thread for each slot # for slot in range(0, slots): _h = LocalHost(host, hostprops, self.queues[host_queue], self) if not _h.validate_settings(): sys.exit(1) self.hosts.append(_h) elif hosttype == 'mounted': for host_queue, slots in host_queues.items(): # # for each queue configured for this host create a dedicated thread for each slot # for slot in range(0, slots): _h = MountedManagedHost(host, hostprops, self.queues[host_queue], self) if not _h.validate_settings(): sys.exit(1) self.hosts.append(_h) elif hosttype == 'streaming': for host_queue, slots in host_queues.items(): # # for each queue configured for this host create a dedicated thread for each slot # for slot in range(0, slots): _h = StreamingManagedHost(host, hostprops, self.queues[host_queue], self) if not _h.validate_settings(): sys.exit(1) self.hosts.append(_h) else: print( crayons.red( f'Unknown cluster host type "{hosttype}" - skipping'))
class Cluster(Thread): """Thread to create host threads and wait for their completion.""" terminal_lock: Lock = Lock() # class-level def __init__(self, name, configs: Dict, config: ConfigFile, ssh: str): """ :param name: Cluster name, used only for thread naming :param configs: The "clusters" section of the global config :param config: The full configuration object :param ssh: Path to local ssh """ super().__init__(name=name, group=None, daemon=True) self.queues: Dict[str, Queue] = dict() self.ssh = ssh self.hosts: List[ManagedHost] = list() self.config = config self.verbose = verbose self.ffmpeg = FFmpeg(config.ffmpeg_path) self.lock = Cluster.terminal_lock self.completed: List = list() for host, props in configs.items(): hostprops = RemoteHostProperties(host, props) if not hostprops.is_enabled: continue hosttype = hostprops.host_type # # make sure Queue exists for name # host_queues: Dict = hostprops.queues if len(host_queues) > 0: for host_queue in host_queues: if host_queue not in self.queues: self.queues[host_queue] = Queue() _h = None if hosttype == 'local': # special case - using pytranscoder host also as cluster host for host_queue, slots in host_queues.items(): # # for each queue configured for this host create a dedicated thread for each slot # for slot in range(0, slots): _h = LocalHost(host, hostprops, self.queues[host_queue], self) if not _h.validate_settings(): sys.exit(1) self.hosts.append(_h) elif hosttype == 'mounted': for host_queue, slots in host_queues.items(): # # for each queue configured for this host create a dedicated thread for each slot # for slot in range(0, slots): _h = MountedManagedHost(host, hostprops, self.queues[host_queue], self) if not _h.validate_settings(): sys.exit(1) self.hosts.append(_h) elif hosttype == 'streaming': for host_queue, slots in host_queues.items(): # # for each queue configured for this host create a dedicated thread for each slot # for slot in range(0, slots): _h = StreamingManagedHost(host, hostprops, self.queues[host_queue], self) if not _h.validate_settings(): sys.exit(1) self.hosts.append(_h) else: print( crayons.red( f'Unknown cluster host type "{hosttype}" - skipping')) def enqueue(self, file, forced_profile: Optional[str]) -> (str, Optional[EncodeJob]): """Add a media file to this cluster queue. This is different than in local mode in that we only care about handling skips here. The profile will be selected once a host is assigned to the work """ path = os.path.abspath( file) # convert to full path so that rule filtering can work if pytranscoder.verbose: print('matching ' + path) media_info = self.ffmpeg.fetch_details(path) if media_info is None: print(crayons.red(f'File not found: {path}')) return None, None if media_info.valid: if pytranscoder.verbose: print(str(media_info)) if forced_profile is None: # # just interested in SKIP rule matches and queue designations here # profile = None rule = self.config.match_rule(media_info) if rule is None: print( crayons.yellow(f'No matching profile found - skipped')) return None, None if rule.is_skip(): basename = os.path.basename(path) print( f'{basename}: Skipping due to profile rule - {rule.name}' ) return None, None profile = self.profiles[rule.profile] else: profile = self.profiles[forced_profile] if pytranscoder.verbose: print("Matched to profile {profile.name}") # not short circuited by a skip rule, add to appropriate queue queue_name = profile.queue_name if profile.queue_name is not None else '_default' if queue_name not in self.queues: print( crayons.red('Error: ') + f'Queue "{queue_name}" referenced in profile "{profile.name}" not defined in any host' ) exit(1) job = EncodeJob(file, media_info, profile.name) self.queues[queue_name].put(job) return queue_name, job return None, None def testrun(self): for host in self.hosts: host.testrun() def run(self): """Start all host threads and wait until queue is drained""" if len(self.hosts) == 0: print(f'No hosts available in cluster "{self.name}"') return for host in self.hosts: host.start() # all hosts running, wait for them to finish for host in self.hosts: host.join() self.completed.extend(host.completed) @property def profiles(self): return self.config.profiles
class QueueThread(Thread): """One transcoding thread associated to a queue""" def __init__(self, queuename, queue: Queue, configfile: ConfigFile, manager): """ :param queuename: Name of the queue, for thread naming purposes only :param queue: Thread-safe queue containing files to be encoded :param configfile: Instance of the parsed configuration (transcode.yml) :param manager: Reference to object that manages this thread """ super().__init__(name=queuename, group=None, daemon=True) self.queue = queue self.config = configfile self._manager = manager self.ffmpeg = FFmpeg(self.config.ffmpeg_path) @property def lock(self): return self._manager.lock def complete(self, path: Path, elapsed_seconds): self._manager.complete.append((str(path), elapsed_seconds)) def start_test(self): self.go() def run(self): self.go() def log(self, *args, **kwargs): self.lock.acquire() print(*args, **kwargs) sys.stdout.flush() self.lock.release() def go(self): while not self.queue.empty(): try: job: LocalJob = self.queue.get() oinput = job.profile.input_options ooutput = job.profile.output_options outpath = job.inpath.with_suffix(job.profile.extension + '.tmp') # # check if we need to exclude any streams # if job.info.is_multistream(): ooutput = ooutput + job.info.ffmpeg_streams(job.profile) cli = [ '-y', *oinput, '-i', str(job.inpath), *ooutput, str(outpath) ] # # display useful information # self.lock.acquire( ) # used to synchronize threads so multiple threads don't create a jumble of output try: print('-' * 40) print('Filename : ' + crayons.green(os.path.basename(str(job.inpath)))) print(f'Profile : {job.profile.name}') print('ffmpeg : ' + ' '.join(cli) + '\n') finally: self.lock.release() if pytranscoder.dry_run: continue basename = job.inpath.name def log_callback(stats): pct_done, pct_comp = calculate_progress(job.info, stats) self.log( f'{basename}: speed: {stats["speed"]}x, comp: {pct_comp}%, done: {pct_done:3}%' ) if job.profile.threshold_check < 100: if pct_done >= job.profile.threshold_check and pct_comp < job.profile.threshold: # compression goal (threshold) not met, kill the job and waste no more time... self.log( f'Encoding of {basename} cancelled and skipped due to threshold not met' ) return True # continue return False job_start = datetime.datetime.now() code = self.ffmpeg.run(cli, log_callback) job_stop = datetime.datetime.now() elapsed = job_stop - job_start if code == 0: if not filter_threshold(job.profile, str(job.inpath), outpath): # oops, this transcode didn't do so well, lets keep the original and scrap this attempt self.log( f'Transcoded file {job.inpath} did not meet minimum savings threshold, skipped' ) self.complete(job.inpath, (job_stop - job_start).seconds) outpath.unlink() continue self.complete(job.inpath, elapsed.seconds) if not pytranscoder.keep_source: if pytranscoder.verbose: self.log(f'replacing {job.inpath} with {outpath}') job.inpath.unlink() outpath.rename( job.inpath.with_suffix(job.profile.extension)) self.log(crayons.green(f'Finished {job.inpath}')) else: self.log( crayons.yellow( f'Finished {outpath}, original file unchanged') ) elif code is not None: self.log( f' Did not complete normally: {self.ffmpeg.last_command}' ) self.log(f'Output can be found in {self.ffmpeg.log_path}') try: outpath.unlink() except: pass finally: self.queue.task_done()
class LocalHost: """Encapsulates functionality for local encoding""" lock: Lock = Lock() complete: List = list() # list of completed files, shared across threads def __init__(self, configfile: ConfigFile): self.queues = dict() self.configfile = configfile self.ffmpeg = FFmpeg(self.configfile.ffmpeg_path) # # initialize the queues # self.queues['_default_'] = Queue() for qname in configfile.queues.keys(): self.queues[qname] = Queue() def start(self): """After initialization this is where processing begins""" # # all files are listed in the queues so start the threads # jobs = list() for name, queue in self.queues.items(): # determine the number of threads to allocate for each queue, minimum of defined max and queued jobs if name == '_default_': concurrent_max = 1 else: concurrent_max = min(self.configfile.queues[name], queue.qsize()) # # Create (n) threads and assign them a queue # for _ in range(concurrent_max): t = QueueThread(name, queue, self.configfile, self) jobs.append(t) t.start() # wait for all queues to drain and all jobs to complete for _, queue in self.queues.items(): queue.join() def enqueue_files(self, files: list): """Add requested files to the appropriate queue :param files: list of (path,profile) tuples :return: """ for path, forced_profile in files: # # do some prechecks... # if forced_profile is not None and not self.configfile.has_profile( forced_profile): print( f'profile "{forced_profile}" referenced from command line not found' ) exit(1) if len(path) == 0: continue if not os.path.isfile(path): print(crayons.red('path not found, skipping: ' + path)) continue if pytranscoder.verbose: print('matching ' + path) media_info = self.ffmpeg.fetch_details(path) if media_info is None: print(crayons.red(f'File not found: {path}')) continue if media_info.valid: if forced_profile is None: rule = self.configfile.match_rule(media_info) if rule is None: print( crayons.yellow( f'No matching profile found - skipped')) continue if rule.is_skip(): print(crayons.green(os.path.basename(path)), f'SKIPPED ({rule.name})') self.complete.append((path, 0)) continue profile_name = rule.profile the_profile = self.configfile.get_profile(profile_name) else: # # looks good, add this file to the thread queue # the_profile = self.configfile.get_profile(forced_profile) profile_name = forced_profile qname = the_profile.queue_name if qname is not None: if not self.configfile.has_queue(the_profile.queue_name): print( crayons.red( f'Profile "{profile_name}" indicated queue "{qname}" that has not been defined' )) sys.exit(1) else: self.queues[qname].put( LocalJob(path, the_profile, media_info)) else: self.queues['_default_'].put( LocalJob(path, the_profile, media_info)) def notify_plex(self): """If plex notifications enabled, tell it to refresh""" if self.configfile.plex_server is not None and not pytranscoder.dry_run: plex_server = self.configfile.plex_server try: from plexapi.server import PlexServer plex = PlexServer('http://{}'.format(plex_server)) plex.library.update() except ModuleNotFoundError: print( 'Library not installed. To use Plex notifications please install the Python 3 Plex API ' + '("pip3 install plexapi")') except Exception as ex2: print(f'Unable to connect to Plex server at {plex_server}') if pytranscoder.verbose: print(str(ex2))