def test_stream_exclude(self): with open('tests/ffmpeg3.out', 'r') as ff: info = MediaInfo.parse_ffmpeg_details('/dev/null', ff.read()) setup = ConfigFile(self.get_setup()) p = setup.get_profile('excl_test_1') streams = info.ffmpeg_streams(p) self.assertEqual(len(streams), 12, 'expected 6 streams (12 elements)')
def test_stream_reassign_default(self): with open('tests/ffmpeg4.out', 'r') as ff: info = MediaInfo.parse_ffmpeg_details('/dev/null', ff.read()) setup = ConfigFile(self.get_setup()) p = setup.get_profile('excl_test_2') streams = info.ffmpeg_streams(p) self.assertEqual(len(streams), 8, 'expected 4 streams (8 elements)')
def test_stream_map_all(self): with open('tests/ffmpeg3.out', 'r') as ff: info = MediaInfo.parse_ffmpeg_details('/dev/null', ff.read()) setup = ConfigFile(self.get_setup()) p = setup.get_profile('qsv') streams = info.ffmpeg_streams(p) self.assertEqual(len(streams), 2, 'expected -map 0')
def test_default_profile(self): info = TranscoderTests.make_media(None, None, None, 720, 45, 3000, 25, None, [], []) config = ConfigFile(self.get_setup()) rule = config.match_rule(info) self.assertIsNotNone(rule, 'expected to match a rule') self.assertEqual(rule.profile, 'hevc_cuda') self.assertEqual(rule.name, 'default')
def test_skip_profile(self): with open('tests/ffmpeg.out', 'r') as ff: info = MediaInfo.parse_ffmpeg_details('/dev/null', ff.read()) info.filesize_mb = 499 config = ConfigFile(self.get_setup()) rule = config.match_rule(info) self.assertIsNotNone(rule, 'Expected rule match') self.assertTrue(rule.is_skip(), 'Expected a SKIP rule')
def test_automap_include(self): info = TranscoderTests.make_media(None, None, None, 720, 45, 3000, 25, None, [{'lang': 'eng', 'stream': '1'}, {'lang': 'ger', 'stream': '2', 'default': True}], []) setup = ConfigFile(self.get_setup()) p = setup.get_profile('hevc_cuda') options = info.ffmpeg_streams(p) self.assertEqual(options, ['-map', '0:0', '-map', '0:1', '-disposition:a:0', 'default'])
def test_include_overides(self): info = TranscoderTests.make_media(None, None, None, 720, 45, 3000, 25, None, [{'lang': 'eng', 'stream': '1'}, {'lang': 'ger', 'stream': '2', 'default': True}], []) setup = ConfigFile(self.get_setup()) p = setup.get_profile('hevc_cuda_8bit') self.assertEqual(p.threshold, 0, 'Threshold should be 0') self.assertEqual(p.threshold_check, 100, 'Threshold check should be 100') self.assertIn("-cq:v 21", p.output_options.as_list(), 'Expected -cq:v 21') self.assertIn('-pix_fmt yuv420p', p.output_options.as_list(), 'expected pix_fmt')
def test_include_overides(self): setup = ConfigFile(self.get_setup()) p = setup.get_profile('hevc_cuda_8bit') self.assertEqual(p.threshold, 0, 'Threshold should be 0') self.assertEqual(p.threshold_check, 100, 'Threshold check should be 100') self.assertIn("-cq:v 21", p.output_options.as_list(), 'Expected -cq:v 21') self.assertIn('-pix_fmt yuv420p', p.output_options.as_list(), 'expected pix_fmt')
def test_cluster_match_default_queue(self, mock_ffmpeg_details, mock_info_parser, mock_os_rename, mock_os_remove, mock_filter_threshold, mock_run_remote): setup = ConfigFile(self.get_setup()) # # setup all mocks # mock_run_remote.return_value = 0 mock_filter_threshold.return_value = True mock_os_rename.return_value = None mock_os_remove.return_value = None info = TranscoderTests.make_media('/dev/null', 'x264', 500, 480, 30 * 60, 420, 24, None, [], []) mock_info_parser.return_value = info mock_ffmpeg_details.return_value = info # # configure the cluster, add the job, and run # cluster = self.setup_cluster1(setup) qname, job = cluster.enqueue('/dev/null.mp4', None) self.assertEqual(qname, '_default', 'Job placed in wrong queue') self.assertEqual(job.profile_name, 'vintage_tv', 'Rule matched to wrong profile') cluster.testrun() for host in cluster.hosts: if host.hostname == 'workstation' and len(host._complete) > 0: dump_stats(host._complete) filename, elapsed = host.completed.pop() self.assertEqual('/dev/null.mp4', filename, 'Completed filename missing from assigned host') break
def test_cluster_streaming_host(self, mock_run_proc, mock_ffmpeg_fetch, mock_move, mock_run, mock_info_parser, mock_os_rename, mock_os_remove, mock_filter_threshold, mock_run_remote): setup = ConfigFile(self.get_setup()) # # setup all mocks # mock_run.return_value = 0, 'ok' mock_run_remote.return_value = 0 mock_move.return_value = 0 mock_filter_threshold.return_value = True mock_os_rename.return_value = None mock_os_remove.return_value = None info = TranscoderTests.make_media('/dev/null', 'x264', 1920, 1080, 110 * 60, 3000, 24, None, [], []) mock_info_parser.return_value = info mock_ffmpeg_fetch.return_value = info # # configure the cluster, add the job, and run # cluster = self.setup_cluster1(setup) qname, job = cluster.enqueue('/dev/null.mp4', None) self.assertEqual(qname, 'q3', 'Job placed in wrong queue') self.assertEqual(job.profile_name, 'qsv', 'Rule matched to wrong profile') cluster.testrun() for host in cluster.hosts: if host.hostname == 'm2' and len(host._complete) > 0: filename, elapsed = host.completed.pop() self.assertEqual('/dev/null.mp4', filename, 'Completed filename missing from assigned host') break
def test_cluster_match_skip(self, mock_ffmpeg_details, mock_os_rename, mock_os_remove, mock_filter_threshold, mock_run_remote): setup = ConfigFile(self.get_setup()) # # setup all mocks # mock_run_remote.return_value = 0 mock_filter_threshold.return_value = True mock_os_rename.return_value = None mock_os_remove.return_value = None info = TranscoderTests.make_media('/dev/null', 'x264', 1920, 1080, 60 * 60, 1800, 24, None, [], []) mock_ffmpeg_details.return_value = info # # configure the cluster, add the job, and run # cluster = self.setup_cluster1(setup) qname, job = cluster.enqueue('/dev/null.mp4', None) self.assertEqual(qname, None, 'Expected to skip')
def get_setup(): return ConfigFile('tests/mixinstest.yml')
def test_rule_match(self): info = TranscoderTests.make_media(None, None, None, 1080, 45, 2300, None, None, [], []) config = ConfigFile(self.get_setup()) rule = config.match_rule(info) self.assertIsNotNone(rule, 'Expected a matched profile')
def start(): if len(sys.argv) == 2 and sys.argv[1] == '-h': print(f'pytrancoder (ver {__version__})') print('usage: pytrancoder [OPTIONS]') print(' or pytrancoder [OPTIONS] --from-file <filename>') print(' or pytrancoder [OPTIONS] file ...') print(' or pytrancoder -c <cluster> file... [--host <name>] -c <cluster> file...') print('No parameters indicates to process the default queue files using profile matching rules.') print( 'The --from-file filename is a file containing a list of full paths to files for transcoding. ') print('OPTIONS:') print(' --host <name> Name of a specific host in your cluster configuration to target, otherwise load-balanced') print(' -s Process files sequentially even if configured for multiple concurrent jobs') print(' --dry-run Run without actually transcoding or modifying anything, useful to test rules and profiles') print(' -v Verbose output, helpful in debugging profiles and rules') print( ' -k Keep source files after transcoding. If used, the transcoded file will have the same ' 'name and .tmp extension') print(' -y <file> Full path to configuration file. Default is ~/.transcode.yml') print(' -p profile to use. If used with --from-file, applies to all listed media in <filename>') print(' -m Add mixins to profile. Separate multiples with a comma') print('\n** PyPi Repo: https://pypi.org/project/pytranscoder-ffmpeg/') print('** Read the docs at https://pytranscoder.readthedocs.io/en/latest/') sys.exit(0) install_sigint_handler() files = list() profile = None mixins = None queue_path = None cluster = None configfile: Optional[ConfigFile] = None host_override = None if len(sys.argv) > 1: files = [] arg = 1 while arg < len(sys.argv): if sys.argv[arg] == '--from-file': # load filenames to encode from given file queue_path = sys.argv[arg + 1] arg += 1 tmpfiles = files_from_file(queue_path) if cluster is None: files.extend([(f, profile) for f in tmpfiles]) else: files.extend([(f, cluster) for f in tmpfiles]) elif sys.argv[arg] == '-p': # specific profile profile = sys.argv[arg + 1] arg += 1 elif sys.argv[arg] == '-y': # specify yaml config file arg += 1 configfile = ConfigFile(sys.argv[arg]) elif sys.argv[arg] == '-k': # keep original pytranscoder.keep_source = True elif sys.argv[arg] == '--dry-run': pytranscoder.dry_run = True elif sys.argv[arg] == '--host': # run all cluster encodes on specific host host_override = sys.argv[arg + 1] arg += 1 elif sys.argv[arg] == '-v': # verbose pytranscoder.verbose = True elif sys.argv[arg] == '-c': # cluster cluster = sys.argv[arg + 1] arg += 1 elif sys.argv[arg] == '-m': # mixins mixins = sys.argv[arg + 1].split(',') arg += 1 else: if os.name == "nt": expanded_files: List = glob.glob(sys.argv[arg]) # handle wildcards in Windows else: expanded_files = [sys.argv[arg]] for f in expanded_files: if cluster is None: files.append((f, profile, mixins)) else: files.append((f, cluster, profile, mixins)) arg += 1 if configfile is None: configfile = ConfigFile(DEFAULT_CONFIG) if not configfile.colorize: crayons.disable() else: crayons.enable() if len(files) == 0 and queue_path is None and configfile.default_queue_file is not None: # # load from list of files # tmpfiles = files_from_file(configfile.default_queue_file) queue_path = configfile.default_queue_file if cluster is None: files.extend([(f, profile, mixins) for f in tmpfiles]) else: files.extend([(f, cluster, profile) for f in tmpfiles]) if len(files) == 0: print(crayons.yellow(f'Nothing to do')) sys.exit(0) if cluster is not None: if host_override is not None: # disable all other hosts in-memory only - to force encodes to the designated host cluster_config = configfile.settings['clusters'] for cluster in cluster_config.values(): for name, this_config in cluster.items(): if name != host_override: this_config['status'] = 'disabled' completed: List = manage_clusters(files, configfile) if len(completed) > 0: qpath = queue_path if queue_path is not None else configfile.default_queue_file pathlist = [p for p, _ in completed] cleanup_queuefile(qpath, set(pathlist)) dump_stats(completed) sys.exit(0) host = LocalHost(configfile) host.enqueue_files(files) # # start all threads and wait for work to complete # host.start() if len(host.complete) > 0: completed_paths = [p for p, _ in host.complete] cleanup_queuefile(queue_path, set(completed_paths)) dump_stats(host.complete) os.system("stty sane")
def test_loadconfig(self): config = ConfigFile('transcode.yml') self.assertIsNotNone(config.settings, 'Config object not loaded') self.assertEqual(len(config.queues), 2, 'Expected 2 queues')
def __init__(self, name, configs: Dict, config: ConfigFile, ssh: str): """ :param name: Cluster name, used only for thread naming :param configs: The "clusters" section of the global config :param config: The full configuration object :param ssh: Path to local ssh """ super().__init__(name=name, group=None, daemon=True) self.queues: Dict[str, Queue] = dict() self.ssh = ssh self.hosts: List[ManagedHost] = list() self.config = config self.verbose = verbose # self.ffmpeg = FFmpeg(config.ffmpeg_path) # self.hbcli = Handbrake(config.hbcli_path) self.lock = Cluster.terminal_lock self.completed: List = list() self.info_processor = config.get_processor() for host, props in configs.items(): hostprops = RemoteHostProperties(host, props) if not hostprops.is_enabled: continue hosttype = hostprops.host_type # # make sure Queue exists for name # host_queues: Dict = hostprops.queues if len(host_queues) > 0: for host_queue in host_queues: if host_queue not in self.queues: self.queues[host_queue] = Queue() _h = None if hosttype == 'local': # special case - using pytranscoder host also as cluster host for host_queue, slots in host_queues.items(): # # for each queue configured for this host create a dedicated thread for each slot # for slot in range(0, slots): _h = LocalHost(host, hostprops, self.queues[host_queue], self) if not _h.validate_settings(): sys.exit(1) self.hosts.append(_h) elif hosttype == 'mounted': for host_queue, slots in host_queues.items(): # # for each queue configured for this host create a dedicated thread for each slot # for slot in range(0, slots): _h = MountedManagedHost(host, hostprops, self.queues[host_queue], self) if not _h.validate_settings(): sys.exit(1) self.hosts.append(_h) elif hosttype == 'streaming': for host_queue, slots in host_queues.items(): # # for each queue configured for this host create a dedicated thread for each slot # for slot in range(0, slots): _h = StreamingManagedHost(host, hostprops, self.queues[host_queue], self) if not _h.validate_settings(): sys.exit(1) self.hosts.append(_h) else: print(crayons.red(f'Unknown cluster host type "{hosttype}" - skipping'))
def test_local_host_setup(self): config: Dict = self.get_setup() host = LocalHost(ConfigFile(config)) self.assertEqual(len(host.queues), 3, 'Expected 3 queues configured')