def test_stream_exclude(self):
with open('tests/ffmpeg3.out', 'r') as ff:
info = MediaInfo.parse_ffmpeg_details('/dev/null', ff.read())
setup = ConfigFile(self.get_setup())
p = setup.get_profile('excl_test_1')
streams = info.ffmpeg_streams(p)
self.assertEqual(len(streams), 12, 'expected 6 streams (12 elements)')
def test_stream_reassign_default(self):
with open('tests/ffmpeg4.out', 'r') as ff:
info = MediaInfo.parse_ffmpeg_details('/dev/null', ff.read())
setup = ConfigFile(self.get_setup())
p = setup.get_profile('excl_test_2')
streams = info.ffmpeg_streams(p)
self.assertEqual(len(streams), 8, 'expected 4 streams (8 elements)')
def test_stream_map_all(self):
with open('tests/ffmpeg3.out', 'r') as ff:
info = MediaInfo.parse_ffmpeg_details('/dev/null', ff.read())
setup = ConfigFile(self.get_setup())
p = setup.get_profile('qsv')
streams = info.ffmpeg_streams(p)
self.assertEqual(len(streams), 2, 'expected -map 0')
def test_default_profile(self):
info = TranscoderTests.make_media(None, None, None, 720, 45, 3000, 25, None, [], [])
config = ConfigFile(self.get_setup())
rule = config.match_rule(info)
self.assertIsNotNone(rule, 'expected to match a rule')
self.assertEqual(rule.profile, 'hevc_cuda')
self.assertEqual(rule.name, 'default')
def test_skip_profile(self):
with open('tests/ffmpeg.out', 'r') as ff:
info = MediaInfo.parse_ffmpeg_details('/dev/null', ff.read())
info.filesize_mb = 499
config = ConfigFile(self.get_setup())
rule = config.match_rule(info)
self.assertIsNotNone(rule, 'Expected rule match')
self.assertTrue(rule.is_skip(), 'Expected a SKIP rule')
def test_automap_include(self):
info = TranscoderTests.make_media(None, None, None, 720, 45, 3000, 25, None,
[{'lang': 'eng', 'stream': '1'},
{'lang': 'ger', 'stream': '2', 'default': True}], [])
setup = ConfigFile(self.get_setup())
p = setup.get_profile('hevc_cuda')
options = info.ffmpeg_streams(p)
self.assertEqual(options, ['-map', '0:0', '-map', '0:1', '-disposition:a:0', 'default'])
def test_include_overides(self):
info = TranscoderTests.make_media(None, None, None, 720, 45, 3000, 25, None,
[{'lang': 'eng', 'stream': '1'},
{'lang': 'ger', 'stream': '2', 'default': True}], [])
setup = ConfigFile(self.get_setup())
p = setup.get_profile('hevc_cuda_8bit')
self.assertEqual(p.threshold, 0, 'Threshold should be 0')
self.assertEqual(p.threshold_check, 100, 'Threshold check should be 100')
self.assertIn("-cq:v 21", p.output_options.as_list(), 'Expected -cq:v 21')
self.assertIn('-pix_fmt yuv420p', p.output_options.as_list(), 'expected pix_fmt')
def test_include_overides(self):
setup = ConfigFile(self.get_setup())
p = setup.get_profile('hevc_cuda_8bit')
self.assertEqual(p.threshold, 0, 'Threshold should be 0')
self.assertEqual(p.threshold_check, 100,
'Threshold check should be 100')
self.assertIn("-cq:v 21", p.output_options.as_list(),
'Expected -cq:v 21')
self.assertIn('-pix_fmt yuv420p', p.output_options.as_list(),
'expected pix_fmt')
def test_cluster_match_default_queue(self, mock_ffmpeg_details, mock_info_parser, mock_os_rename, mock_os_remove,
mock_filter_threshold, mock_run_remote):
setup = ConfigFile(self.get_setup())
#
# setup all mocks
#
mock_run_remote.return_value = 0
mock_filter_threshold.return_value = True
mock_os_rename.return_value = None
mock_os_remove.return_value = None
info = TranscoderTests.make_media('/dev/null', 'x264', 500, 480, 30 * 60, 420, 24, None, [], [])
mock_info_parser.return_value = info
mock_ffmpeg_details.return_value = info
#
# configure the cluster, add the job, and run
#
cluster = self.setup_cluster1(setup)
qname, job = cluster.enqueue('/dev/null.mp4', None)
self.assertEqual(qname, '_default', 'Job placed in wrong queue')
self.assertEqual(job.profile_name, 'vintage_tv', 'Rule matched to wrong profile')
cluster.testrun()
for host in cluster.hosts:
if host.hostname == 'workstation' and len(host._complete) > 0:
dump_stats(host._complete)
filename, elapsed = host.completed.pop()
self.assertEqual('/dev/null.mp4', filename, 'Completed filename missing from assigned host')
break
def test_cluster_streaming_host(self, mock_run_proc, mock_ffmpeg_fetch, mock_move, mock_run, mock_info_parser,
mock_os_rename, mock_os_remove,
mock_filter_threshold, mock_run_remote):
setup = ConfigFile(self.get_setup())
#
# setup all mocks
#
mock_run.return_value = 0, 'ok'
mock_run_remote.return_value = 0
mock_move.return_value = 0
mock_filter_threshold.return_value = True
mock_os_rename.return_value = None
mock_os_remove.return_value = None
info = TranscoderTests.make_media('/dev/null', 'x264', 1920, 1080, 110 * 60, 3000, 24, None, [], [])
mock_info_parser.return_value = info
mock_ffmpeg_fetch.return_value = info
#
# configure the cluster, add the job, and run
#
cluster = self.setup_cluster1(setup)
qname, job = cluster.enqueue('/dev/null.mp4', None)
self.assertEqual(qname, 'q3', 'Job placed in wrong queue')
self.assertEqual(job.profile_name, 'qsv', 'Rule matched to wrong profile')
cluster.testrun()
for host in cluster.hosts:
if host.hostname == 'm2' and len(host._complete) > 0:
filename, elapsed = host.completed.pop()
self.assertEqual('/dev/null.mp4', filename,
'Completed filename missing from assigned host')
break
def test_cluster_match_skip(self, mock_ffmpeg_details, mock_os_rename, mock_os_remove,
mock_filter_threshold, mock_run_remote):
setup = ConfigFile(self.get_setup())
#
# setup all mocks
#
mock_run_remote.return_value = 0
mock_filter_threshold.return_value = True
mock_os_rename.return_value = None
mock_os_remove.return_value = None
info = TranscoderTests.make_media('/dev/null', 'x264', 1920, 1080, 60 * 60, 1800, 24, None, [], [])
mock_ffmpeg_details.return_value = info
#
# configure the cluster, add the job, and run
#
cluster = self.setup_cluster1(setup)
qname, job = cluster.enqueue('/dev/null.mp4', None)
self.assertEqual(qname, None, 'Expected to skip')
def get_setup():
return ConfigFile('tests/mixinstest.yml')
def test_rule_match(self):
info = TranscoderTests.make_media(None, None, None, 1080, 45, 2300, None, None, [], [])
config = ConfigFile(self.get_setup())
rule = config.match_rule(info)
self.assertIsNotNone(rule, 'Expected a matched profile')
def start():
if len(sys.argv) == 2 and sys.argv[1] == '-h':
print(f'pytrancoder (ver {__version__})')
print('usage: pytrancoder [OPTIONS]')
print(' or pytrancoder [OPTIONS] --from-file <filename>')
print(' or pytrancoder [OPTIONS] file ...')
print(' or pytrancoder -c <cluster> file... [--host <name>] -c <cluster> file...')
print('No parameters indicates to process the default queue files using profile matching rules.')
print(
'The --from-file filename is a file containing a list of full paths to files for transcoding. ')
print('OPTIONS:')
print(' --host <name> Name of a specific host in your cluster configuration to target, otherwise load-balanced')
print(' -s Process files sequentially even if configured for multiple concurrent jobs')
print(' --dry-run Run without actually transcoding or modifying anything, useful to test rules and profiles')
print(' -v Verbose output, helpful in debugging profiles and rules')
print(
' -k Keep source files after transcoding. If used, the transcoded file will have the same '
'name and .tmp extension')
print(' -y <file> Full path to configuration file. Default is ~/.transcode.yml')
print(' -p profile to use. If used with --from-file, applies to all listed media in <filename>')
print(' -m Add mixins to profile. Separate multiples with a comma')
print('\n** PyPi Repo: https://pypi.org/project/pytranscoder-ffmpeg/')
print('** Read the docs at https://pytranscoder.readthedocs.io/en/latest/')
sys.exit(0)
install_sigint_handler()
files = list()
profile = None
mixins = None
queue_path = None
cluster = None
configfile: Optional[ConfigFile] = None
host_override = None
if len(sys.argv) > 1:
files = []
arg = 1
while arg < len(sys.argv):
if sys.argv[arg] == '--from-file': # load filenames to encode from given file
queue_path = sys.argv[arg + 1]
arg += 1
tmpfiles = files_from_file(queue_path)
if cluster is None:
files.extend([(f, profile) for f in tmpfiles])
else:
files.extend([(f, cluster) for f in tmpfiles])
elif sys.argv[arg] == '-p': # specific profile
profile = sys.argv[arg + 1]
arg += 1
elif sys.argv[arg] == '-y': # specify yaml config file
arg += 1
configfile = ConfigFile(sys.argv[arg])
elif sys.argv[arg] == '-k': # keep original
pytranscoder.keep_source = True
elif sys.argv[arg] == '--dry-run':
pytranscoder.dry_run = True
elif sys.argv[arg] == '--host': # run all cluster encodes on specific host
host_override = sys.argv[arg + 1]
arg += 1
elif sys.argv[arg] == '-v': # verbose
pytranscoder.verbose = True
elif sys.argv[arg] == '-c': # cluster
cluster = sys.argv[arg + 1]
arg += 1
elif sys.argv[arg] == '-m': # mixins
mixins = sys.argv[arg + 1].split(',')
arg += 1
else:
if os.name == "nt":
expanded_files: List = glob.glob(sys.argv[arg]) # handle wildcards in Windows
else:
expanded_files = [sys.argv[arg]]
for f in expanded_files:
if cluster is None:
files.append((f, profile, mixins))
else:
files.append((f, cluster, profile, mixins))
arg += 1
if configfile is None:
configfile = ConfigFile(DEFAULT_CONFIG)
if not configfile.colorize:
crayons.disable()
else:
crayons.enable()
if len(files) == 0 and queue_path is None and configfile.default_queue_file is not None:
#
# load from list of files
#
tmpfiles = files_from_file(configfile.default_queue_file)
queue_path = configfile.default_queue_file
if cluster is None:
files.extend([(f, profile, mixins) for f in tmpfiles])
else:
files.extend([(f, cluster, profile) for f in tmpfiles])
if len(files) == 0:
print(crayons.yellow(f'Nothing to do'))
sys.exit(0)
if cluster is not None:
if host_override is not None:
# disable all other hosts in-memory only - to force encodes to the designated host
cluster_config = configfile.settings['clusters']
for cluster in cluster_config.values():
for name, this_config in cluster.items():
if name != host_override:
this_config['status'] = 'disabled'
completed: List = manage_clusters(files, configfile)
if len(completed) > 0:
qpath = queue_path if queue_path is not None else configfile.default_queue_file
pathlist = [p for p, _ in completed]
cleanup_queuefile(qpath, set(pathlist))
dump_stats(completed)
sys.exit(0)
host = LocalHost(configfile)
host.enqueue_files(files)
#
# start all threads and wait for work to complete
#
host.start()
if len(host.complete) > 0:
completed_paths = [p for p, _ in host.complete]
cleanup_queuefile(queue_path, set(completed_paths))
dump_stats(host.complete)
os.system("stty sane")
def test_loadconfig(self):
config = ConfigFile('transcode.yml')
self.assertIsNotNone(config.settings, 'Config object not loaded')
self.assertEqual(len(config.queues), 2, 'Expected 2 queues')
def __init__(self, name, configs: Dict, config: ConfigFile, ssh: str):
"""
:param name: Cluster name, used only for thread naming
:param configs: The "clusters" section of the global config
:param config: The full configuration object
:param ssh: Path to local ssh
"""
super().__init__(name=name, group=None, daemon=True)
self.queues: Dict[str, Queue] = dict()
self.ssh = ssh
self.hosts: List[ManagedHost] = list()
self.config = config
self.verbose = verbose
# self.ffmpeg = FFmpeg(config.ffmpeg_path)
# self.hbcli = Handbrake(config.hbcli_path)
self.lock = Cluster.terminal_lock
self.completed: List = list()
self.info_processor = config.get_processor()
for host, props in configs.items():
hostprops = RemoteHostProperties(host, props)
if not hostprops.is_enabled:
continue
hosttype = hostprops.host_type
#
# make sure Queue exists for name
#
host_queues: Dict = hostprops.queues
if len(host_queues) > 0:
for host_queue in host_queues:
if host_queue not in self.queues:
self.queues[host_queue] = Queue()
_h = None
if hosttype == 'local':
# special case - using pytranscoder host also as cluster host
for host_queue, slots in host_queues.items():
#
# for each queue configured for this host create a dedicated thread for each slot
#
for slot in range(0, slots):
_h = LocalHost(host, hostprops, self.queues[host_queue], self)
if not _h.validate_settings():
sys.exit(1)
self.hosts.append(_h)
elif hosttype == 'mounted':
for host_queue, slots in host_queues.items():
#
# for each queue configured for this host create a dedicated thread for each slot
#
for slot in range(0, slots):
_h = MountedManagedHost(host, hostprops, self.queues[host_queue], self)
if not _h.validate_settings():
sys.exit(1)
self.hosts.append(_h)
elif hosttype == 'streaming':
for host_queue, slots in host_queues.items():
#
# for each queue configured for this host create a dedicated thread for each slot
#
for slot in range(0, slots):
_h = StreamingManagedHost(host, hostprops, self.queues[host_queue], self)
if not _h.validate_settings():
sys.exit(1)
self.hosts.append(_h)
else:
print(crayons.red(f'Unknown cluster host type "{hosttype}" - skipping'))
def test_local_host_setup(self):
config: Dict = self.get_setup()
host = LocalHost(ConfigFile(config))
self.assertEqual(len(host.queues), 3, 'Expected 3 queues configured')
