def go(self):
while not self.queue.empty():
try:
job: EncodeJob = self.queue.get()
inpath = job.inpath
#
# Got to do the rule matching again.
# This time we narrow down the available profiles based on host definition
#
if pytranscoder.verbose:
self.log('matching ' + inpath)
_profile: Profile = self.match_profile(job, self.name)
if _profile is None:
continue
#
# calculate paths
#
outpath = inpath[0:inpath.rfind('.')] + _profile.extension + '.tmp'
#
# build command line
#
oinput = _profile.input_options.as_shell_params()
ooutput = _profile.output_options.as_shell_params()
remote_inpath = self.converted_path(inpath)
remote_outpath = self.converted_path(outpath)
processor = self.props.get_processor_by_name(_profile.processor)
if _profile.is_ffmpeg:
if job.media_info.is_multistream() and self.configfile.automap and _profile.automap:
ooutput = ooutput + job.media_info.ffmpeg_streams(_profile)
cli = ['-y', *oinput, '-i', remote_inpath, *ooutput, remote_outpath]
else:
cli = ['-i', remote_inpath, *oinput, *ooutput, '-o', remote_outpath]
#
# display useful information
#
self.lock.acquire() # used to synchronize threads so multiple threads don't create a jumble of output
try:
print('-' * 40)
print(f'Host : {self.hostname} (local)')
print('Filename : ' + crayons.green(os.path.basename(remote_inpath)))
print(f'Profile : {_profile.name}')
print('ffmpeg : ' + ' '.join(cli) + '\n')
finally:
self.lock.release()
if pytranscoder.dry_run:
continue
basename = os.path.basename(job.inpath)
def log_callback(stats):
pct_done, pct_comp = calculate_progress(job.media_info, stats)
self.log(f'{basename}: speed: {stats["speed"]}x, comp: {pct_comp}%, done: {pct_done:3}%')
if _profile.threshold_check < 100:
if pct_done >= _profile.threshold_check and pct_comp < _profile.threshold:
# compression goal (threshold) not met, kill the job and waste no more time...
self.log(f'Encoding of {basename} cancelled and skipped due to threshold not met')
return True
return False
def hb_log_callback(stats):
self.log(f'{basename}: avg fps: {stats["fps"]}, ETA: {stats["eta"]}')
return False
#
# Start process
#
job_start = datetime.datetime.now()
if processor.is_ffmpeg():
code = processor.run(cli, log_callback)
else:
code = processor.run(cli, hb_log_callback)
job_stop = datetime.datetime.now()
#
# process completed, check results and finish
#
if code == 0:
if not filter_threshold(_profile, inpath, outpath):
self.log(
f'Transcoded file {inpath} did not meet minimum savings threshold, skipped')
self.complete(inpath, (job_stop - job_start).seconds)
os.remove(outpath)
continue
if not pytranscoder.keep_source:
if verbose:
self.log('removing ' + inpath)
os.remove(inpath)
if verbose:
self.log('renaming ' + outpath)
os.rename(outpath, outpath[0:-4])
self.complete(inpath, (job_stop - job_start).seconds)
self.log(crayons.green(f'Finished {job.inpath}'))
elif code is not None:
self.log(f' Did not complete normally: {processor.last_command}')
self.log(f'Output can be found in {processor.log_path}')
try:
os.remove(outpath)
except:
pass
except Exception as ex:
self.log(ex)
finally:
self.queue.task_done()
def go(self):
ssh_cmd = [self._manager.ssh, self.props.user + '@' + self.props.ip]
#
# Keep pulling items from the queue until done. Other threads will be pulling from the same queue
# if multiple hosts configured on the same cluster.
#
while not self.queue.empty():
try:
job: EncodeJob = self.queue.get()
inpath = job.inpath
#
# Convert escaped spaces back to normal. Typical for bash to escape spaces and special characters
# in filenames.
#
inpath = inpath.replace('\\ ', ' ')
#
# Got to do the rule matching again.
# This time we narrow down the available profiles based on host definition
#
_profile: Profile = self.match_profile(job, self.name)
if _profile is None:
continue
#
# calculate full input and output paths
#
remote_working_dir = self.props.working_dir
remote_inpath = os.path.join(remote_working_dir, os.path.basename(inpath))
remote_outpath = os.path.join(remote_working_dir, os.path.basename(inpath) + '.tmp')
#
# build remote commandline
#
oinput = _profile.input_options.as_shell_params()
ooutput = _profile.output_options.as_shell_params()
processor = self.props.get_processor_by_name(_profile.processor)
if _profile.is_ffmpeg:
if job.media_info.is_multistream() and self.configfile.automap and _profile.automap:
ooutput = ooutput + job.media_info.ffmpeg_streams(_profile)
cmd = ['-y', *oinput, '-i', self.converted_path(remote_inpath),
*ooutput, self.converted_path(remote_outpath)]
else:
cmd = ['-i', self.converted_path(remote_inpath), *oinput,
*ooutput, '-o', self.converted_path(remote_outpath)]
cli = [*ssh_cmd, *cmd]
#
# display useful information
#
self.lock.acquire() # used to synchronize threads so multiple threads don't create a jumble of output
try:
print('-' * 40)
print(f'Host : {self.hostname} (streaming)')
print('Filename : ' + crayons.green(os.path.basename(remote_inpath)))
print(f'Profile : {job.profile_name}')
print('ssh : ' + ' '.join(cli) + '\n')
finally:
self.lock.release()
if pytranscoder.dry_run:
continue
#
# Copy source file to remote
#
target_dir = remote_working_dir
if self.props.is_windows():
# trick to make scp work on the Windows side
target_dir = '/' + remote_working_dir
scp = ['scp', inpath, self.props.user + '@' + self.props.ip + ':' + target_dir]
self.log(' '.join(scp))
code, output = run(scp)
if code != 0:
self.log(crayons.red('Unknown error copying source to remote - media skipped'))
if self._manager.verbose:
self.log(output)
continue
basename = os.path.basename(job.inpath)
def log_callback(stats):
pct_done, pct_comp = calculate_progress(job.media_info, stats)
self.log(f'{basename}: speed: {stats["speed"]}x, comp: {pct_comp}%, done: {pct_done:3}%')
if _profile.threshold_check < 100:
if pct_done >= _profile.threshold_check and pct_comp < _profile.threshold:
# compression goal (threshold) not met, kill the job and waste no more time...
self.log(f'Encoding of {basename} cancelled and skipped due to threshold not met')
return True
# continue
return False
def hb_log_callback(stats):
self.log(f'{basename}: avg fps: {stats["fps"]}, ETA: {stats["eta"]}')
return False
#
# Start remote
#
job_start = datetime.datetime.now()
if processor.is_ffmpeg():
code = processor.run_remote(self._manager.ssh, self.props.user, self.props.ip, cmd, log_callback)
else:
code = processor.run_remote(self._manager.ssh, self.props.user, self.props.ip, cmd, hb_log_callback)
job_stop = datetime.datetime.now()
if code != 0:
self.log(crayons.red('Unknown error encoding on remote'))
continue
#
# copy results back to local
#
retrieved_copy_name = os.path.join(gettempdir(), os.path.basename(remote_outpath))
cmd = ['scp', self.props.user + '@' + self.props.ip + ':' + remote_outpath, retrieved_copy_name]
self.log(' '.join(cmd))
code, output = run(cmd)
#
# process completed, check results and finish
#
if code == 0:
if not filter_threshold(_profile, inpath, retrieved_copy_name):
self.log(
f'Transcoded file {inpath} did not meet minimum savings threshold, skipped')
self.complete(inpath, (job_stop - job_start).seconds)
os.remove(retrieved_copy_name)
continue
self.complete(inpath, (job_stop - job_start).seconds)
if not pytranscoder.keep_source:
os.rename(retrieved_copy_name, retrieved_copy_name[0:-4])
retrieved_copy_name = retrieved_copy_name[0:-4]
if verbose:
self.log(f'moving media to {inpath}')
shutil.move(retrieved_copy_name, inpath)
self.log(crayons.green(f'Finished {inpath}'))
elif code is not None:
self.log(crayons.red(f'error during remote transcode of {inpath}'))
self.log(f' Did not complete normally: {processor.last_command}')
self.log(f'Output can be found in {processor.log_path}')
# self.log(f'Removing temporary media copies from {remote_working_dir}')
if self.props.is_windows():
remote_outpath = self.converted_path(remote_outpath)
remote_inpath = self.converted_path(remote_inpath)
self.run_process([*ssh_cmd, f'"del {remote_outpath}"'])
self.run_process([*ssh_cmd, f'"del {remote_inpath}"'])
else:
self.run_process([*ssh_cmd, f'"rm {remote_outpath}"'])
self.run_process([*ssh_cmd, f'"rm {remote_inpath}"'])
finally:
self.queue.task_done()
def go(self):
while not self.queue.empty():
try:
job: LocalJob = self.queue.get()
oinput = job.profile.input_options
ooutput = job.profile.output_options
outpath = job.inpath.with_suffix(job.profile.extension +
'.tmp')
#
# check if we need to exclude any streams
#
if job.info.is_multistream():
ooutput = ooutput + job.info.ffmpeg_streams(job.profile)
cli = [
'-y', *oinput, '-i',
str(job.inpath), *ooutput,
str(outpath)
]
#
# display useful information
#
self.lock.acquire(
) # used to synchronize threads so multiple threads don't create a jumble of output
try:
print('-' * 40)
print('Filename : ' +
crayons.green(os.path.basename(str(job.inpath))))
print(f'Profile : {job.profile.name}')
print('ffmpeg : ' + ' '.join(cli) + '\n')
finally:
self.lock.release()
if pytranscoder.dry_run:
continue
basename = job.inpath.name
def log_callback(stats):
pct_done, pct_comp = calculate_progress(job.info, stats)
self.log(
f'{basename}: speed: {stats["speed"]}x, comp: {pct_comp}%, done: {pct_done:3}%'
)
if job.profile.threshold_check < 100:
if pct_done >= job.profile.threshold_check and pct_comp < job.profile.threshold:
# compression goal (threshold) not met, kill the job and waste no more time...
self.log(
f'Encoding of {basename} cancelled and skipped due to threshold not met'
)
return True
# continue
return False
job_start = datetime.datetime.now()
code = self.ffmpeg.run(cli, log_callback)
job_stop = datetime.datetime.now()
elapsed = job_stop - job_start
if code == 0:
if not filter_threshold(job.profile, str(job.inpath),
outpath):
# oops, this transcode didn't do so well, lets keep the original and scrap this attempt
self.log(
f'Transcoded file {job.inpath} did not meet minimum savings threshold, skipped'
)
self.complete(job.inpath,
(job_stop - job_start).seconds)
outpath.unlink()
continue
self.complete(job.inpath, elapsed.seconds)
if not pytranscoder.keep_source:
if pytranscoder.verbose:
self.log(f'replacing {job.inpath} with {outpath}')
job.inpath.unlink()
outpath.rename(
job.inpath.with_suffix(job.profile.extension))
self.log(crayons.green(f'Finished {job.inpath}'))
else:
self.log(
crayons.yellow(
f'Finished {outpath}, original file unchanged')
)
elif code is not None:
self.log(
f' Did not complete normally: {self.ffmpeg.last_command}'
)
self.log(f'Output can be found in {self.ffmpeg.log_path}')
try:
outpath.unlink()
except:
pass
finally:
self.queue.task_done()
def go(self):
while not self.queue.empty():
try:
job: LocalJob = self.queue.get()
input_opt = job.profile.input_options.as_shell_params()
output_opt = self.config.output_from_profile(job.profile, job.mixins)
fls = False
if self.config.fls_path():
# lets write output to local storage, for efficiency
outpath = PurePath(self.config.fls_path(), job.inpath.with_suffix(job.profile.extension).name)
fls = True
else:
outpath = job.inpath.with_suffix(job.profile.extension + '.tmp')
#
# check if we need to exclude any streams
#
processor = self.config.get_processor_by_name(job.profile.processor)
if job.profile.is_ffmpeg:
if job.info.is_multistream() and self.config.automap and job.profile.automap:
output_opt = output_opt + job.info.ffmpeg_streams(job.profile)
cli = ['-y', *input_opt, '-i', str(job.inpath), *output_opt, str(outpath)]
else:
cli = ['-i', str(job.inpath), *input_opt, *output_opt, '-o', str(outpath)]
#
# display useful information
#
self.lock.acquire() # used to synchronize threads so multiple threads don't create a jumble of output
try:
print('-' * 40)
print('Filename : ' + crayons.green(os.path.basename(str(job.inpath))))
print(f'Profile : {job.profile.name}')
print('{:<6} : '.format(job.profile.processor) + ' '.join(cli) + '\n')
finally:
self.lock.release()
if pytranscoder.dry_run:
continue
basename = job.inpath.name
def log_callback(stats):
pct_done, pct_comp = calculate_progress(job.info, stats)
pytranscoder.status_queue.put({ 'host': 'local',
'file': basename,
'speed': stats['speed'],
'comp': pct_comp,
'done': pct_done})
#self.log(f'{basename}: speed: {stats["speed"]}x, comp: {pct_comp}%, done: {pct_done:3}%')
if job.profile.threshold_check < 100:
if pct_done >= job.profile.threshold_check and pct_comp < job.profile.threshold:
# compression goal (threshold) not met, kill the job and waste no more time...
self.log(f'Encoding of {basename} cancelled and skipped due to threshold not met')
return True
return False
def hbcli_callback(stats):
self.log(f'{basename}: avg fps: {stats["fps"]}, ETA: {stats["eta"]}')
return False
job_start = datetime.datetime.now()
if processor.is_ffmpeg():
code = processor.run(cli, log_callback)
else:
code = processor.run(cli, hbcli_callback)
job_stop = datetime.datetime.now()
elapsed = job_stop - job_start
if code == 0:
if not filter_threshold(job.profile, str(job.inpath), outpath):
# oops, this transcode didn't do so well, lets keep the original and scrap this attempt
self.log(f'Transcoded file {job.inpath} did not meet minimum savings threshold, skipped')
self.complete(job.inpath, (job_stop - job_start).seconds)
os.unlink(str(outpath))
continue
self.complete(job.inpath, elapsed.seconds)
if not pytranscoder.keep_source:
if pytranscoder.verbose:
self.log(f'replacing {job.inpath} with {outpath}')
job.inpath.unlink()
if fls:
shutil.move(outpath, job.inpath.with_suffix(job.profile.extension))
else:
outpath.rename(job.inpath.with_suffix(job.profile.extension))
self.log(crayons.green(f'Finished {job.inpath}'))
else:
self.log(crayons.yellow(f'Finished {outpath}, original file unchanged'))
elif code is not None:
self.log(f' Did not complete normally: {processor.last_command}')
self.log(f'Output can be found in {processor.log_path}')
try:
outpath.unlink()
except:
pass
finally:
self.queue.task_done()
def go(self):
while not self.queue.empty():
try:
job: EncodeJob = self.queue.get()
inpath = job.inpath
#
# Got to do the rule matching again.
# This time we narrow down the available profiles based on host definition
#
if pytranscoder.verbose:
self.log('matching ' + inpath)
_profile: Profile = self.match_profile(job, self.name)
if _profile is None:
continue
#
# calculate paths
#
outpath = inpath[0:inpath.
rfind('.')] + '.h265' + _profile.extension
remote_inpath = inpath
remote_outpath = outpath
if self.props.has_path_subst:
#
# fix the input path to match what the remote machine expects
#
remote_inpath, remote_outpath = self.props.substitute_paths(
inpath, outpath)
#
# build command line
#
oinput = _profile.input_options.as_shell_params()
ooutput = _profile.output_options.as_shell_params()
# quiet = ['-nostats', '-hide_banner']
remote_inpath = self.converted_path(remote_inpath)
remote_outpath = self.converted_path(remote_outpath)
if job.media_info.is_multistream(
) and self.configfile.automap and _profile.automap:
ooutput = ooutput + job.media_info.ffmpeg_streams(_profile)
cmd = [
'-y', *oinput, '-i', f'"{remote_inpath}"', *ooutput,
f'"{remote_outpath}"'
]
#
# display useful information
#
self.lock.acquire(
) # used to synchronize threads so multiple threads don't create a jumble of output
try:
print('-' * 40)
print(f'Host : {self.hostname} (mounted)')
print('Filename : ' +
crayons.green(os.path.basename(remote_inpath)))
print(f'Profile : {_profile.name}')
print('ffmeg : ' + ' '.join(cmd) + '\n')
finally:
self.lock.release()
if pytranscoder.dry_run:
continue
basename = os.path.basename(job.inpath)
def log_callback(stats):
pct_done, pct_comp = calculate_progress(
job.media_info, stats)
self.log_stats(basename, stats['speed'], pct_comp,
pct_done)
if _profile.threshold_check < 100:
if pct_done >= _profile.threshold_check and pct_comp < _profile.threshold:
# compression goal (threshold) not met, kill the job and waste no more time...
self.log(
f'Encoding of {basename} cancelled and skipped due to threshold not met'
)
return True
# continue
return False
#
# Start remote ffmpeg
#
job_start = datetime.datetime.now()
code = self.ffmpeg.run_remote(self._manager.ssh,
self.props.user, self.props.ip,
cmd, log_callback)
job_stop = datetime.datetime.now()
#
# process completed, check results and finish
#
if code == 0:
if not filter_threshold(_profile, inpath, outpath):
self.log(
f'Transcoded file {inpath} did not meet minimum savings threshold, skipped'
)
self.complete(inpath, (job_stop - job_start).seconds)
try:
os.remove(outpath)
except FileNotFoundError:
pass
continue
if not pytranscoder.keep_source:
if verbose:
self.log('removing ' + inpath)
try:
os.remove(inpath)
except FileNotFoundError:
pass
if verbose:
self.log('renaming ' + outpath)
os.rename(outpath, outpath[0:-4])
self.complete(inpath, (job_stop - job_start).seconds)
self.log(crayons.green(f'Finished {job.inpath}'))
elif code is not None:
self.log(
f'Did not complete normally: {self.ffmpeg.last_command}'
)
self.log(f'Output can be found in {self.ffmpeg.log_path}')
try:
os.remove(outpath)
except FileNotFoundError:
pass
except Exception as ex:
self.log(ex)
finally:
self.queue.task_done()
