def go(self): while not self.queue.empty(): try: job: EncodeJob = self.queue.get() inpath = job.inpath # # Got to do the rule matching again. # This time we narrow down the available profiles based on host definition # if pytranscoder.verbose: self.log('matching ' + inpath) _profile: Profile = self.match_profile(job, self.name) if _profile is None: continue # # calculate paths # outpath = inpath[0:inpath.rfind('.')] + _profile.extension + '.tmp' # # build command line # oinput = _profile.input_options.as_shell_params() ooutput = _profile.output_options.as_shell_params() remote_inpath = self.converted_path(inpath) remote_outpath = self.converted_path(outpath) processor = self.props.get_processor_by_name(_profile.processor) if _profile.is_ffmpeg: if job.media_info.is_multistream() and self.configfile.automap and _profile.automap: ooutput = ooutput + job.media_info.ffmpeg_streams(_profile) cli = ['-y', *oinput, '-i', remote_inpath, *ooutput, remote_outpath] else: cli = ['-i', remote_inpath, *oinput, *ooutput, '-o', remote_outpath] # # display useful information # self.lock.acquire() # used to synchronize threads so multiple threads don't create a jumble of output try: print('-' * 40) print(f'Host : {self.hostname} (local)') print('Filename : ' + crayons.green(os.path.basename(remote_inpath))) print(f'Profile : {_profile.name}') print('ffmpeg : ' + ' '.join(cli) + '\n') finally: self.lock.release() if pytranscoder.dry_run: continue basename = os.path.basename(job.inpath) def log_callback(stats): pct_done, pct_comp = calculate_progress(job.media_info, stats) self.log(f'{basename}: speed: {stats["speed"]}x, comp: {pct_comp}%, done: {pct_done:3}%') if _profile.threshold_check < 100: if pct_done >= _profile.threshold_check and pct_comp < _profile.threshold: # compression goal (threshold) not met, kill the job and waste no more time... self.log(f'Encoding of {basename} cancelled and skipped due to threshold not met') return True return False def hb_log_callback(stats): self.log(f'{basename}: avg fps: {stats["fps"]}, ETA: {stats["eta"]}') return False # # Start process # job_start = datetime.datetime.now() if processor.is_ffmpeg(): code = processor.run(cli, log_callback) else: code = processor.run(cli, hb_log_callback) job_stop = datetime.datetime.now() # # process completed, check results and finish # if code == 0: if not filter_threshold(_profile, inpath, outpath): self.log( f'Transcoded file {inpath} did not meet minimum savings threshold, skipped') self.complete(inpath, (job_stop - job_start).seconds) os.remove(outpath) continue if not pytranscoder.keep_source: if verbose: self.log('removing ' + inpath) os.remove(inpath) if verbose: self.log('renaming ' + outpath) os.rename(outpath, outpath[0:-4]) self.complete(inpath, (job_stop - job_start).seconds) self.log(crayons.green(f'Finished {job.inpath}')) elif code is not None: self.log(f' Did not complete normally: {processor.last_command}') self.log(f'Output can be found in {processor.log_path}') try: os.remove(outpath) except: pass except Exception as ex: self.log(ex) finally: self.queue.task_done()
def go(self): ssh_cmd = [self._manager.ssh, self.props.user + '@' + self.props.ip] # # Keep pulling items from the queue until done. Other threads will be pulling from the same queue # if multiple hosts configured on the same cluster. # while not self.queue.empty(): try: job: EncodeJob = self.queue.get() inpath = job.inpath # # Convert escaped spaces back to normal. Typical for bash to escape spaces and special characters # in filenames. # inpath = inpath.replace('\\ ', ' ') # # Got to do the rule matching again. # This time we narrow down the available profiles based on host definition # _profile: Profile = self.match_profile(job, self.name) if _profile is None: continue # # calculate full input and output paths # remote_working_dir = self.props.working_dir remote_inpath = os.path.join(remote_working_dir, os.path.basename(inpath)) remote_outpath = os.path.join(remote_working_dir, os.path.basename(inpath) + '.tmp') # # build remote commandline # oinput = _profile.input_options.as_shell_params() ooutput = _profile.output_options.as_shell_params() processor = self.props.get_processor_by_name(_profile.processor) if _profile.is_ffmpeg: if job.media_info.is_multistream() and self.configfile.automap and _profile.automap: ooutput = ooutput + job.media_info.ffmpeg_streams(_profile) cmd = ['-y', *oinput, '-i', self.converted_path(remote_inpath), *ooutput, self.converted_path(remote_outpath)] else: cmd = ['-i', self.converted_path(remote_inpath), *oinput, *ooutput, '-o', self.converted_path(remote_outpath)] cli = [*ssh_cmd, *cmd] # # display useful information # self.lock.acquire() # used to synchronize threads so multiple threads don't create a jumble of output try: print('-' * 40) print(f'Host : {self.hostname} (streaming)') print('Filename : ' + crayons.green(os.path.basename(remote_inpath))) print(f'Profile : {job.profile_name}') print('ssh : ' + ' '.join(cli) + '\n') finally: self.lock.release() if pytranscoder.dry_run: continue # # Copy source file to remote # target_dir = remote_working_dir if self.props.is_windows(): # trick to make scp work on the Windows side target_dir = '/' + remote_working_dir scp = ['scp', inpath, self.props.user + '@' + self.props.ip + ':' + target_dir] self.log(' '.join(scp)) code, output = run(scp) if code != 0: self.log(crayons.red('Unknown error copying source to remote - media skipped')) if self._manager.verbose: self.log(output) continue basename = os.path.basename(job.inpath) def log_callback(stats): pct_done, pct_comp = calculate_progress(job.media_info, stats) self.log(f'{basename}: speed: {stats["speed"]}x, comp: {pct_comp}%, done: {pct_done:3}%') if _profile.threshold_check < 100: if pct_done >= _profile.threshold_check and pct_comp < _profile.threshold: # compression goal (threshold) not met, kill the job and waste no more time... self.log(f'Encoding of {basename} cancelled and skipped due to threshold not met') return True # continue return False def hb_log_callback(stats): self.log(f'{basename}: avg fps: {stats["fps"]}, ETA: {stats["eta"]}') return False # # Start remote # job_start = datetime.datetime.now() if processor.is_ffmpeg(): code = processor.run_remote(self._manager.ssh, self.props.user, self.props.ip, cmd, log_callback) else: code = processor.run_remote(self._manager.ssh, self.props.user, self.props.ip, cmd, hb_log_callback) job_stop = datetime.datetime.now() if code != 0: self.log(crayons.red('Unknown error encoding on remote')) continue # # copy results back to local # retrieved_copy_name = os.path.join(gettempdir(), os.path.basename(remote_outpath)) cmd = ['scp', self.props.user + '@' + self.props.ip + ':' + remote_outpath, retrieved_copy_name] self.log(' '.join(cmd)) code, output = run(cmd) # # process completed, check results and finish # if code == 0: if not filter_threshold(_profile, inpath, retrieved_copy_name): self.log( f'Transcoded file {inpath} did not meet minimum savings threshold, skipped') self.complete(inpath, (job_stop - job_start).seconds) os.remove(retrieved_copy_name) continue self.complete(inpath, (job_stop - job_start).seconds) if not pytranscoder.keep_source: os.rename(retrieved_copy_name, retrieved_copy_name[0:-4]) retrieved_copy_name = retrieved_copy_name[0:-4] if verbose: self.log(f'moving media to {inpath}') shutil.move(retrieved_copy_name, inpath) self.log(crayons.green(f'Finished {inpath}')) elif code is not None: self.log(crayons.red(f'error during remote transcode of {inpath}')) self.log(f' Did not complete normally: {processor.last_command}') self.log(f'Output can be found in {processor.log_path}') # self.log(f'Removing temporary media copies from {remote_working_dir}') if self.props.is_windows(): remote_outpath = self.converted_path(remote_outpath) remote_inpath = self.converted_path(remote_inpath) self.run_process([*ssh_cmd, f'"del {remote_outpath}"']) self.run_process([*ssh_cmd, f'"del {remote_inpath}"']) else: self.run_process([*ssh_cmd, f'"rm {remote_outpath}"']) self.run_process([*ssh_cmd, f'"rm {remote_inpath}"']) finally: self.queue.task_done()
def go(self): while not self.queue.empty(): try: job: LocalJob = self.queue.get() oinput = job.profile.input_options ooutput = job.profile.output_options outpath = job.inpath.with_suffix(job.profile.extension + '.tmp') # # check if we need to exclude any streams # if job.info.is_multistream(): ooutput = ooutput + job.info.ffmpeg_streams(job.profile) cli = [ '-y', *oinput, '-i', str(job.inpath), *ooutput, str(outpath) ] # # display useful information # self.lock.acquire( ) # used to synchronize threads so multiple threads don't create a jumble of output try: print('-' * 40) print('Filename : ' + crayons.green(os.path.basename(str(job.inpath)))) print(f'Profile : {job.profile.name}') print('ffmpeg : ' + ' '.join(cli) + '\n') finally: self.lock.release() if pytranscoder.dry_run: continue basename = job.inpath.name def log_callback(stats): pct_done, pct_comp = calculate_progress(job.info, stats) self.log( f'{basename}: speed: {stats["speed"]}x, comp: {pct_comp}%, done: {pct_done:3}%' ) if job.profile.threshold_check < 100: if pct_done >= job.profile.threshold_check and pct_comp < job.profile.threshold: # compression goal (threshold) not met, kill the job and waste no more time... self.log( f'Encoding of {basename} cancelled and skipped due to threshold not met' ) return True # continue return False job_start = datetime.datetime.now() code = self.ffmpeg.run(cli, log_callback) job_stop = datetime.datetime.now() elapsed = job_stop - job_start if code == 0: if not filter_threshold(job.profile, str(job.inpath), outpath): # oops, this transcode didn't do so well, lets keep the original and scrap this attempt self.log( f'Transcoded file {job.inpath} did not meet minimum savings threshold, skipped' ) self.complete(job.inpath, (job_stop - job_start).seconds) outpath.unlink() continue self.complete(job.inpath, elapsed.seconds) if not pytranscoder.keep_source: if pytranscoder.verbose: self.log(f'replacing {job.inpath} with {outpath}') job.inpath.unlink() outpath.rename( job.inpath.with_suffix(job.profile.extension)) self.log(crayons.green(f'Finished {job.inpath}')) else: self.log( crayons.yellow( f'Finished {outpath}, original file unchanged') ) elif code is not None: self.log( f' Did not complete normally: {self.ffmpeg.last_command}' ) self.log(f'Output can be found in {self.ffmpeg.log_path}') try: outpath.unlink() except: pass finally: self.queue.task_done()
def go(self): while not self.queue.empty(): try: job: LocalJob = self.queue.get() input_opt = job.profile.input_options.as_shell_params() output_opt = self.config.output_from_profile(job.profile, job.mixins) fls = False if self.config.fls_path(): # lets write output to local storage, for efficiency outpath = PurePath(self.config.fls_path(), job.inpath.with_suffix(job.profile.extension).name) fls = True else: outpath = job.inpath.with_suffix(job.profile.extension + '.tmp') # # check if we need to exclude any streams # processor = self.config.get_processor_by_name(job.profile.processor) if job.profile.is_ffmpeg: if job.info.is_multistream() and self.config.automap and job.profile.automap: output_opt = output_opt + job.info.ffmpeg_streams(job.profile) cli = ['-y', *input_opt, '-i', str(job.inpath), *output_opt, str(outpath)] else: cli = ['-i', str(job.inpath), *input_opt, *output_opt, '-o', str(outpath)] # # display useful information # self.lock.acquire() # used to synchronize threads so multiple threads don't create a jumble of output try: print('-' * 40) print('Filename : ' + crayons.green(os.path.basename(str(job.inpath)))) print(f'Profile : {job.profile.name}') print('{:<6} : '.format(job.profile.processor) + ' '.join(cli) + '\n') finally: self.lock.release() if pytranscoder.dry_run: continue basename = job.inpath.name def log_callback(stats): pct_done, pct_comp = calculate_progress(job.info, stats) pytranscoder.status_queue.put({ 'host': 'local', 'file': basename, 'speed': stats['speed'], 'comp': pct_comp, 'done': pct_done}) #self.log(f'{basename}: speed: {stats["speed"]}x, comp: {pct_comp}%, done: {pct_done:3}%') if job.profile.threshold_check < 100: if pct_done >= job.profile.threshold_check and pct_comp < job.profile.threshold: # compression goal (threshold) not met, kill the job and waste no more time... self.log(f'Encoding of {basename} cancelled and skipped due to threshold not met') return True return False def hbcli_callback(stats): self.log(f'{basename}: avg fps: {stats["fps"]}, ETA: {stats["eta"]}') return False job_start = datetime.datetime.now() if processor.is_ffmpeg(): code = processor.run(cli, log_callback) else: code = processor.run(cli, hbcli_callback) job_stop = datetime.datetime.now() elapsed = job_stop - job_start if code == 0: if not filter_threshold(job.profile, str(job.inpath), outpath): # oops, this transcode didn't do so well, lets keep the original and scrap this attempt self.log(f'Transcoded file {job.inpath} did not meet minimum savings threshold, skipped') self.complete(job.inpath, (job_stop - job_start).seconds) os.unlink(str(outpath)) continue self.complete(job.inpath, elapsed.seconds) if not pytranscoder.keep_source: if pytranscoder.verbose: self.log(f'replacing {job.inpath} with {outpath}') job.inpath.unlink() if fls: shutil.move(outpath, job.inpath.with_suffix(job.profile.extension)) else: outpath.rename(job.inpath.with_suffix(job.profile.extension)) self.log(crayons.green(f'Finished {job.inpath}')) else: self.log(crayons.yellow(f'Finished {outpath}, original file unchanged')) elif code is not None: self.log(f' Did not complete normally: {processor.last_command}') self.log(f'Output can be found in {processor.log_path}') try: outpath.unlink() except: pass finally: self.queue.task_done()
def go(self): while not self.queue.empty(): try: job: EncodeJob = self.queue.get() inpath = job.inpath # # Got to do the rule matching again. # This time we narrow down the available profiles based on host definition # if pytranscoder.verbose: self.log('matching ' + inpath) _profile: Profile = self.match_profile(job, self.name) if _profile is None: continue # # calculate paths # outpath = inpath[0:inpath. rfind('.')] + '.h265' + _profile.extension remote_inpath = inpath remote_outpath = outpath if self.props.has_path_subst: # # fix the input path to match what the remote machine expects # remote_inpath, remote_outpath = self.props.substitute_paths( inpath, outpath) # # build command line # oinput = _profile.input_options.as_shell_params() ooutput = _profile.output_options.as_shell_params() # quiet = ['-nostats', '-hide_banner'] remote_inpath = self.converted_path(remote_inpath) remote_outpath = self.converted_path(remote_outpath) if job.media_info.is_multistream( ) and self.configfile.automap and _profile.automap: ooutput = ooutput + job.media_info.ffmpeg_streams(_profile) cmd = [ '-y', *oinput, '-i', f'"{remote_inpath}"', *ooutput, f'"{remote_outpath}"' ] # # display useful information # self.lock.acquire( ) # used to synchronize threads so multiple threads don't create a jumble of output try: print('-' * 40) print(f'Host : {self.hostname} (mounted)') print('Filename : ' + crayons.green(os.path.basename(remote_inpath))) print(f'Profile : {_profile.name}') print('ffmeg : ' + ' '.join(cmd) + '\n') finally: self.lock.release() if pytranscoder.dry_run: continue basename = os.path.basename(job.inpath) def log_callback(stats): pct_done, pct_comp = calculate_progress( job.media_info, stats) self.log_stats(basename, stats['speed'], pct_comp, pct_done) if _profile.threshold_check < 100: if pct_done >= _profile.threshold_check and pct_comp < _profile.threshold: # compression goal (threshold) not met, kill the job and waste no more time... self.log( f'Encoding of {basename} cancelled and skipped due to threshold not met' ) return True # continue return False # # Start remote ffmpeg # job_start = datetime.datetime.now() code = self.ffmpeg.run_remote(self._manager.ssh, self.props.user, self.props.ip, cmd, log_callback) job_stop = datetime.datetime.now() # # process completed, check results and finish # if code == 0: if not filter_threshold(_profile, inpath, outpath): self.log( f'Transcoded file {inpath} did not meet minimum savings threshold, skipped' ) self.complete(inpath, (job_stop - job_start).seconds) try: os.remove(outpath) except FileNotFoundError: pass continue if not pytranscoder.keep_source: if verbose: self.log('removing ' + inpath) try: os.remove(inpath) except FileNotFoundError: pass if verbose: self.log('renaming ' + outpath) os.rename(outpath, outpath[0:-4]) self.complete(inpath, (job_stop - job_start).seconds) self.log(crayons.green(f'Finished {job.inpath}')) elif code is not None: self.log( f'Did not complete normally: {self.ffmpeg.last_command}' ) self.log(f'Output can be found in {self.ffmpeg.log_path}') try: os.remove(outpath) except FileNotFoundError: pass except Exception as ex: self.log(ex) finally: self.queue.task_done()