def encode(jobid=None, db=None, job=None, procqueue=None, preset='slow', vbitrate_param='-crf:v 18', abitrate_param='-c:a libfdk_aac -b:a 128k', tmpfile=None, outfile=None, statusfile=None): # task = System(path=transcoder, db=db) task = System(path='nice', db=db) try: output = task( '-n %s' % NICELEVEL, '%s' % transcoder, '-i "%s"' % tmpfile, # parameter to overwrite output file if present without prompt '-y', # parameter de-interlacing filter '-filter:v yadif=0:-1:1', # parameter to allow streaming content '-movflags faststart', # parameter needed when hdhomerun prime mpeg2 files sometime repeat timestamps '-vsync passthrough', # change video resolution scale_param, # h264 video codec '-c:v libx264', # presets for h264 encode that effect encode speed/output filesize '-preset:v %s' % preset, # ########## IMPORTANT ############ # ffmpeg versions after 08-18-2015 include a change to force explicit IDR frames, # setting this flag helps/corrects myth seektable indexing h264-encoded files # uncomment the line below if you have a recent version of ffmpeg that supports this option # '-forced-idr 1', # parameters to determine video encode target bitrate vbitrate_param, # parameters to determine audio encode target bitrate abitrate_param, # parameter to encode all input audio streams into the output # '-map 0:a', # parameters to set the first output audio stream # to be an audio stream having the specified language (default=eng -> English) # '-metadata:s:a:0', # 'language=%s' % language, # parameter to copy input subtitle streams into the output '-c:s copy', # '-c:s mov_text', # parameters to set the first output subtitle stream # to be an english subtitle stream # '-metadata:s:s:0', # 'language=%s' % language, # we can control the number of encode threads (disabled) # '-threads 2', # output file parameter '"%s"' % outfile, # redirection of output to temporaryfile '> %s 2>&1 < /dev/null' % statusfile) except MythError, e: print 'Command failed with output:\n%s' % e.stderr if jobid: job.update({'status':job.ERRORED, 'comment':'Transcoding to mp4 failed'}) procqueue.put(CleanExit) sys.exit(e.retcode)
def test_Methodheap_MythXML_002_010(self): """Test MythXML.getRecorded() during standard time and daylight saving time. """ preview_cet_is_pic = False preview_cest_is_pic = False with add_log_flags(): m_instance = MythXML() progs = m_instance.getRecorded() try: found_cet = False found_cest = False while True: p = next(progs) if not found_cet: if (p.starttime > self.t1_cet and p.starttime < self.t2_cet): pcet = p found_cet = True if not found_cest: if (p.starttime > self.t1_cest and p.starttime < self.t2_cest): pcest = p found_cest = True if (found_cet and found_cest): break except StopIteration: raise preview_cet = m_instance.getPreviewImage(str(pcet.chanid), pcet.recstartts) with open('/tmp/preview_cet', 'wb') as f: f.write(preview_cet) out_cet = System.system('file /tmp/preview_cet') preview_cet_is_pic = (len( tailandgrep('/tmp/my_logfile', 2, 'JPEG|PNG')) > 0) preview_cest = m_instance.getPreviewImage(str(pcest.chanid), pcest.recstartts) with open('/tmp/preview_cest', 'wb') as f: f.write(preview_cest) out_cest = System.system('file /tmp/preview_cest') preview_cest_is_pic = (len( tailandgrep('/tmp/my_logfile', 2, 'JPEG|PNG')) > 0) self.assertTrue(preview_cet_is_pic) self.assertTrue(preview_cest_is_pic)
def test_Methodheap_MythXML_001_010(self): """Test MythXML.getRecorded() during standard time and daylight saving time. """ preview_cet_is_pic = False preview_cest_is_pic = False m_instance = MythXML() progs = m_instance.getRecorded() try: found_cet = False found_cest = False while True: p = next(progs) if not found_cet: if (p.starttime > self.t1_cet and p.starttime < self.t2_cet): pcet = p found_cet = True if not found_cest: if (p.starttime > self.t1_cest and p.starttime < self.t2_cest): pcest = p found_cest = True if (found_cet and found_cest): break except StopIteration: raise preview_cet = m_instance.getPreviewImage(str(pcet.chanid), pcet.recstartts) with open('/tmp/preview_cet', 'wb') as f: f.write(preview_cet) s_cet = System(path='file') out_cet = s_cet('/tmp/preview_cet') preview_cet_is_pic = ((b'PNG' in out_cet) or (b'JPEG' in out_cet)) preview_cest = m_instance.getPreviewImage(str(pcest.chanid), pcest.recstartts) with open('/tmp/preview_cest', 'wb') as f: f.write(preview_cest) s_cest = System(path='file') out_cest = s_cest('/tmp/preview_cest') preview_cest_is_pic = ((b'PNG' in out_cest) or (b'JPEG' in out_cest)) self.assertTrue(preview_cet_is_pic) self.assertTrue(preview_cest_is_pic)
def get_duration(db=None, rec=None, transcoder='/usr/bin/ffmpeg', filename=None): task = System(path=transcoder, db=db) if filename is None: return -1 try: output = task('-i "%s"' % filename, '1>&2') except MythError, e: pass
def test_repr_005_01(self): """ Test '__repr__' and '__str__' methods of 'MythBE' class. Note: MythBE inherits from 'FileOps', which is inherited from 'BeCache'. """ s = System(path='echo') print() print(repr(s)) print(str(s))
def encode(jobid=None, db=None, job=None, procqueue=None, preset='slow', scaling='', burncc='', usemkv=0, vbitrate_param='-q 21', abitrate_param='--aencoder copy:ac3', tmpfile=None, outfile=None, statusfile=None): # task = System(path=transcoder, db=db) task = System(path='nice', db=db) if debug: script = 'nice -n {} {} -i "{}"'.format(NICELEVEL, transcoder, tmpfile) # parameter to allow streaming content script = '{} -O {} --markers --detelecine --strict-anamorphic'.format( script, scaling) # h264 video codec script = '{} --large-file --encoder x264'.format(script) if usemkv == 1: # use the Normal preset for MKV script = '{} -Z Normal'.format(script) # parameter to copy input subtitle streams into the output script = '{} -s 1'.format(script) else: # presets for h264 encode that effect encode speed/output filesize script = '{} --encopts {}'.format(script, preset) # parameters to determine video encode target bitrate script = '{} {}'.format(script, vbitrate_param) # parameters to determine audio encode target bitrate script = '{} {}'.format(script, abitrate_param) # parameter to copy input subtitle streams into the output script = '{} -s 1 {}'.format(script, burncc) # output file parameter script = '{} -o "{}"'.format(script, outfile) # redirection of output to temporaryfile script = '{} > {} 2>&1 < /dev/null'.format(script, statusfile) print 'Executing Transcoder: \n{}'.format(script) try: output = task('{}'.format(script)) except MythError, e: PrintException() #print 'Command failed with output:\n{}'.format(e.ename,e.args,e.stderr) #TODO: if failure, need to return a failure code to abort the rest of the script. if jobid: job.update({ 'status': job.ERRORED, 'comment': 'Transcoding to {} failed'.format(filetype) }) procqueue.put(CleanExit) sys.exit(e.retcode)
def test_Methodheap_MythXML_002_09(self): """Test MythXML.getPreviewImage().""" a = False with add_log_flags(): m_instance = MythXML() rec_chanid = self.testenv['DOWNCHANID'] rec_starttime = self.testenv['DOWNSTARTTIME'] preview = m_instance.getPreviewImage(str(rec_chanid), rec_starttime) with open('/tmp/preview', 'wb') as f: f.write(preview) out1 = System.system('file /tmp/preview') a = (len(tailandgrep('/tmp/my_logfile', 8, 'JPEG|PNG')) > 0) self.assertTrue(a)
def runjob(rec): db = MythDB() timestr = time.strftime("%m-%d-%y %H:%M:%S") title_san = re.sub("\s", ".", rec.title) trans_log_file = os.path.join(log_dir, "%s_transcode_log_%s.hb.txt" % (title_san, timestr)) commflag_log_file = os.path.join(log_dir, "%s_transcode_log_%s.cf.txt" % (title_san, timestr)) mythical_log_file = os.path.join(log_dir, "%s_transcode_log_%s.ml.txt" % (title_san, timestr)) sg = findfile('/'+rec.basename, rec.storagegroup, db=db) if sg is None: print 'Local access to recording not found.' sys.exit(1) infile = os.path.join(sg.dirname, rec.basename) tmpfile = '%s.tmp' % infile.rsplit('.',1)[0] outfile = '%s.mp4' % infile.rsplit('.',1)[0] print "Infile: %s" % infile print "Outfile: %s" % outfile # reformat 'starttime' for use with mythtranscode/ffmpeg/mythcommflag starttime = str(rec.starttime.utcisoformat().replace(u':', '').replace(u' ', '').replace(u'T', '').replace('-', '')) chanid = rec.chanid # Lossless transcode to strip cutlist if rec.cutlist == 1 and false: if job: job.update({'status':4, 'comment':'Removing Cutlist'}) task = System(path='mythtranscode', db=db) try: output = task('--chanid "%s"' % chanid, '--starttime "%s"' % starttime, '--mpeg2', '--honorcutlist', '-o "%s"' % tmpfile, '2> /dev/null') except MythError, e: print 'Command failed with output:\n%s' % e.stderr if job: job.update({'status':304, 'comment':'Removing Cutlist failed'}) sys.exit(e.retcode)
def runjob(jobid=None, chanid=None, starttime=None): db = MythDB() if jobid: job = Job(jobid, db=db) chanid = job.chanid starttime = job.starttime rec = Recorded((chanid, starttime), db=db) sg = findfile('/' + rec.basename, rec.storagegroup, db=db) if sg is None: print 'Local access to recording not found.' sys.exit(1) infile = os.path.join(sg.dirname, rec.basename) tmpfile = '%s.tmp' % infile.rsplit('.', 1)[0] outfile = '%s.mp4' % infile.rsplit('.', 1)[0] # reformat 'starttime' for use with mythtranscode/ffmpeg/mythcommflag starttime = str(starttime.utcisoformat().replace(u':', '').replace( u' ', '').replace(u'T', '').replace('-', '')) # Lossless transcode to strip cutlist if rec.cutlist == 1: if jobid: job.update({'status': 4, 'comment': 'Removing Cutlist'}) task = System(path='mythtranscode', db=db) try: output = task('--chanid "%s"' % chanid, '--starttime "%s"' % starttime, '--mpeg2', '--honorcutlist', '-o "%s"' % tmpfile, '2> /dev/null') except MythError, e: print 'Command failed with output:\n%s' % e.stderr if jobid: job.update({ 'status': 304, 'comment': 'Removing Cutlist failed' }) sys.exit(e.retcode)
def get_duration(db=None, rec=None, transcoder='/usr/bin/ffmpeg', filename=None): task = System(path=transcoder, db=db) if filename is None: return -1 try: output = task('-i "%s"' % filename, '1>&2') except MythError as e: err = e pass r = re.compile('Duration: (.*?), start') m = r.search(err.stderr.decode('utf-8')) if m: duration = m.group(1).split(':') duration_secs = float((int(duration[0])*60+int(duration[1]))*60+float(duration[2])) duration_msecs = int(1000*duration_secs) if debug: print('Duration %s' % m.group(1)) print('Duration %s' % duration) print('Duration in seconds "%s"' % duration_secs) print('Duration in milliseconds "%s"' % duration_msecs) return duration_secs, err return -1, err
except MythError, e: print 'Command failed with output:\n%s' % e.stderr if jobid: job.update({ 'status': 304, 'comment': 'Removing Cutlist failed' }) sys.exit(e.retcode) else: copyfile('%s' % infile, '%s' % tmpfile) # Transcode to mp4 if jobid: job.update({'status': 4, 'comment': 'Transcoding to mp4'}) task = System(path=transcoder, db=db) try: output = task('-i "%s"' % tmpfile, '-filter:v yadif=0:-1:1', '-c:v libx264', '-preset:v slow', '-crf:v 18', '-strict -2', '-metadata:s:a:0', 'language="eng"', '"%s"' % outfile, '2> /dev/null') except MythError, e: print 'Command failed with output:\n%s' % e.stderr if jobid: job.update({'status': 304, 'comment': 'Transcoding to mp4 failed'}) sys.exit(e.retcode) rec.basename = os.path.basename(outfile) os.remove(infile) # Cleanup the old *.png files for filename in glob('%s*.png' % infile):
print transcoder print '-v' print '-q 20.0' print '-e x264' print '-r 25' print '--crop 0:0:0:0' print '-d' print '-m' print '-x b-adapt=2:rc-lookahead=50:ref=3:bframes=3:me=umh:subme=8:trellis=1:merange=20:direct=auto' print '-i "%s"' % tmpfile print '-o "%s"' % outfile print '-4' print '--optimize 2 >> "%s"' % trans_log_file task = System(path=transcoder, db=db) try: output = task('-v', '-q 20.0', '-e x264', '-r 25', '--crop 0:0:0:0', '-d', '-m', '-x b-adapt=2:rc-lookahead=50:ref=3:bframes=3:me=umh:subme=8:trellis=1:merange=20:direct=auto', '-i "%s"' % tmpfile, '-o "%s"' % outfile, '-4', '--optimize 2 >> "%s"' % trans_log_file) except MythError, e: print 'Command failed with output:\n%s' % e.stderr
rec.basename = outfile os.remove(infile) rec.filesize = os.path.getsize(outfile) rec.transcoded = 1 rec.seek.clean() if flush_commskip: for index,mark in reversed(list(enumerate(rec.markup))): if mark.type in (rec.markup.MARK_COMM_START, rec.markup.MARK_COMM_END): del rec.markup[index] rec.bookmark = 0 rec.cutlist = 0 rec.markup.commit() if build_seektable: task = System(path='mythcommflag') task.command('--chanid %s' % chanid, '--starttime %s' % starttime, '--rebuild') rec.update() def main(): parser = OptionParser(usage="usage: %prog [options] [jobid]") parser.add_option('--chanid', action='store', type='int', dest='chanid', help='Use chanid for manual operation') parser.add_option('--starttime', action='store', type='int', dest='starttime', help='Use starttime for manual operation') parser.add_option('-v', '--verbose', action='store', type='string', dest='verbose',
rec.seek.clean() if flush_commskip: for index,mark in reversed(list(enumerate(rec.markup))): if mark.type in (rec.markup.MARK_COMM_START, rec.markup.MARK_COMM_END): del rec.markup[index] rec.bookmark = 0 rec.cutlist = 0 rec.markup.commit() rec.update() if build_seektable: try: task = System(path='mythcommflag', db=self.db()) task.command('--chanid %s' % chanid, '--starttime %s' % starttime) except MythError, e: self.log('Mythcommflag --chanid %s --starttime %s failed: %s' % (chanid, starttime, str(e)), LOGLEVEL.ERR) if jobid: job.update({'status':272, 'comment':'Transcode Completed'}) def main(): parser = OptionParser(usage="usage: %prog [options] [jobid]") parser.add_option('--chanid', action='store', type='int', dest='chanid', help='Use chanid for manual operation') parser.add_option('--starttime', action='store', type='int',
def add_metadata(db=None, jobid=None, debug=False, job=None, rec=None, filetype='mkv', filename=None, metaapp='/usr/bin/AtomicParsley'): if debug: print 'Adding metadata to the file.' if jobid: progress_str = 'Adding metadata to the file.' job.update({'status': job.RUNNING, 'comment': progress_str}) # Use AtomicParsley for metadata work on the file if it's not a MKV if filetype != 'mkv': tmptitle = rec.title.encode('utf-8').strip() castmembers = '' director = '' if rec.programid[0:2] == 'MV': # TODO: Add Actors and Director - need to loop over rec.cast object and reference rec.cast[x].role to determine if actor/director for castmember in rec.cast: if castmember.role == 'director': if len(director) == 0: director = [castmember.name] else: director.append(castmember.name) if len(castmembers) < 5 and castmember.role == 'actor': if len(castmembers) == 0: castmembers = [castmember.name] else: castmembers.append(castmember.name) if debug: print 'directors: {}'.format(','.join(director)) print 'cast members: {}'.format(','.join(castmembers)) tmptitle = '{}'.format(rec.title.encode('utf-8').strip()) if rec.season > 0 and rec.episode > 0: tmptitle = '{0:s} S{1:d} E{2:02d}'.format( rec.title.encode('utf-8').strip(), rec.season, rec.episode) atomicparams = '"{}" --title "{}" --genre "{}" --year "{}" --TVShowName "{}" --TVSeasonNum "{}" --TVEpisodeNum "{}" --TVEpisode "{}" --comment "{}" --description "{}" --longdesc "{}" --artist "{}" --albumArtist "{}" --overWrite'.format( os.path.realpath(filename), tmptitle, rec.category, rec.originalairdate, rec.title.encode('utf-8').strip(), rec.season, rec.episode, rec.programid, rec.subtitle.encode('utf-8').strip(), rec.subtitle.encode('utf-8').strip(), rec.description.encode('utf-8').strip(), ','.join(castmembers), ','.join(director)) try: # Set a time limit on how long to try and write the atomicparsley metadata to the file. max_time = POLL_INTERVAL start_time = time.time() while (time.time() - start_time) < max_time: metatask = System(path='nice', db=db) metatask('-n {} {} {}'.format(NICELEVEL, metaapp, atomicparams)) except Exception as e: if debug: PrintException() print 'Adding metadata to the filename failed. Run this manually: /usr/bin/AtomicParsley {}'.format( atomicparams) if jobid: job.update({ 'status': job.FINISHED, 'comment': 'Adding metadata to the filename failed. Run this manually: /usr/bin/AtomicParsley {}' .format(atomicparams) })
print "Outfile: %s" % outfile if os.path.splitext(infile)[1] == '.mp4': print 'Infile is already mp4! Dont do anything!' sys.exit(0) # reformat 'starttime' for use with mythtranscode/ffmpeg/mythcommflag starttime = str(rec.starttime.utcisoformat().replace(u':', '').replace(u' ', '').replace(u'T', '').replace('-', '')) # Lossless transcode to strip cutlist if rec.cutlist == 1: print 'Removing cutlist...' if jobid: job.update({'status':4, 'comment':'Removing Cutlist'}) task = System(path='mythtranscode', db=db) try: output = task('--chanid "%s"' % chanid, '--starttime "%s"' % starttime, '--mpeg2', '--honorcutlist', '-o "%s"' % tmpfile, '2> /dev/null') except MythError, e: print 'Removing cutlist failure: Command failed with output:\n%s' % e.stderr if jobid: job.update({'status':304, 'comment':'Removing Cutlist failed'}) sys.exit(e.retcode) print 'Removing cutlist...done' else: print 'No cutlist found, skipping'
def runjob(jobid=None, chanid=None, starttime=None, tzoffset=None): global estimateBitrate db = MythDB() if jobid: job = Job(jobid, db=db) chanid = job.chanid utcstarttime = job.starttime else: job=None; utcstarttime = datetime.strptime(starttime, "%Y%m%d%H%M%S") utcstarttime = utcstarttime + timedelta(hours=tzoffset) if debug: print 'chanid "%s"' % chanid print 'utcstarttime "%s"' % utcstarttime rec = Recorded((chanid, utcstarttime), db=db); utcstarttime = rec.starttime; starttime_datetime = utcstarttime # reformat 'starttime' for use with mythtranscode/ffmpeg/mythcommflag starttime = str(utcstarttime.utcisoformat().replace(u':', '').replace(u' ', '').replace(u'T', '').replace('-', '')) if debug: print 'mythtv format starttime "%s"' % starttime input_filesize = rec.filesize if rec.commflagged: if debug: print 'Recording has been scanned to detect commerical breaks.' waititer=1 keepWaiting = True while keepWaiting == True: keepWaiting=False; for index,jobitem in reversed(list(enumerate(db.searchJobs(chanid=chanid, starttime=starttime_datetime)))): if jobitem.type == jobitem.COMMFLAG: # Commercial flagging job if debug: print 'Commercial flagging job detected with status %s' % jobitem.status if jobitem.status == jobitem.RUNNING: # status = RUNNING? job.update({'status':job.PAUSED, 'comment':'Waited %d secs for the commercial flagging job' % (waititer*POLL_INTERVAL) \ + ' currently running on this recording to complete.'}) if debug: print 'Waited %d secs for the commercial flagging job' % (waititer*POLL_INTERVAL) \ + ' currently running on this recording to complete.' time.sleep(POLL_INTERVAL); keepWaiting=True waititer = waititer + 1 break else: if debug: print 'Recording has not been scanned to detect/remove commercial breaks.' if require_commflagged: if jobid: job.update({'status':job.RUNNING, 'comment':'Required commercial flagging for this file is not found.' + 'Flagging commercials and cancelling any queued commercial flagging.'}) # cancel any queued job to flag commercials for this recording and run commercial flagging in this script for index,jobitem in reversed(list(enumerate(db.searchJobs(chanid=chanid,starttime=starttime_datetime)))): if debug: if index==0: print jobitem.keys() print index,jobitem.id,jobitem.chanid if jobitem.type == jobitem.COMMFLAG: # Commercial flagging job if jobitem.status == jobitem.RUNNING: # status = RUNNING? jobitem.cmds = jobitem.STOP # stop command from the frontend to stop the commercial flagging job #jobitem.setStatus(jobitem.CANCELLED) #jobitem.setComment('Cancelled: Transcode command ran commercial flagging for this recording.') jobitem.update({'status':jobitem.CANCELLED, 'comment':'A user transcode job ran commercial flagging for' + ' this recording and cancelled this job.'}) if debug: print 'Flagging Commercials...' # Call "mythcommflag --chanid $CHANID --starttime $STARTTIME" task = System(path='mythcommflag', db=db) try: output = task('--chanid "%s"' % chanid, '--starttime "%s"' % starttime, '2> /dev/null') except MythError, e: # it seems mythcommflag always exits with an decoding error "eno: Unknown error 541478725 (541478725)" pass
infile = os.path.join(sg.dirname, rec.basename) tmpfile = '%s.tmp' % infile.rsplit('.',1)[0] # tmpfile = infile outfile = '%s.mp4' % infile.rsplit('.',1)[0] outfile = '%s.mp4' % os.path.join(sg.dirname, "proxy", rec.basename).rsplit('.',1)[0] if debug: print 'tmpfile "%s"' % tmpfile clipped_bytes=0; # If selected, create a cutlist to remove commercials via mythtranscode by running: # mythutil --gencutlist --chanid $CHANID --starttime $STARTTIME if generate_commcutlist: if jobid: job.update({'status':job.RUNNING, 'comment':'Generating Cutlist for commercial removal'}) task = System(path='mythutil', db=db) try: output = task('--gencutlist', '--chanid "%s"' % chanid, '--starttime "%s"' % starttime) # '--loglevel debug', # '2> /dev/null') except MythError, e: print 'Command "mythutil --gencutlist" failed with output:\n%s' % e.stderr if jobid: job.update({'status':job.ERRORED, 'comment':'Generation of commercial Cutlist failed'}) sys.exit(e.retcode) # Lossless transcode to strip cutlist if generate_commcutlist or rec.cutlist==1: if jobid:
def runjob(jobid=None, chanid=None, starttime=None, tzoffset=None, maxWidth=maxWidth, maxHeight=maxHeight, sdonly=0, burncc=0, usemkv=0, overwrite=1): global estimateBitrate db = MythDB() try: if jobid: job = Job(jobid, db=db) chanid = job.chanid utcstarttime = job.starttime else: job = None utcstarttime = datetime.strptime(starttime, "%Y%m%d%H%M%S") utcstarttime = utcstarttime + timedelta(hours=tzoffset) if debug: print 'chanid "%s"' % chanid print 'utcstarttime "%s"' % utcstarttime rec = Recorded((chanid, utcstarttime), db=db) utcstarttime = rec.starttime starttime_datetime = utcstarttime # reformat 'starttime' for use with mythtranscode/HandBrakeCLI/mythcommflag starttime = str(utcstarttime.utcisoformat().replace(u':', '').replace( u' ', '').replace(u'T', '').replace('-', '')) if debug: print 'mythtv format starttime "%s"' % starttime input_filesize = rec.filesize if rec.commflagged: if debug: print 'Recording has been scanned to detect commerical breaks.' waititer = 1 keepWaiting = True while keepWaiting == True: keepWaiting = False for index, jobitem in reversed( list( enumerate( db.searchJobs(chanid=chanid, starttime=starttime_datetime)))): if jobitem.type == jobitem.COMMFLAG: # Commercial flagging job if debug: print 'Commercial flagging job detected with status %s' % jobitem.status if jobitem.status == jobitem.RUNNING: # status = RUNNING? job.update({'status':job.PAUSED, 'comment':'Waited %d secs for the commercial flagging job' % (waititer*POLL_INTERVAL) \ + ' currently running on this recording to complete.'}) if debug: print 'Waited %d secs for the commercial flagging job' % (waititer*POLL_INTERVAL) \ + ' currently running on this recording to complete.' time.sleep(POLL_INTERVAL) keepWaiting = True waititer = waititer + 1 break else: if debug: print 'Recording has not been scanned to detect/remove commercial breaks.' if require_commflagged: if jobid: job.update({ 'status': job.RUNNING, 'comment': 'Required commercial flagging for this file is not found.' + 'Flagging commercials and cancelling any queued commercial flagging.' }) # cancel any queued job to flag commercials for this recording and run commercial flagging in this script for index, jobitem in reversed( list( enumerate( db.searchJobs(chanid=chanid, starttime=starttime_datetime)))): if debug: if index == 0: print jobitem.keys() print index, jobitem.id, jobitem.chanid if jobitem.type == jobitem.COMMFLAG: # Commercial flagging job if jobitem.status == jobitem.RUNNING: # status = RUNNING? jobitem.cmds = jobitem.STOP # stop command from the frontend to stop the commercial flagging job #jobitem.setStatus(jobitem.CANCELLED) #jobitem.setComment('Cancelled: Transcode command ran commercial flagging for this recording.') jobitem.update({ 'status': jobitem.CANCELLED, 'comment': 'A user transcode job ran commercial flagging for' + ' this recording and cancelled this job.' }) if debug: print 'Flagging Commercials...' # Call "mythcommflag --chanid $CHANID --starttime $STARTTIME" task = System(path='mythcommflag', db=db) try: output = task('--chanid "%s"' % chanid, '--starttime "%s"' % starttime, '2> /dev/null') except MythError, e: # it seems mythcommflag always exits with an decoding error "eno: Unknown error 541478725 (541478725)" pass #print 'Command failed with output:\n%s' % e.stderr #if jobid: # job.update({'status':304, 'comment':'Flagging commercials failed'}) #sys.exit(e.retcode) sg = findfile('/' + rec.basename, rec.storagegroup, db=db) if sg is None: print 'Local access to recording not found.' sys.exit(1) infile = os.path.join(sg.dirname, rec.basename) #TODO: set overWrite to 0 if infile is m4v or mkv (already converted) #tmpfile = '%s.tmp' % infile.rsplit('.',1)[0] outtitle = rec.title.replace("&", "and") outtitle = re.sub('[^A-Za-z0-9 ]+', '', outtitle) filetype = 'm4v' #DEBUG CODE TO FIND OBJECT STRUCT: #print '{}'.format(dir(rec.getProgram())) #print '{}'.format(rec.getProgram().year) if usemkv == 1: filetype = 'mkv' #print '!{}!'.format(rec.programid[0:2]) if rec.season > 0 and rec.episode > 0: #if there are seasons and episode numbers in the recording data outtitle = '{0:s} S{1:d} E{2:02d}'.format(outtitle, rec.season, rec.episode) elif rec.programid[0:2] == 'MV' and str(rec.getProgram().year).isdigit( ): #if it's a movie and has an original air date for when it came out outtitle = '{} ({})'.format(outtitle, rec.getProgram().year) elif rec.programid[ 0: 2] == 'MV' and rec.originalairdate != None and rec.originalairdate > datetime.date( datetime(1, 1, 1, 0, 0) ): #if it's a movie and has an original air date for when it came out outtitle = '{} ({})'.format(outtitle, rec.originalairdate.year) elif 'Sports' in rec.category: #if it's sports outtitle = '{}-{}-{}'.format( outtitle, re.sub('[^A-Za-z0-9 ]+', '', rec.subtitle), str(rec.starttime.strftime("%Y%m%d"))) elif rec.programid[0:2] == 'SH' and (' News ' in rec.title or rec.category == 'News'): #if it's a news show outtitle = '{}-{}'.format(outtitle, str(rec.starttime.strftime("%Y%m%d"))) elif rec.originalairdate != None and rec.originalairdate > datetime.date( datetime(1, 1, 1, 0, 0)): #if it has an original air date outtitle = '{} {}'.format( outtitle, str(rec.originalairdate.strftime("%Y%m%d"))) else: outtitle = '{} {}'.format(outtitle, str(rec.starttime.strftime("%Y%m%d"))) outtitle = '{}.{}'.format(outtitle, filetype) outfile = os.path.join(sg.dirname, outtitle) tmpfile = '{}.{}'.format( outfile.rsplit('.', 1)[0], infile.rsplit('.', 1)[1]) if tmpfile == infile: tmpfile = '{}.tmp'.format(infile.rsplit('.', 1)[0]) if (overwrite == 0): # If not overwritting the file, use the export folder outfile = os.path.join(exportFolder, outtitle) if debug: print 'overwrite is 0. outfile "{}"'.format(outfile) if os.path.isfile(outfile) or infile == outfile: # If outfile exists already, create a new name for the file. outfile = '{}-{}.{}'.format( outfile.rsplit('.', 1)[0], str(rec.starttime.strftime("%Y%m%d")), filetype) if os.path.isfile(tmpfile): # If the infile and tmpfile are the same, create a new name for the tmpfile tmpfile = '{}-{}.tmp'.format( outfile.rsplit('.', 1)[0], str(rec.starttime.strftime("%Y%m%d"))) if os.path.isfile(tmpfile): # If tmp exists already, create a new name for the file. outfile = '{}-{}.tmp'.format( tmpfile.rsplit('.', 1)[0], str(rec.starttime.strftime("%Y%m%d"))) if debug: print 'tmp exists. outfile "{}"'.format(outfile) if debug: print 'infile "{}"'.format(infile) print 'tmpfile "{}"'.format(tmpfile) print 'outfile "{}"'.format(outfile) #add_metadata(db, jobid, debug, job, rec, filetype, tmpfile) clipped_bytes = 0 # If selected, create a cutlist to remove commercials via mythtranscode by running: # mythutil --gencutlist --chanid $CHANID --starttime $STARTTIME if generate_commcutlist: if jobid: job.update({ 'status': job.RUNNING, 'comment': 'Generating Cutlist for commercial removal' }) task = System(path='mythutil', db=db) try: output = task('--gencutlist', '--chanid "%s"' % chanid, '--starttime "%s"' % starttime) except MythError, e: print 'Command "mythutil --gencutlist" failed with output:\n%s' % e.stderr if jobid: job.update({ 'status': job.ERRORED, 'comment': 'Generation of commercial Cutlist failed' }) sys.exit(e.retcode)
def runjob(jobid=None, chanid=None, starttime=None, tzoffset=None): global estimateBitrate db = MythDB() if jobid: job = Job(jobid, db=db) chanid = job.chanid utcstarttime = job.starttime else: job=None; #utcstarttime = datetime.strptime(starttime, "%Y%m%d%H%M%S%z") utcstarttime = parse(starttime) utcstarttime = utcstarttime + timedelta(hours=tzoffset) if debug: print('chanid "%s"' % chanid) print('utcstarttime "%s"' % utcstarttime) rec = Recorded((chanid, utcstarttime), db=db); utcstarttime = rec.starttime; starttime_datetime = utcstarttime # reformat 'starttime' for use with mythtranscode/ffmpeg/mythcommflag starttime = str(utcstarttime.utcisoformat().replace(':', '').replace(' ', '').replace('T', '').replace('-', '')) if debug: print('mythtv format starttime "%s"' % starttime) input_filesize = rec.filesize if rec.commflagged: if debug: print('Recording has been scanned to detect commerical breaks.') waititer=1 keepWaiting = True while keepWaiting == True: keepWaiting=False; for index,jobitem in reversed(list(enumerate(db.searchJobs(chanid=chanid, starttime=starttime_datetime)))): if jobitem.type == jobitem.COMMFLAG: # Commercial flagging job if debug: print('Commercial flagging job detected with status %s' % jobitem.status) if jobitem.status == jobitem.RUNNING: # status = RUNNING? job.update({'status':job.PAUSED, 'comment':'Waited %d secs for the commercial flagging job' % (waititer*POLL_INTERVAL) \ + ' currently running on this recording to complete.'}) if debug: print('Waited %d secs for the commercial flagging job' % (waititer*POLL_INTERVAL) \ + ' currently running on this recording to complete.') time.sleep(POLL_INTERVAL); keepWaiting=True waititer = waititer + 1 break else: if debug: print('Recording has not been scanned to detect/remove commercial breaks.') if require_commflagged: if jobid: job.update({'status':job.RUNNING, 'comment':'Required commercial flagging for this file is not found.' + 'Flagging commercials and cancelling any queued commercial flagging.'}) # cancel any queued job to flag commercials for this recording and run commercial flagging in this script for index,jobitem in reversed(list(enumerate(db.searchJobs(chanid=chanid,starttime=starttime_datetime)))): if debug: if index==0: print(list(jobitem.keys())) print(index,jobitem.id,jobitem.chanid) if jobitem.type == jobitem.COMMFLAG: # Commercial flagging job if jobitem.status == jobitem.RUNNING: # status = RUNNING? jobitem.cmds = jobitem.STOP # stop command from the frontend to stop the commercial flagging job #jobitem.setStatus(jobitem.CANCELLED) #jobitem.setComment('Cancelled: Transcode command ran commercial flagging for this recording.') jobitem.update({'status':jobitem.CANCELLED, 'comment':'A user transcode job ran commercial flagging for' + ' this recording and cancelled this job.'}) if debug: print('Flagging Commercials...') # Call "mythcommflag --chanid $CHANID --starttime $STARTTIME" task = System(path='mythcommflag', db=db) try: output = task('--chanid "%s"' % chanid, '--starttime "%s"' % starttime, '2> /dev/null') except MythError as e: # it seems mythcommflag always exits with an decoding error "eno: Unknown error 541478725 (541478725)" pass #print 'Command failed with output:\n%s' % e.stderr #if jobid: # job.update({'status':304, 'comment':'Flagging commercials failed'}) #sys.exit(e.retcode) sg = findfile(rec.basename, rec.storagegroup, db=db) if sg is None: print('Local access to recording not found.') sys.exit(1) infile = os.path.join(sg.dirname, rec.basename) tmpfile = '%s.tmp' % infile.rsplit('.',1)[0] # tmpfile = infile outfile = '%s.mp4' % infile.rsplit('.',1)[0] if debug: print('tmpfile "%s"' % tmpfile) clipped_bytes=0; # If selected, create a cutlist to remove commercials via mythtranscode by running: # mythutil --gencutlist --chanid $CHANID --starttime $STARTTIME if generate_commcutlist: if jobid: job.update({'status':job.RUNNING, 'comment':'Generating Cutlist for commercial removal'}) task = System(path='mythutil', db=db) try: output = task('--gencutlist', '--chanid "%s"' % chanid, '--starttime "%s"' % starttime) # '--loglevel debug', # '2> /dev/null') except MythError as e: print('Command "mythutil --gencutlist" failed with output:\n%s' % e.stderr) if jobid: job.update({'status':job.ERRORED, 'comment':'Generation of commercial Cutlist failed'}) sys.exit(e.retcode) # Lossless transcode to strip cutlist if generate_commcutlist or rec.cutlist==1: if jobid: job.update({'status':job.RUNNING, 'comment':'Removing Cutlist'}) task = System(path='mythtranscode', db=db) try: output = task('--chanid "%s"' % chanid, '--starttime "%s"' % starttime, '--mpeg2', '--honorcutlist', '-o "%s"' % tmpfile, '1>&2') # '2> /dev/null') clipped_filesize = os.path.getsize(tmpfile) clipped_bytes = input_filesize - clipped_filesize clipped_compress_pct = float(clipped_bytes)/input_filesize rec.commflagged = 0 except MythError as e: print('Command "mythtranscode --honorcutlist" failed with output:\n%s' % e.stderr) if jobid: job.update({'status':job.ERRORED, 'comment':'Removing Cutlist failed. Copying file instead.'}) # sys.exit(e.retcode) copyfile('%s' % infile, '%s' % tmpfile) clipped_filesize = input_filesize clipped_bytes = 0 clipped_compress_pct = 0 pass else: if jobid: job.update({'status':job.RUNNING, 'comment':'Creating temporary file for transcoding.'}) copyfile('%s' % infile, '%s' % tmpfile) clipped_filesize = input_filesize clipped_bytes = 0 clipped_compress_pct = 0 duration_secs = 0 # Estimate bitrate, and detect duration and number of frames if estimateBitrate: if jobid: job.update({'status':job.RUNNING, 'comment':'Estimating bitrate; detecting frames per second, and resolution.'}) duration_secs, e = get_duration(db, rec, transcoder, tmpfile); if duration_secs>0: bitrate = int(clipped_filesize*8/(1024*duration_secs)) else: print('Estimate bitrate failed falling back to constant rate factor encoding.\n') estimateBitrate = False duration_secs = 0 print(e.stderr.decode('utf-8')) # get framerate of mpeg2 video stream and detect if stream is HD r = re.compile('mpeg2video (.*?) fps,') m = r.search(e.stderr.decode('utf-8')) strval = m.group(1) if debug: print(strval) isHD = False if "1920x1080" in strval or "1280x720" in strval or "2560x1440" in strval: if debug: print('Stream is HD') isHD = True else: if debug: print('Stream is not HD') framerate = float(m.group(1).split(' ')[-1]) if debug: print('Framerate %s' % framerate) # Setup transcode video bitrate and quality parameters # if estimateBitrate is true and the input content is HD: # encode 'medium' preset and vbitrate = inputfile_bitrate*compressionRatio # else: # encode at user default preset and constant rate factor ('slow' and 20) preset = preset_nonHD if estimateBitrate: if isHD: h264_bitrate = int(bitrate*compressionRatio) # HD coding with specified target bitrate (CRB encoding) if hdvideo_tgt_bitrate > 0 and h264_bitrate > hdvideo_tgt_bitrate: h264_bitrate = hdvideo_tgt_bitrate; vbitrate_param = '-b:v %dk' % h264_bitrate else: # HD coding with disabled or acceptable target bitrate (CRF encoding) vbitrate_param = '-crf:v %s' % crf preset = preset_HD else: # non-HD encoding (CRF encoding) vbitrate_param = '-crf:v %s' % crf else: vbitrate_param = '-crf:v %s' % crf if hdvideo_min_bitrate > 0: vbitrate_param = vbitrate_param + ' -minrate %sk' % hdvideo_min_bitrate if hdvideo_max_bitrate > 0: vbitrate_param = vbitrate_param + ' -maxrate %sk' % hdvideo_max_bitrate if hdvideo_max_bitrate > 0 or hdvideo_min_bitrate > 0: vbitrate_param = vbitrate_param + ' -bufsize %sk' % device_bufsize if debug: print('Video bitrate parameter "%s"' % vbitrate_param) print('Video h264 preset parameter "%s"' % preset) # Setup transcode audio bitrate and quality parameters # Right now, the setup is as follows: # if input is HD: # copy audio streams to output, i.e., input=output audio # else: # output is libfdk_aac encoded at 128kbps if isHD: abitrate_param = abitrate_param_HD # preserve 5.1 audio else: abitrate_param = abitrate_param_nonHD if debug: print('Audio bitrate parameter "%s"' % abitrate_param) # Transcode to mp4 # if jobid: # job.update({'status':4, 'comment':'Transcoding to mp4'}) # ffmpeg output is redirected to the temporary file tmpstatusfile and # a second thread continuously reads this file while # the transcode is in-process. see while loop below for the monitoring thread tf = tempfile.NamedTemporaryFile() tmpstatusfile = tf.name # tmpstatusfile = '/tmp/ffmpeg-transcode.txt' if debug: print('Using temporary file "%s" for ffmpeg status updates.' % tmpstatusfile) res = [] # create a thread to perform the encode ipq = queue.Queue() t = threading.Thread(target=wrapper, args=(encode, (jobid, db, job, ipq, preset, vbitrate_param, abitrate_param, tmpfile, outfile, tmpstatusfile,), res)) t.start() # wait for ffmpeg to open the file and emit its initialization information # before we start the monitoring process time.sleep(1) # open the temporary file having the ffmeg output text and process it to generate status updates hangiter=0; with open(tmpstatusfile) as f: # read all the opening ffmpeg status/analysis lines lines = f.readlines() # set initial progress to -1 prev_progress=-1 framenum=0 fps=1.0 while t.is_alive(): # read all output since last readline() call lines = f.readlines() if len(lines) > 0: # every ffmpeg output status line ends with a carriage return '\r' # split the last read line at these locations lines=lines[-1].split('\r') # if debug: # print lines; hangiter=0 if len(lines) > 1 and lines[-2].startswith('frame'): # since typical reads will have the last line ending with \r the last status # message is at index=[-2] start processing this line # replace multiple spaces with one space lines[-2] = re.sub(' +',' ',lines[-2]) # remove any spaces after equals signs lines[-2] = re.sub('= +','=',lines[-2]) # split the fields at the spaces the first two fields for typical # status lines will be framenum=XXXX and fps=YYYY parse the values values = lines[-2].split(' ') if len(values) > 1: if debug: print('values %s' % values) prev_framenum = framenum prev_fps = fps try: # framenum = current frame number being encoded framenum = int(values[0].split('=')[1]) # fps = frames per second for the encoder fps = float(values[1].split('=')[1]) except ValueError as e: print('ffmpeg status parse exception: "%s"' % e) framenum = prev_framenum fps = prev_fps pass # progress = 0-100 represent percent complete for the transcode progress = int((100*framenum)/(duration_secs*framerate)) # eta_secs = estimated number of seconds until transcoding is complete eta_secs = int((float(duration_secs*framerate)-framenum)/fps) # pct_realtime = how many real seconds it takes to encode 1 second of video pct_realtime = float(fps/framerate) if debug: print('framenum = %d fps = %.2f' % (framenum, fps)) if progress != prev_progress: if debug: print('Progress %d%% encoding %.1f frames per second ETA %d mins' \ % ( progress, fps, float(eta_secs)/60)) if jobid: progress_str = 'Transcoding to mp4 %d%% complete ETA %d mins fps=%.1f.' \ % ( progress, float(eta_secs)/60, fps) job.update({'status':job.RUNNING, 'comment': progress_str}) prev_progress = progress elif len(lines) > 1: if debug: print('Read pathological output %s' % lines[-2]) else: if debug: print('Read no lines of ffmpeg output for %s secs. Possible hang?' % (POLL_INTERVAL*hangiter)) hangiter = hangiter + 1 if jobid: progress_str = 'Read no lines of ffmpeg output for %s secs. Possible hang?' % (POLL_INTERVAL*hangiter) job.update({'status':job.RUNNING, 'comment': progress_str}) time.sleep(POLL_INTERVAL) if debug: print('res = "%s"' % res) t.join(1) try: if ipq.get_nowait() == CleanExit: sys.exit() except queue.Empty: pass if flush_commskip: task = System(path='mythutil') task.command('--chanid %s' % chanid, '--starttime %s' % starttime, '--clearcutlist', '2> /dev/null') task = System(path='mythutil') task.command('--chanid %s' % chanid, '--starttime %s' % starttime, '--clearskiplist', '2> /dev/null') if flush_commskip: for index,mark in reversed(list(enumerate(rec.markup))): if mark.type in (rec.markup.MARK_COMM_START, rec.markup.MARK_COMM_END): del rec.markup[index] rec.bookmark = 0 rec.cutlist = 0 rec.markup.commit() # tf.close(); # os.remove(tmpstatusfile); rec.basename = os.path.basename(outfile) rec.filesize = os.path.getsize(outfile) # rec.commflagged = 0 rec.transcoded = 1 rec.seek.clean() rec.update() os.remove(infile) # Cleanup the old *.png files for filename in glob('%s*.png' % infile): os.remove(filename) os.remove(tmpfile) try: os.remove('%s.map' % tmpfile) except OSError: pass output_filesize = rec.filesize if duration_secs > 0: output_bitrate = int(output_filesize*8/(1024*duration_secs)) # kbps actual_compression_ratio = 1 - float(output_filesize)/clipped_filesize compressed_pct = 1 - float(output_filesize)/input_filesize if build_seektable: if jobid: job.update({'status':job.RUNNING, 'comment':'Rebuilding seektable'}) task = System(path='mythcommflag') task.command('--chanid %s' % chanid, '--starttime %s' % starttime, '--rebuild', '2> /dev/null') # fix during in the recorded markup table this will be off if commercials are removed duration_msecs, e = get_duration(db, rec, transcoder, outfile) duration_msecs = 1000*duration_msecs for index,mark in reversed(list(enumerate(rec.markup))): # find the duration markup entry and correct any error in the video duration that might be there if mark.type == 33: if debug: print('Markup Duration in milliseconds "%s"' % mark.data) error = mark.data - duration_msecs if error != 0: if debug: print('Markup Duration error is "%s"msecs' % error) mark.data = duration_msecs #rec.bookmark = 0 #rec.cutlist = 0 rec.markup.commit() if jobid: if output_bitrate: job.update({'status':job.FINISHED, 'comment':'Transcode Completed @ %dkbps, compressed file by %d%% (clipped %d%%, transcoder compressed %d%%)' % (output_bitrate,int(compressed_pct*100),int(clipped_compress_pct*100),int(actual_compression_ratio*100))}) else: job.update({'status':job.FINISHED, 'comment':'Transcode Completed'})
framerate = float(fps_pattern.search(str(avconv_fps)).groups()[0]) else: print "Cannot determine framerate... abort" sys.exit(1) print "Video frame rate: %s" % str(framerate) # reformat 'starttime' for use with mythtranscode/ffmpeg/mythcommflag starttime = str(rec.starttime.utcisoformat().replace(u':', '').replace( u' ', '').replace(u'T', '').replace('-', '')) # Get skip list if jobid: job.update({'status': 4, 'comment': 'Getting skip list'}) task = System(path='mythutil', db=db) try: output = task('-q', '--getskiplist', '--chanid %d' % chanid, '--starttime %s' % starttime) except MythError, e: print 'Command failed with output:\n%s' % e.stderr if jobid: job.update({'status': 304, 'comment': 'Getting skip list failed'}) sys.exit(e.retcode) print 'Skip list:' print output output = output.split(':')[1].strip() cuts = [tuple([int(z) for z in y.split('-')]) for y in output.split(',')] print cuts starts_with_commercial = False
infile = os.path.join(sg.dirname, rec.basename) tmpfile = '%s.tmp' % infile.rsplit('.',1)[0] outfile = '%s.mp4' % infile.rsplit('.',1)[0] print "Infile: %s" % infile print "Outfile: %s" % outfile # reformat 'starttime' for use with mythtranscode/ffmpeg/mythcommflag starttime = str(rec.starttime.utcisoformat().replace(u':', '').replace(u' ', '').replace(u'T', '').replace('-', '')) # Lossless transcode to strip cutlist if rec.cutlist == 1: if jobid: job.update({'status':4, 'comment':'Removing Cutlist'}) task = System(path='mythtranscode', db=db) try: output = task('--chanid "%s"' % chanid, '--starttime "%s"' % starttime, '--mpeg2', '--honorcutlist', '-o "%s"' % tmpfile, '2> /dev/null') except MythError, e: print 'Command failed with output:\n%s' % e.stderr if jobid: job.update({'status':304, 'comment':'Removing Cutlist failed'}) sys.exit(e.retcode) else: tmpfile = infile # copyfile('%s' % infile, '%s' % tmpfile)
if os.path.splitext(infile)[1] == '.mp4': print 'Infile is already mp4! Dont do anything!' sys.exit(0) # reformat 'starttime' for use with mythtranscode/ffmpeg/mythcommflag starttime = str(rec.starttime.utcisoformat().replace(u':', '').replace( u' ', '').replace(u'T', '').replace('-', '')) # Lossless transcode to strip cutlist if rec.cutlist == 1: print 'Removing cutlist...' if jobid: job.update({'status': 4, 'comment': 'Removing Cutlist'}) task = System(path='mythtranscode', db=db) try: output = task('--chanid "%s"' % chanid, '--starttime "%s"' % starttime, '--mpeg2', '--honorcutlist', '-o "%s"' % tmpfile, '2> /dev/null') except MythError, e: print 'Removing cutlist failure: Command failed with output:\n%s' % e.stderr if jobid: job.update({ 'status': 304, 'comment': 'Removing Cutlist failed' }) sys.exit(e.retcode) print 'Removing cutlist...done' else: