def get_link_details(rec, channels):
sg = findfile(rec.basename, rec.storagegroup, rec._db)
if sg is None:
return
source = os.path.join(sg.dirname, rec.basename)
target = rec.formatPath(format)
target = target.replace(u"\xa3", "")
for cid in channels:
target = target.replace("(%i)" % (cid, ), "(%s)" % (channels[cid], ))
return source, target.split("/")[0], target.split("/")[1]
def get_link_details(rec, channels):
sg = findfile(rec.basename, rec.storagegroup, rec._db)
if sg is None:
return
source = os.path.join(sg.dirname, rec.basename)
target = rec.formatPath(format)
target = target.replace(u"\xa3", "")
for cid in channels:
target = target.replace("(%i)" % (cid, ), "(%s)" % (channels[cid], ))
return source, target.split("/")[0], target.split("/")[1]
def gen_link(rec):
sg = findfile(rec.basename, rec.storagegroup, rec._db)
source = os.path.join(sg.dirname, rec.basename)
dest = os.path.join(opts.dest, rec.formatPath(format))
if opts.underscores:
dest = dest.replace(' ','_')
sdest = dest.split('/')
for i in range(2,len(sdest)):
tmppath = os.path.join(*sdest[:i])
if not os.access(tmppath, os.F_OK):
os.mkdir(tmppath)
os.symlink(source, dest)
def gen_link(rec):
sg = findfile(rec.basename, rec.storagegroup, rec._db)
source = os.path.join(sg.dirname, rec.basename)
dest = os.path.join(opts.dest, rec.formatPath(format))
if opts.underscores:
dest = dest.replace(' ','_')
sdest = dest.split('/')
for i in range(2,len(sdest)):
tmppath = os.path.join(*sdest[:i])
if not os.access(tmppath, os.F_OK):
os.mkdir(tmppath)
os.symlink(source, dest)
def runjob(rec):
db = MythDB()
timestr = time.strftime("%m-%d-%y %H:%M:%S")
title_san = re.sub("\s", ".", rec.title)
trans_log_file = os.path.join(log_dir, "%s_transcode_log_%s.hb.txt" % (title_san, timestr))
commflag_log_file = os.path.join(log_dir, "%s_transcode_log_%s.cf.txt" % (title_san, timestr))
mythical_log_file = os.path.join(log_dir, "%s_transcode_log_%s.ml.txt" % (title_san, timestr))
sg = findfile('/'+rec.basename, rec.storagegroup, db=db)
if sg is None:
print 'Local access to recording not found.'
sys.exit(1)
infile = os.path.join(sg.dirname, rec.basename)
tmpfile = '%s.tmp' % infile.rsplit('.',1)[0]
outfile = '%s.mp4' % infile.rsplit('.',1)[0]
print "Infile: %s" % infile
print "Outfile: %s" % outfile
# reformat 'starttime' for use with mythtranscode/ffmpeg/mythcommflag
starttime = str(rec.starttime.utcisoformat().replace(u':', '').replace(u' ', '').replace(u'T', '').replace('-', ''))
chanid = rec.chanid
# Lossless transcode to strip cutlist
if rec.cutlist == 1 and false:
if job:
job.update({'status':4, 'comment':'Removing Cutlist'})
task = System(path='mythtranscode', db=db)
try:
output = task('--chanid "%s"' % chanid,
'--starttime "%s"' % starttime,
'--mpeg2',
'--honorcutlist',
'-o "%s"' % tmpfile,
'2> /dev/null')
except MythError, e:
print 'Command failed with output:\n%s' % e.stderr
if job:
job.update({'status':304, 'comment':'Removing Cutlist failed'})
sys.exit(e.retcode)
def runjob(jobid=None, chanid=None, starttime=None):
db = MythDB()
if jobid:
job = Job(jobid, db=db)
chanid = job.chanid
starttime = job.starttime
rec = Recorded((chanid, starttime), db=db)
sg = findfile('/' + rec.basename, rec.storagegroup, db=db)
if sg is None:
print 'Local access to recording not found.'
sys.exit(1)
infile = os.path.join(sg.dirname, rec.basename)
tmpfile = '%s.tmp' % infile.rsplit('.', 1)[0]
outfile = '%s.mp4' % infile.rsplit('.', 1)[0]
# reformat 'starttime' for use with mythtranscode/ffmpeg/mythcommflag
starttime = str(starttime.utcisoformat().replace(u':', '').replace(
u' ', '').replace(u'T', '').replace('-', ''))
# Lossless transcode to strip cutlist
if rec.cutlist == 1:
if jobid:
job.update({'status': 4, 'comment': 'Removing Cutlist'})
task = System(path='mythtranscode', db=db)
try:
output = task('--chanid "%s"' % chanid,
'--starttime "%s"' % starttime, '--mpeg2',
'--honorcutlist', '-o "%s"' % tmpfile,
'2> /dev/null')
except MythError, e:
print 'Command failed with output:\n%s' % e.stderr
if jobid:
job.update({
'status': 304,
'comment': 'Removing Cutlist failed'
})
sys.exit(e.retcode)
print title_san
try:
os.mkdir(log_dir)
except OSError, e:
pass
log_file = os.path.join(log_dir, "%s_transcode_log_%s.txt" % (title_san, timestr))
trans_log_file = os.path.join(log_dir, "%s_transcode_log_%s.hb.txt" % (title_san, timestr))
commflag_log_file = os.path.join(log_dir, "%s_transcode_log_%s.cf.txt" % (title_san, timestr))
print 'Capturing log in %s...' % log_file
lfp = open(log_file, 'w')
sys.stdout = tee(sys.stdout, lfp)
sys.stderr = tee(sys.stderr, lfp)
print 'Logging into %s' % log_file
sg = findfile('/'+rec.basename, rec.storagegroup, db=db)
if sg is None:
print 'Local access to recording not found.'
sys.exit(1)
infile = os.path.join(sg.dirname, rec.basename)
tmpfile = '%s.tmp' % infile.rsplit('.',1)[0]
outfile = '%s.mp4' % infile.rsplit('.',1)[0]
print "Infile: %s" % infile
print "Outfile: %s" % outfile
# reformat 'starttime' for use with mythtranscode/ffmpeg/mythcommflag
starttime = str(rec.starttime.utcisoformat().replace(u':', '').replace(u' ', '').replace(u'T', '').replace('-', ''))
# Lossless transcode to strip cutlist
# Call "mythcommflag --chanid $CHANID --starttime $STARTTIME"
task = System(path='mythcommflag', db=db)
try:
output = task('--chanid "%s"' % chanid,
'--starttime "%s"' % starttime,
'2> /dev/null')
except MythError, e:
# it seems mythcommflag always exits with an decoding error "eno: Unknown error 541478725 (541478725)"
pass
#print 'Command failed with output:\n%s' % e.stderr
#if jobid:
# job.update({'status':304, 'comment':'Flagging commercials failed'})
#sys.exit(e.retcode)
sg = findfile('/'+rec.basename, rec.storagegroup, db=db)
if sg is None:
print 'Local access to recording not found.'
sys.exit(1)
infile = os.path.join(sg.dirname, rec.basename)
tmpfile = '%s.tmp' % infile.rsplit('.',1)[0]
# tmpfile = infile
outfile = '%s.mp4' % infile.rsplit('.',1)[0]
outfile = '%s.mp4' % os.path.join(sg.dirname, "proxy", rec.basename).rsplit('.',1)[0]
if debug:
print 'tmpfile "%s"' % tmpfile
clipped_bytes=0;
# If selected, create a cutlist to remove commercials via mythtranscode by running:
def runjob(jobid=None, chanid=None, starttime=None, tzoffset=None):
global estimateBitrate
db = MythDB()
if jobid:
job = Job(jobid, db=db)
chanid = job.chanid
utcstarttime = job.starttime
else:
job=None;
#utcstarttime = datetime.strptime(starttime, "%Y%m%d%H%M%S%z")
utcstarttime = parse(starttime)
utcstarttime = utcstarttime + timedelta(hours=tzoffset)
if debug:
print('chanid "%s"' % chanid)
print('utcstarttime "%s"' % utcstarttime)
rec = Recorded((chanid, utcstarttime), db=db);
utcstarttime = rec.starttime;
starttime_datetime = utcstarttime
# reformat 'starttime' for use with mythtranscode/ffmpeg/mythcommflag
starttime = str(utcstarttime.utcisoformat().replace(':', '').replace(' ', '').replace('T', '').replace('-', ''))
if debug:
print('mythtv format starttime "%s"' % starttime)
input_filesize = rec.filesize
if rec.commflagged:
if debug:
print('Recording has been scanned to detect commerical breaks.')
waititer=1
keepWaiting = True
while keepWaiting == True:
keepWaiting=False;
for index,jobitem in reversed(list(enumerate(db.searchJobs(chanid=chanid, starttime=starttime_datetime)))):
if jobitem.type == jobitem.COMMFLAG: # Commercial flagging job
if debug:
print('Commercial flagging job detected with status %s' % jobitem.status)
if jobitem.status == jobitem.RUNNING: # status = RUNNING?
job.update({'status':job.PAUSED,
'comment':'Waited %d secs for the commercial flagging job' % (waititer*POLL_INTERVAL) \
+ ' currently running on this recording to complete.'})
if debug:
print('Waited %d secs for the commercial flagging job' % (waititer*POLL_INTERVAL) \
+ ' currently running on this recording to complete.')
time.sleep(POLL_INTERVAL);
keepWaiting=True
waititer = waititer + 1
break
else:
if debug:
print('Recording has not been scanned to detect/remove commercial breaks.')
if require_commflagged:
if jobid:
job.update({'status':job.RUNNING, 'comment':'Required commercial flagging for this file is not found.'
+ 'Flagging commercials and cancelling any queued commercial flagging.'})
# cancel any queued job to flag commercials for this recording and run commercial flagging in this script
for index,jobitem in reversed(list(enumerate(db.searchJobs(chanid=chanid,starttime=starttime_datetime)))):
if debug:
if index==0:
print(list(jobitem.keys()))
print(index,jobitem.id,jobitem.chanid)
if jobitem.type == jobitem.COMMFLAG: # Commercial flagging job
if jobitem.status == jobitem.RUNNING: # status = RUNNING?
jobitem.cmds = jobitem.STOP # stop command from the frontend to stop the commercial flagging job
#jobitem.setStatus(jobitem.CANCELLED)
#jobitem.setComment('Cancelled: Transcode command ran commercial flagging for this recording.')
jobitem.update({'status':jobitem.CANCELLED,
'comment':'A user transcode job ran commercial flagging for'
+ ' this recording and cancelled this job.'})
if debug:
print('Flagging Commercials...')
# Call "mythcommflag --chanid $CHANID --starttime $STARTTIME"
task = System(path='mythcommflag', db=db)
try:
output = task('--chanid "%s"' % chanid,
'--starttime "%s"' % starttime,
'2> /dev/null')
except MythError as e:
# it seems mythcommflag always exits with an decoding error "eno: Unknown error 541478725 (541478725)"
pass
#print 'Command failed with output:\n%s' % e.stderr
#if jobid:
# job.update({'status':304, 'comment':'Flagging commercials failed'})
#sys.exit(e.retcode)
sg = findfile(rec.basename, rec.storagegroup, db=db)
if sg is None:
print('Local access to recording not found.')
sys.exit(1)
infile = os.path.join(sg.dirname, rec.basename)
tmpfile = '%s.tmp' % infile.rsplit('.',1)[0]
# tmpfile = infile
outfile = '%s.mp4' % infile.rsplit('.',1)[0]
if debug:
print('tmpfile "%s"' % tmpfile)
clipped_bytes=0;
# If selected, create a cutlist to remove commercials via mythtranscode by running:
# mythutil --gencutlist --chanid $CHANID --starttime $STARTTIME
if generate_commcutlist:
if jobid:
job.update({'status':job.RUNNING, 'comment':'Generating Cutlist for commercial removal'})
task = System(path='mythutil', db=db)
try:
output = task('--gencutlist',
'--chanid "%s"' % chanid,
'--starttime "%s"' % starttime)
# '--loglevel debug',
# '2> /dev/null')
except MythError as e:
print('Command "mythutil --gencutlist" failed with output:\n%s' % e.stderr)
if jobid:
job.update({'status':job.ERRORED, 'comment':'Generation of commercial Cutlist failed'})
sys.exit(e.retcode)
# Lossless transcode to strip cutlist
if generate_commcutlist or rec.cutlist==1:
if jobid:
job.update({'status':job.RUNNING, 'comment':'Removing Cutlist'})
task = System(path='mythtranscode', db=db)
try:
output = task('--chanid "%s"' % chanid,
'--starttime "%s"' % starttime,
'--mpeg2',
'--honorcutlist',
'-o "%s"' % tmpfile,
'1>&2')
# '2> /dev/null')
clipped_filesize = os.path.getsize(tmpfile)
clipped_bytes = input_filesize - clipped_filesize
clipped_compress_pct = float(clipped_bytes)/input_filesize
rec.commflagged = 0
except MythError as e:
print('Command "mythtranscode --honorcutlist" failed with output:\n%s' % e.stderr)
if jobid:
job.update({'status':job.ERRORED, 'comment':'Removing Cutlist failed. Copying file instead.'})
# sys.exit(e.retcode)
copyfile('%s' % infile, '%s' % tmpfile)
clipped_filesize = input_filesize
clipped_bytes = 0
clipped_compress_pct = 0
pass
else:
if jobid:
job.update({'status':job.RUNNING, 'comment':'Creating temporary file for transcoding.'})
copyfile('%s' % infile, '%s' % tmpfile)
clipped_filesize = input_filesize
clipped_bytes = 0
clipped_compress_pct = 0
duration_secs = 0
# Estimate bitrate, and detect duration and number of frames
if estimateBitrate:
if jobid:
job.update({'status':job.RUNNING, 'comment':'Estimating bitrate; detecting frames per second, and resolution.'})
duration_secs, e = get_duration(db, rec, transcoder, tmpfile);
if duration_secs>0:
bitrate = int(clipped_filesize*8/(1024*duration_secs))
else:
print('Estimate bitrate failed falling back to constant rate factor encoding.\n')
estimateBitrate = False
duration_secs = 0
print(e.stderr.decode('utf-8'))
# get framerate of mpeg2 video stream and detect if stream is HD
r = re.compile('mpeg2video (.*?) fps,')
m = r.search(e.stderr.decode('utf-8'))
strval = m.group(1)
if debug:
print(strval)
isHD = False
if "1920x1080" in strval or "1280x720" in strval or "2560x1440" in strval:
if debug:
print('Stream is HD')
isHD = True
else:
if debug:
print('Stream is not HD')
framerate = float(m.group(1).split(' ')[-1])
if debug:
print('Framerate %s' % framerate)
# Setup transcode video bitrate and quality parameters
# if estimateBitrate is true and the input content is HD:
# encode 'medium' preset and vbitrate = inputfile_bitrate*compressionRatio
# else:
# encode at user default preset and constant rate factor ('slow' and 20)
preset = preset_nonHD
if estimateBitrate:
if isHD:
h264_bitrate = int(bitrate*compressionRatio)
# HD coding with specified target bitrate (CRB encoding)
if hdvideo_tgt_bitrate > 0 and h264_bitrate > hdvideo_tgt_bitrate:
h264_bitrate = hdvideo_tgt_bitrate;
vbitrate_param = '-b:v %dk' % h264_bitrate
else: # HD coding with disabled or acceptable target bitrate (CRF encoding)
vbitrate_param = '-crf:v %s' % crf
preset = preset_HD
else: # non-HD encoding (CRF encoding)
vbitrate_param = '-crf:v %s' % crf
else:
vbitrate_param = '-crf:v %s' % crf
if hdvideo_min_bitrate > 0:
vbitrate_param = vbitrate_param + ' -minrate %sk' % hdvideo_min_bitrate
if hdvideo_max_bitrate > 0:
vbitrate_param = vbitrate_param + ' -maxrate %sk' % hdvideo_max_bitrate
if hdvideo_max_bitrate > 0 or hdvideo_min_bitrate > 0:
vbitrate_param = vbitrate_param + ' -bufsize %sk' % device_bufsize
if debug:
print('Video bitrate parameter "%s"' % vbitrate_param)
print('Video h264 preset parameter "%s"' % preset)
# Setup transcode audio bitrate and quality parameters
# Right now, the setup is as follows:
# if input is HD:
# copy audio streams to output, i.e., input=output audio
# else:
# output is libfdk_aac encoded at 128kbps
if isHD:
abitrate_param = abitrate_param_HD # preserve 5.1 audio
else:
abitrate_param = abitrate_param_nonHD
if debug:
print('Audio bitrate parameter "%s"' % abitrate_param)
# Transcode to mp4
# if jobid:
# job.update({'status':4, 'comment':'Transcoding to mp4'})
# ffmpeg output is redirected to the temporary file tmpstatusfile and
# a second thread continuously reads this file while
# the transcode is in-process. see while loop below for the monitoring thread
tf = tempfile.NamedTemporaryFile()
tmpstatusfile = tf.name
# tmpstatusfile = '/tmp/ffmpeg-transcode.txt'
if debug:
print('Using temporary file "%s" for ffmpeg status updates.' % tmpstatusfile)
res = []
# create a thread to perform the encode
ipq = queue.Queue()
t = threading.Thread(target=wrapper, args=(encode,
(jobid, db, job, ipq, preset, vbitrate_param, abitrate_param,
tmpfile, outfile, tmpstatusfile,), res))
t.start()
# wait for ffmpeg to open the file and emit its initialization information
# before we start the monitoring process
time.sleep(1)
# open the temporary file having the ffmeg output text and process it to generate status updates
hangiter=0;
with open(tmpstatusfile) as f:
# read all the opening ffmpeg status/analysis lines
lines = f.readlines()
# set initial progress to -1
prev_progress=-1
framenum=0
fps=1.0
while t.is_alive():
# read all output since last readline() call
lines = f.readlines()
if len(lines) > 0:
# every ffmpeg output status line ends with a carriage return '\r'
# split the last read line at these locations
lines=lines[-1].split('\r')
# if debug:
# print lines;
hangiter=0
if len(lines) > 1 and lines[-2].startswith('frame'):
# since typical reads will have the last line ending with \r the last status
# message is at index=[-2] start processing this line
# replace multiple spaces with one space
lines[-2] = re.sub(' +',' ',lines[-2])
# remove any spaces after equals signs
lines[-2] = re.sub('= +','=',lines[-2])
# split the fields at the spaces the first two fields for typical
# status lines will be framenum=XXXX and fps=YYYY parse the values
values = lines[-2].split(' ')
if len(values) > 1:
if debug:
print('values %s' % values)
prev_framenum = framenum
prev_fps = fps
try:
# framenum = current frame number being encoded
framenum = int(values[0].split('=')[1])
# fps = frames per second for the encoder
fps = float(values[1].split('=')[1])
except ValueError as e:
print('ffmpeg status parse exception: "%s"' % e)
framenum = prev_framenum
fps = prev_fps
pass
# progress = 0-100 represent percent complete for the transcode
progress = int((100*framenum)/(duration_secs*framerate))
# eta_secs = estimated number of seconds until transcoding is complete
eta_secs = int((float(duration_secs*framerate)-framenum)/fps)
# pct_realtime = how many real seconds it takes to encode 1 second of video
pct_realtime = float(fps/framerate)
if debug:
print('framenum = %d fps = %.2f' % (framenum, fps))
if progress != prev_progress:
if debug:
print('Progress %d%% encoding %.1f frames per second ETA %d mins' \
% ( progress, fps, float(eta_secs)/60))
if jobid:
progress_str = 'Transcoding to mp4 %d%% complete ETA %d mins fps=%.1f.' \
% ( progress, float(eta_secs)/60, fps)
job.update({'status':job.RUNNING, 'comment': progress_str})
prev_progress = progress
elif len(lines) > 1:
if debug:
print('Read pathological output %s' % lines[-2])
else:
if debug:
print('Read no lines of ffmpeg output for %s secs. Possible hang?' % (POLL_INTERVAL*hangiter))
hangiter = hangiter + 1
if jobid:
progress_str = 'Read no lines of ffmpeg output for %s secs. Possible hang?' % (POLL_INTERVAL*hangiter)
job.update({'status':job.RUNNING, 'comment': progress_str})
time.sleep(POLL_INTERVAL)
if debug:
print('res = "%s"' % res)
t.join(1)
try:
if ipq.get_nowait() == CleanExit:
sys.exit()
except queue.Empty:
pass
if flush_commskip:
task = System(path='mythutil')
task.command('--chanid %s' % chanid,
'--starttime %s' % starttime,
'--clearcutlist',
'2> /dev/null')
task = System(path='mythutil')
task.command('--chanid %s' % chanid,
'--starttime %s' % starttime,
'--clearskiplist',
'2> /dev/null')
if flush_commskip:
for index,mark in reversed(list(enumerate(rec.markup))):
if mark.type in (rec.markup.MARK_COMM_START, rec.markup.MARK_COMM_END):
del rec.markup[index]
rec.bookmark = 0
rec.cutlist = 0
rec.markup.commit()
# tf.close();
# os.remove(tmpstatusfile);
rec.basename = os.path.basename(outfile)
rec.filesize = os.path.getsize(outfile)
# rec.commflagged = 0
rec.transcoded = 1
rec.seek.clean()
rec.update()
os.remove(infile)
# Cleanup the old *.png files
for filename in glob('%s*.png' % infile):
os.remove(filename)
os.remove(tmpfile)
try:
os.remove('%s.map' % tmpfile)
except OSError:
pass
output_filesize = rec.filesize
if duration_secs > 0:
output_bitrate = int(output_filesize*8/(1024*duration_secs)) # kbps
actual_compression_ratio = 1 - float(output_filesize)/clipped_filesize
compressed_pct = 1 - float(output_filesize)/input_filesize
if build_seektable:
if jobid:
job.update({'status':job.RUNNING, 'comment':'Rebuilding seektable'})
task = System(path='mythcommflag')
task.command('--chanid %s' % chanid,
'--starttime %s' % starttime,
'--rebuild',
'2> /dev/null')
# fix during in the recorded markup table this will be off if commercials are removed
duration_msecs, e = get_duration(db, rec, transcoder, outfile)
duration_msecs = 1000*duration_msecs
for index,mark in reversed(list(enumerate(rec.markup))):
# find the duration markup entry and correct any error in the video duration that might be there
if mark.type == 33:
if debug:
print('Markup Duration in milliseconds "%s"' % mark.data)
error = mark.data - duration_msecs
if error != 0:
if debug:
print('Markup Duration error is "%s"msecs' % error)
mark.data = duration_msecs
#rec.bookmark = 0
#rec.cutlist = 0
rec.markup.commit()
if jobid:
if output_bitrate:
job.update({'status':job.FINISHED, 'comment':'Transcode Completed @ %dkbps, compressed file by %d%% (clipped %d%%, transcoder compressed %d%%)' % (output_bitrate,int(compressed_pct*100),int(clipped_compress_pct*100),int(actual_compression_ratio*100))})
else:
job.update({'status':job.FINISHED, 'comment':'Transcode Completed'})
def runjob(jobid=None, chanid=None, starttime=None, tzoffset=None):
global estimateBitrate
db = MythDB()
if jobid:
job = Job(jobid, db=db)
chanid = job.chanid
utcstarttime = job.starttime
else:
print 'Job id not found.'
sys.exit(1)
if debug:
print 'chanid "%s"' % chanid
print 'utcstarttime "%s"' % utcstarttime
rec = Recorded((chanid, utcstarttime), db=db)
utcstarttime = rec.starttime
starttime_datetime = utcstarttime
# reformat 'starttime' for use with mythtranscode/ffmpeg/mythcommflag
starttime = str(utcstarttime.utcisoformat().replace(u':', '').replace(
u' ', '').replace(u'T', '').replace('-', ''))
if debug:
print 'mythtv format starttime "%s"' % starttime
input_filesize = rec.filesize
#sys.exit(e.retcode)
print 'Transcoding. ' + rec.basename + " " + rec.storagegroup
sg = findfile('/' + rec.basename, rec.storagegroup, db=db)
if sg is None:
print 'Local access to recording not found.'
sys.exit(1)
infile = os.path.join(sg.dirname, rec.basename)
outfile = '%s.mp4' % os.path.join(sg.dirname, "proxy",
rec.basename).rsplit('.', 1)[0]
# outfile = '%s.mp4' % infile.rsplit('.',1)[0]
framerate = 59.94
clipped_bytes = 0
duration_secs, e = get_duration(db, rec, transcoder, infile)
abitrate_param = abitrate_param_HD # preserve 5.1 audio
if debug:
print 'Audio bitrate parameter "%s"' % abitrate_param
# Transcode to mp4
#if jobid:
# job.update({'status':4, 'comment':'Transcoding to mp4'})
# ffmpeg output is redirected to the temporary file tmpstatusfile and
# a second thread continuously reads this file while
# the transcode is in-process. see while loop below for the monitoring thread
tf = tempfile.NamedTemporaryFile()
tmpstatusfile = tf.name
# tmpstatusfile = '/tmp/ffmpeg-transcode.txt'
if debug:
print 'Using temporary file "%s" for ffmpeg status updates.' % tmpstatusfile
res = []
# create a thread to perform the encode
ipq = Queue.Queue()
t = threading.Thread(target=wrapper,
args=(encode, (
jobid,
db,
job,
ipq,
preset,
vbitrate_param,
abitrate_param,
infile,
outfile,
tmpstatusfile,
), res))
t.start()
# wait for ffmpeg to open the file and emit its initialization information
# before we start the monitoring process
time.sleep(1)
# open the temporary file having the ffmeg output text and process it to generate status updates
hangiter = 0
with open(tmpstatusfile) as f:
# read all the opening ffmpeg status/analysis lines
lines = f.readlines()
# set initial progress to -1
prev_progress = -1
framenum = 0
fps = 1.0
while t.is_alive():
# read all output since last readline() call
lines = f.readlines()
if len(lines) > 0:
# every ffmpeg output status line ends with a carriage return '\r'
# split the last read line at these locations
lines = lines[-1].split('\r')
hangiter = 0
if len(lines) > 1 and lines[-2].startswith('frame'):
# since typical reads will have the last line ending with \r the last status
# message is at index=[-2] start processing this line
# replace multiple spaces with one space
lines[-2] = re.sub(' +', ' ', lines[-2])
# remove any spaces after equals signs
lines[-2] = re.sub('= +', '=', lines[-2])
# split the fields at the spaces the first two fields for typical
# status lines will be framenum=XXXX and fps=YYYY parse the values
values = lines[-2].split(' ')
if len(values) > 1:
if debug:
print 'values %s' % values
prev_framenum = framenum
prev_fps = fps
try:
# framenum = current frame number being encoded
framenum = int(values[0].split('=')[1])
# fps = frames per second for the encoder
fps = float(values[1].split('=')[1])
except ValueError, e:
print 'ffmpeg status parse exception: "%s"' % e
framenum = prev_framenum
fps = prev_fps
pass
# progress = 0-100 represent percent complete for the transcode
progress = int(
(100 * framenum) / (duration_secs * framerate))
# eta_secs = estimated number of seconds until transcoding is complete
eta_secs = int(
(float(duration_secs * framerate) - framenum) / fps)
# pct_realtime = how many real seconds it takes to encode 1 second of video
pct_realtime = float(fps / framerate)
if debug:
print 'framenum = %d fps = %.2f' % (framenum, fps)
if progress != prev_progress:
if debug:
print 'Progress %d%% encoding %.1f frames per second ETA %d mins' \
% ( progress, fps, float(eta_secs)/60)
if jobid:
progress_str = 'Transcoding to mp4 %d%% complete ETA %d mins fps=%.1f.' \
% ( progress, float(eta_secs)/60, fps)
job.update({
'status': job.RUNNING,
'comment': progress_str
})
prev_progress = progress
elif len(lines) > 1:
if debug:
print 'Read pathological output %s' % lines[-2]
else:
if debug:
print 'Read no lines of ffmpeg output for %s secs. Possible hang?' % (
POLL_INTERVAL * hangiter)
hangiter = hangiter + 1
if jobid:
progress_str = 'Read no lines of ffmpeg output for %s secs. Possible hang?' % (
POLL_INTERVAL * hangiter)
job.update({
'status': job.RUNNING,
'comment': progress_str
})
time.sleep(POLL_INTERVAL)
if debug:
print 'res = "%s"' % res
def runjob(jobid=None,
chanid=None,
starttime=None,
tzoffset=None,
maxWidth=maxWidth,
maxHeight=maxHeight,
sdonly=0,
burncc=0,
usemkv=0,
overwrite=1):
global estimateBitrate
db = MythDB()
try:
if jobid:
job = Job(jobid, db=db)
chanid = job.chanid
utcstarttime = job.starttime
else:
job = None
utcstarttime = datetime.strptime(starttime, "%Y%m%d%H%M%S")
utcstarttime = utcstarttime + timedelta(hours=tzoffset)
if debug:
print 'chanid "%s"' % chanid
print 'utcstarttime "%s"' % utcstarttime
rec = Recorded((chanid, utcstarttime), db=db)
utcstarttime = rec.starttime
starttime_datetime = utcstarttime
# reformat 'starttime' for use with mythtranscode/HandBrakeCLI/mythcommflag
starttime = str(utcstarttime.utcisoformat().replace(u':', '').replace(
u' ', '').replace(u'T', '').replace('-', ''))
if debug:
print 'mythtv format starttime "%s"' % starttime
input_filesize = rec.filesize
if rec.commflagged:
if debug:
print 'Recording has been scanned to detect commerical breaks.'
waititer = 1
keepWaiting = True
while keepWaiting == True:
keepWaiting = False
for index, jobitem in reversed(
list(
enumerate(
db.searchJobs(chanid=chanid,
starttime=starttime_datetime)))):
if jobitem.type == jobitem.COMMFLAG: # Commercial flagging job
if debug:
print 'Commercial flagging job detected with status %s' % jobitem.status
if jobitem.status == jobitem.RUNNING: # status = RUNNING?
job.update({'status':job.PAUSED,
'comment':'Waited %d secs for the commercial flagging job' % (waititer*POLL_INTERVAL) \
+ ' currently running on this recording to complete.'})
if debug:
print 'Waited %d secs for the commercial flagging job' % (waititer*POLL_INTERVAL) \
+ ' currently running on this recording to complete.'
time.sleep(POLL_INTERVAL)
keepWaiting = True
waititer = waititer + 1
break
else:
if debug:
print 'Recording has not been scanned to detect/remove commercial breaks.'
if require_commflagged:
if jobid:
job.update({
'status':
job.RUNNING,
'comment':
'Required commercial flagging for this file is not found.'
+
'Flagging commercials and cancelling any queued commercial flagging.'
})
# cancel any queued job to flag commercials for this recording and run commercial flagging in this script
for index, jobitem in reversed(
list(
enumerate(
db.searchJobs(chanid=chanid,
starttime=starttime_datetime)))):
if debug:
if index == 0:
print jobitem.keys()
print index, jobitem.id, jobitem.chanid
if jobitem.type == jobitem.COMMFLAG: # Commercial flagging job
if jobitem.status == jobitem.RUNNING: # status = RUNNING?
jobitem.cmds = jobitem.STOP # stop command from the frontend to stop the commercial flagging job
#jobitem.setStatus(jobitem.CANCELLED)
#jobitem.setComment('Cancelled: Transcode command ran commercial flagging for this recording.')
jobitem.update({
'status':
jobitem.CANCELLED,
'comment':
'A user transcode job ran commercial flagging for'
+ ' this recording and cancelled this job.'
})
if debug:
print 'Flagging Commercials...'
# Call "mythcommflag --chanid $CHANID --starttime $STARTTIME"
task = System(path='mythcommflag', db=db)
try:
output = task('--chanid "%s"' % chanid,
'--starttime "%s"' % starttime,
'2> /dev/null')
except MythError, e:
# it seems mythcommflag always exits with an decoding error "eno: Unknown error 541478725 (541478725)"
pass
#print 'Command failed with output:\n%s' % e.stderr
#if jobid:
# job.update({'status':304, 'comment':'Flagging commercials failed'})
#sys.exit(e.retcode)
sg = findfile('/' + rec.basename, rec.storagegroup, db=db)
if sg is None:
print 'Local access to recording not found.'
sys.exit(1)
infile = os.path.join(sg.dirname, rec.basename)
#TODO: set overWrite to 0 if infile is m4v or mkv (already converted)
#tmpfile = '%s.tmp' % infile.rsplit('.',1)[0]
outtitle = rec.title.replace("&", "and")
outtitle = re.sub('[^A-Za-z0-9 ]+', '', outtitle)
filetype = 'm4v'
#DEBUG CODE TO FIND OBJECT STRUCT:
#print '{}'.format(dir(rec.getProgram()))
#print '{}'.format(rec.getProgram().year)
if usemkv == 1:
filetype = 'mkv'
#print '!{}!'.format(rec.programid[0:2])
if rec.season > 0 and rec.episode > 0: #if there are seasons and episode numbers in the recording data
outtitle = '{0:s} S{1:d} E{2:02d}'.format(outtitle, rec.season,
rec.episode)
elif rec.programid[0:2] == 'MV' and str(rec.getProgram().year).isdigit(
): #if it's a movie and has an original air date for when it came out
outtitle = '{} ({})'.format(outtitle, rec.getProgram().year)
elif rec.programid[
0:
2] == 'MV' and rec.originalairdate != None and rec.originalairdate > datetime.date(
datetime(1, 1, 1, 0, 0)
): #if it's a movie and has an original air date for when it came out
outtitle = '{} ({})'.format(outtitle, rec.originalairdate.year)
elif 'Sports' in rec.category: #if it's sports
outtitle = '{}-{}-{}'.format(
outtitle, re.sub('[^A-Za-z0-9 ]+', '', rec.subtitle),
str(rec.starttime.strftime("%Y%m%d")))
elif rec.programid[0:2] == 'SH' and (' News ' in rec.title
or rec.category
== 'News'): #if it's a news show
outtitle = '{}-{}'.format(outtitle,
str(rec.starttime.strftime("%Y%m%d")))
elif rec.originalairdate != None and rec.originalairdate > datetime.date(
datetime(1, 1, 1, 0, 0)): #if it has an original air date
outtitle = '{} {}'.format(
outtitle, str(rec.originalairdate.strftime("%Y%m%d")))
else:
outtitle = '{} {}'.format(outtitle,
str(rec.starttime.strftime("%Y%m%d")))
outtitle = '{}.{}'.format(outtitle, filetype)
outfile = os.path.join(sg.dirname, outtitle)
tmpfile = '{}.{}'.format(
outfile.rsplit('.', 1)[0],
infile.rsplit('.', 1)[1])
if tmpfile == infile:
tmpfile = '{}.tmp'.format(infile.rsplit('.', 1)[0])
if (overwrite == 0):
# If not overwritting the file, use the export folder
outfile = os.path.join(exportFolder, outtitle)
if debug:
print 'overwrite is 0. outfile "{}"'.format(outfile)
if os.path.isfile(outfile) or infile == outfile:
# If outfile exists already, create a new name for the file.
outfile = '{}-{}.{}'.format(
outfile.rsplit('.', 1)[0],
str(rec.starttime.strftime("%Y%m%d")), filetype)
if os.path.isfile(tmpfile):
# If the infile and tmpfile are the same, create a new name for the tmpfile
tmpfile = '{}-{}.tmp'.format(
outfile.rsplit('.', 1)[0],
str(rec.starttime.strftime("%Y%m%d")))
if os.path.isfile(tmpfile):
# If tmp exists already, create a new name for the file.
outfile = '{}-{}.tmp'.format(
tmpfile.rsplit('.', 1)[0],
str(rec.starttime.strftime("%Y%m%d")))
if debug:
print 'tmp exists. outfile "{}"'.format(outfile)
if debug:
print 'infile "{}"'.format(infile)
print 'tmpfile "{}"'.format(tmpfile)
print 'outfile "{}"'.format(outfile)
#add_metadata(db, jobid, debug, job, rec, filetype, tmpfile)
clipped_bytes = 0
# If selected, create a cutlist to remove commercials via mythtranscode by running:
# mythutil --gencutlist --chanid $CHANID --starttime $STARTTIME
if generate_commcutlist:
if jobid:
job.update({
'status':
job.RUNNING,
'comment':
'Generating Cutlist for commercial removal'
})
task = System(path='mythutil', db=db)
try:
output = task('--gencutlist', '--chanid "%s"' % chanid,
'--starttime "%s"' % starttime)
except MythError, e:
print 'Command "mythutil --gencutlist" failed with output:\n%s' % e.stderr
if jobid:
job.update({
'status':
job.ERRORED,
'comment':
'Generation of commercial Cutlist failed'
})
sys.exit(e.retcode)