def run_shell_exec(self, cmd, track=True):
"""
Run the shell exec command.
@param cmd String Command to be run
@return int, string
"""
cmd = 'ulimit -f ' + escape_shellarg(background_size_limit) + ';' + \
'ulimit -v ' + escape_shellarg(background_memory_limit) + ';' + \
'ulimit -t ' + escape_shellarg(background_time_limit) + ';' + \
'ulimit -a;' + \
'nice -n ' + escape_shellarg(background_priority) + ' ' + cmd + \
' 2>&1'
# Adapted from https://gist.github.com/marazmiki/3015621
process = subprocess.Popen(cmd,
stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True,
shell=True)
re_duration = re.compile(r'Duration: (\d{2}:\d{2}:\d{2})')
re_position = re.compile(r'time=(\d{2}:\d{2}:\d{2})', re.I)
duration = None
position = None
newpercentage = percentage = -1
while process.poll() is None:
# for line in process.stdout.readlines():
# http://bugs.python.org/issue3907
while True:
line = process.stdout.readline()
if not line:
break
print line,
if track:
if duration is None:
duration_match = re_duration.search(line)
if duration_match:
duration = time_to_seconds(duration_match.group(1))
else:
position_match = re_position.search(line)
if position_match:
position = time_to_seconds(position_match.group(1))
if duration and position:
newpercentage = min(
int(math.floor(100 * position / duration)),
100)
if newpercentage != percentage:
percentage = newpercentage
self.statuscallback(None, percentage)
time.sleep(2)
return process.returncode, ''
def run_shell_exec(self, cmd, track=True):
"""
Run the shell exec command.
@param cmd String Command to be run
@return int, string
"""
cmd = 'ulimit -f ' + escape_shellarg(background_size_limit) + ';' + \
'ulimit -v ' + escape_shellarg(background_memory_limit) + ';' + \
'ulimit -t ' + escape_shellarg(background_time_limit) + ';' + \
'ulimit -a;' + \
'nice -n ' + escape_shellarg(background_priority) + ' ' + cmd + \
' 2>&1'
# Adapted from https://gist.github.com/marazmiki/3015621
process = subprocess.Popen(
cmd, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
universal_newlines=True, shell=True
)
re_duration = re.compile(r'Duration: (\d{2}:\d{2}:\d{2})')
re_position = re.compile(r'time=(\d{2}:\d{2}:\d{2})', re.I)
duration = None
position = None
newpercentage = percentage = -1
while process.poll() is None:
# for line in process.stdout.readlines():
# http://bugs.python.org/issue3907
while True:
line = process.stdout.readline()
if not line:
break
print line,
if track:
if duration is None:
duration_match = re_duration.search(line)
if duration_match:
duration = time_to_seconds(duration_match.group(1))
else:
position_match = re_position.search(line)
if position_match:
position = time_to_seconds(position_match.group(1))
if duration and position:
newpercentage = min(int(
math.floor(100 * position / duration)
), 100)
if newpercentage != percentage:
percentage = newpercentage
self.statuscallback(None, percentage)
time.sleep(2)
return process.returncode, ''
def ffmpeg_add_audio_options(self, options, p):
"""
Add ffmpeg shell options for audio.
@param options array
@param p
@return string
"""
cmd = ''
if 'audioQuality' in options:
cmd += " -aq " + escape_shellarg(options['audioQuality'])
if 'audioBitrate' in options:
cmd += ' -ab ' + str(options['audioBitrate']) * 1000
if 'samplerate' in options:
cmd += " -ar " + escape_shellarg(options['samplerate'])
if 'channels' in options:
cmd += " -ac " + escape_shellarg(options['channels'])
if 'audioCodec' in options:
encoders = {
'vorbis': 'libvorbis',
'opus': 'libopus',
'mp3': 'libmp3lame',
}
if options['audioCodec'] in encoders:
codec = encoders[options['audioCodec']]
else:
codec = options['audioCodec']
cmd += " -acodec " + escape_shellarg(codec)
if codec == 'aac':
# the aac encoder is currently "experimental" in libav 9? :P
cmd += ' -strict experimental'
else:
# if no audio codec set use vorbis :
cmd += " -acodec libvorbis "
return cmd
def ffmpeg_add_theora_video_options(self, options, p):
"""
Add ffmpeg shell options for ogg.
Warning: does not create Ogg skeleton metadata track.
@param options
@param p
@return string
"""
cmd = ' -threads ' + str(ffmpeg_threads)
# Check for video quality:
if 'videoQuality' in options and int(options['videoQuality']) >= 0:
cmd += " -q:v " + escape_shellarg(options['videoQuality'])
# Check for video bitrate:
if 'videoBitrate' in options:
cmd += " -qmin 1 -qmax 51"
cmd += " -vb " + escape_shellarg(
int(options['videoBitrate']) * 1000)
# Set the codec:
cmd += " -vcodec theora"
# Check for keyframeInterval
if 'keyframeInterval' in options:
cmd += ' -g ' + escape_shellarg(options['keyframeInterval'])
cmd += ' -keyint_min ' + \
escape_shellarg(options['keyframeInterval'])
if 'deinterlace' in options:
cmd += ' -deinterlace'
if 'framerate' in options:
cmd += ' -r ' + escape_shellarg(options['framerate'])
# Output Ogg
cmd += " -f ogg"
return cmd
def ffmpeg_add_audio_options(self, options, p):
"""
Add ffmpeg shell options for audio.
@param options array
@param p
@return string
"""
cmd = ''
if 'audioQuality' in options:
cmd += " -aq " + escape_shellarg(options['audioQuality'])
if 'audioBitrate' in options:
cmd += ' -ab ' + str(options['audioBitrate']) * 1000
if 'samplerate' in options:
cmd += " -ar " + escape_shellarg(options['samplerate'])
if 'channels' in options:
cmd += " -ac " + escape_shellarg(options['channels'])
if 'audioCodec' in options:
encoders = {
'vorbis': 'libvorbis',
'opus': 'libopus',
'mp3': 'libmp3lame',
}
if options['audioCodec'] in encoders:
codec = encoders[options['audioCodec']]
else:
codec = options['audioCodec']
cmd += " -acodec " + escape_shellarg(codec)
if codec == 'aac':
# the aac encoder is currently "experimental" in libav 9? :P
cmd += ' -strict experimental'
else:
# if no audio codec set use vorbis :
cmd += " -acodec libvorbis "
return cmd
def ffmpeg_add_theora_video_options(self, options, p):
"""
Add ffmpeg shell options for ogg.
Warning: does not create Ogg skeleton metadata track.
@param options
@param p
@return string
"""
cmd = ' -threads ' + str(ffmpeg_threads)
# Check for video quality:
if 'videoQuality' in options and options['videoQuality'] >= 0:
cmd += " -q:v " + escape_shellarg(options['videoQuality'])
# Check for video bitrate:
if 'videoBitrate' in options:
cmd += " -qmin 1 -qmax 51"
cmd += " -vb " + escape_shellarg(options['videoBitrate'] * 1000)
# Set the codec:
cmd += " -vcodec theora"
# Check for keyframeInterval
if 'keyframeInterval' in options:
cmd += ' -g ' + escape_shellarg(options['keyframeInterval'])
cmd += ' -keyint_min ' + \
escape_shellarg(options['keyframeInterval'])
if 'deinterlace' in options:
cmd += ' -deinterlace'
if 'framerate' in options:
cmd += ' -r ' + escape_shellarg(options['framerate'])
# Output Ogg
cmd += " -f ogg"
return cmd
def ffmpeg_add_theora_video_options(self, options, p):
"""
Add ffmpeg shell options for ogg.
Warning: does not create Ogg skeleton metadata track.
@param options
@param p
@return string
"""
cmd = " -threads " + str(ffmpeg_threads)
# Check for video quality:
if "videoQuality" in options and options["videoQuality"] >= 0:
cmd += " -q:v " + escape_shellarg(options["videoQuality"])
# Check for video bitrate:
if "videoBitrate" in options:
cmd += " -qmin 1 -qmax 51"
cmd += " -vb " + escape_shellarg(options["videoBitrate"] * 1000)
# Set the codec:
cmd += " -vcodec theora"
# Check for keyframeInterval
if "keyframeInterval" in options:
cmd += " -g " + escape_shellarg(options["keyframeInterval"])
cmd += " -keyint_min " + escape_shellarg(options["keyframeInterval"])
if "deinterlace" in options:
cmd += " -deinterlace"
if "framerate" in options:
cmd += " -r " + escape_shellarg(options["framerate"])
# Output Ogg
cmd += " -f ogg"
return cmd
def ffmpeg_add_audio_options(self, options, p):
"""
Add ffmpeg shell options for audio.
@param options array
@param p
@return string
"""
cmd = ""
if "audioQuality" in options:
cmd += " -aq " + escape_shellarg(options["audioQuality"])
if "audioBitrate" in options:
cmd += " -ab " + str(options["audioBitrate"]) * 1000
if "samplerate" in options:
cmd += " -ar " + escape_shellarg(options["samplerate"])
if "channels" in options:
cmd += " -ac " + escape_shellarg(options["channels"])
if "audioCodec" in options:
encoders = {"vorbis": "libvorbis", "opus": "libopus", "mp3": "libmp3lame"}
if options["audioCodec"] in encoders:
codec = encoders[options["audioCodec"]]
else:
codec = options["audioCodec"]
cmd += " -acodec " + escape_shellarg(codec)
if codec == "aac":
# the aac encoder is currently "experimental" in libav 9? :P
cmd += " -strict experimental"
else:
# if no audio codec set use vorbis :
cmd += " -acodec libvorbis "
return cmd
def ffmpeg_add_webm_video_options(self, options, p):
"""
Add ffmpeg shell options for webm.
@param options
@param p
@return string
"""
cmd = ' -threads ' + str(ffmpeg_threads)
# check for presets:
if 'preset' in options:
if options['preset'] == "360p":
cmd += " -vpre libvpx-360p"
elif options['preset'] == "720p":
cmd += " -vpre libvpx-720p"
elif options['preset'] == "1080p":
cmd += " -vpre libvpx-1080p"
# Add the boiler plate vp8 ffmpeg command:
cmd += " -skip_threshold 0 -bufsize 6000k -rc_init_occupancy 4000"
# Check for video quality:
if 'videoQuality' in options and options['videoQuality'] >= 0:
# Map 0-10 to 63-0, higher values worse quality
quality = 63 - int(int(options['videoQuality']) / 10.0 * 63)
cmd += " -qmin " + escape_shellarg(quality)
cmd += " -qmax " + escape_shellarg(quality)
# Check for video bitrate:
if 'videoBitrate' in options:
cmd += " -qmin 1 -qmax 51"
cmd += " -vb " + escape_shellarg(options['videoBitrate'] * 1000)
# Set the codec:
if options['videoCodec'] == 'vp9':
cmd += " -vcodec libvpx-vp9"
if 'tileColumns' in options:
cmd += ' -tile-columns ' + \
escape_shellarg(options['tileColumns'])
else:
cmd += " -vcodec libvpx"
# Check for keyframeInterval
if 'keyframeInterval' in options:
cmd += ' -g ' + escape_shellarg(options['keyframeInterval'])
cmd += ' -keyint_min ' + \
escape_shellarg(options['keyframeInterval'])
if 'deinterlace' in options:
cmd += ' -deinterlace'
# Output WebM
cmd += " -f webm"
return cmd
def ffmpeg_add_webm_video_options(self, options, p):
"""
Add ffmpeg shell options for webm.
@param options
@param p
@return string
"""
cmd = ' -threads ' + str(ffmpeg_threads)
# check for presets:
if 'preset' in options:
if options['preset'] == "360p":
cmd += " -vpre libvpx-360p"
elif options['preset'] == "720p":
cmd += " -vpre libvpx-720p"
elif options['preset'] == "1080p":
cmd += " -vpre libvpx-1080p"
# Add the boiler plate vp8 ffmpeg command:
cmd += " -skip_threshold 0 -bufsize 6000k -rc_init_occupancy 4000"
# Check for video quality:
if 'videoQuality' in options and options['videoQuality'] >= 0:
# Map 0-10 to 63-0, higher values worse quality
quality = 63 - int(int(options['videoQuality']) / 10.0 * 63)
cmd += " -qmin " + escape_shellarg(quality)
cmd += " -qmax " + escape_shellarg(quality)
# Check for video bitrate:
if 'videoBitrate' in options:
cmd += " -qmin 1 -qmax 51"
cmd += " -vb " + escape_shellarg(options['videoBitrate'] * 1000)
# Set the codec:
if options['videoCodec'] == 'vp9':
cmd += " -vcodec libvpx-vp9"
if 'tileColumns' in options:
cmd += ' -tile-columns ' + \
escape_shellarg(options['tileColumns'])
else:
cmd += " -vcodec libvpx"
# Check for keyframeInterval
if 'keyframeInterval' in options:
cmd += ' -g ' + escape_shellarg(options['keyframeInterval'])
cmd += ' -keyint_min ' + \
escape_shellarg(options['keyframeInterval'])
if 'deinterlace' in options:
cmd += ' -deinterlace'
# Output WebM
cmd += " -f webm"
return cmd
def ffmpeg_add_h264_video_options(self, options, p):
"""
Add ffmpeg shell options for h264.
@param options
@param p
@return string
"""
# Set the codec:
cmd = " -threads " + str(ffmpeg_threads) + " -vcodec libx264"
if 'videoBitrate' in options:
cmd += " -b " + escape_shellarg(options['videoBitrate'])
# Output mp4
cmd += " -f mp4"
return cmd
def ffmpeg_add_h264_video_options(self, options, p):
"""
Add ffmpeg shell options for h264.
@param options
@param p
@return string
"""
# Set the codec:
cmd = " -threads " + str(ffmpeg_threads) + " -vcodec libx264"
if 'videoBitrate' in options:
cmd += " -b " + escape_shellarg(options['videoBitrate'])
# Output mp4
cmd += " -f mp4"
return cmd
def ffmpeg_encode(self, options, p=0):
"""
Utility helper for ffmpeg and ffmpeg2theora mapping.
@param options array
@param p int
@return bool|string
"""
if not os.path.isfile(self.get_source_path()):
return "source file is missing, " + self.get_source_path() + \
". Encoding failed."
# Set up the base command
cmd = escape_shellarg(ffmpeg_location) + ' -y -i ' + \
escape_shellarg(self.get_source_path())
if 'vpre' in options:
cmd += ' -vpre ' + escape_shellarg(options['vpre'])
if 'novideo' in options:
cmd += " -vn "
elif self.preserve['video']:
cmd += " -vcodec copy"
elif options['videoCodec'] == 'vp8' or options['videoCodec'] == 'vp9':
cmd += self.ffmpeg_add_webm_video_options(options, p)
elif options['videoCodec'] == 'h264':
cmd += self.ffmpeg_add_h264_video_options(options, p)
elif options['videoCodec'] == 'theora':
cmd += self.ffmpeg_add_theora_video_options(options, p)
# Check for start time
if 'starttime' in options:
cmd += ' -ss ' + escape_shellarg(options['starttime'])
else:
options['starttime'] = 0
# Check for end time:
if 'endtime' in options:
cmd += ' -t ' + str(options['endtime']) - str(options['starttime'])
if p == 1 or 'noaudio' in options:
cmd += ' -an'
elif self.preserve['audio']:
cmd += " -acodec copy"
else:
cmd += self.ffmpeg_add_audio_options(options, p)
if p != 0:
cmd += " -pass " + escape_shellarg(p)
cmd += " -passlogfile " + \
escape_shellarg(self.get_target_path() + '.log')
# And the output target:
if p == 1:
cmd += ' /dev/null'
else:
cmd += " " + escape_shellarg(self.get_target_path())
self.output("Running cmd: " + cmd + "\n")
# Right before we output remove the old file
retval, shellOutput = self.run_shell_exec(cmd, track=p != 1)
if int(retval) != 0:
return cmd + \
"\nExitcode: " + str(retval)
return True
def ffmpeg_encode(self, options, p=0):
"""
Utility helper for ffmpeg and ffmpeg2theora mapping.
@param options array
@param p int
@return bool|string
"""
if not os.path.isfile(self.get_source_path()):
return "source file is missing, " + self.get_source_path() + ". Encoding failed."
# Set up the base command
cmd = escape_shellarg(ffmpeg_location) + " -y -i " + escape_shellarg(self.get_source_path())
if "vpre" in options:
cmd += " -vpre " + escape_shellarg(options["vpre"])
if "novideo" in options:
cmd += " -vn "
elif self.preserve["video"]:
cmd += " -vcodec copy"
elif options["videoCodec"] == "vp8" or options["videoCodec"] == "vp9":
cmd += self.ffmpeg_add_webm_video_options(options, p)
elif options["videoCodec"] == "h264":
cmd += self.ffmpeg_add_h264_video_options(options, p)
elif options["videoCodec"] == "theora":
cmd += self.ffmpeg_add_theora_video_options(options, p)
# Check for start time
if "starttime" in options:
cmd += " -ss " + escape_shellarg(options["starttime"])
else:
options["starttime"] = 0
# Check for end time:
if "endtime" in options:
cmd += " -t " + str(options["endtime"]) - str(options["starttime"])
if p == 1 or "noaudio" in options:
cmd += " -an"
elif self.preserve["audio"]:
cmd += " -acodec copy"
else:
cmd += self.ffmpeg_add_audio_options(options, p)
if p != 0:
cmd += " -pass " + escape_shellarg(p)
cmd += " -passlogfile " + escape_shellarg(self.get_target_path() + ".log")
# And the output target:
if p == 1:
cmd += " /dev/null"
else:
cmd += " " + escape_shellarg(self.get_target_path())
self.output("Running cmd: " + cmd + "\n")
# Right before we output remove the old file
retval, shellOutput = self.run_shell_exec(cmd, track=p != 1)
if int(retval) != 0:
return cmd + "\nExitcode: " + str(retval)
return True
def ffmpeg_encode(self, options, p=0):
"""
Utility helper for ffmpeg and ffmpeg2theora mapping.
@param options array
@param p int
@return bool|string
"""
if not os.path.isfile(self.get_source_path()):
return "source file is missing, " + self.get_source_path() + \
". Encoding failed."
# Set up the base command
cmd = escape_shellarg(ffmpeg_location) + ' -y -i ' + \
escape_shellarg(self.get_source_path())
if 'vpre' in options:
cmd += ' -vpre ' + escape_shellarg(options['vpre'])
if 'novideo' in options:
cmd += " -vn "
elif self.preserve['video']:
cmd += " -vcodec copy"
elif options['videoCodec'] == 'vp8' or options['videoCodec'] == 'vp9':
cmd += self.ffmpeg_add_webm_video_options(options, p)
elif options['videoCodec'] == 'h264':
cmd += self.ffmpeg_add_h264_video_options(options, p)
elif options['videoCodec'] == 'theora':
cmd += self.ffmpeg_add_theora_video_options(options, p)
# Check for start time
if 'starttime' in options:
cmd += ' -ss ' + escape_shellarg(options['starttime'])
else:
options['starttime'] = 0
# Check for end time:
if 'endtime' in options:
cmd += ' -t ' + str(options['endtime']) - str(options['starttime'])
if p == 1 or 'noaudio' in options:
cmd += ' -an'
elif self.preserve['audio']:
cmd += " -acodec copy"
else:
cmd += self.ffmpeg_add_audio_options(options, p)
if p != 0:
cmd += " -pass " + escape_shellarg(p)
cmd += " -passlogfile " + \
escape_shellarg(self.get_target_path() + '.log')
# And the output target:
if p == 1:
cmd += ' /dev/null'
else:
cmd += " " + escape_shellarg(self.get_target_path())
self.output("Running cmd: " + cmd + "\n")
# Right before we output remove the old file
retval, shellOutput = self.run_shell_exec(cmd, track=p != 1)
if int(retval) != 0:
return cmd + \
"\nExitcode: " + str(retval)
return True
def ffmpeg_add_webm_video_options(self, options, p):
"""
Add ffmpeg shell options for webm.
@param options
@param p
@return string
"""
cmd = ' -threads ' + str(ffmpeg_threads)
if options['videoCodec'] == 'vp9':
cmd += ' -row-mt 1'
# check for presets:
if 'preset' in options:
if options['preset'] == "360p":
cmd += " -vpre libvpx-360p"
elif options['preset'] == "720p":
cmd += " -vpre libvpx-720p"
elif options['preset'] == "1080p":
cmd += " -vpre libvpx-1080p"
# Check for video quality:
if 'videoQuality' in options and int(options['videoQuality']) >= 0:
# Map 0-10 to 63-0, higher values worse quality
quality = 63 - int(int(options['videoQuality']) / 10.0 * 63)
cmd += " -qmin " + escape_shellarg(quality)
cmd += " -qmax " + escape_shellarg(quality)
# Check for video bitrate:
if 'videoBitrate' in options:
cmd += " -qmin 1 -qmax 51"
cmd += " -vb " + escape_shellarg(int(options['videoBitrate']) * 1000)
# Set the codec:
if options['videoCodec'] == 'vp9':
cmd += " -vcodec libvpx-vp9"
if 'tileColumns' in options:
cmd += ' -tile-columns ' + \
escape_shellarg(options['tileColumns'])
else:
cmd += " -vcodec libvpx"
if 'altref' in options:
cmd += ' -auto-alt-ref 1'
cmd += ' -lag-in-frames 25'
# Check for keyframeInterval
if 'keyframeInterval' in options:
cmd += ' -g ' + escape_shellarg(options['keyframeInterval'])
cmd += ' -keyint_min ' + \
escape_shellarg(options['keyframeInterval'])
if 'deinterlace' in options:
cmd += ' -deinterlace'
if p == 1:
# Make first pass faster...
cmd += ' -speed 4'
elif 'speed' in options:
cmd += ' -speed ' + escape_shellarg(options['speed'])
# Output WebM
cmd += " -f webm"
return cmd
def run_shell_exec(self, cmd, track=True):
"""
Run the shell exec command.
@param cmd String Command to be run
@return int, string
"""
cmd = (
"ulimit -f "
+ escape_shellarg(background_size_limit)
+ ";"
+ "ulimit -v "
+ escape_shellarg(background_memory_limit)
+ ";"
+ "ulimit -t "
+ escape_shellarg(background_time_limit)
+ ";"
+ "ulimit -a;"
+ "nice -n "
+ escape_shellarg(background_priority)
+ " "
+ cmd
+ " 2>&1"
)
# Adapted from https://gist.github.com/marazmiki/3015621
process = subprocess.Popen(
cmd,
stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True,
shell=True,
preexec_fn=os.setsid,
)
re_duration = re.compile(r"Duration: (\d{2}:\d{2}:\d{2})")
re_position = re.compile(r"time=(\d{2}:\d{2}:\d{2})", re.I)
duration = None
position = None
newpercentage = percentage = -1
while process.poll() is None:
# for line in process.stdout.readlines():
# http://bugs.python.org/issue3907
while True:
line = process.stdout.readline()
if not line:
break
print line,
if track:
if duration is None:
duration_match = re_duration.search(line)
if duration_match:
duration = time_to_seconds(duration_match.group(1))
else:
position_match = re_position.search(line)
if position_match:
position = time_to_seconds(position_match.group(1))
if duration and position:
newpercentage = min(int(math.floor(100 * position / duration)), 100)
if newpercentage != percentage:
percentage = newpercentage
try:
self.statuscallback(None, percentage)
except TaskAbort:
os.killpg(os.getpgid(process.pid), signal.SIGTERM)
raise
time.sleep(2)
return process.returncode, ""
