def _loop(self):
executable = self.get_executable()
args = self.get_parameters()
self._start_logging(executable, args)
args.insert(0, executable)
logging.debug("Conversion: (%s)", " ".join(args))
kwargs = {
"bufsize": 1,
"stdout": subprocess.PIPE,
"stderr": subprocess.STDOUT,
"stdin": subprocess.PIPE,
"close_fds": True
}
try:
self.process_handle = Popen(args, **kwargs)
self.process_output(line_reader(self.process_handle.stdout))
self.process_handle.wait()
except OSError, ose:
if ose.errno == errno.ENOENT:
self.error = _("%(program)s does not exist.",
{"program": self.get_executable()})
else:
logging.exception("Exception in conversion loop: %s %s", args,
kwargs)
def call_command(*args, **kwargs):
"""Call an external command. If the command doesn't exit with
status 0, or if it outputs to stderr, an exception will be raised.
Returns stdout.
:param ignore_stderr: boolean. defaults to False. If True and
the command returns stderr, it's ignored rather than raising
an exception.
:param return_everything: boolean. defaults to False. If True,
then this returns (retcode, stdout, stderr). This implies
ignore_stderr is True, so you don't need to explicitly state
that, too.
:param env: dict. Environment to pass to subprocess.Popen
"""
ignore_stderr = kwargs.pop("ignore_stderr", False)
return_everything = kwargs.pop("return_everything", False)
env = kwargs.pop("env", None)
if kwargs:
raise TypeError("extra keyword arguments: %s" % kwargs)
pipe = Popen(args, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
stdout, stderr = pipe.communicate()
if return_everything:
return (pipe.returncode, stdout, stderr)
elif pipe.returncode != 0:
raise OSError(
"call_command with %s has return code %s\n" "stdout:%s\nstderr:%s" % (args, pipe.returncode, stdout, stderr)
)
elif stderr and not ignore_stderr:
raise OSError("call_command with %s outputed error text:\n%s" % (args, stderr))
else:
return stdout
def call_command(*args, **kwargs):
"""Call an external command. If the command doesn't exit with
status 0, or if it outputs to stderr, an exception will be raised.
Returns stdout.
:param ignore_stderr: boolean. defaults to False. If True and
the command returns stderr, it's ignored rather than raising
an exception.
:param return_everything: boolean. defaults to False. If True,
then this returns (retcode, stdout, stderr). This implies
ignore_stderr is True, so you don't need to explicitly state
that, too.
"""
ignore_stderr = kwargs.pop('ignore_stderr', False)
return_everything = kwargs.pop('return_everything', False)
if kwargs:
raise TypeError('extra keyword arguments: %s' % kwargs)
pipe = Popen(args,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = pipe.communicate()
if return_everything:
return (pipe.returncode, stdout, stderr)
elif pipe.returncode != 0:
raise OSError("call_command with %s has return code %s\n"
"stdout:%s\nstderr:%s" %
(args, pipe.returncode, stdout, stderr))
elif stderr and not ignore_stderr:
raise OSError("call_command with %s outputed error text:\n%s" %
(args, stderr))
else:
return stdout
def diskutil(cmd, path_or_disk, use_plist=True):
args = ['/usr/sbin/diskutil', cmd]
if use_plist:
args.append('-plist')
if path_or_disk:
args.append(path_or_disk)
proc = Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = proc.communicate()
if not use_plist:
return stdout
try:
return plistlib.readPlistFromString(stdout)
except:
logging.warn('error parsing plist for command: %s\n%s' % (
' '.join(args), stdout))
def launch_download_daemon(oldpid, env):
kill_process(oldpid)
# FIXME - wtf is going on here between os.environ, env and
# environ?
for key, value in env.items():
os.environ[key] = value
environ = os.environ.copy()
environ['DEMOCRACY_DOWNLOADER_LOG'] = app.config.get(
prefs.DOWNLOADER_LOG_PATHNAME)
environ['MIRO_APP_VERSION'] = app.config.get(prefs.APP_VERSION)
if hasattr(app, 'in_unit_tests'):
environ['MIRO_IN_UNIT_TESTS'] = '1'
environ.update(env)
# start the downloader. We use the subprocess module to turn off
# the console. One slightly awkward thing is that the current
# process might not have a valid stdin/stdout/stderr, so we create
# a pipe to it that we never actually use.
# note that we use "Miro" instead of the app name here, so custom
# versions will work
downloader_path = (os.path.join(resources.app_root(),
"Miro_Downloader.exe"),)
return Popen(downloader_path, stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
env=environ)
def _loop(self):
executable = self.get_executable()
args = self.get_parameters()
self._start_logging(executable, args)
args.insert(0, executable)
logging.debug("Conversion: (%s)", " ".join(args))
kwargs = {"bufsize": 1,
"stdout": subprocess.PIPE,
"stderr": subprocess.STDOUT,
"stdin": subprocess.PIPE,
"close_fds": True}
try:
self.process_handle = Popen(args, **kwargs)
self.process_output(line_reader(self.process_handle.stdout))
self.process_handle.wait()
except OSError, ose:
if ose.errno == errno.ENOENT:
self.error = _("%(program)s does not exist.",
{"program": self.get_executable()})
else:
logging.exception("Exception in conversion loop: %s %s",
args, kwargs)
def _start_subprocess(self):
cmd_line, env = miro_helper_program_info()
kwargs = {
"stdout": subprocess.PIPE,
"stdin": subprocess.PIPE,
"stderr": open(os.devnull, 'wb'),
"env": env,
"close_fds": True
}
process = Popen(cmd_line, **kwargs)
return process
def _app_command_line():
"""Get the command line to lanch the Miro.app bundle. """
exe = NSBundle.mainBundle().executablePath()
os_info = os.uname()
os_version = int(os_info[2].split('.')[0])
if os_version < 9:
return [exe]
else:
arch = Popen("/usr/bin/arch",
stdout=subprocess.PIPE).communicate()[0].strip()
return ['/usr/bin/arch', '-%s' % arch, exe]
def launch_download_daemon(oldpid, env):
kill_process(oldpid)
# FIXME - wtf is going on here between os.environ, env and
# environ?
for key, value in env.items():
os.environ[key] = value
environ = os.environ.copy()
environ['DEMOCRACY_DOWNLOADER_LOG'] = app.config.get(
prefs.DOWNLOADER_LOG_PATHNAME)
environ['MIRO_APP_VERSION'] = app.config.get(prefs.APP_VERSION)
if hasattr(app, 'in_unit_tests'):
environ['MIRO_IN_UNIT_TESTS'] = '1'
environ.update(env)
# note that we use "Miro" instead of the app name here, so custom
# versions will work
downloader_path = (os.path.join(resources.app_root(),
"Miro_Downloader.exe"), )
return Popen(downloader_path, close_fds=True, env=environ)
class ConversionTask(object):
def __init__(self, converter_info, item_info, target_folder, create_item):
self.item_info = item_info
self.converter_info = converter_info
self.input_path = item_info.filename
self.final_output_path, self.temp_output_path = build_output_paths(
item_info, target_folder, converter_info)
self.create_item = create_item
self.key = "%s->%s" % (self.input_path, self.final_output_path)
self.thread = None
self.duration = None
self.progress = 0
self.log_path = None
self.log_file = None
self.process_handle = None
self.error = None
self.start_time = time.time()
def get_executable(self):
raise NotImplementedError()
def get_parameters(self):
raise NotImplementedError()
def get_output_size_guess(self):
if self.item_info.duration and self.converter_info.bit_rate:
return self.converter_info.bit_rate * self.item_info.duration / 8
return self.item_info.size
def get_display_name(self):
return self.converter_info.displayname
def run(self):
logging.debug("temp_output_path: [%s] final_output_path: [%s]",
self.temp_output_path, self.final_output_path)
self.progress = 0
self.thread = threading.Thread(target=utils.thread_body,
args=[self._loop],
name="Conversion Task")
self.thread.setDaemon(True)
self.thread.start()
def get_eta(self):
"""Calculates the eta for this conversion to be completed.
:returns: None if progress is <= 0, otherwise returns number
of seconds until this is complete
"""
if self.progress <= 0:
return None
progress = self.progress * 100
duration = time.time() - self.start_time
time_per_percent = duration / progress
return int(time_per_percent * (100 - progress))
def is_pending(self):
return self.thread is None
def is_running(self):
return self.thread is not None and self.thread.isAlive()
def done_running(self):
return self.thread is not None and not self.thread.isAlive()
def is_finished(self):
return self.done_running() and not self.is_failed()
def is_failed(self):
return (self.error or (self.process_handle is not None
and self.process_handle.returncode is not None
and self.process_handle.returncode != 0))
def _loop(self):
executable = self.get_executable()
args = self.get_parameters()
self._start_logging(executable, args)
args.insert(0, executable)
logging.debug("Conversion: (%s)", " ".join(args))
kwargs = {
"bufsize": 1,
"stdout": subprocess.PIPE,
"stderr": subprocess.STDOUT,
"stdin": subprocess.PIPE,
"close_fds": True
}
try:
self.process_handle = Popen(args, **kwargs)
self.process_output(line_reader(self.process_handle.stdout))
self.process_handle.wait()
except OSError, ose:
if ose.errno == errno.ENOENT:
self.error = _("%(program)s does not exist.",
{"program": self.get_executable()})
else:
logging.exception("Exception in conversion loop: %s %s", args,
kwargs)
finally:
def transcode(self):
rc = True
try:
ffmpeg_exe = get_ffmpeg_executable_path()
kwargs = {"stdin": open(os.devnull, 'rb'),
"stdout": subprocess.PIPE,
"stderr": open(os.devnull, 'wb'),
"close_fds": True}
args = [ffmpeg_exe, "-i", self.media_file]
if self.time_offset:
logging.debug('transcode: start job @ %d' % self.time_offset)
args += TranscodeObject.time_offset_args + [
str(self.time_offset)]
video_needs_trancode = False
if self.has_video:
logging.debug('Video codec: %s', self.video_codec)
logging.debug('Video size: %s', self.video_size)
if video_can_copy(self.video_codec, self.video_size):
args += get_transcode_video_copy_options()
else:
args += get_transcode_video_options()
video_needs_transcode = True
if self.has_audio:
logging.debug('Audio codec: %s', self.audio_codec)
logging.debug('Audio sample rate: %s', self.audio_sample_rate)
if (valid_av_combo(self.video_codec, self.audio_codec) and
audio_can_copy(self.audio_codec, self.audio_sample_rate)):
args += get_transcode_audio_copy_options()
else:
args += get_transcode_audio_options()
else:
raise ValueError('no video or audio stream present')
args += TranscodeObject.output_args
logging.debug('Running command %s' % ' '.join(args))
self.ffmpeg_handle = Popen(args, **kwargs)
segmenter_exe = get_segmenter_executable_path()
args = [segmenter_exe]
address, port = self.sink.server_address
args += TranscodeObject.segmenter_args + [str(port)]
# Can't use close_fds here because we need to pass the fds to
# the child
kwargs = {"stdout": open(os.devnull, 'rb'),
"stdin": self.ffmpeg_handle.stdout,
"stderr": open(os.devnull, 'wb')}
logging.debug('Running command %s' % ' '.join(args))
self.segmenter_handle = Popen(args, **kwargs)
self.sink_thread = threading.Thread(target=thread_body,
args=[self.segmenter_consumer],
name="Segmenter Consumer")
self.sink_thread.daemon = True
self.sink_thread.start()
except StandardError:
(typ, value, tb) = sys.exc_info()
logging.error('ERROR: %s %s' % (str(typ), str(value)))
rc = False
self.transcode_gate.set()
return rc
def needs_transcode(media_file):
"""needs_transcode()
Returns (False, None) if no need to transcode.
Returns (True, info) if there is a need to transcode.
where info is
(duration, has_audio, has_video)
The duration is transmitted for transcoding purposes because we need to
build a m3u8 playlist and what we get out of the Miro database may be
unreliable (does not exist).
May throw exception if ffmpeg not found. Remember to catch."""
ffmpeg_exe = get_ffmpeg_executable_path()
kwargs = {"stdout": subprocess.PIPE,
"stderr": subprocess.PIPE,
"stdin": subprocess.PIPE,
"close_fds": True}
args = [ffmpeg_exe, "-i", media_file]
handle = Popen(args, **kwargs)
# XXX unbounded read here but should be okay, ffmpeg output is finite.
# note that we need to read from stderr, since that's what ffmpeg spits
# out.
text = handle.stderr.read()
# Initial determination based on the file type - need to drill down
# to see if the resolution, etc are within parameters.
if container_regex.search(text):
transcode = False
else:
transcode = True
# " Duration: XX:XX:XX.XX, ..."
match = duration_regex.search(text)
start, end = match.span()
duration_start = text[end:]
duration = duration_start[:duration_start.index(',')]
# Convert to seconds. We can't handle fractions of a second so
# skip over those bits.
hours, minutes, seconds = duration.split(':')
# Strip the fractional seconds. Always round up, then we won't miss
# any data.
seconds = int(float(seconds) + 0.5)
seconds += int(minutes) * 60
seconds += int(hours) * 3600
has_audio = has_audio_regex.search(text)
has_video = has_video_regex.search(text)
vcodec = acodec = size = sample_rate = None
vcopy = acopy = False
if has_video:
vcodec, size = get_video_parameters(has_video)
vcopy = video_can_copy(vcodec, size)
if not vcopy:
transcode = False
if has_audio:
acodec, sample_rate = get_audio_parameters(has_audio)
acopy = audio_can_copy(acodec, sample_rate)
if not acopy:
transcode = False
return (transcode, (seconds, has_audio, acodec, sample_rate,
has_video, vcodec, size))
class ConversionTask(object):
def __init__(self, converter_info, item_info, target_folder,
create_item):
self.item_info = item_info
self.converter_info = converter_info
self.input_path = item_info.filename
self.final_output_path, self.temp_output_path = build_output_paths(
item_info, target_folder, converter_info)
self.create_item = create_item
self.key = "%s->%s" % (self.input_path, self.final_output_path)
self.thread = None
self.duration = None
self.progress = 0
self.log_path = None
self.log_file = None
self.process_handle = None
self.error = None
self.start_time = time.time()
def get_executable(self):
raise NotImplementedError()
def get_parameters(self):
raise NotImplementedError()
def get_output_size_guess(self):
if self.item_info.duration and self.converter_info.bit_rate:
return self.converter_info.bit_rate * self.item_info.duration / 8
return self.item_info.size
def get_display_name(self):
return self.converter_info.displayname
def run(self):
logging.debug("temp_output_path: [%s] final_output_path: [%s]",
self.temp_output_path, self.final_output_path)
self.progress = 0
self.thread = threading.Thread(target=utils.thread_body,
args=[self._loop],
name="Conversion Task")
self.thread.setDaemon(True)
self.thread.start()
def get_eta(self):
"""Calculates the eta for this conversion to be completed.
:returns: None if progress is <= 0, otherwise returns number
of seconds until this is complete
"""
if self.progress <= 0:
return None
progress = self.progress * 100
duration = time.time() - self.start_time
time_per_percent = duration / progress
return int(time_per_percent * (100 - progress))
def is_pending(self):
return self.thread is None
def is_running(self):
return self.thread is not None and self.thread.isAlive()
def done_running(self):
return self.thread is not None and not self.thread.isAlive()
def is_finished(self):
return self.done_running() and not self.is_failed()
def is_failed(self):
return (self.error or
(self.process_handle is not None and
self.process_handle.returncode is not None and
self.process_handle.returncode != 0))
def _loop(self):
executable = self.get_executable()
args = self.get_parameters()
self._start_logging(executable, args)
args.insert(0, executable)
logging.debug("Conversion: (%s)", " ".join(args))
kwargs = {"bufsize": 1,
"stdout": subprocess.PIPE,
"stderr": subprocess.STDOUT,
"stdin": subprocess.PIPE,
"close_fds": True}
try:
self.process_handle = Popen(args, **kwargs)
self.process_output(line_reader(self.process_handle.stdout))
self.process_handle.wait()
except OSError, ose:
if ose.errno == errno.ENOENT:
self.error = _("%(program)s does not exist.",
{"program": self.get_executable()})
else:
logging.exception("Exception in conversion loop: %s %s",
args, kwargs)
finally:
