def _collect(self):
"""Polls for messages to collect data."""
start = time.time()
timeout_s = self._timeout_ms / float(1000)
tags = {}
while time.time() - start < timeout_s:
if not self._bus.have_pending():
continue
message = self._bus.pop()
if message.type == gst.MESSAGE_ERROR:
raise exceptions.ScannerError(
encoding.locale_decode(message.parse_error()[0]))
elif message.type == gst.MESSAGE_EOS:
return tags
elif message.type == gst.MESSAGE_ASYNC_DONE:
if message.src == self._pipe:
return tags
elif message.type == gst.MESSAGE_TAG:
# Taglists are not really dicts, hence the lack of .items() and
# explicit .keys. We only keep the last tag for each key, as we
# assume this is the best, some formats will produce multiple
# taglists. Lastly we force everything to lists for conformity.
taglist = message.parse_tag()
for key in taglist.keys():
value = taglist[key]
if not isinstance(value, list):
value = [value]
tags[key] = value
raise exceptions.ScannerError('Timeout after %dms' % self._timeout_ms)
def _collect(self):
"""Polls for messages to collect data."""
start = time.time()
timeout_s = self.timeout_ms / float(1000)
poll_timeout_ns = 1000
data = {}
while time.time() - start < timeout_s:
message = self.bus.poll(gst.MESSAGE_ANY, poll_timeout_ns)
if message is None:
pass # polling the bus timed out.
elif message.type == gst.MESSAGE_ERROR:
raise exceptions.ScannerError(message.parse_error()[0])
elif message.type == gst.MESSAGE_EOS:
return data
elif message.type == gst.MESSAGE_ASYNC_DONE:
if message.src == self.pipe:
return data
elif message.type == gst.MESSAGE_TAG:
taglist = message.parse_tag()
for key in taglist.keys():
data[key] = taglist[key]
raise exceptions.ScannerError('Timeout after %dms' % self.timeout_ms)
def scan(self, uri):
try:
info = self.discoverer.discover_uri(uri)
except gobject.GError as e:
# Loosing traceback is non-issue since this is from C code.
raise exceptions.ScannerError(e)
data = {}
audio_streams = info.get_audio_streams()
if not audio_streams:
raise exceptions.ScannerError('Did not find any audio streams.')
for stream in audio_streams:
taglist = stream.get_tags()
if not taglist:
continue
for key in taglist.keys():
# XXX: For some crazy reason some wma files spit out lists
# here, not sure if this is due to better data in headers or
# wma being stupid. So ugly hack for now :/
if type(taglist[key]) is list:
data[key] = taglist[key][0]
else:
data[key] = taglist[key]
# Never trust metadata for these fields:
data[b'uri'] = uri
data[b'duration'] = info.get_duration() // gst.MSECOND
if data[b'duration'] < 100:
raise exceptions.ScannerError(
'Rejecting file with less than 100ms audio data.')
return data
def _process(pipeline, timeout_ms):
bus = pipeline.get_bus()
tags = {}
mime = None
have_audio = False
missing_message = None
types = (Gst.MessageType.ELEMENT | Gst.MessageType.APPLICATION
| Gst.MessageType.ERROR | Gst.MessageType.EOS
| Gst.MessageType.ASYNC_DONE | Gst.MessageType.TAG)
timeout = timeout_ms
previous = int(time.time() * 1000)
while timeout > 0:
message = bus.timed_pop_filtered(timeout * Gst.MSECOND, types)
if message is None:
break
elif message.type == Gst.MessageType.ELEMENT:
if GstPbutils.is_missing_plugin_message(message):
missing_message = message
elif message.type == Gst.MessageType.APPLICATION:
if message.get_structure().get_name() == 'have-type':
mime = message.get_structure().get_value('caps').get_name()
if mime and (mime.startswith('text/')
or mime == 'application/xml'):
return tags, mime, have_audio
elif message.get_structure().get_name() == 'have-audio':
have_audio = True
elif message.type == Gst.MessageType.ERROR:
error = encoding.locale_decode(message.parse_error()[0])
if missing_message and not mime:
caps = missing_message.get_structure().get_value('detail')
mime = caps.get_structure(0).get_name()
return tags, mime, have_audio
raise exceptions.ScannerError(error)
elif message.type == Gst.MessageType.EOS:
return tags, mime, have_audio
elif message.type == Gst.MessageType.ASYNC_DONE:
if message.src == pipeline:
return tags, mime, have_audio
elif message.type == Gst.MessageType.TAG:
taglist = message.parse_tag()
# Note that this will only keep the last tag.
tags.update(tags_lib.convert_taglist(taglist))
now = int(time.time() * 1000)
timeout -= now - previous
previous = now
raise exceptions.ScannerError('Timeout after %dms' % timeout_ms)
def _process(pipeline, timeout_ms):
clock = pipeline.get_clock()
bus = pipeline.get_bus()
timeout = timeout_ms * gst.MSECOND
tags = {}
mime = None
have_audio = False
missing_message = None
types = (gst.MESSAGE_ELEMENT | gst.MESSAGE_APPLICATION | gst.MESSAGE_ERROR
| gst.MESSAGE_EOS | gst.MESSAGE_ASYNC_DONE | gst.MESSAGE_TAG)
previous = clock.get_time()
while timeout > 0:
message = bus.timed_pop_filtered(timeout, types)
if message is None:
break
elif message.type == gst.MESSAGE_ELEMENT:
if gst.pbutils.is_missing_plugin_message(message):
missing_message = message
elif message.type == gst.MESSAGE_APPLICATION:
if message.structure.get_name() == 'have-type':
mime = message.structure['caps'].get_name()
if mime.startswith('text/') or mime == 'application/xml':
return tags, mime, have_audio
elif message.structure.get_name() == 'have-audio':
have_audio = True
elif message.type == gst.MESSAGE_ERROR:
error = encoding.locale_decode(message.parse_error()[0])
if missing_message and not mime:
caps = missing_message.structure['detail']
mime = caps.get_structure(0).get_name()
return tags, mime, have_audio
raise exceptions.ScannerError(error)
elif message.type == gst.MESSAGE_EOS:
return tags, mime, have_audio
elif message.type == gst.MESSAGE_ASYNC_DONE:
if message.src == pipeline:
return tags, mime, have_audio
elif message.type == gst.MESSAGE_TAG:
taglist = message.parse_tag()
# Note that this will only keep the last tag.
tags.update(utils.convert_taglist(taglist))
now = clock.get_time()
timeout -= now - previous
previous = now
raise exceptions.ScannerError('Timeout after %dms' % timeout_ms)
def scan(self, uri):
"""
Scan the given uri collecting relevant metadata.
:param uri: URI of the resource to scan.
:type event: string
:return: Dictionary of tags, duration, mtime and uri information.
"""
try:
self._setup(uri)
tags = self._collect() # Ensure collect before queries.
data = {
'uri': uri,
'tags': tags,
'mtime': self._query_mtime(uri),
'duration': self._query_duration()
}
finally:
self._reset()
if self._min_duration_ms is None:
return data
elif data['duration'] >= self._min_duration_ms * gst.MSECOND:
return data
raise exceptions.ScannerError('Rejecting file with less than %dms '
'audio data.' % self._min_duration_ms)
def test_translate_uri_when_scanner_fails(scanner, provider, caplog):
scanner.scan.side_effect = exceptions.ScannerError('foo failed')
result = provider.translate_uri('bar')
assert result is None
assert 'Problem scanning URI bar: foo failed' in caplog.text()
def _setup_pipeline(uri, proxy_config=None):
src = Gst.Element.make_from_uri(Gst.URIType.SRC, uri)
if not src:
raise exceptions.ScannerError('GStreamer can not open: %s' % uri)
if proxy_config:
utils.setup_proxy(src, proxy_config)
signals = utils.Signals()
pipeline = Gst.ElementFactory.make('pipeline')
pipeline.add(src)
if _has_src_pads(src):
_setup_decodebin(src, src.get_static_pad('src'), pipeline, signals)
elif _has_dynamic_src_pad(src):
signals.connect(src, 'pad-added', _setup_decodebin, pipeline, signals)
else:
raise exceptions.ScannerError('No pads found in source element.')
return pipeline, signals
def test_scan_fails_and_playlist_parsing_fails(self, scanner, provider,
caplog):
scanner.scan.side_effect = exceptions.ScannerError('some failure')
responses.add(responses.GET,
STREAM_URI,
body=b'some audio data',
content_type='audio/mpeg')
result = provider.translate_uri(STREAM_URI)
assert 'Unwrapping stream from URI: %s' % STREAM_URI
assert ('GStreamer failed scanning URI (%s)' % STREAM_URI
in caplog.text())
assert (
'Failed parsing URI (%s) as playlist; found potential stream.' %
STREAM_URI in caplog.text())
assert result == STREAM_URI
def _setup_pipeline(uri, proxy_config=None):
src = gst.element_make_from_uri(gst.URI_SRC, uri)
if not src:
raise exceptions.ScannerError('GStreamer can not open: %s' % uri)
typefind = gst.element_factory_make('typefind')
decodebin = gst.element_factory_make('decodebin2')
pipeline = gst.element_factory_make('pipeline')
pipeline.add_many(src, typefind, decodebin)
gst.element_link_many(src, typefind, decodebin)
if proxy_config:
utils.setup_proxy(src, proxy_config)
typefind.connect('have-type', _have_type, decodebin)
decodebin.connect('pad-added', _pad_added, pipeline)
return pipeline
def scan(self, uri):
"""
Scan the given uri collecting relevant metadata.
:param uri: URI of the resource to scan.
:type event: string
:return: Dictionary of tags, duration, mtime and uri information.
"""
try:
self._setup(uri)
data = self._collect()
# Make sure uri and duration does not come from tags.
data[b'uri'] = uri
data[b'mtime'] = self._query_mtime(uri)
data[gst.TAG_DURATION] = self._query_duration()
finally:
self._reset()
if data[gst.TAG_DURATION] < self.min_duration_ms * gst.MSECOND:
raise exceptions.ScannerError('Rejecting file with less than %dms '
'audio data.' % self.min_duration_ms)
return data
def test_scan_fails_but_playlist_parsing_succeeds(self, scanner, provider,
caplog):
scanner.scan.side_effect = [
# Scanning playlist
exceptions.ScannerError('some failure'),
# Scanning stream
mock.Mock(mime='audio/mpeg', playable=True),
]
responses.add(responses.GET,
PLAYLIST_URI,
body=BODY,
content_type='audio/x-mpegurl')
result = provider.translate_uri(PLAYLIST_URI)
assert 'Unwrapping stream from URI: %s' % PLAYLIST_URI
assert ('GStreamer failed scanning URI (%s)' % PLAYLIST_URI
in caplog.text())
assert 'Parsed playlist (%s)' % PLAYLIST_URI in caplog.text()
assert ('Unwrapped potential audio/mpeg stream: %s' % STREAM_URI
in caplog.text())
assert result == STREAM_URI
def _setup_pipeline(uri, proxy_config=None):
src = Gst.Element.make_from_uri(Gst.URIType.SRC, uri)
if not src:
raise exceptions.ScannerError('GStreamer can not open: %s' % uri)
typefind = Gst.ElementFactory.make('typefind')
decodebin = Gst.ElementFactory.make('decodebin')
pipeline = Gst.ElementFactory.make('pipeline')
for e in (src, typefind, decodebin):
pipeline.add(e)
src.link(typefind)
typefind.link(decodebin)
if proxy_config:
utils.setup_proxy(src, proxy_config)
signals = utils.Signals()
signals.connect(typefind, 'have-type', _have_type, decodebin)
signals.connect(decodebin, 'pad-added', _pad_added, pipeline)
signals.connect(decodebin, 'autoplug-select', _autoplug_select)
return pipeline, signals
def _process(pipeline, timeout_ms):
bus = pipeline.get_bus()
tags = {}
mime = None
have_audio = False
missing_message = None
duration = None
types = (Gst.MessageType.ELEMENT | Gst.MessageType.APPLICATION
| Gst.MessageType.ERROR | Gst.MessageType.EOS
| Gst.MessageType.ASYNC_DONE | Gst.MessageType.DURATION_CHANGED
| Gst.MessageType.TAG)
timeout = timeout_ms
start = int(time.time() * 1000)
while timeout > 0:
msg = bus.timed_pop_filtered(timeout * Gst.MSECOND, types)
if msg is None:
break
if logger.isEnabledFor(log.TRACE_LOG_LEVEL) and msg.get_structure():
debug_text = msg.get_structure().to_string()
if len(debug_text) > 77:
debug_text = debug_text[:77] + '...'
_trace('element %s: %s', msg.src.get_name(), debug_text)
if msg.type == Gst.MessageType.ELEMENT:
if GstPbutils.is_missing_plugin_message(msg):
missing_message = msg
elif msg.type == Gst.MessageType.APPLICATION:
if msg.get_structure().get_name() == 'have-type':
mime = msg.get_structure().get_value('caps').get_name()
if mime and (mime.startswith('text/')
or mime == 'application/xml'):
return tags, mime, have_audio, duration
elif msg.get_structure().get_name() == 'have-audio':
have_audio = True
elif msg.type == Gst.MessageType.ERROR:
error = encoding.locale_decode(msg.parse_error()[0])
if missing_message and not mime:
caps = missing_message.get_structure().get_value('detail')
mime = caps.get_structure(0).get_name()
return tags, mime, have_audio, duration
raise exceptions.ScannerError(error)
elif msg.type == Gst.MessageType.EOS:
return tags, mime, have_audio, duration
elif msg.type == Gst.MessageType.ASYNC_DONE:
success, duration = _query_duration(pipeline)
if tags and success:
return tags, mime, have_audio, duration
# Don't try workaround for non-seekable sources such as mmssrc:
if not _query_seekable(pipeline):
return tags, mime, have_audio, duration
# Workaround for upstream bug which causes tags/duration to arrive
# after pre-roll. We get around this by starting to play the track
# and then waiting for a duration change.
# https://bugzilla.gnome.org/show_bug.cgi?id=763553
logger.debug('Using workaround for duration missing before play.')
result = pipeline.set_state(Gst.State.PLAYING)
if result == Gst.StateChangeReturn.FAILURE:
return tags, mime, have_audio, duration
elif msg.type == Gst.MessageType.DURATION_CHANGED and tags:
# VBR formats sometimes seem to not have a duration by the time we
# go back to paused. So just try to get it right away.
success, duration = _query_duration(pipeline)
pipeline.set_state(Gst.State.PAUSED)
if success:
return tags, mime, have_audio, duration
elif msg.type == Gst.MessageType.TAG:
taglist = msg.parse_tag()
# Note that this will only keep the last tag.
tags.update(tags_lib.convert_taglist(taglist))
timeout = timeout_ms - (int(time.time() * 1000) - start)
raise exceptions.ScannerError('Timeout after %dms' % timeout_ms)
def test_playlistlibraryprovider_scan_track_error(backend, scanner, caplog):
caplog.set_level(logging.DEBUG)
scanner.scan.side_effect = exceptions.ScannerError(message="error")
track = backend.library._scan_track("http://example.com/track1.mp3")
assert track is None
def test_simplifies_track_name_when_scan_fails(self):
scan.Scanner.scan = MagicMock(
side_effect=exceptions.ScannerError('error'))
prov = MplayerLibraryProvider(backend=None)
track = prov.lookup('mplayer:http://test_uri')[0]
self.assertEquals(track.name, 'test_uri')
def _process(pipeline, timeout_ms):
bus = pipeline.get_bus()
tags = {}
mime = None
have_audio = False
missing_message = None
duration = None
types = (Gst.MessageType.ELEMENT | Gst.MessageType.APPLICATION
| Gst.MessageType.ERROR | Gst.MessageType.EOS
| Gst.MessageType.ASYNC_DONE | Gst.MessageType.DURATION_CHANGED
| Gst.MessageType.TAG)
timeout = timeout_ms
start = int(time.time() * 1000)
while timeout > 0:
message = bus.timed_pop_filtered(timeout * Gst.MSECOND, types)
if message is None:
break
elif message.type == Gst.MessageType.ELEMENT:
if GstPbutils.is_missing_plugin_message(message):
missing_message = message
elif message.type == Gst.MessageType.APPLICATION:
if message.get_structure().get_name() == 'have-type':
mime = message.get_structure().get_value('caps').get_name()
if mime and (mime.startswith('text/')
or mime == 'application/xml'):
return tags, mime, have_audio, duration
elif message.get_structure().get_name() == 'have-audio':
have_audio = True
elif message.type == Gst.MessageType.ERROR:
error = encoding.locale_decode(message.parse_error()[0])
if missing_message and not mime:
caps = missing_message.get_structure().get_value('detail')
mime = caps.get_structure(0).get_name()
return tags, mime, have_audio, duration
raise exceptions.ScannerError(error)
elif message.type == Gst.MessageType.EOS:
return tags, mime, have_audio, duration
elif message.type == Gst.MessageType.ASYNC_DONE:
success, duration = _query_duration(pipeline)
if tags and success:
return tags, mime, have_audio, duration
# Workaround for upstream bug which causes tags/duration to arrive
# after pre-roll. We get around this by starting to play the track
# and then waiting for a duration change.
# https://bugzilla.gnome.org/show_bug.cgi?id=763553
result = pipeline.set_state(Gst.State.PLAYING)
if result == Gst.StateChangeReturn.FAILURE:
return tags, mime, have_audio, duration
elif message.type == Gst.MessageType.DURATION_CHANGED:
# duration will be read after ASYNC_DONE received; for now
# just give it a non-None value to flag that we have a duration:
duration = 0
elif message.type == Gst.MessageType.TAG:
taglist = message.parse_tag()
# Note that this will only keep the last tag.
tags.update(tags_lib.convert_taglist(taglist))
timeout = timeout_ms - (int(time.time() * 1000) - start)
# workaround for https://bugzilla.gnome.org/show_bug.cgi?id=763553:
# if we got what we want then stop playing (and wait for ASYNC_DONE)
if tags and duration is not None:
pipeline.set_state(Gst.State.PAUSED)
raise exceptions.ScannerError('Timeout after %dms' % timeout_ms)
