def lookup(self, uri: str) -> models.Track: song_blob = self._songs.get_blob_client(blob_for_uri(uri)) song_etag = song_blob.get_blob_properties().etag public_uri = self.backend.get_public_uri_for(uri) try: cached_data = self._get_cached_metadata(etag=song_etag, song_uri=uri) if cached_data is not None: (song_tags, duration) = cached_data else: result = self._scanner.scan(public_uri) song_tags = result.tags duration = result.duration track = tags.convert_tags_to_track(song_tags).replace( uri=uri, length=duration) if cached_data is None: self._store_cached_metadata(etag=song_etag, song_uri=uri, song_tags=song_tags, duration=duration) except exceptions.ScannerError as e: logger.warning("Failed looking up %s at %s: %s", uri, public_uri, e) track = models.Track(uri=uri) return [track]
def on_receive(self, message): file_uri = message['file_uri'] file_producer = message['file_producer'] try: result = self._scanner.scan(uri=file_producer().download_link) logger.debug('Tagging result for file %s: %s', file_uri, result) if result.playable: track = convert_tags_to_track(result.tags).replace( uri=file_uri, length=result.duration) self._parent_ref.tell({'file_uri': file_uri, 'track': track}) except ScannerError as e: logger.debug('Couldn\'t get track info for file %s: %s', file_uri, e)
def _scan_metadata( self, *, media_dir, file_mtimes, files, library, timeout, flush_threshold, limit ): logger.info("Scanning...") files = sorted(files)[:limit] scanner = scan.Scanner(timeout) progress = _ScanProgress(batch_size=flush_threshold, total=len(files)) for absolute_path in files: try: file_uri = absolute_path.as_uri() result = scanner.scan(file_uri) if not result.playable: logger.warning( f"Failed scanning {file_uri}: No audio found in file" ) elif result.duration is None: logger.warning( f"Failed scanning {file_uri}: " "No duration information found in file" ) elif result.duration < MIN_DURATION_MS: logger.warning( f"Failed scanning {file_uri}: " f"Track shorter than {MIN_DURATION_MS}ms" ) else: local_uri = translator.path_to_local_track_uri( absolute_path, media_dir ) mtime = file_mtimes.get(absolute_path) track = tags.convert_tags_to_track(result.tags).replace( uri=local_uri, length=result.duration, last_modified=mtime ) library.add(track, result.tags, result.duration) logger.debug(f"Added {track.uri}") except Exception as error: logger.warning(f"Failed scanning {file_uri}: {error}") if progress.increment(): progress.log() if library.flush(): logger.debug("Progress flushed") progress.log() logger.info("Done scanning")
def lookup(self, uri): logger.debug("Looking up file URI: %s", uri) local_path = path.uri_to_path(uri) try: result = self._scanner.scan(uri) track = tags.convert_tags_to_track(result.tags).replace( uri=uri, length=result.duration) except exceptions.ScannerError as e: logger.warning("Failed looking up %s: %s", uri, e) track = models.Track(uri=uri) if not track.name: track = track.replace(name=local_path.name) return [track]
def lookup(self, uri): if urllib.parse.urlsplit(uri).scheme not in self.backend.uri_schemes: return [] if self._blacklist_re.match(uri): logger.debug('URI matched metadata lookup blacklist: %s', uri) return [Track(uri=uri)] try: result = self._scanner.scan(uri) track = tags.convert_tags_to_track(result.tags).replace( uri=uri, length=result.duration) except exceptions.ScannerError as e: logger.warning('Problem looking up %s: %s', uri, e) track = Track(uri=uri) return [track]
def lookup(self, uri): logger.debug('Looking up file URI: %s', uri) local_path = path.uri_to_path(uri) try: result = self._scanner.scan(uri) track = tags.convert_tags_to_track(result.tags).copy( uri=uri, length=result.duration) except exceptions.ScannerError as e: logger.warning('Failed looking up %s: %s', uri, e) track = models.Track(uri=uri) if not track.name: filename = os.path.basename(local_path) name = urllib2.unquote(filename).decode(FS_ENCODING, 'replace') track = track.copy(name=name) return [track]
def lookup(self, uri): if urllib.parse.urlsplit(uri).scheme not in self.backend.uri_schemes: return [] if self.backend._blacklist_re.match(uri): logger.debug("URI matched metadata lookup blacklist: %s", uri) return [Track(uri=uri)] _, scan_result = _unwrap_stream( uri, timeout=self.backend._timeout, scanner=self.backend._scanner, requests_session=self.backend._session ) if scan_result: track = tags.convert_tags_to_track(scan_result.tags).replace(uri=uri, length=scan_result.duration) else: logger.warning("Problem looking up %s", uri) track = Track(uri=uri) return [track]
def lookup(self, uri): if urllib.parse.urlsplit(uri).scheme not in self.backend.uri_schemes: return [] if self.backend._blacklist_re.match(uri): logger.debug('URI matched metadata lookup blacklist: %s', uri) return [Track(uri=uri)] _, scan_result = _unwrap_stream( uri, timeout=self.backend._timeout, scanner=self.backend._scanner, requests_session=self.backend._session) if scan_result: track = tags.convert_tags_to_track(scan_result.tags).replace( uri=uri, length=scan_result.duration) else: logger.warning('Problem looking up %s', uri) track = Track(uri=uri) return [track]
def _scan_track(self, uri): """Identify playable tracks.""" try: scan_result = self.backend.scanner.scan( uri, timeout=self.backend.timeout) except exceptions.ScannerError as exc: logger.warning( "GStreamer failed scanning URI (%s): %s %s", uri, exc, self.backend.timeout, ) return has_interesting_mime = ( scan_result.mime is not None and not scan_result.mime.startswith("text/") and not scan_result.mime.startswith("application/")) if scan_result.playable or has_interesting_mime: logger.debug( "Unwrapped potential %s stream: %s", scan_result.mime, uri, ) name = os.path.basename(urllib.parse.urlparse(uri).path) track = tags.convert_tags_to_track(scan_result.tags).replace( uri=uri, length=scan_result.duration) if not track.name: track = track.replace(name=name) return track else: logger.warning( "Unwrapped potential %s non-playable stream: %s", scan_result.mime, uri, ) return
def lookup(self, uri): local_path = mpath.uri_to_path(uri) # check if it's playlist if uri.endswith(('.m3u', '.m3u8')): tracks = [] logger.debug('Medialib... we have playlist: %s', uri) track = models.Track(uri=uri) items = self.backend.playlists.get_items(uri) logger.debug(str(items)) if not items: logger.warn("Playlist '%s' does not exist", str(uri)) return None for i in items: logger.debug('we will add to tracklist: ' + str(i)) if i.type == models.Ref.TRACK: tracks.append( models.Track(uri=i.uri, name=i.name) ) return tracks elif uri.endswith(('.opml')): logger.debug('Medialib... we have opml: %s', uri) tracks = feeds.get_tracks_form_opml(uri) return tracks else: track = None try: result = self._scanner.scan(uri) track = tags.convert_tags_to_track(result.tags).copy( uri=uri, length=result.duration) except exceptions.ScannerError as e: logger.warning('Failed looking up %s: %s', uri, e) track = models.Track(uri=uri) if not track.name: filename = os.path.basename(local_path) name = urllib2.unquote(filename).decode(FS_ENCODING, 'replace') track = track.copy(name=name) return [track]
def run(self, args, config): media_dir = config['local']['media_dir'] scan_timeout = config['local']['scan_timeout'] flush_threshold = config['local']['scan_flush_threshold'] excluded_file_extensions = config['local']['excluded_file_extensions'] excluded_file_extensions = tuple( bytes(file_ext.lower()) for file_ext in excluded_file_extensions) library = _get_library(args, config) if library is None: return 1 file_mtimes, file_errors = path.find_mtimes( media_dir, follow=config['local']['scan_follow_symlinks']) logger.info('Found %d files in media_dir.', len(file_mtimes)) if file_errors: logger.warning('Encountered %d errors while scanning media_dir.', len(file_errors)) for name in file_errors: logger.debug('Scan error %r for %r', file_errors[name], name) num_tracks = library.load() logger.info('Checking %d tracks from library.', num_tracks) uris_to_update = set() uris_to_remove = set() uris_in_library = set() for track in library.begin(): abspath = translator.local_track_uri_to_path(track.uri, media_dir) mtime = file_mtimes.get(abspath) if mtime is None: logger.debug('Missing file %s', track.uri) uris_to_remove.add(track.uri) elif mtime > track.last_modified or args.force: uris_to_update.add(track.uri) uris_in_library.add(track.uri) logger.info('Removing %d missing tracks.', len(uris_to_remove)) for uri in uris_to_remove: library.remove(uri) for abspath in file_mtimes: relpath = os.path.relpath(abspath, media_dir) uri = translator.path_to_local_track_uri(relpath) if b'/.' in relpath or relpath.startswith(b'.'): logger.debug('Skipped %s: Hidden directory/file.', uri) elif relpath.lower().endswith(excluded_file_extensions): logger.debug('Skipped %s: File extension excluded.', uri) elif uri not in uris_in_library: uris_to_update.add(uri) logger.info( 'Found %d tracks which need to be updated.', len(uris_to_update)) logger.info('Scanning...') uris_to_update = sorted(uris_to_update, key=lambda v: v.lower()) uris_to_update = uris_to_update[:args.limit] scanner = scan.Scanner(scan_timeout) progress = _Progress(flush_threshold, len(uris_to_update)) for uri in uris_to_update: try: relpath = translator.local_track_uri_to_path(uri, media_dir) file_uri = path.path_to_uri(os.path.join(media_dir, relpath)) result = scanner.scan(file_uri) if not result.playable: logger.warning('Failed %s: No audio found in file.', uri) elif result.duration < MIN_DURATION_MS: logger.warning('Failed %s: Track shorter than %dms', uri, MIN_DURATION_MS) else: mtime = file_mtimes.get(os.path.join(media_dir, relpath)) track = tags.convert_tags_to_track(result.tags).replace( uri=uri, length=result.duration, last_modified=mtime) if library.add_supports_tags_and_duration: library.add( track, tags=result.tags, duration=result.duration) else: library.add(track) logger.debug('Added %s', track.uri) except exceptions.ScannerError as error: logger.warning('Failed %s: %s', uri, error) if progress.increment(): progress.log() if library.flush(): logger.debug('Progress flushed.') progress.log() library.close() logger.info('Done scanning.') return 0
def check(self, expected): actual = tags.convert_tags_to_track(self.tags) self.assertEqual(expected, actual)
def check(self, expected): actual = tags.convert_tags_to_track(self.tags) assert expected == actual
def run(self, args, config): media_dir = config['local']['media_dir'] scan_timeout = config['local']['scan_timeout'] flush_threshold = config['local']['scan_flush_threshold'] excluded_file_extensions = config['local']['excluded_file_extensions'] excluded_file_extensions = tuple( bytes(file_ext.lower()) for file_ext in excluded_file_extensions) library = _get_library(args, config) if library is None: return 1 file_mtimes, file_errors = path.find_mtimes( media_dir, follow=config['local']['scan_follow_symlinks']) logger.info('Found %d files in media_dir.', len(file_mtimes)) if file_errors: logger.warning('Encountered %d errors while scanning media_dir.', len(file_errors)) for name in file_errors: logger.debug('Scan error %r for %r', file_errors[name], name) num_tracks = library.load() logger.info('Checking %d tracks from library.', num_tracks) uris_to_update = set() uris_to_remove = set() uris_in_library = set() for track in library.begin(): abspath = translator.local_track_uri_to_path(track.uri, media_dir) mtime = file_mtimes.get(abspath) if mtime is None: logger.debug('Missing file %s', track.uri) uris_to_remove.add(track.uri) elif mtime > track.last_modified or args.force: uris_to_update.add(track.uri) uris_in_library.add(track.uri) logger.info('Removing %d missing tracks.', len(uris_to_remove)) for uri in uris_to_remove: library.remove(uri) for abspath in file_mtimes: relpath = os.path.relpath(abspath, media_dir) uri = translator.path_to_local_track_uri(relpath) if b'/.' in relpath: logger.debug('Skipped %s: Hidden directory/file.', uri) elif relpath.lower().endswith(excluded_file_extensions): logger.debug('Skipped %s: File extension excluded.', uri) elif uri not in uris_in_library: uris_to_update.add(uri) logger.info('Found %d tracks which need to be updated.', len(uris_to_update)) logger.info('Scanning...') uris_to_update = sorted(uris_to_update, key=lambda v: v.lower()) uris_to_update = uris_to_update[:args.limit] scanner = scan.Scanner(scan_timeout) progress = _Progress(flush_threshold, len(uris_to_update)) for uri in uris_to_update: try: relpath = translator.local_track_uri_to_path(uri, media_dir) file_uri = path.path_to_uri(os.path.join(media_dir, relpath)) result = scanner.scan(file_uri) if not result.playable: logger.warning('Failed %s: No audio found in file.', uri) elif result.duration < MIN_DURATION_MS: logger.warning('Failed %s: Track shorter than %dms', uri, MIN_DURATION_MS) else: mtime = file_mtimes.get(os.path.join(media_dir, relpath)) track = tags.convert_tags_to_track(result.tags).replace( uri=uri, length=result.duration, last_modified=mtime) if library.add_supports_tags_and_duration: library.add(track, tags=result.tags, duration=result.duration) else: library.add(track) logger.debug('Added %s', track.uri) except exceptions.ScannerError as error: logger.warning('Failed %s: %s', uri, error) if progress.increment(): progress.log() if library.flush(): logger.debug('Progress flushed.') progress.log() library.close() logger.info('Done scanning.') return 0