def scan(self, paths, exclude=[], cofuncid=None):
def need_yield(last_yield=[0]):
current = time.time()
if abs(current - last_yield[0]) > 0.015:
last_yield[0] = current
return True
return False
def need_added(last_added=[0]):
current = time.time()
if abs(current - last_added[0]) > 1.0:
last_added[0] = current
return True
return False
# first scan each path for new files
paths_to_load = []
for scan_path in paths:
print_d("Scanning %r." % scan_path)
desc = _("Scanning %s") % (fsn2text(unexpand(scan_path)))
with Task(_("Library"), desc) as task:
if cofuncid:
task.copool(cofuncid)
for real_path in iter_paths(scan_path, exclude=exclude):
if need_yield():
task.pulse()
yield
# skip unknown file extensions
if not formats.filter(real_path):
continue
# already loaded
if self.contains_filename(real_path):
continue
paths_to_load.append(real_path)
yield
# then (try to) load all new files
with Task(_("Library"), _("Loading files")) as task:
if cofuncid:
task.copool(cofuncid)
added = []
for real_path in task.gen(paths_to_load):
item = self.add_filename(real_path, False)
if item is not None:
added.append(item)
if len(added) > 100 or need_added():
self.add(added)
added = []
yield
if added and need_yield():
yield
if added:
self.add(added)
added = []
yield True
def rebuild(self,
paths,
force=False,
exclude=[],
scan_dots=False,
cofuncid=None):
"""Reload or remove songs if they have changed or been deleted.
This generator rebuilds the library over the course of iteration.
Any paths given will be scanned for new files, using the 'scan'
method.
Only items present in the library when the rebuild is started
will be checked.
If this function is copooled, set "cofuncid" to enable pause/stop
buttons in the UI.
"""
print_d("Rebuilding, force is %s." % force, self)
task = Task(_("Library"), _("Checking mount points"))
if cofuncid:
task.copool(cofuncid)
for i, (point, items) in task.list(enumerate(self._masked.items())):
if os.path.ismount(point):
self._contents.update(items)
del (self._masked[point])
self.emit('added', items.values())
yield True
task = Task(_("Library"), _("Scanning library"))
if cofuncid:
task.copool(cofuncid)
changed, removed = set(), set()
for i, (key, item) in task.list(enumerate(sorted(self.items()))):
if key in self._contents and force or not item.valid():
self.reload(item, changed, removed)
# These numbers are pretty empirical. We should yield more
# often than we emit signals; that way the main loop stays
# interactive and doesn't get bogged down in updates.
if len(changed) > 100:
self.emit('changed', changed)
changed = set()
if len(removed) > 100:
self.emit('removed', removed)
removed = set()
if len(changed) > 5 or i % 100 == 0:
yield True
print_d("Removing %d, changing %d." % (len(removed), len(changed)),
self)
if removed:
self.emit('removed', removed)
if changed:
self.emit('changed', changed)
for value in self.scan(paths, exclude, scan_dots, cofuncid):
yield value
def __set_inhibit_play(self, inhibit):
"""Change the inhibit state"""
if inhibit == self._inhibit_play:
return
self._inhibit_play = inhibit
# task management
if inhibit:
if not self._task:
def stop_buf(*args):
self._player.paused = True
self._task = Task(_("Stream"), _("Buffering"), stop=stop_buf)
elif self._task:
self._task.finish()
self._task = None
# state management
if inhibit:
# save the current state
status, state, pending = self.bin.get_state(
timeout=STATE_CHANGE_TIMEOUT)
if status == Gst.StateChangeReturn.SUCCESS and \
state == Gst.State.PLAYING:
self._wanted_state = state
else:
# no idea, at least don't play
self._wanted_state = Gst.State.PAUSED
self.bin.set_state(Gst.State.PAUSED)
else:
# restore the old state
self.bin.set_state(self._wanted_state)
self._wanted_state = None
def move_root(self,
old_root: str,
new_root: fsnative,
write_files: bool = True) -> Generator[None, None, None]:
"""
Move the root for all songs in a given (scan) directory.
We avoid dereferencing the destination, to allow users things like:
1. Symlink new_path -> old_root
2. Move QL root to new_path
3. Remove symlink
4. Move audio files: old_root -> new_path
"""
# TODO: only move primary library
old_path = Path(normalize_path(old_root, canonicalise=True)).expanduser()
new_path = Path(normalize_path(new_root)).expanduser()
if not old_path.is_dir():
print_w(f"Source dir {str(old_path)!r} doesn't exist, assuming that's OK",
self._name)
if not new_path.is_dir():
raise ValueError(f"Destination {new_path!r} is not a directory")
print_d(f"Checking entire library for {str(old_path)!r}", self._name)
missing: Set[AudioFile] = set()
changed = set()
total = len(self)
if not total:
return
with Task(_("Library"), _("Moving library files")) as task:
yield
for i, song in enumerate(list(self.values())):
task.update(i / total)
key = normalize_path(song.key)
path = Path(key)
if old_path in path.parents:
# TODO: more Pathlib-friendly dir replacement...
new_key = key.replace(str(old_path), str(new_path), 1)
new_key = normalize_path(new_key, canonicalise=False)
if new_key == key:
print_w(f"Substitution failed for {key!r}", self._name)
# We need to update ~filename and ~mountpoint
song.sanitize()
if write_files:
song.write()
if self.move_song(song, new_key):
changed.add(song)
else:
missing.add(song)
elif not (i % 1000):
print_d(f"Not moved, for example: {key!r}", self._name)
if not i % 100:
yield
self.changed(changed)
if missing:
print_w(f"Couldn't find {len(list(missing))} files: {missing}",
self._name)
yield
self.save()
print_d(f"Done moving {len(changed)} track(s) (of {total}) "
f"to {str(new_path)!r}.", self._name)
def remove_roots(self,
old_roots: Iterable[str]) -> Generator[None, None, None]:
"""Remove library roots (scandirs) entirely, and all their songs"""
old_paths = [
Path(normalize_path(root, canonicalise=True)).expanduser()
for root in old_roots
]
total = len(self)
removed = set()
print_d(
f"Removing library roots {old_roots} from {self._name} library")
yield
with Task(_("Library"), _("Removing library files")) as task:
for i, song in enumerate(list(self.values())):
task.update(i / total)
key = normalize_path(song.key)
song_path = Path(key)
if any(path in song_path.parents for path in old_paths):
removed.add(song)
if not i % 100:
yield
if removed:
self.remove(removed)
else:
print_d(f"No tracks in {old_roots} to remove from {self._name}")
def plugin_playlist(self, playlist):
# TODO - only get coordinator nodes, somehow
self.device: SoCo = soco.discovery.any_soco()
device = self.device
if not device:
qltk.ErrorMessage(
app.window, _("Error finding Sonos device(s)"),
_("Error finding Sonos. Please check settings")).run()
else:
sonos_pls = device.get_sonos_playlists()
pl_id_to_name = {pl.item_id: pl.title for pl in sonos_pls}
print_d("Sonos playlists: %s" % pl_id_to_name)
ret = GetSonosPlaylistDialog(pl_id_to_name).run(playlist.name)
if ret:
spl_id, name = ret
if spl_id:
spl: DidlPlaylistContainer = next(s for s in sonos_pls
if s.item_id == spl_id)
print_w(f"Replacing existing Sonos playlist {spl!r}")
device.remove_sonos_playlist(spl)
print_d(f"Creating new playlist {name!r}")
spl = device.create_sonos_playlist(name)
task = Task("Sonos",
_("Export to Sonos playlist"),
stop=self.__cancel_add)
copool.add(self.__add_songs,
task,
playlist.songs,
spl,
funcid="sonos-playlist-save")
def test_multiple_tasks(self):
self.assertEquals(self.c.active_tasks, [])
self.assertNotEqual(self.c.source, "")
t1 = Task("src", "desc", controller=self.c)
self.assertEquals(self.c.source, "src")
self.assertEquals(self.c.active_tasks, [t1])
t1.update(0.5)
self.assertEquals(self.c.frac, 0.5)
t2 = Task("src2", "desc2", controller=self.c)
self.assertEquals(self.c.source, _("Active tasks"))
self.assertEquals(self.c.frac, 0.25)
Task("src3", "desc3", controller=self.c, known_length=False)
self.assertAlmostEqual(self.c.frac, 0.5 / 3)
t1.finish()
t2.finish()
self.assertEquals(self.c.desc, "desc3")
self.assertEquals(self.c.frac, None)
def check_songs():
with Task(_("Refresh songs"), _("%d songs") % len(songs)) as task:
task.copool(check_songs)
for i, song in enumerate(songs):
song = song._song
if song in app.library:
app.library.reload(song)
task.update((float(i) + 1) / len(songs))
yield
def check_songs():
desc = numeric_phrase("%d song", "%d songs", len(songs))
with Task(_("Rescan songs"), desc) as task:
task.copool(check_songs)
for i, song in enumerate(songs):
song = song._song
if song in app.library:
app.library.reload(song)
task.update((float(i) + 1) / len(songs))
yield
def scan(self, paths, exclude=[], cofuncid=None):
added = []
exclude = [expanduser(path) for path in exclude if path]
def need_yield(last_yield=[0]):
current = time.time()
if abs(current - last_yield[0]) > 0.015:
last_yield[0] = current
return True
return False
def need_added(last_added=[0]):
current = time.time()
if abs(current - last_added[0]) > 1.0:
last_added[0] = current
return True
return False
for fullpath in paths:
print_d("Scanning %r." % fullpath, self)
desc = _("Scanning %s") % (unexpand(fsdecode(fullpath)))
with Task(_("Library"), desc) as task:
if cofuncid:
task.copool(cofuncid)
fullpath = expanduser(fullpath)
if filter(fullpath.startswith, exclude):
continue
for path, dnames, fnames in os.walk(fullpath):
for filename in fnames:
fullfilename = os.path.join(path, filename)
if filter(fullfilename.startswith, exclude):
continue
if fullfilename not in self._contents:
fullfilename = os.path.realpath(fullfilename)
# skip unknown file extensions
if not formats.filter(fullfilename):
continue
if filter(fullfilename.startswith, exclude):
continue
if fullfilename not in self._contents:
item = self.add_filename(fullfilename, False)
if item is not None:
added.append(item)
if len(added) > 100 or need_added():
self.add(added)
added = []
task.pulse()
yield
if added and need_yield():
yield
if added:
self.add(added)
added = []
task.pulse()
yield True
def _get_stream_urls(self, songs):
# Pre-cache. It's horrible, but at least you can play things immediately
with Task(_("Soundcloud"), "Pre-fetching stream URLs") as task:
total = len(songs)
for i, song in enumerate(songs):
# Only update ones without streaming URLs
# TODO: But yes these will time out...
if "~uri" not in song or "api.soundcloud.com" in song["~uri"]:
self.client.get_stream_url(song)
task.update(i / total)
yield
def plugin_playlist(self, playlist):
pattern_text = CONFIG.default_pattern
dialog = ExportToFolderDialog(self.plugin_window, pattern_text)
if dialog.run() == Gtk.ResponseType.OK:
directory = dialog.directory_chooser.get_filename()
pattern = FileFromPattern(dialog.pattern_entry.get_text())
task = Task("Export", _("Export Playlist to Folder"),
stop=self.__cancel_copy)
copool.add(self.__copy_songs, task,
playlist.songs, directory, pattern,
funcid="export-playlist-folder")
dialog.destroy()
def download_taglist(callback, cofuncid, step=1024 * 10):
"""Generator for loading the bz2 compressed tag list.
Calls callback with the decompressed data or None in case of
an error."""
with Task(_("Internet Radio"), _("Downloading station list")) as task:
if cofuncid:
task.copool(cofuncid)
try:
response = urlopen(STATION_LIST_URL)
except (EnvironmentError, HTTPException) as e:
print_e("Failed fetching from %s" % STATION_LIST_URL, e)
GLib.idle_add(callback, None)
return
try:
size = int(response.info().get("content-length", 0))
except ValueError:
size = 0
decomp = bz2.BZ2Decompressor()
data = b""
temp = b""
read = 0
while temp or not data:
read += len(temp)
if size:
task.update(float(read) / size)
else:
task.pulse()
yield True
try:
data += decomp.decompress(temp)
temp = response.read(step)
except (IOError, EOFError):
data = None
break
response.close()
yield True
stations = None
if data:
stations = parse_taglist(data)
GLib.idle_add(callback, stations)
def plugin_playlist(self, playlist):
self.init_server()
if not self.server.is_connected:
qltk.ErrorMessage(
None,
_("Error finding Squeezebox server"),
_("Error finding %s. Please check settings") %
self.server.config
).run()
else:
name = self.__get_playlist_name(name=playlist.name)
if name:
task = Task("Squeezebox", _("Export to Squeezebox playlist"),
stop=self.__cancel_add)
copool.add(self.__add_songs, task, playlist.songs, name,
funcid="squeezebox-playlist-save")
def _resume_after_delay(delay, refresh_rate=20):
if delay <= 0:
return
app.player.paused = True
delay_timer = GLib.timeout_add(1000 * delay, app.player.play)
task = Task(_("Shuffle by Grouping"),
_("Waiting to start new group…"),
stop=lambda: GLib.source_remove(delay_timer))
def countdown():
for i in range(int(refresh_rate * delay)):
task.update(i / (refresh_rate * delay))
yield True
task.finish()
yield False
GLib.timeout_add(1000 / refresh_rate, next, countdown())
def _get_stations_from(uri: str,
on_done: Callable[[Iterable[IRFile], str], None])\
-> None:
"""Fetches the URI content and extracts IRFiles
Called from thread - so no direct GTK+ interaction
:param uri: URI of station
:param on_done: a callback taking files when done (or none if errored)
"""
with Task(_("Internet Radio"), _("Add stations")) as task:
irfs: Collection[IRFile] = []
GLib.idle_add(task.pulse)
if (uri.lower().endswith(".pls")
or uri.lower().endswith(".m3u")
or uri.lower().endswith(".m3u8")):
if not re.match('^([^/:]+)://', uri):
# Assume HTTP if no protocol given. See #2731
uri = 'http://' + uri
print_d("Assuming http: %s" % uri)
# Error handling outside
sock = None
GLib.idle_add(task.pulse)
_fn, ext = splitext(uri.lower())
try:
sock = urlopen(uri, timeout=6)
if ext == ".pls":
irfs = parse_pls(sock)
elif ext in (".m3u", ".m3u8"):
irfs = parse_m3u(sock)
GLib.idle_add(task.pulse)
except IOError as e:
print_e(f"Couldn't download from {uri} ({e})")
finally:
if sock:
sock.close()
else:
try:
irfs = [IRFile(uri)]
except ValueError as e:
print_e("Can't add URI %s" % uri, e)
on_done(irfs, uri)
def watching_producer():
# TODO: integrate this better with scanning.
for fullpath in paths:
desc = _("Adding watches for %s") % (fsn2text(unexpand(fullpath)))
with Task(_("Library"), desc) as task:
normalised = Path(normalize_path(fullpath, True)).expanduser()
if any(Path(exclude) in normalised.parents
for exclude in exclude_dirs):
continue
unpulsed = 0
self.monitor_dir(normalised)
for path, dirs, files in os.walk(normalised):
normalised = Path(normalize_path(path, True))
for d in dirs:
self.monitor_dir(normalised / d)
unpulsed += len(dirs)
if unpulsed > 50:
task.pulse()
unpulsed = 0
yield
def emit_signal(songs, signal="changed", block_size=50, name=None,
cofuncid=None):
"""
A generator that signals `signal` on the library
in blocks of `block_size`. Useful for copools.
"""
i = 0
with Task(_("Library"), name or signal) as task:
if cofuncid:
task.copool(cofuncid)
total = len(songs)
while i < total:
more = songs[i:i + block_size]
if not more:
return
if 0 == ((i / block_size) % 10):
print_d("Signalling '%s' (%d/%d songs)"
% (signal, i, total))
task.update(float(i) / total)
app.library.emit(signal, more)
i += block_size
yield
def scan(self, paths, exclude=[], cofuncid=None):
added = []
exclude = [expanduser(path) for path in exclude if path]
for fullpath in paths:
print_d("Scanning %r." % fullpath, self)
desc = _("Scanning %s") % (unexpand(fsdecode(fullpath)))
with Task(_("Library"), desc) as task:
if cofuncid:
task.copool(cofuncid)
fullpath = expanduser(fullpath)
if filter(fullpath.startswith, exclude):
continue
for path, dnames, fnames in os.walk(util.fsnative(fullpath)):
for filename in fnames:
fullfilename = os.path.join(path, filename)
if filter(fullfilename.startswith, exclude):
continue
if fullfilename not in self._contents:
fullfilename = os.path.realpath(fullfilename)
# skip unknown file extensions
if not formats.filter(fullfilename):
continue
if filter(fullfilename.startswith, exclude):
continue
if fullfilename not in self._contents:
item = self.add_filename(fullfilename, False)
if item is not None:
added.append(item)
if len(added) > 20:
self.add(added)
added = []
task.pulse()
yield True
if added:
self.add(added)
added = []
task.pulse()
yield True
def __init__(self, songs: Collection[AudioFile], task=None) -> None:
super().__init__()
self.songs = songs
self.successful: Set[AudioFile] = set()
self.failed: Set[AudioFile] = set()
self.task = task or Task(_("Browser"), _("Downloading files"))
def search_cover(self, cancellable, songs):
"""Search for all the covers applicable to `songs` across all providers
Every successful image result emits a 'covers-found' signal
(unless cancelled)."""
sources = [source for source in self.sources if not source.embedded]
processed = {}
all_groups = {}
task = Task(_("Cover Art"),
_("Querying album art providers"),
stop=cancellable.cancel)
def finished(provider, success):
processed[provider] = success
total = self._total_groupings(all_groups)
frac = len(processed) / total
print_d("%s is finished: %d / %d" %
(provider, len(processed), total))
task.update(frac)
if frac >= 1:
task.finish()
self.emit('searches-complete', processed)
def search_complete(provider, results):
name = provider.name
if not results:
print_d('No covers from {0}'.format(name))
finished(provider, False)
return
finished(provider, True)
if not (cancellable and cancellable.is_cancelled()):
covers = {
CoverData(url=res['cover'],
source=name,
dimensions=res.get('dimensions', None))
for res in results
}
self.emit('covers-found', provider, covers)
provider.disconnect_by_func(search_complete)
def failure(provider, result):
finished(provider, False)
name = provider.__class__.__name__
print_d('Failed to get cover from {name} ({msg})'.format(
name=name, msg=result))
provider.disconnect_by_func(failure)
def song_groups(songs, sources):
all_groups = {}
for plugin in sources:
groups = {}
for song in songs:
group = plugin.group_by(song) or ''
groups.setdefault(group, []).append(song)
all_groups[plugin] = groups
return all_groups
all_groups = song_groups(songs, sources)
print_d("Got %d plugin groupings" % self._total_groupings(all_groups))
for plugin, groups in all_groups.items():
print_d("Getting covers from %s" % plugin)
for key, group in sorted(groups.items()):
song = sorted(group, key=lambda s: s.key)[0]
artists = {s.comma('artist') for s in group}
if len(artists) > 1:
print_d("%d artist groups in %s - probably a compilation. "
"Using provider to search for compilation" %
(len(artists), key))
song = AudioFile(song)
try:
del song['artist']
except KeyError:
# Artist(s) from other grouped songs, never mind.
pass
provider = plugin(song)
provider.connect('search-complete', search_complete)
provider.connect('fetch-failure', failure)
provider.search()
return all_groups
