def test_inequality_equalish_performance(self):
t0 = time.time()
repeats = 2000
for i in xrange(repeats):
assert Query("album!=foo the bar").search(self.s1)
ineq_time = (time.time() - t0)
t1 = time.time()
for i in xrange(repeats):
assert Query("album=!foo the bar").search(self.s1)
not_val_time = (time.time() - t1)
self.assertAlmostEqual(ineq_time, not_val_time, places=1)
def test_inequality_equalish_performance(self):
t0 = time.time()
repeats = 2000
for i in xrange(repeats):
assert Query("album!=foo the bar").search(self.s1)
ineq_time = (time.time() - t0)
t1 = time.time()
for i in xrange(repeats):
assert Query("album=!foo the bar").search(self.s1)
not_val_time = (time.time() - t1)
self.assertAlmostEqual(ineq_time, not_val_time, places=1)
def utitle(string):
"""Title-case a string using a less destructive method than str.title."""
new_string = string[0].capitalize()
# It's possible we need to capitalise the second character...
cap = iswbound(string[0])
for i in xrange(1, len(string)):
s = string[i]
prev = string[i - 1]
# Special case apostrophe in the middle of a word.
# Also, extra case to deal with Irish-style names (eg O'Conner)
if u"'" == s \
and string[i - 1].isalpha() \
and not (i > 1 and string[i - 2].isspace() and
prev.lower() == u"o"):
cap = False
elif iswbound(s):
cap = True
elif cap and s.isalpha():
cap = False
s = s.capitalize()
else:
cap = False
new_string += s
return new_string
def diacritic_for_letters(regenerate=False):
"""Returns a mapping for combining diacritic mark to ascii characters
for which they can be used to combine to a single unicode char.
(actually not ascii, but unicode from the Lu/Ll/Lt categories,
but mainly ascii)
Since this is quite expensive to compute, the result is a cached version
unless regenerate != True. regenerate = True is used for unittests
to validate the cache.
"""
if not regenerate:
return _DIACRITIC_CACHE
d = {}
for i in xrange(sys.maxunicode):
u = unichr(i)
n = unicodedata.normalize("NFKD", u)
if len(n) <= 1:
continue
if unicodedata.category(u) not in ("Lu", "Ll", "Lt"):
continue
if not all(map(unicodedata.combining, n[1:])):
continue
d.setdefault(n[1:], set()).add(n[0])
for k, v in d.items():
d[k] = u"".join(sorted(v))
return d
def __key_pressed(self, widget, event):
# alt+X switches to page X
for i in xrange(self.get_n_pages()):
if is_accel(event, "<alt>%d" % (i + 1)):
self.set_current_page(i)
return True
return False
def utitle(string):
"""Title-case a string using a less destructive method than str.title."""
new_string = string[0].capitalize()
# It's possible we need to capitalise the second character...
cap = iswbound(string[0])
for i in xrange(1, len(string)):
s = string[i]
prev = string[i - 1]
# Special case apostrophe in the middle of a word.
# Also, extra case to deal with Irish-style names (eg O'Conner)
if u"'" == s \
and string[i - 1].isalpha() \
and not (i > 1 and string[i - 2].isspace() and
prev.lower() == u"o"):
cap = False
elif iswbound(s):
cap = True
elif cap and s.isalpha():
cap = False
s = s.capitalize()
else:
cap = False
new_string += s
return new_string
def __key_pressed(self, widget, event):
# alt+X switches to page X
for i in xrange(self.get_n_pages()):
if is_accel(event, "<alt>%d" % (i + 1)):
self.set_current_page(i)
return True
return False
def diacritic_for_letters(regenerate=False):
"""Returns a mapping for combining diacritic mark to ascii characters
for which they can be used to combine to a single unicode char.
(actually not ascii, but unicode from the Lu/Ll/Lt categories,
but mainly ascii)
Since this is quite expensive to compute, the result is a cached version
unless regenerate != True. regenerate = True is used for unittests
to validate the cache.
"""
if not regenerate:
return _DIACRITIC_CACHE
d = {}
for i in xrange(sys.maxunicode):
u = unichr(i)
n = unicodedata.normalize("NFKD", u)
if len(n) <= 1:
continue
if unicodedata.category(u) not in ("Lu", "Ll", "Lt"):
continue
if not all(map(unicodedata.combining, n[1:])):
continue
d.setdefault(n[1:], set()).add(n[0])
for k, v in d.items():
d[k] = u"".join(sorted(v))
return d
def clear(self):
"""Remove all entries.
Thread safe.
"""
with self._iter_lock:
for i in xrange(len(self._log)):
self._log.popleft()
def __select_children(self, iter_, model, selection):
nchildren = model.iter_n_children(iter_)
last = model.get_path(iter_)
for i in xrange(nchildren):
child = model.iter_nth_child(iter_, i)
self.expand_row(model.get_path(child), False)
last = self.__select_children(child, model, selection)
return last
def __select_children(self, iter_, model, selection):
nchildren = model.iter_n_children(iter_)
last = model.get_path(iter_)
for i in xrange(nchildren):
child = model.iter_nth_child(iter_, i)
self.expand_row(model.get_path(child), False)
last = self.__select_children(child, model, selection)
return last
def _fixup_range(start, end, mapping):
extra = []
for i in xrange(start, end + 1):
u = unichr(i)
if u in mapping:
extra.append(re_escape(u"".join(mapping[u])))
start = re_escape(unichr(start))
end = re_escape(unichr(end))
return u"%s%s-%s" % ("".join(extra), start, end)
def test_inequality_performance(self):
t = time.time()
for i in xrange(500):
# Native assert is a bit lighter...
assert Query("album!=foo the bar").search(self.s1)
assert Query("album=foo the bar").search(self.s2)
assert Query("foo the bar").search(self.s2)
assert not Query("foo the bar").search(self.s1)
us = (time.time() - t) * 1000000 / ((i + 1) * 4)
print("Blended Query searches average %.0f μs" % us)
def test_inequality_performance(self):
t = time.time()
for i in xrange(500):
# Native assert is a bit lighter...
assert Query("album!=foo the bar").search(self.s1)
assert Query("album=foo the bar").search(self.s2)
assert Query("foo the bar").search(self.s2)
assert not Query("foo the bar").search(self.s1)
us = (time.time() - t) * 1000000 / ((i + 1) * 4)
print("Blended Query searches average %.0f μs" % us)
def _save_iter(self):
# only pop/append/len are threadsafe, implement iter with them
with self._iter_lock:
temp = collections.deque()
for i in xrange(len(self._log)):
item = self._log.popleft()
yield item
temp.append(item)
while temp:
self._log.appendleft(temp.pop())
def __song_updated(self, librarian, songs):
"""Only update rows that are currently displayed.
Warning: This makes the row-changed signal useless.
"""
vrange = self.get_visible_range()
if vrange is None:
return
(start, ), (end, ) = vrange
model = self.get_model()
for path in xrange(start, end + 1):
row = model[path]
if row[0] in songs:
model.row_changed(row.path, row.iter)
def __song_updated(self, librarian, songs):
"""Only update rows that are currently displayed.
Warning: This makes the row-changed signal useless.
"""
vrange = self.get_visible_range()
if vrange is None:
return
(start,), (end,) = vrange
model = self.get_model()
for path in xrange(start, end + 1):
row = model[path]
if row[0] in songs:
model.row_changed(row.path, row.iter)
def __on_scroll(self, indicator, steps, direction):
# If direction here is always UP you're hitting
# https://bugs.launchpad.net/indicator-application/+bug/1075152
modifier_swap = pconfig.getboolean("modifier_swap")
for step in xrange(steps):
if direction == Gdk.ScrollDirection.UP:
if modifier_swap:
app.player.previous()
else:
app.player.volume += 0.05
elif direction == Gdk.ScrollDirection.DOWN:
if modifier_swap:
app.player.next()
else:
app.player.volume -= 0.05
def start(self, query, limit=5):
"""Start the search and returns the covers"""
self.page_count = 0
self.covers = []
self.limit = limit
self.__parse_page(1, query)
if len(self.covers) < limit:
for page in xrange(2, self.page_count + 1):
self.__parse_page(page, query)
if len(self.covers) >= limit:
break
return self.covers
def start(self, query, limit=5):
"""Start the search and returns the covers"""
self.page_count = 0
self.covers = []
self.limit = limit
self.__parse_page(1, query)
if len(self.covers) < limit:
for page in xrange(2, self.page_count + 1):
self.__parse_page(page, query)
if len(self.covers) >= limit:
break
return self.covers
def __on_scroll(self, indicator, steps, direction):
# If direction here is always UP you're hitting
# https://bugs.launchpad.net/indicator-application/+bug/1075152
modifier_swap = pconfig.getboolean("modifier_swap")
for step in xrange(steps):
if direction == Gdk.ScrollDirection.UP:
if modifier_swap:
app.player.previous()
else:
app.player.volume += 0.05
elif direction == Gdk.ScrollDirection.DOWN:
if modifier_swap:
app.player.next()
else:
app.player.volume -= 0.05
def _fill_view(view):
"""Adds a model with 100 text rows and a column to display them.
Returns the model.
"""
model = Gtk.ListStore(str)
column = Gtk.TreeViewColumn("foo")
title = Gtk.CellRendererText()
column.pack_start(title, True)
column.add_attribute(title, "text", 0)
view.append_column(column)
for x in xrange(100):
model.append(row=["foo"])
view.set_model(model)
return model
def _fill_view(view):
"""Adds a model with 100 text rows and a column to display them.
Returns the model.
"""
model = Gtk.ListStore(str)
column = Gtk.TreeViewColumn("foo")
title = Gtk.CellRendererText()
column.pack_start(title, True)
column.add_attribute(title, "text", 0)
view.append_column(column)
for x in xrange(100):
model.append(row=["foo"])
view.set_model(model)
return model
def _humanise(text):
"""Reverts a title-cased string to a more natural (English) title-casing.
Intended for use after util.title() only"""
def previous_real_word(ws, idx):
"""Returns the first non-null word from words before position `idx`"""
while idx > 0:
idx -= 1
if ws[idx] != "":
break
return ws[idx]
words = text.split(" ") # Yes: to preserve double spacing (!)
for i in xrange(1, len(words) - 1):
word = words[i]
if word in ENGLISH_INCORRECTLY_CAPITALISED_WORDS:
prev = previous_real_word(words, i)
# Add an exception for would-be ellipses...
if (prev and
(not prev[-1] in ENGLISH_SENTENCE_ENDS or prev[-3:] == '...')):
words[i] = word.lower()
return u" ".join(words)
def _humanise(text):
"""Reverts a title-cased string to a more natural (English) title-casing.
Intended for use after util.title() only"""
def previous_real_word(ws, idx):
"""Returns the first non-null word from words before position `idx`"""
while idx > 0:
idx -= 1
if ws[idx] != "":
break
return ws[idx]
words = text.split(" ") # Yes: to preserve double spacing (!)
for i in xrange(1, len(words) - 1):
word = words[i]
if word in ENGLISH_INCORRECTLY_CAPITALISED_WORDS:
prev = previous_real_word(words, i)
# Add an exception for would-be ellipses...
if (prev and (not prev[-1] in ENGLISH_SENTENCE_ENDS
or prev[-3:] == '...')):
words[i] = word.lower()
return u" ".join(words)
def test_signal_count(self):
m = ObjectStore()
def handler(model, path, iter_, result):
result[0] += 1
inserted = [0]
m.connect("row-inserted", handler, inserted)
changed = [0]
m.connect("row-changed", handler, changed)
m.append([1])
m.prepend([8])
m.insert(0, [1])
m.insert_before(None, [1])
m.insert_after(None, [1])
m.insert_many(0, [1, 2, 3])
m.append_many([1, 2, 3])
list(m.iter_append_many([1, 2, 3]))
list(m.iter_append_many(xrange(3)))
self.assertEqual(changed[0], 0)
self.assertEqual(inserted[0], len(m))
def test_signal_count(self):
m = ObjectStore()
def handler(model, path, iter_, result):
result[0] += 1
inserted = [0]
m.connect("row-inserted", handler, inserted)
changed = [0]
m.connect("row-changed", handler, changed)
m.append([1])
m.prepend([8])
m.insert(0, [1])
m.insert_before(None, [1])
m.insert_after(None, [1])
m.insert_many(0, [1, 2, 3])
m.append_many([1, 2, 3])
list(m.iter_append_many([1, 2, 3]))
list(m.iter_append_many(xrange(3)))
self.assertEqual(changed[0], 0)
self.assertEqual(inserted[0], len(m))
class Duplicates(SongsMenuPlugin, PluginConfigMixin):
PLUGIN_ID = 'Duplicates'
PLUGIN_NAME = _('Duplicates Browser')
PLUGIN_DESC = _('Finds and displays similarly tagged versions of songs.')
PLUGIN_ICON = Icons.EDIT_SELECT_ALL
MIN_GROUP_SIZE = 2
_CFG_KEY_KEY = "key_expression"
__DEFAULT_KEY_VALUE = "~artist~title~version"
_CFG_REMOVE_WHITESPACE = 'remove_whitespace'
_CFG_REMOVE_DIACRITICS = 'remove_diacritics'
_CFG_REMOVE_PUNCTUATION = 'remove_punctuation'
_CFG_CASE_INSENSITIVE = 'case_insensitive'
plugin_handles = any_song(is_finite)
# Cached values
key_expression = None
__cfg_cache = {}
__remove_punctuation_trans = tbl = dict.fromkeys(
i for i in xrange(sys.maxunicode)
if unicodedata.category(unichr(i)).startswith('P'))
"""Lookup all Unicode punctuation, and remove it"""
@classmethod
def get_key_expression(cls):
if not cls.key_expression:
cls.key_expression = (cls.config_get(cls._CFG_KEY_KEY,
cls.__DEFAULT_KEY_VALUE))
return cls.key_expression
@classmethod
def PluginPreferences(cls, window):
def key_changed(entry):
cls.key_expression = None
cls.config_set(cls._CFG_KEY_KEY, entry.get_text().strip())
vb = Gtk.VBox(spacing=10)
vb.set_border_width(0)
hbox = Gtk.HBox(spacing=6)
# TODO: construct a decent validator and use ValidatingEntry
e = UndoEntry()
e.set_text(cls.get_key_expression())
e.connect("changed", key_changed)
e.set_tooltip_markup(
_("Accepts QL tag expressions like "
"<tt>~artist~title</tt> or <tt>musicbrainz_track_id</tt>"))
lbl = Gtk.Label(label=_("_Group duplicates by:"))
lbl.set_mnemonic_widget(e)
lbl.set_use_underline(True)
hbox.pack_start(lbl, False, True, 0)
hbox.pack_start(e, True, True, 0)
frame = qltk.Frame(label=_("Duplicate Key"), child=hbox)
vb.pack_start(frame, True, True, 0)
# Matching Option
toggles = [
(cls._CFG_REMOVE_WHITESPACE, _("Remove _Whitespace")),
(cls._CFG_REMOVE_DIACRITICS, _("Remove _Diacritics")),
(cls._CFG_REMOVE_PUNCTUATION, _("Remove _Punctuation")),
(cls._CFG_CASE_INSENSITIVE, _("Case _Insensitive")),
]
vb2 = Gtk.VBox(spacing=6)
for key, label in toggles:
ccb = ConfigCheckButton(label, 'plugins', cls._config_key(key))
ccb.set_active(cls.config_get_bool(key))
vb2.pack_start(ccb, True, True, 0)
frame = qltk.Frame(label=_("Matching options"), child=vb2)
vb.pack_start(frame, False, True, 0)
vb.show_all()
return vb
@staticmethod
def remove_accents(s):
return "".join(c for c in unicodedata.normalize('NFKD', text_type(s))
if not unicodedata.combining(c))
@classmethod
def get_key(cls, song):
key = song(cls.get_key_expression())
if cls.config_get_bool(cls._CFG_REMOVE_DIACRITICS):
key = cls.remove_accents(key)
if cls.config_get_bool(cls._CFG_CASE_INSENSITIVE):
key = key.lower()
if cls.config_get_bool(cls._CFG_REMOVE_PUNCTUATION):
key = (key.translate(cls.__remove_punctuation_trans))
if cls.config_get_bool(cls._CFG_REMOVE_WHITESPACE):
key = "_".join(key.split())
return key
def plugin_songs(self, songs):
model = DuplicatesTreeModel()
self.__cfg_cache = {}
# Index all songs by our custom key
# TODO: make this cache-friendly
print_d("Calculating duplicates for %d song(s)..." % len(songs))
groups = {}
for song in songs:
key = self.get_key(song)
if key and key in groups:
print_d("Found duplicate based on '%s'" % key)
groups[key].add(song._song)
elif key:
groups[key] = {song._song}
for song in app.library:
key = self.get_key(song)
if key in groups:
groups[key].add(song)
# Now display the grouped duplicates
for (key, children) in groups.items():
if len(children) < self.MIN_GROUP_SIZE:
continue
# The parent (group) label
model.add_group(key, children)
dialog = DuplicateDialog(model)
dialog.show()
def test_iter_append_many_iterable_int(self):
m = ObjectStore()
for x in m.iter_append_many((i for i in xrange(10))):
pass
self.failUnlessEqual([r[0] for r in m], list(range(10)))
def test_iter_append_many_iterable_object(self):
objects = [object() for i in xrange(10)]
m = ObjectStore()
for x in m.iter_append_many((i for i in objects)):
pass
self.failUnlessEqual([r[0] for r in m], objects)
from quodlibet.compat import xrange, text_type, number_types, string_types, \
swap_to_string, listmap
from collections import Iterable
from quodlibet.util.path import escape_filename, unescape_filename
from quodlibet.util.dprint import print_d
from quodlibet.util.misc import total_ordering, hashable
from .collections import HashedList
PEOPLE = list(_PEOPLE)
# Collections value albumartist more than song artist (Issue 1034)
PEOPLE.remove("albumartist")
PEOPLE.insert(0, "albumartist")
ELPOEP = list(reversed(PEOPLE))
PEOPLE_SCORE = [100 ** i for i in xrange(len(PEOPLE))]
def avg(nums):
"""Returns the average (arithmetic mean) of a list of numbers"""
return float(sum(nums)) / len(nums)
def bayesian_average(nums, c=None, m=None):
"""Returns the Bayesian average of an iterable of numbers,
with parameters defaulting to config specific to ~#rating."""
m = m or config.RATINGS.default
c = c or config.getfloat("settings", "bayesian_rating_factor", 0.0)
ret = float(m * c + sum(nums)) / (c + len(nums))
return ret
def _remove_punctuation_trans():
"""Lookup all Unicode punctuation, and remove it"""
return dict.fromkeys(i for i in xrange(sys.maxunicode)
if unicodedata.category(unichr(i)).startswith('P'))
def create_pipelines(self):
# create as many pipelines as threads
self.pipes = [ReplayGainPipeline() for _ in xrange(get_num_threads())]
def test_iter_append_many_iterable_int(self):
m = ObjectStore()
for x in m.iter_append_many((i for i in xrange(10))):
pass
self.failUnlessEqual([r[0] for r in m], list(range(10)))
from quodlibet import config
from quodlibet.formats._audio import TAG_TO_SORT, INTERN_NUM_DEFAULT
from quodlibet.formats._audio import PEOPLE as _PEOPLE
from quodlibet.compat import xrange
from collections import Iterable
from quodlibet.util.path import escape_filename, unescape_filename
from quodlibet.util.path import bytes2fsnative, is_fsnative, fsnative2bytes
from .collections import HashedList
PEOPLE = list(_PEOPLE)
# Collections value albumartist more than song artist (Issue 1034)
PEOPLE.remove("albumartist")
PEOPLE.insert(0, "albumartist")
ELPOEP = list(reversed(PEOPLE))
PEOPLE_SCORE = [100**i for i in xrange(len(PEOPLE))]
def avg(nums):
"""Returns the average (arithmetic mean) of a list of numbers"""
return float(sum(nums)) / len(nums)
def bayesian_average(nums, c=None, m=None):
"""Returns the Bayesian average of an iterable of numbers,
with parameters defaulting to config specific to ~#rating."""
m = m or config.RATINGS.default
c = c or config.getfloat("settings", "bayesian_rating_factor", 0.0)
ret = float(m * c + sum(nums)) / (c + len(nums))
return ret
def test_iter_append_many_iterable_object(self):
objects = [object() for i in xrange(10)]
m = ObjectStore()
for x in m.iter_append_many((i for i in objects)):
pass
self.failUnlessEqual([r[0] for r in m], objects)
def __iter__(self):
for i in xrange(len(self)):
yield self._list.nth_tag_name(i)
def create_pipelines(self):
# create as many pipelines as threads
self.pipes = [ReplayGainPipeline() for _ in xrange(get_num_threads())]
def _remove_punctuation_trans():
"""Lookup all Unicode punctuation, and remove it"""
return dict.fromkeys(
i for i in xrange(sys.maxunicode)
if unicodedata.category(unichr(i)).startswith('P'))
