class Library(Worker):
_config = None
_library_filename= None
# 5 seconds
_queue_timeout = 5
_queue = None
_running = False
_update_queued = False
_db_version = 1
# general purpose dict
_client_data = {}
_history= deque()
_lastplayed_track= None
_playing_track= None
_sync= Condition()
_factor_ranking = {
'relation' : 0.98,
'rating' : 0.01,
'random' : 0.01,
}
_relation_decay = 0.5
_ranking_updated = False
# default rating of unplayed tracks. set > 0.5 to favor unplayed < 0.5 to
# inhibit unplayed
_unplayed_rating = 0.5
_relation_resetted = False
_queue_update_factor = 0.5
# thresholds in seconds
_thres_track_lastplayed= 60 * 60 * 24 * 7 * 4 # 4weeks
_thres_track_laststarted= 60 * 60 * 24 # 1day
_thres_track_lastqueued= 10 * 60 # 10min
_thres_artist_laststarted = 10 * 60 # 10min
_thres_artist_lastplayed = 10 * 60 # 10min
_thres_artist_lastqueued= 60
# 6 months
_thres_lastfm_lookup = 60 * 60 * 24 * 7 * 4 * 6
_thres_echonest_lookup = 60 * 60 * 24 * 7 * 4 * 6
# apply relations to maxhistory entries:
# def fac(x, m=maxhistory): return -sqrt(1.0*x/m) + 1.0
# e.g. 5: 1.00 0.55 0.36 0.22 0.10
# e.g. 10: 1.00 0.68 0.55 0.45 0.37 0.29 0.23 0.16 0.11 0.05
maxhistory= 5
_title= 'library'
_ranking_base = 1000
_lastfm = None
_echonest = None
_queue_lookup_results = True
_seed_tracks = set()
_seed_artists = set()
def __init__(self, config):
Log.__init__(self, self._title)
self._logger.info(u"init")
Worker.__init__(self, lifo=False)
self._config = config
self._library_filename = self._config.get('Library', 'path',
join(self._config.configdir(), 'library.pkl'))
Factories.clear()
Logger.set_logger(self._logger)
if self._config.get('TrackRelation', 'use_db', True):
TrackRelationFactory.use_db()
TrackRelationFactory.set_path(self._config.get('TrackRelation',
'path', join(self._config.configdir(), '')))
else:
TrackRelationFactory.use_fs()
TrackRelationFactory.set_path(self._config.get('TrackRelation',
'path', join(self._config.configdir(), 'track')))
TrackRelationFactory.set_maxentries(
self._config.get('TrackRelation', 'maxentries', 500))
if self._config.get('ArtistRelation', 'use_db', True):
ArtistRelationFactory.use_db()
ArtistRelationFactory.set_path(self._config.get('ArtistRelation',
'path', join(self._config.configdir(), '')))
else:
ArtistRelationFactory.use_fs()
ArtistRelationFactory.set_path(self._config.get('ArtistRelation',
'path', join(self._config.configdir(), 'artist')))
ArtistRelationFactory.set_maxentries(
self._config.get('ArtistRelation', 'maxentries', 500))
if self._config.get('Lookup', 'QueueResults', 'False') == 'True':
self._queue_lookup_results = True
else:
self._queue_lookup_results = False
# FIXME: log if one of those libs is not present
if self._config.get('Lookup', 'UseLastFM', 'True') == 'True':
self._lastfm = LastFM(config)
self._lastfm.start()
self._thres_lastfm_lookup = self._config.getint('Lookup',
'ThresholdLastFM', self._thres_lastfm_lookup)
if self._config.get('Lookup', 'UseEchoNest', 'True') == 'True':
self._echonest = EchoNest(config)
self._echonest.start()
self._thres_echonest_lookup = self._config.getint('Lookup',
'ThresholdEchoNest', self._thres_echonest_lookup)
# read, normalize and update ranking factors
factor = 0.0
for k in self._factor_ranking.keys():
self._factor_ranking[k] = self._config.getfloat('Ranking',
"factor%s" % k, self._factor_ranking[k])
factor += self._factor_ranking[k]
for k in self._factor_ranking.keys():
self._factor_ranking[k] /= factor
self._config.set('Ranking', "factor%s" % k, self._factor_ranking[k])
self._relation_decay = self._config.getfloat('Ranking', 'RelationDecay',
self._relation_decay)
self._queue_update_factor = self._config.getfloat('Rating',
'QueueUpdateFactor', self._queue_update_factor)
self._unplayed_rating = self._config.getfloat('Ranking',
'UnplayedRating', self._unplayed_rating)
self._thres_track_lastplayed = self._config.getint('Ranking',
'ThresholdTrackLastPlayed', self._thres_track_lastplayed)
self._thres_track_laststarted= self._config.getint('Ranking',
'ThresholdTrackLastStarted', self._thres_track_laststarted)
self._thres_track_lastqueued= self._config.getint('Ranking',
'ThresholdTrackLastQueued', self._thres_track_lastqueued)
self._thres_artist_lastplayed = self._config.getint('Ranking',
'ThresholdArtistLastPlayed', self._thres_artist_lastplayed)
self._thres_artist_laststarted= self._config.getint('Ranking',
'ThresholdArtistLastStarted', self._thres_artist_laststarted)
self._thres_artist_lastqueued= self._config.getint('Ranking',
'ThresholdArtistLastQueued', self._thres_artist_lastqueued)
def set_artist_threshold(self, played=-1, started=-1, queued=-1):
if played != -1:
self._thres_artist_lastplayed = played
self._config.set('Ranking', 'ThresholdArtistLastPlayed', played)
if started != -1:
self._thres_artist_laststarted= started
self._config.set('Ranking', 'ThresholdArtistLastStarted', started)
if queue != -1:
self._thres_artist_lastqueued= queued
self._config.set('Ranking', 'ThresholdArtistLastQueued', queued)
def set_artist_threshold(self, played=-1, started=-1, queued=-1):
if played != -1:
self._thres_artist_lastplayed = played
self._config.set('Ranking', 'ThresholdArtistLastPlayed', played)
if started != -1:
self._thres_artist_laststarted= started
self._config.set('Ranking', 'ThresholdArtistLastStarted', started)
if queue != -1:
self._thres_artist_lastqueued= queued
self._config.set('Ranking', 'ThresholdArtistLastQueued', queued)
def deactivate(self):
self.queue_function(self._deactivate, None)
def _deactivate(self, dummy=None):
""" set all artists, albums, tracks and files to be not active """
self.acquire()
FileFactory.deactivate()
self.release()
def activate(self):
self.queue_function(self._activate, None)
def _activate(self, dummy=None):
self.acquire()
TrackRelationFactory.check_active()
ArtistRelationFactory.check_active()
self.release()
def stop(self):
if self._running:
Worker.stop(self)
self._save()
if self._lastfm:
self._lastfm.stop()
if self._echonest:
self._echonest.stop()
self._running = False
def acquire(self): self._sync.acquire()
def release(self): self._sync.release()
def load(self, locked=False):
if not path.exists(self._library_filename):
self._logger.info(u"not loading: %s not found" % self._library_filename)
else:
if not locked: self.acquire()
self._logger.info(u"loading from %s" % self._library_filename)
Factories.clear()
try:
timestamp = datetime.utcnow()
input = open(self._library_filename, 'rb')
Factories.load(input)
self._client_data = pickle.load(input)
input.close()
self._logger.info(u"loading took %s" % ( (datetime.utcnow() - timestamp)))
self.dump_stats()
except IOError:
self._logger.error('IOError')
self._logger.error(format_exc())
except EOFError:
self._logger.error('EOFError')
self._logger.error(format_exc())
Factories.clear()
except AttributeError:
self._logger.error('AttributeError: Format changed?')
self._logger.error(format_exc())
Factories.clear()
if not locked: self.release()
def save(self):
self._logger.info(u'queueing save')
self.queue_function(self._save)
def _save(self):
self.acquire()
self._logger.info(u"saving %s~" % self._library_filename)
level = -1 # -1 for latest method, 0 for ascii
timestamp = datetime.utcnow()
output = open(u"%s~" % self._library_filename, 'wb')
Factories.dump(output, level)
pickle.dump(self._client_data, output, level)
output.close()
self._logger.info(u"saved %s~" % self._library_filename)
if path.exists(self._library_filename):
try:
self._logger.info(u"removing %s" % self._library_filename)
remove(self._library_filename)
except OSError:
self._logger.error('OSError')
self._logger.error(format_exc())
self._logger.info(u"moving %s~ to %s" % (self._library_filename,
self._library_filename))
rename(u"%s~" % self._library_filename, self._library_filename)
self._logger.info(u"saving took %s" % ( (datetime.utcnow() - timestamp)))
# save configuration
self._config.save()
self.release()
def add_file(self, path, artist_name, track_title):
self.queue_function(self._add_file, path, artist_name, track_title)
def _add_file(self, path, artist_name, track_title):
self.acquire()
f = FileFactory.get(path, artist_name, track_title)
f.activate()
self.release()
""" similar inputs """
def similar_artists(self, artist_nameA, artist_nameB, match, source):
#self._logger.debug(u"similar_artists %s %s %2.2f %s" % (artist_name0,
# artist_name1, match, source))
self.queue_function(self._similar_artists, artist_nameA,
artist_nameB, match, source)
def _similar_artists(self, artist_nameA, artist_nameB, match, source,
locked=False):
if not locked: self.acquire()
#self._logger.debug(u"%s: [%s]-[%s] %2.2f" % (source, artist_nameA,
# artist_nameB, match))
artistA = ArtistFactory.by_key(artist_nameA)
artistB = ArtistFactory.by_key(artist_nameB)
#if not artistA: self._logger.debug(u"similar_artists[%s]: not found" %
# artist_nameA)
#if not artistB: self._logger.debug(u"similar_artists[%s]: not found" %
# artist_nameB)
if artistA and artistB:
relation = ArtistRelationFactory.get(artistA, artistB)
old_rating = relation.rating
relation.rate(0.75 + 0.25 * match)
self._logger.debug(u"%s [%s]-[%s] m(%2.2f) r(%2.2f|%2.2f)" % (source,
artistA, artistB, match, relation.rating, old_rating))
if self._queue_lookup_results:
if self._lastfm:
self._lastfm.similar_artists_low(self.similar_artists, artist_nameB,
self._thres_lastfm_lookup)
if self._echonest:
self._echonest.similar_artists_low(self.similar_artists, artist_nameB,
self._thres_lastfm_lookup)
if not locked: self.release()
def similar_tracks(self, artist_name0, track_title0,
artist_name1, track_title1, match, source):
#self._logger.info(u"similar_tracks %s %s %s %s %2.2f %s" % (artist_name0,
# track_title0, artist_name1, track_title1, match, source))
self.queue_function(self._similar_tracks, artist_name0,
track_title0, artist_name1, track_title1, match, source)
def _similar_tracks(self, artist_nameA, track_titleA,
artist_nameB, track_titleB, match, source, locked=False):
if not locked: self.acquire()
#self._logger.debug(u"%s: [%s-%s]-[%s-%s] %2.2f" % (source, artist_nameA,
# track_titleA, artist_nameB, track_titleB, match))
trackA = TrackFactory.by_key(TrackFactory.get_key(artist_nameA,
track_titleA))
trackB = TrackFactory.by_key(TrackFactory.get_key(artist_nameB,
track_titleB))
#if not trackA: self._logger.debug(u"similar_tracks[%s-%s]: not found" %
# (artist_nameA, track_titleA))
#if not trackB: self._logger.debug(u"similar_tracks[%s-%s]: not found" %
# (artist_nameB, track_titleB))
if trackA and trackB:
relation = TrackRelationFactory.get(trackA, trackB)
old_rating = relation.rating
relation.rate(0.75 + 0.25 * match)
self._logger.debug(u"%s [%s]-[%s] m(%2.2f) r(%2.2f|%2.2f)" % (source,
trackA, trackB, match, relation.rating, old_rating))
if self._queue_lookup_results:
if self._lastfm:
self._lastfm.similar_tracks_low(self.similar_tracks, artist_nameB,
track_titleB, self._thres_lastfm_lookup)
if self._echonest:
self._echonest.similar_tracks_low(self.similar_tracks, artist_nameB,
track_titleB, self._thres_lastfm_lookup)
if not locked: self.release()
""" statistics """
def nr_of_artists(self):
return len(self._artists)
def nr_of_albums(self):
return len(self._albums)
def nr_of_tracks(self):
return len(self._tracks)
def nr_of_files(self):
return len(self._files)
def nr_of_relations(self):
return len(self._relations)
def nr_of_active_tracks(self):
nr_of_active_tracks= 0
for track in self.tracks():
if track['active']:
nr_of_active_tracks+= 1
return nr_of_active_tracks
def __factor_relation(self, nr, maxnr): return -sqrt(1.0*nr/maxnr) + 1.0
def __rating(self, struct, inc=True, mix=0.7, factor=1.0):
"""
kept for docstring...
mixed rating function. updates the values of the structs
'rolling_rating', 'stable_rating' and 'rating' based on 'playcount' and
'skipcount' (or 'playfactor' and 'skipfactor' if present) and the last
"action" (play or skip). the '*rating' is between 0.0 and 1.0 with 0.5
being unrated / unsure and > 0.5 being a positive and < 0.5 being a
negativ rating. the mix factor (0.0-1.0) scales the rating to be more
stable (>0.5) or more jumpy (<0.5). the factor determines how much of the
old rating for rolling ratings should be "kept" with 1.0 being all new and
0.0 being all old. usefull for history relations.
some thoughts:
the goal is to determine the rating of track, artist, albums and
relations of the three.
depending on the subject (track, artist, album), the rating should adapt
to plays and skips slow or fast.
tracks are not played that often (compared to artists and albums). rating
should adapt fast here. example: i play my new weekly favorite track
around 10 times this week. next week i do not want to hear it anymore.
how many skips to push it below 0.5? keep in mind, that it might just be
played in the wrong mix, so the first skip should not be too penalizing.
i guess ~3 skips should be enough to but it below or near 0.5. mix 0.7
should work.
initially i wanted to calculate the artist and album rating similarily
(with a more stable mix of ~0.9), but: given that the shuffle algorithm
only spits out stuff i like and ignores the stuff i hate. let's take an
artist which has a lot of songs that i don't like, but just this one track
i absolutly love. over time i played this track 40 times and skipped the
other 11 tracks just once. taking my mix factor of 0.9 i would end up
with an artist rating of ~.8. so the chances are high that i get
suggested all tracks by this artist and need to skip them at least once.
so artist rating would be near useless. average rating of all tracks of
this artist would be ~0.47. with ~6 bad songs required to counter the
high runner (average < 0.5).
rolling rating which adapts fast to plays and skips:
7 plays @0.5 (.75 .88 .94 .97 .98 .99 1)
-> 7 skips @1.0 (.50 .25 .13 .06 .03 .02 .01)
< 0.5 after 2nd skip, this is too bouncy
7 skips @0.5 (.25 .13 .06 .03 .02 .01 0)
-> 7 plays @0.0 (.50 .75 .88 .94 .97 .98 .99)
> 0.5 after 2nd play, this is too bouncy
stable rating which accounts for overall plays and skips:
7 plays @0.5 (.67 .75 .80 .83 .86 .88 .89)
-> 7 skips @0.89 (.80 .73 .67 .62 .57 .53 .50)
< 0.5 after 8th skip, this is too slow
7 skips @0.5 (.33 .25 .20 .17 .14 .13 .11)
-> 7 plays @0.11 (.20 .27 .33 .38 .43 .47 .50)
> 0.5 after 8th play, this is too slow
mixing it together to get the best of both worlds (mix=0.7)
7 plays @0.5 (0.6917 0.7875 0.8413 0.8740 0.8953 0.9102 0.9211)
> 7 skips @0.92 (0.7094 0.5838 0.5040 0.4494 0.4093 0.3780 0.3523)
< 0.5 after 4th skip, good penalty after 2nd skip
"""
playfactor = struct.get('playfactor', struct['playcount'] * 1.0)
skipfactor = struct.get('skipfactor', struct['skipcount'] * 1.0)
if inc:
new_rolling = (struct['rolling_rating'] +
(1.0 - struct['rolling_rating']) / 2.0)
else:
new_rolling = (struct['rolling_rating'] -
(struct['rolling_rating'] / 2.0))
struct['rolling_rating'] = (
factor * new_rolling + (1.0 - factor) * struct['rolling_rating'])
new_stable = ((playfactor + 1.0) /
(playfactor + skipfactor + 2.0))
struct['stable_rating'] = (
factor * new_stable + (1.0 - factor) * struct['stable_rating'])
struct['rating'] = (struct['stable_rating'] * mix +
struct['rolling_rating'] * (1.0 - mix))
def get_file(self, path):
f = FileFactory.by_path(path)
if not f:
self._logger.error(u"file not recognized %s" % path)
return None
return f
def file_started(self, file):
self.queue_function(self._file_started, file)
def _file_started(self, file):
self.acquire()
file.started()
self.track_started(file.track)
self.release()
def file_stopped(self, file):
self.queue_function(self._file_stopped, file)
def _file_stopped(self, file):
self.acquire()
file.stopped()
self.track_stopped(file.track)
self.release()
def file_skipped(self, file):
self.queue_function(self._file_skipped, file)
def _file_skipped(self, file):
self.acquire()
file.skipped()
self.track_played(file.track, skipped=True, locked=True)
self.release()
def file_played(self, file):
self.queue_function(self._file_played, file)
def _file_played(self, file):
self.acquire()
file.played()
self.track_played(file.track, skipped=False, locked=True)
self.release()
def file_queued(self, file):
self.queue_function(self._file_queued, file)
def _file_queued(self, file):
self.acquire()
file.queued()
self.track_queued(file.track, locked=True)
self.release()
def lookup(self, track, locked=False):
if not locked: self.acquire()
ArtistRelationFactory.load_artist(track.artist)
TrackRelationFactory.load_track(track)
# queue similar artists / tracks lookup from lastfm
if self._lastfm:
self._lastfm.similar_artists(self.similar_artists, track.artist.name,
self._thres_lastfm_lookup)
self._lastfm.similar_tracks(self.similar_tracks, track.artist.name,
track.title, self._thres_lastfm_lookup)
# queue similar artists / tracks lookup from echonest
if self._echonest:
self._echonest.similar_artists(self.similar_artists, track.artist.name,
self._thres_echonest_lookup)
self._echonest.similar_tracks(self.similar_tracks, track.artist.name,
track.title, self._thres_echonest_lookup)
if not locked: self.release()
def track_queued(self, track, locked=False):
if not locked: self.acquire()
self.lookup(track, locked=True)
if not locked: self.release()
def track_started(self, track, locked=False):
if not locked: self.acquire()
self._logger.info(u"started: %s" % track)
timestamp= now()
self._playing_track = track
# add this track to the history
self._history.extendleft([{'track': track,'skipped':False}])
self.lookup(track, locked=True)
self.update_ranking(locked=True)
# CONFIG: max history length?
if len(self._history) > 100: self._history.pop()
if not locked: self.release()
def track_stopped(self, track, locked=False):
if not locked: self.acquire()
self._playing_track = None
if len(self._history) == 0 or self._history[0]['track'] != track:
self._logger.error("stopped: no record of starting this track")
else:
self._history.popleft()
if not locked: self.release()
def track_skipped(self, track, locked=False):
self.track_played(track, skipped=True, locked=locked)
def file_queue_skip(self, file, locked=False):
file.queued()
return self.track_queue_skip(file.track, locked)
def file_queue_add(self, file, locked=False):
file.queued()
return self.track_queue_add(file.track, locked)
def track_queue_skip(self, track, locked=False):
self.track_queue_updated(track, skipped=True, locked=locked)
def track_queue_add(self, track, locked=False):
self.track_queue_updated(track, skipped=False, locked=locked)
def track_queue_updated(self, track, skipped, locked=False):
# FIXME: locked?
self.queue_function(self._track_queue_updated, track, skipped)
def _track_queue_updated(self, track, skipped, locked=False):
if self._playing_track:
reference_track = self._playing_track
elif self._lastplayed_track:
reference_track = self._lastplayed_track
else:
return
if not locked: self.acquire()
relation = TrackRelationFactory.get(track, reference_track)
relation.update(not skipped, self._queue_update_factor)
self._logger.debug(relation)
self.update_ranking()
if not locked: self.release()
def track_played(self, track, skipped= False, locked=False):
if not locked: self.acquire()
if len(self._history) == 0 or self._history[0]['track'] != track:
self._logger.error(u"played: no record of starting this track")
if not locked: self.release()
return
if not skipped: self._lastplayed_track = track
timestamp= now()
self._history[0]['skipped']= skipped
if len(self._history) > 1:
for i in range(min(self.maxhistory, len(self._history) - 1)):
if skipped and self._history[1+i]['skipped']: continue
factor= self.__factor_relation(i, self.maxhistory)
hist_track= self._history[1+i]['track']
hist_skipped = self._history[1+i]['skipped']
track_relation = TrackRelationFactory.get(track, hist_track)
track_relation.update(not hist_skipped, factor)
self._logger.info(u"relation updated: %s" % track_relation)
self._relation_resetted = False
self.update_ranking()
if not locked: self.release()
def next_file(self, locked=False):
self._logger.info(u"next file: start")
if not locked: self.acquire()
if not self._ranking_updated: self.update_ranking()
best_ranking = 0
best_tracks = []
tt = []
timestamp = now()
# calculate last_*_timestamps (played / queued / started)
track_lastplayed_timestamp = timestamp - self._thres_track_lastplayed
track_laststarted_timestamp = timestamp - self._thres_track_laststarted
track_lastqueued_timestamp = timestamp - self._thres_track_lastqueued
artist_lastplayed_timestamp = timestamp - self._thres_artist_lastplayed
artist_laststarted_timestamp = timestamp - self._thres_artist_laststarted
artist_lastqueued_timestamp = timestamp - self._thres_artist_lastqueued
has_active_tracks = False
for track in TrackFactory.active_tracks():
has_active_tracks = True
artist = track.artist
factor = 1.0
if (track.lastplayed > track_lastplayed_timestamp):
factor = min(factor,
1.0 - ((1.0 * track.lastplayed - track_lastplayed_timestamp) /
self._thres_track_lastplayed))
if (artist.lastplayed > artist_lastplayed_timestamp):
factor = min(factor,
1.0 - ((1.0 * artist.lastplayed - artist_lastplayed_timestamp) /
self._thres_artist_lastplayed))
if (track.laststarted > track_laststarted_timestamp):
factor = min(factor,
1.0 - ((1.0 * track.laststarted - track_laststarted_timestamp) /
self._thres_track_laststarted))
if (artist.laststarted > artist_laststarted_timestamp):
factor = min(factor,
1.0 - ((1.0 * artist.laststarted - artist_laststarted_timestamp) /
self._thres_artist_laststarted))
if (track.lastqueued > track_lastqueued_timestamp):
factor = min(factor,
1.0 - ((1.0 * track.lastqueued - track_lastqueued_timestamp) /
self._thres_track_lastqueued))
if (artist.lastqueued > artist_lastqueued_timestamp):
factor = min(factor,
1.0 - ((1.0 * artist.lastqueued - artist_lastqueued_timestamp) /
self._thres_artist_lastqueued))
ranking = int(self._ranking_base * factor * track.ranking)
if ranking > best_ranking:
self._logger.debug("%2.2f (best=): %s" % (ranking, track))
best_ranking = ranking
best_tracks = [track]
elif ranking == best_ranking:
self._logger.debug("%2.2f (best+): %s" % (ranking, track))
best_tracks.append(track)
top_ten(tt, track, ranking)
if not has_active_tracks:
self._logger.error(u"No active tracks")
if not locked: self.release()
return None
top_ten_dump(tt, self._logger.info, u"rank")
self._logger.info("best tracks: %d" % (len(best_tracks)))
best_track = choice(best_tracks)
best_track.started()
# pick the best file
best_rating = 0.0
best_files = []
for file in best_track.files():
if not file.active: continue
t = file.playcount / (1.0 + file.skipcount)
if t > best_rating:
best_rating = t
best_files = [file]
elif t == best_rating:
best_files.append(file)
if not locked: self.release()
self._logger.info(u"next file: stop")
return choice(best_files)
def reset(self, full=False):
self._logger.info(u'queueing reset')
self.queue_function(self._reset, full)
def _reset(self, full=False):
""" clear recent play history + unset last played track """
self.acquire()
self._history.clear()
self._lastplayed_track = None
self._playing_track = None
for track in TrackFactory.active_tracks():
track.ranking = 0.5
track.relation = 0.5
track.relation_old = 0.5
track.relation_cnt = 0
if full:
for track in TrackFactory.active_tracks():
track.lastplayed = 0
track.laststarted = 0
track.lastqueued = 0
for artist in ArtistFactory.active_artists():
artist.lastplayed = 0
artist.laststarted = 0
artist.lastqueued = 0
self.release()
def reset_ban(self, locked=False):
if not locked: self.acquire()
for track in TrackFactory.active_tracks():
track.ban = False
if not locked: self.release()
def reset_boost(self, locked=False):
if not locked: self.acquire()
for track in TrackFactory.active_tracks():
track.boost = False
if not locked: self.release()
def reset_seeds(self, locked=False):
if not locked: self.acquire()
self._seed_tracks = set()
self._seed_artists = set()
if not locked: self.release()
def add_seed(self, track, locked=False):
if not locked: self.acquire()
self.lookup(track, True)
self._seed_tracks.add(track)
self._seed_artists.add(track.artist)
if not locked: self.release()
def seed(self, track, locked=False):
"""Calculate relations based on track as seed.
"""
if not locked: self.acquire()
benchmark = Benchmark()
timestamp = now()
seed_track = set()
seed_artist = set()
if track:
seed_track.add(track)
seed_artist.add(track.artist)
self.lookup(track, True)
# check artist relations
cnt = 0
benchmark.start()
tt = []
for seed_a in seed_artist.union(self._seed_artists):
self._logger.info(u'check artist relations for {}'.format(seed_a))
for artist_relation in ArtistRelationFactory.by_artist(seed_a):
cnt += 1
other_artist = artist_relation.artistA
if artist_relation.artistA.name == seed_a.name:
other_artist = artist_relation.artistB
other_artist.relation_sum += artist_relation.rating
other_artist.relation_cnt += 1
other_artist.relation = (other_artist.relation_sum /
other_artist.relation_cnt)
top_ten(tt, u'artist related with {}({}/{}={}) to {}'.format(
scale_rating(artist_relation.rating),
scale_rating(other_artist.relation_sum),
scale_rating(other_artist.relation_cnt),
scale_rating(other_artist.relation),
other_artist), artist_relation.rating)
artist_relation.lastused = timestamp
top_ten_dump(tt, self._logger.info)
self._logger.info(u"update ranking: check artist took %s" % benchmark)
self._logger.info(u"updated %d artist(s)" % cnt)
cnt = 0
benchmark.start()
tt = []
for seed_t in seed_track.union(self._seed_tracks):
self._logger.info(u'check track relations for {}'.format(seed_t))
for track_relation in TrackRelationFactory.by_track(seed_t):
other_track = track_relation.trackA
if track_relation.trackA.title == seed_t.title and \
track_relation.trackA.artist.name == seed_t.artist.name:
other_track = track_relation.trackB
cnt += 1
if not track.ban:
other_track.relation_sum += track_relation.rating
other_track.relation_cnt += 1
other_track.relation = (other_track.relation_sum /
other_track.relation_cnt)
top_ten(tt, u'track related with {} to {}'.format(
scale_rating(track_relation.rating), other_track),
track_relation.rating)
track_relation.lastused = timestamp
top_ten_dump(tt, self._logger.info)
self._logger.info(u"update ranking: check track took %s" % benchmark)
self._logger.info(u"updated %d track(s)" % cnt)
if not locked: self.release()
def update_ranking(self, locked=False):
if not self._update_queued:
if not locked: self.acquire()
self._update_queued = True
self.queue_function(self._update_ranking, None)
if not locked: self.release()
def _update_ranking(self, locked=False):
benchmark = Benchmark()
if not locked: self.acquire()
seed_track = None
if self._lastplayed_track:
seed_track = self._lastplayed_track
elif self._playing_track:
seed_track = self._playing_track
if not self._relation_resetted:
benchmark.start()
self._relation_resetted = True
for track in TrackFactory.active_tracks():
track.relation_old = track.relation
track.relation_sum = 0.0
track.relation_cnt = 0
for artist in ArtistFactory.active_artists():
artist.relation_old = artist.relation
artist.relation_sum = 0.0
artist.relation_cnt = 0
self._logger.info(u"update ranking: resetting took %s" % benchmark)
has_active_tracks = False
for track in TrackFactory.active_tracks():
has_active_tracks = True
break
benchmark.start()
# new relation = old relation * decay
# e.g. for decay = 0.5 (0.75)
# decfacA = 0.5 * 0.5 = 0.25 (0.75 * 0.5 = 0.375)
# decfacB = 1.0 - 0.5 = 0.5 (1.0 - 0.75 = 0.25)
# relation_old=0.75 -> 0.25+0.5*0.75=0.625 (0.375+0.25*0.75=0.5625)
decfacA = self._relation_decay * 0.5
decfacB = 1.0 - self._relation_decay
for track in TrackFactory.active_tracks():
if (track.relation_old > 0.501) or (track.relation_old < 0.499):
track.relation = (decfacA + decfacB * track.relation_old)
else:
track.relation = 0.5
for artist in ArtistFactory.active_artists():
if (artist.relation_old > 0.501) or (artist.relation_old < 0.499):
artist.relation = (decfacA + decfacB * artist.relation_old)
else:
artist.relation = 0.5
self._logger.info(u"update ranking: set old relation + decay took %s" %
benchmark)
if has_active_tracks:
self._ranking_updated = True
self.seed(seed_track, True)
benchmark.start()
at = []
tt = []
for track in TrackFactory.active_tracks():
"""
so we have:
ranking [0-1.0] (old)
rating [0-1.0]
relation [0-1.0]
random [0-1.0]
and:
factor min(track_lastplayed/started,artist_lastplayed/started) [0-1.0]
? moved to next_file()
"""
artist = track.artist
r = random()
# calculate new ranking
if track.boost:
self._logger.info(u"pre boost: %s" % track)
track.relation = (track.relation + 99.0) / 100.0
self._logger.info(u"post boost: %s" % track)
elif track.ban:
track.relation = 0.0
# mix with artist relation if we don't have a track relation
if track.relation_cnt == 0:
if artist.relation_cnt > 0:
track.relation = (0.75 * track.relation + 0.25 * artist.relation)
top_ten(at, u'relation cnt = {} with {} now {} to {}'.format(
artist.relation_cnt, scale_rating(artist.relation),
scale_rating(track.relation), artist), track.relation)
else:
top_ten(tt, u'relation cnt = {} with {} to {}'.format(
track.relation_cnt, scale_rating(track.relation), track),
track.relation)
track.ranking = (
self._factor_ranking['rating'] * track.get_rating() +
self._factor_ranking['relation'] * track.relation +
self._factor_ranking['random'] * r
)
self._logger.info(u"update ranking: took %s" % benchmark)
self._update_queued = False
if not locked: self.release()
def dump_stats(self):
self._logger.info(u"%d artists, %d tracks %d files" % (
ArtistFactory.len(), TrackFactory.len(), FileFactory.len()))
def __init__(self, config):
Log.__init__(self, self._title)
self._logger.info(u"init")
Worker.__init__(self, lifo=False)
self._config = config
self._library_filename = self._config.get('Library', 'path',
join(self._config.configdir(), 'library.pkl'))
Factories.clear()
Logger.set_logger(self._logger)
if self._config.get('TrackRelation', 'use_db', True):
TrackRelationFactory.use_db()
TrackRelationFactory.set_path(self._config.get('TrackRelation',
'path', join(self._config.configdir(), '')))
else:
TrackRelationFactory.use_fs()
TrackRelationFactory.set_path(self._config.get('TrackRelation',
'path', join(self._config.configdir(), 'track')))
TrackRelationFactory.set_maxentries(
self._config.get('TrackRelation', 'maxentries', 500))
if self._config.get('ArtistRelation', 'use_db', True):
ArtistRelationFactory.use_db()
ArtistRelationFactory.set_path(self._config.get('ArtistRelation',
'path', join(self._config.configdir(), '')))
else:
ArtistRelationFactory.use_fs()
ArtistRelationFactory.set_path(self._config.get('ArtistRelation',
'path', join(self._config.configdir(), 'artist')))
ArtistRelationFactory.set_maxentries(
self._config.get('ArtistRelation', 'maxentries', 500))
if self._config.get('Lookup', 'QueueResults', 'False') == 'True':
self._queue_lookup_results = True
else:
self._queue_lookup_results = False
# FIXME: log if one of those libs is not present
if self._config.get('Lookup', 'UseLastFM', 'True') == 'True':
self._lastfm = LastFM(config)
self._lastfm.start()
self._thres_lastfm_lookup = self._config.getint('Lookup',
'ThresholdLastFM', self._thres_lastfm_lookup)
if self._config.get('Lookup', 'UseEchoNest', 'True') == 'True':
self._echonest = EchoNest(config)
self._echonest.start()
self._thres_echonest_lookup = self._config.getint('Lookup',
'ThresholdEchoNest', self._thres_echonest_lookup)
# read, normalize and update ranking factors
factor = 0.0
for k in self._factor_ranking.keys():
self._factor_ranking[k] = self._config.getfloat('Ranking',
"factor%s" % k, self._factor_ranking[k])
factor += self._factor_ranking[k]
for k in self._factor_ranking.keys():
self._factor_ranking[k] /= factor
self._config.set('Ranking', "factor%s" % k, self._factor_ranking[k])
self._relation_decay = self._config.getfloat('Ranking', 'RelationDecay',
self._relation_decay)
self._queue_update_factor = self._config.getfloat('Rating',
'QueueUpdateFactor', self._queue_update_factor)
self._unplayed_rating = self._config.getfloat('Ranking',
'UnplayedRating', self._unplayed_rating)
self._thres_track_lastplayed = self._config.getint('Ranking',
'ThresholdTrackLastPlayed', self._thres_track_lastplayed)
self._thres_track_laststarted= self._config.getint('Ranking',
'ThresholdTrackLastStarted', self._thres_track_laststarted)
self._thres_track_lastqueued= self._config.getint('Ranking',
'ThresholdTrackLastQueued', self._thres_track_lastqueued)
self._thres_artist_lastplayed = self._config.getint('Ranking',
'ThresholdArtistLastPlayed', self._thres_artist_lastplayed)
self._thres_artist_laststarted= self._config.getint('Ranking',
'ThresholdArtistLastStarted', self._thres_artist_laststarted)
self._thres_artist_lastqueued= self._config.getint('Ranking',
'ThresholdArtistLastQueued', self._thres_artist_lastqueued)