def updateRules(self): raw = tsv.parse_all(datadir("rules"), "string", "string", "string", "string") self.rules_belongtogether = [ b for [a, b, c, d] in raw if a == "belongtogether" ] self.rules_notanartist = [ b for [a, b, c, d] in raw if a == "notanartist" ] self.rules_replacetitle = { b.lower(): c for [a, b, c, d] in raw if a == "replacetitle" } self.rules_replaceartist = { b.lower(): c for [a, b, c, d] in raw if a == "replaceartist" } self.rules_ignoreartist = [ b.lower() for [a, b, c, d] in raw if a == "ignoreartist" ] self.rules_addartists = { c.lower(): (b.lower(), d) for [a, b, c, d] in raw if a == "addartists" } self.rules_fixartists = { c.lower(): b for [a, b, c, d] in raw if a == "fixartists" } self.rules_artistintitle = { b.lower(): c for [a, b, c, d] in raw if a == "artistintitle" }
def build_db(): log("Building database...") global SCROBBLES, ARTISTS, TRACKS global TRACKS_NORMALIZED_SET, TRACKS_NORMALIZED, ARTISTS_NORMALIZED_SET, ARTISTS_NORMALIZED global SCROBBLESDICT, STAMPS SCROBBLES = [] ARTISTS = [] TRACKS = [] STAMPS = [] SCROBBLESDICT = {} TRACKS_NORMALIZED = [] ARTISTS_NORMALIZED = [] ARTISTS_NORMALIZED_SET = set() TRACKS_NORMALIZED_SET = set() # parse files db = tsv.parse_all(datadir("scrobbles"),"int","string","string",comments=False) #db = parseAllTSV("scrobbles","int","string","string",escape=False) for sc in db: artists = sc[1].split("␟") title = sc[2] time = sc[0] readScrobble(artists,title,time) # optimize database SCROBBLES.sort(key = lambda tup: tup[1]) #SCROBBLESDICT = {obj[1]:obj for obj in SCROBBLES} STAMPS = [t for t in SCROBBLESDICT] STAMPS.sort() # inform malojatime module about earliest scrobble if len(STAMPS) > 0: register_scrobbletime(STAMPS[0]) # NOT NEEDED BECAUSE WE DO THAT ON ADDING EVERY ARTIST ANYWAY # get extra artists with no real scrobbles from countas rules #for artist in coa.getAllArtists(): #for artist in coa.getCreditedList(ARTISTS): # if artist not in ARTISTS: # log(artist + " is added to database because of countas rules",module="debug") # ARTISTS.append(artist) # coa.updateIDs(ARTISTS) #start regular tasks utilities.update_medals() utilities.update_weekly() utilities.send_stats() global ISSUES ISSUES = check_issues() log("Database fully built!")
def updateRules(self): raw = tsv.parse_all(data_dir["rules"](),"string","string","string") self.rules_countas = {b:c for [a,b,c] in raw if a=="countas"} self.rules_countas_id = {} self.rules_include = {} #Twice the memory, double the performance! # (Yes, we're saving redundant information here, but it's not unelegant if it's within a closed object!) for a in self.rules_countas: self.rules_include[self.rules_countas[a]] = self.rules_include.setdefault(self.rules_countas[a],[]) + [a]
def updateRules(self): raw = tsv.parse_all("rules","string","string","string") self.rules_belongtogether = [b for [a,b,c] in raw if a=="belongtogether"] self.rules_notanartist = [b for [a,b,c] in raw if a=="notanartist"] self.rules_replacetitle = {b.lower():c for [a,b,c] in raw if a=="replacetitle"} self.rules_replaceartist = {b.lower():c for [a,b,c] in raw if a=="replaceartist"} # we always need to be able to tell if our current database is made with the current rules self.checksums = utilities.checksumTSV("rules")
def updateRules(self): raw = tsv.parse_all(datadir("rules"), "string", "string", "string", "string") self.rules_belongtogether = [ b for [a, b, c, d] in raw if a == "belongtogether" ] self.rules_notanartist = [ b for [a, b, c, d] in raw if a == "notanartist" ] self.rules_replacetitle = { b.lower(): c for [a, b, c, d] in raw if a == "replacetitle" } self.rules_replaceartist = { b.lower(): c for [a, b, c, d] in raw if a == "replaceartist" } self.rules_ignoreartist = [ b.lower() for [a, b, c, d] in raw if a == "ignoreartist" ] self.rules_addartists = { c.lower(): (b.lower(), d) for [a, b, c, d] in raw if a == "addartists" } self.rules_fixartists = { c.lower(): b for [a, b, c, d] in raw if a == "fixartists" } self.rules_artistintitle = { b.lower(): c for [a, b, c, d] in raw if a == "artistintitle" } #self.rules_regexartist = [[b,c] for [a,b,c,d] in raw if a=="regexartist"] #self.rules_regextitle = [[b,c] for [a,b,c,d] in raw if a=="regextitle"] # TODO #self.plugin_artistparsers = [] #self.plugin_titleparsers = [] #if settings.get_settings("USE_PARSE_PLUGINS"): # for ep in pkg_resources.iter_entry_points(group='maloja.artistparsers'): # self.plugin_artistparsers.append(ep.load()) # for ep in pkg_resources.iter_entry_points(group='maloja.titleparsers'): # self.plugin_titleparsers.append(ep.load()) # we always need to be able to tell if our current database is made with the current rules self.checksums = utilities.checksumTSV(datadir("rules"))
def build_db(): global dbstatus dbstatus['healthy'] = False dbstatus['rebuildinprogress'] = True log("Building database...") global SCROBBLES, ARTISTS, TRACKS global TRACKS_NORMALIZED_SET, TRACKS_NORMALIZED, ARTISTS_NORMALIZED_SET, ARTISTS_NORMALIZED global SCROBBLESDICT, STAMPS SCROBBLES = [] ARTISTS = [] TRACKS = [] STAMPS = [] SCROBBLESDICT = {} TRACKS_NORMALIZED = [] ARTISTS_NORMALIZED = [] ARTISTS_NORMALIZED_SET = set() TRACKS_NORMALIZED_SET = set() # parse files db = tsv.parse_all(data_dir['scrobbles'](), "int", "string", "string", comments=False) scrobblenum = len(db) log(f"Found {scrobblenum} scrobbles...") usebar = not settings.get_settings("CLEAN_OUTPUT") if usebar: pbar = ProgressBar(max=scrobblenum, prefix="Loading scrobbles") else: n = 0 m = max(int(scrobblenum / 25), 20) #db = parseAllTSV("scrobbles","int","string","string",escape=False) for sc in db: artists = sc[1].split("␟") title = sc[2] time = sc[0] readScrobble(artists, title, time) if usebar: pbar.progress() else: n += 1 if n % m == 0: log(f"Loaded {n}/{scrobblenum}...") if usebar: pbar.done() log("Database loaded, optimizing...") # optimize database SCROBBLES.sort(key=lambda tup: tup[1]) #SCROBBLESDICT = {obj[1]:obj for obj in SCROBBLES} STAMPS = [t for t in SCROBBLESDICT] STAMPS.sort() # inform malojatime module about earliest scrobble if len(STAMPS) > 0: register_scrobbletime(STAMPS[0]) # NOT NEEDED BECAUSE WE DO THAT ON ADDING EVERY ARTIST ANYWAY # get extra artists with no real scrobbles from countas rules #for artist in coa.getAllArtists(): #for artist in coa.getCreditedList(ARTISTS): # if artist not in ARTISTS: # log(artist + " is added to database because of countas rules",module="debug") # ARTISTS.append(artist) # coa.updateIDs(ARTISTS) #start regular tasks utilities.update_medals() utilities.update_weekly() utilities.send_stats() global ISSUES ISSUES = check_issues() dbstatus['healthy'] = True dbstatus['rebuildinprogress'] = False log("Database fully built!")