def updateRules(self):
raw = tsv.parse_all(datadir("rules"), "string", "string", "string",
"string")
self.rules_belongtogether = [
b for [a, b, c, d] in raw if a == "belongtogether"
]
self.rules_notanartist = [
b for [a, b, c, d] in raw if a == "notanartist"
]
self.rules_replacetitle = {
b.lower(): c
for [a, b, c, d] in raw if a == "replacetitle"
}
self.rules_replaceartist = {
b.lower(): c
for [a, b, c, d] in raw if a == "replaceartist"
}
self.rules_ignoreartist = [
b.lower() for [a, b, c, d] in raw if a == "ignoreartist"
]
self.rules_addartists = {
c.lower(): (b.lower(), d)
for [a, b, c, d] in raw if a == "addartists"
}
self.rules_fixartists = {
c.lower(): b
for [a, b, c, d] in raw if a == "fixartists"
}
self.rules_artistintitle = {
b.lower(): c
for [a, b, c, d] in raw if a == "artistintitle"
}
def build_db():
log("Building database...")
global SCROBBLES, ARTISTS, TRACKS
global TRACKS_NORMALIZED_SET, TRACKS_NORMALIZED, ARTISTS_NORMALIZED_SET, ARTISTS_NORMALIZED
global SCROBBLESDICT, STAMPS
SCROBBLES = []
ARTISTS = []
TRACKS = []
STAMPS = []
SCROBBLESDICT = {}
TRACKS_NORMALIZED = []
ARTISTS_NORMALIZED = []
ARTISTS_NORMALIZED_SET = set()
TRACKS_NORMALIZED_SET = set()
# parse files
db = tsv.parse_all(datadir("scrobbles"),"int","string","string",comments=False)
#db = parseAllTSV("scrobbles","int","string","string",escape=False)
for sc in db:
artists = sc[1].split("␟")
title = sc[2]
time = sc[0]
readScrobble(artists,title,time)
# optimize database
SCROBBLES.sort(key = lambda tup: tup[1])
#SCROBBLESDICT = {obj[1]:obj for obj in SCROBBLES}
STAMPS = [t for t in SCROBBLESDICT]
STAMPS.sort()
# inform malojatime module about earliest scrobble
if len(STAMPS) > 0: register_scrobbletime(STAMPS[0])
# NOT NEEDED BECAUSE WE DO THAT ON ADDING EVERY ARTIST ANYWAY
# get extra artists with no real scrobbles from countas rules
#for artist in coa.getAllArtists():
#for artist in coa.getCreditedList(ARTISTS):
# if artist not in ARTISTS:
# log(artist + " is added to database because of countas rules",module="debug")
# ARTISTS.append(artist)
# coa.updateIDs(ARTISTS)
#start regular tasks
utilities.update_medals()
utilities.update_weekly()
utilities.send_stats()
global ISSUES
ISSUES = check_issues()
log("Database fully built!")
def updateRules(self):
raw = tsv.parse_all(data_dir["rules"](),"string","string","string")
self.rules_countas = {b:c for [a,b,c] in raw if a=="countas"}
self.rules_countas_id = {}
self.rules_include = {} #Twice the memory, double the performance!
# (Yes, we're saving redundant information here, but it's not unelegant if it's within a closed object!)
for a in self.rules_countas:
self.rules_include[self.rules_countas[a]] = self.rules_include.setdefault(self.rules_countas[a],[]) + [a]
def updateRules(self):
raw = tsv.parse_all("rules","string","string","string")
self.rules_belongtogether = [b for [a,b,c] in raw if a=="belongtogether"]
self.rules_notanartist = [b for [a,b,c] in raw if a=="notanartist"]
self.rules_replacetitle = {b.lower():c for [a,b,c] in raw if a=="replacetitle"}
self.rules_replaceartist = {b.lower():c for [a,b,c] in raw if a=="replaceartist"}
# we always need to be able to tell if our current database is made with the current rules
self.checksums = utilities.checksumTSV("rules")
def updateRules(self):
raw = tsv.parse_all(datadir("rules"), "string", "string", "string",
"string")
self.rules_belongtogether = [
b for [a, b, c, d] in raw if a == "belongtogether"
]
self.rules_notanartist = [
b for [a, b, c, d] in raw if a == "notanartist"
]
self.rules_replacetitle = {
b.lower(): c
for [a, b, c, d] in raw if a == "replacetitle"
}
self.rules_replaceartist = {
b.lower(): c
for [a, b, c, d] in raw if a == "replaceartist"
}
self.rules_ignoreartist = [
b.lower() for [a, b, c, d] in raw if a == "ignoreartist"
]
self.rules_addartists = {
c.lower(): (b.lower(), d)
for [a, b, c, d] in raw if a == "addartists"
}
self.rules_fixartists = {
c.lower(): b
for [a, b, c, d] in raw if a == "fixartists"
}
self.rules_artistintitle = {
b.lower(): c
for [a, b, c, d] in raw if a == "artistintitle"
}
#self.rules_regexartist = [[b,c] for [a,b,c,d] in raw if a=="regexartist"]
#self.rules_regextitle = [[b,c] for [a,b,c,d] in raw if a=="regextitle"]
# TODO
#self.plugin_artistparsers = []
#self.plugin_titleparsers = []
#if settings.get_settings("USE_PARSE_PLUGINS"):
# for ep in pkg_resources.iter_entry_points(group='maloja.artistparsers'):
# self.plugin_artistparsers.append(ep.load())
# for ep in pkg_resources.iter_entry_points(group='maloja.titleparsers'):
# self.plugin_titleparsers.append(ep.load())
# we always need to be able to tell if our current database is made with the current rules
self.checksums = utilities.checksumTSV(datadir("rules"))
def build_db():
global dbstatus
dbstatus['healthy'] = False
dbstatus['rebuildinprogress'] = True
log("Building database...")
global SCROBBLES, ARTISTS, TRACKS
global TRACKS_NORMALIZED_SET, TRACKS_NORMALIZED, ARTISTS_NORMALIZED_SET, ARTISTS_NORMALIZED
global SCROBBLESDICT, STAMPS
SCROBBLES = []
ARTISTS = []
TRACKS = []
STAMPS = []
SCROBBLESDICT = {}
TRACKS_NORMALIZED = []
ARTISTS_NORMALIZED = []
ARTISTS_NORMALIZED_SET = set()
TRACKS_NORMALIZED_SET = set()
# parse files
db = tsv.parse_all(data_dir['scrobbles'](),
"int",
"string",
"string",
comments=False)
scrobblenum = len(db)
log(f"Found {scrobblenum} scrobbles...")
usebar = not settings.get_settings("CLEAN_OUTPUT")
if usebar: pbar = ProgressBar(max=scrobblenum, prefix="Loading scrobbles")
else:
n = 0
m = max(int(scrobblenum / 25), 20)
#db = parseAllTSV("scrobbles","int","string","string",escape=False)
for sc in db:
artists = sc[1].split("␟")
title = sc[2]
time = sc[0]
readScrobble(artists, title, time)
if usebar: pbar.progress()
else:
n += 1
if n % m == 0: log(f"Loaded {n}/{scrobblenum}...")
if usebar: pbar.done()
log("Database loaded, optimizing...")
# optimize database
SCROBBLES.sort(key=lambda tup: tup[1])
#SCROBBLESDICT = {obj[1]:obj for obj in SCROBBLES}
STAMPS = [t for t in SCROBBLESDICT]
STAMPS.sort()
# inform malojatime module about earliest scrobble
if len(STAMPS) > 0: register_scrobbletime(STAMPS[0])
# NOT NEEDED BECAUSE WE DO THAT ON ADDING EVERY ARTIST ANYWAY
# get extra artists with no real scrobbles from countas rules
#for artist in coa.getAllArtists():
#for artist in coa.getCreditedList(ARTISTS):
# if artist not in ARTISTS:
# log(artist + " is added to database because of countas rules",module="debug")
# ARTISTS.append(artist)
# coa.updateIDs(ARTISTS)
#start regular tasks
utilities.update_medals()
utilities.update_weekly()
utilities.send_stats()
global ISSUES
ISSUES = check_issues()
dbstatus['healthy'] = True
dbstatus['rebuildinprogress'] = False
log("Database fully built!")
