def get_all_records(
self,
game: str,
version: Optional[int] = None,
userlist: Optional[List[UserID]] = None,
locationlist: Optional[List[int]] = None,
) -> List[Tuple[UserID, Score]]:
# First, pass off to local-only if this was called with parameters we don't support
if (version is None or userlist is not None
or locationlist is not None):
return self.music.get_all_records(game, version, userlist,
locationlist)
# Now, fetch all records remotely and locally
localcards, localscores, remotescores = Parallel.execute([
self.user.get_all_cards,
lambda: self.music.get_all_records(game, version, userlist,
locationlist),
lambda: Parallel.flatten(
Parallel.call(
[client.get_records for client in self.clients],
game,
version,
APIConstants.ID_TYPE_SERVER,
[],
)),
])
return self.__merge_global_scores(game, version, localcards,
localscores, remotescores)
def test_empty(self) -> None:
results = Parallel.execute([])
self.assertEqual(results, [])
results = Parallel.map(lambda x: x, [])
self.assertEqual(results, [])
results = Parallel.call([])
self.assertEqual(results, [])
results = Parallel.flatten([])
self.assertEqual(results, [])
def get_scores_for_card(cardid: str) -> List[Score]:
return Parallel.flatten(
Parallel.call(
[client.get_records for client in self.clients],
game,
version,
APIConstants.ID_TYPE_INSTANCE,
[songid, songchart, cardid],
))
def get_all_songs(
self,
game: str,
version: Optional[int] = None,
) -> List[Song]:
"""
Given a game and a version, look up all song/chart combos associated with that game.
Parameters:
game - String representing a game series.
version - Integer representing which version of the game.
Returns:
A list of Song objects detailing the song information for each song.
"""
if version is None:
# We could do a ton of work to support this by iterating over all versions
# and combining, but this isn't going to be used in that manner, so lets
# skip that for now.
return []
catalogs: List[Dict[str, List[Dict[str, Any]]]] = Parallel.call(
[client.get_catalog for client in self.clients], game, version)
retval: List[Song] = []
seen: Set[str] = set()
for catalog in catalogs:
for entry in catalog.get('songs', []):
song = self.__format_song(
game,
version,
int(entry['song']),
int(entry['chart']),
str(entry['title'] if entry['title'] is not None else "")
or None,
str(entry['artist'] if entry['artist'] is not None else "")
or None,
str(entry['genre'] if entry['genre'] is not None else "")
or None,
entry,
)
if song is None:
continue
key = f"{song.id}_{song.chart}"
if key in seen:
continue
retval.append(song)
seen.add(key)
return retval
def get_items(self, game: str, version: int) -> List[Item]:
"""
Given a game/userid, find all items in the catalog.
Parameters:
game - String identifier of the game looking up the catalog.
version - Integer identifier of the version looking up this catalog.
Returns:
A list of item objects.
"""
catalogs: List[Dict[str, List[Dict[str, Any]]]] = Parallel.call(
[client.get_catalog for client in self.clients], game, version)
retval: List[Item] = []
seen: Set[str] = set()
for catalog in catalogs:
for catalogtype in catalog:
# Simple LUT for now, might need to be complicated later
if game == GameConstants.SDVX:
translation = {
"purchases": self.__translate_sdvx_song_unlock,
"appealcards": self.__translate_sdvx_appealcard,
}.get(catalogtype, None)
elif game == GameConstants.JUBEAT:
translation = {
"emblems": self.__translate_jubeat_emblems,
}.get(catalogtype, None)
elif game == GameConstants.IIDX:
translation = {
"qpros": self.__translate_iidx_qpros,
}.get(catalogtype, None)
else:
translation = None
# If we don't have a mapping for this, ignore it
if translation is None:
continue
for entry in catalog[catalogtype]:
# Translate the entry
item = translation(entry)
# Now, see if it is unique, and if so, remember it
key = f"{item.type}_{item.id}"
if key in seen:
continue
retval.append(item)
seen.add(key)
return retval
def get_all_profiles(self, game: str, version: int) -> List[Tuple[UserID, ValidatedDict]]:
# Fetch local and remote profiles, and then merge by adding remote profiles to local
# profiles when we don't have a profile for that user ID yet.
local_cards, local_profiles, remote_profiles = Parallel.execute([
self.user.get_all_cards,
lambda: self.user.get_all_profiles(game, version),
lambda: Parallel.flatten(Parallel.call(
[client.get_profiles for client in self.clients],
game,
version,
APIConstants.ID_TYPE_SERVER,
[],
)),
])
card_to_id = {cardid: userid for (cardid, userid) in local_cards}
id_to_profile = {userid: profile for (userid, profile) in local_profiles}
for profile in remote_profiles:
cardids = sorted([card.upper() for card in profile.get('cards', [])])
if len(cardids) == 0:
# We don't care about anonymous profiles
continue
local_cards = [cardid for cardid in cardids if cardid in card_to_id]
if len(local_cards) > 0:
# We have a local version of this profile!
continue
# Create a fake user with this profile
del profile['cards']
exact_match = profile.get('match', 'partial') == 'exact'
if not exact_match:
continue
userid = RemoteUser.card_to_userid(cardids[0])
refid = self.user.get_refid(game, version, userid)
extid = self.user.get_extid(game, version, userid)
# Add in our defaults we always provide
profile['game'] = game
profile['version'] = version
profile['refid'] = refid
profile['extid'] = extid
id_to_profile[userid] = self.__format_profile(ValidatedDict(profile))
return [(userid, id_to_profile[userid]) for userid in id_to_profile]
def test_call(self) -> None:
def fun1(x: int) -> int:
return x * 10
def fun2(x: int) -> int:
return -x * 10
def fun3(x: int) -> int:
return x * 2
def fun4(x: int) -> int:
return -x * 2
def fun5(x: int) -> int:
return x
results = Parallel.call([fun1, fun2, fun3, fun4, fun5], 2)
self.assertEqual(results, [20, -20, 4, -4, 2])
def get_all_scores(
self,
game: str,
version: Optional[int] = None,
userid: Optional[UserID] = None,
songid: Optional[int] = None,
songchart: Optional[int] = None,
since: Optional[int] = None,
until: Optional[int] = None,
) -> List[Tuple[UserID, Score]]:
# First, pass off to local-only if this was called with parameters we don't support
if (version is None or userid is not None or songid is None):
return self.music.get_all_scores(game, version, userid, songid,
songchart, since, until)
# Now, figure out the request key based on passed in parameters
if songchart is None:
songkey = [songid]
else:
songkey = [songid, songchart]
# Now, fetch all the scores remotely and locally
localcards, localscores, remotescores = Parallel.execute([
self.user.get_all_cards,
lambda: self.music.get_all_scores(game, version, userid, songid,
songchart, since, until),
lambda: Parallel.flatten(
Parallel.call(
[client.get_records for client in self.clients],
game,
version,
APIConstants.ID_TYPE_SONG,
songkey,
since,
until,
)),
])
return self.__merge_global_scores(game, version, localcards,
localscores, remotescores)
def __profile_request(self, game: str, version: int, userid: UserID, exact: bool) -> Optional[ValidatedDict]:
# First, get or create the extid/refid for this virtual user
cardid = RemoteUser.userid_to_card(userid)
refid = self.user.get_refid(game, version, userid)
extid = self.user.get_extid(game, version, userid)
profiles = Parallel.flatten(Parallel.call(
[client.get_profiles for client in self.clients],
game,
version,
APIConstants.ID_TYPE_CARD,
[cardid],
))
for profile in profiles:
cards = [card.upper() for card in profile.get('cards', [])]
if cardid in cards:
# Sanitize the returned data
profile = copy.deepcopy(profile)
del profile['cards']
exact_match = profile.get('match', 'partial') == 'exact'
if exact and (not exact_match):
# This is a partial match, not for this game/version
continue
if 'match' in profile:
del profile['match']
# Add in our defaults we always provide
profile['game'] = game
profile['version'] = version if exact_match else 0
profile['refid'] = refid
profile['extid'] = extid
return self.__format_profile(ValidatedDict(profile))
return None
def test_class(self) -> None:
class A:
def fun(self, x: int) -> int:
return x * 10
class B:
def fun(self, x: int) -> int:
return x * 20
class C:
def fun(self, x: int) -> int:
return x * 30
class D:
def fun(self, x: int) -> int:
return x * 40
class E:
def fun(self, x: int) -> int:
return x * 50
classes = [A(), B(), C(), D(), E()]
results = Parallel.call([c.fun for c in classes], 2)
self.assertEqual(results, [20, 40, 60, 80, 100])
def get_clear_rates(
self,
game: str,
version: int,
songid: Optional[int] = None,
songchart: Optional[int] = None,
) -> Dict[int, Dict[int, Dict[str, int]]]:
"""
Given an optional songid, or optional songid and songchart, looks up clear rates
in remote servers that are connected to us. If neither id or chart is given, looks
up global clear rates. If songid is given, looks up clear rates for each chart for
the song. If songid and chart is given, looks up clear rates for that song/chart.
Returns a dictionary keyed by songid, whos values are a dictionary keyed by chart,
whos values are a dictionary containing integer counts keyed by 'plays', 'clears',
and 'combos'. An example is as follows:
{
musicid: {
chart: {
plays: total plays,
clears: total clears,
combos: total full combos,
},
},
}
"""
if songid is None and songchart is None:
statistics = Parallel.flatten(
Parallel.call(
[client.get_statistics for client in self.clients],
game,
version,
APIConstants.ID_TYPE_SERVER,
[],
))
elif songid is not None:
if songchart is None:
ids = [songid]
else:
ids = [songid, songchart]
statistics = Parallel.flatten(
Parallel.call(
[client.get_statistics for client in self.clients],
game,
version,
APIConstants.ID_TYPE_SONG,
ids,
))
else:
statistics = []
retval: Dict[int, Dict[int, Dict[str, int]]] = {}
for stat in statistics:
songid = stat.get('song')
songchart = stat.get('chart')
if songid is None or songchart is None:
continue
songid = int(songid)
songchart = int(songchart)
if songid not in retval:
retval[songid] = {}
if songchart not in retval[songid]:
retval[songid][songchart] = {
'plays': 0,
'clears': 0,
'combos': 0,
}
def get_val(v: str) -> int:
out = stat.get(v, -1)
if out < 0:
out = 0
return out
retval[songid][songchart]['plays'] += get_val('plays')
retval[songid][songchart]['clears'] += get_val('clears')
retval[songid][songchart]['combos'] += get_val('combos')
return retval
def get_scores(
self,
game: str,
version: int,
userid: UserID,
since: Optional[int] = None,
until: Optional[int] = None,
) -> List[Score]:
relevant_cards = self.__get_cardids(userid)
if RemoteUser.is_remote(userid):
# No need to look up local score for this user
scores = Parallel.flatten(
Parallel.call(
[client.get_records for client in self.clients],
game,
version,
APIConstants.ID_TYPE_CARD,
relevant_cards,
since,
until,
))
localscores: List[Score] = []
else:
localscores, scores = Parallel.execute([
lambda: self.music.get_scores(game, version, userid, since,
until),
lambda: Parallel.flatten(
Parallel.call(
[client.get_records for client in self.clients],
game,
version,
APIConstants.ID_TYPE_CARD,
relevant_cards,
since,
until,
)),
])
allscores: Dict[int, Dict[int, Score]] = {}
def add_score(score: Score) -> None:
if score.id not in allscores:
allscores[score.id] = {}
allscores[score.id][score.chart] = score
def get_score(songid: int, songchart: int) -> Optional[Score]:
return allscores.get(songid, {}).get(songchart)
# First, seed with local scores
for score in localscores:
add_score(score)
# Second, merge in remote scorse
for remotescore in scores:
songid = int(remotescore['song'])
chart = int(remotescore['chart'])
newscore = self.__format_score(game, version, songid, chart,
remotescore)
oldscore = get_score(songid, chart)
if oldscore is None:
add_score(newscore)
else:
add_score(self.__merge_score(game, version, oldscore,
newscore))
# Finally, flatten and return
finalscores: List[Score] = []
for songid in allscores:
for chart in allscores[songid]:
finalscores.append(allscores[songid][chart])
return finalscores
def get_any_profiles(self, game: str, version: int, userids: List[UserID]) -> List[Tuple[UserID, Optional[ValidatedDict]]]:
if len(userids) == 0:
return []
remote_ids = [
userid for userid in userids
if RemoteUser.is_remote(userid)
]
local_ids = [
userid for userid in userids
if not RemoteUser.is_remote(userid)
]
if len(remote_ids) == 0:
# We only have local profiles here, just pass on to the underlying layer
return self.user.get_any_profiles(game, version, local_ids)
else:
# We have to fetch some local profiles and some remote profiles, and then
# merge them together
card_to_userid = {
RemoteUser.userid_to_card(userid): userid
for userid in remote_ids
}
local_profiles, remote_profiles = Parallel.execute([
lambda: self.user.get_any_profiles(game, version, local_ids),
lambda: Parallel.flatten(Parallel.call(
[client.get_profiles for client in self.clients],
game,
version,
APIConstants.ID_TYPE_CARD,
[RemoteUser.userid_to_card(userid) for userid in remote_ids],
))
])
for profile in remote_profiles:
cards = [card.upper() for card in profile.get('cards', [])]
for card in cards:
# Map it back to the requested user
userid = card_to_userid.get(card)
if userid is None:
continue
# Sanitize the returned data
profile = copy.deepcopy(profile)
del profile['cards']
exact_match = profile.get('match', 'partial') == 'exact'
if 'match' in profile:
del profile['match']
refid = self.user.get_refid(game, version, userid)
extid = self.user.get_extid(game, version, userid)
# Add in our defaults we always provide
profile['game'] = game
profile['version'] = version if exact_match else 0
profile['refid'] = refid
profile['extid'] = extid
local_profiles.append(
(userid, self.__format_profile(ValidatedDict(profile))),
)
# Mark that we saw this card/user
del card_to_userid[card]
# Finally, mark all missing remote profiles as None
for card in card_to_userid:
local_profiles.append((card_to_userid[card], None))
return local_profiles
