def test_empty(self) -> None:
results = Parallel.execute([])
self.assertEqual(results, [])
results = Parallel.map(lambda x: x, [])
self.assertEqual(results, [])
results = Parallel.call([])
self.assertEqual(results, [])
results = Parallel.flatten([])
self.assertEqual(results, [])
def get_scores_for_card(cardid: str) -> List[Score]:
return Parallel.flatten(
Parallel.call(
[client.get_records for client in self.clients],
game,
version,
APIConstants.ID_TYPE_INSTANCE,
[songid, songchart, cardid],
))
def get_score(self, game: str, version: int, userid: UserID, songid: int,
songchart: int) -> Optional[Score]:
# Helper function so we can iterate over all servers for a single card
def get_scores_for_card(cardid: str) -> List[Score]:
return Parallel.flatten(
Parallel.call(
[client.get_records for client in self.clients],
game,
version,
APIConstants.ID_TYPE_INSTANCE,
[songid, songchart, cardid],
))
relevant_cards = self.__get_cardids(userid)
if RemoteUser.is_remote(userid):
# No need to look up local score for this user
scores = Parallel.flatten(
Parallel.map(
get_scores_for_card,
relevant_cards,
))
localscore = None
else:
localscore, scores = Parallel.execute([
lambda: self.music.get_score(game, version, userid, songid,
songchart),
lambda: Parallel.flatten(
Parallel.map(
get_scores_for_card,
relevant_cards,
)),
])
topscore = localscore
for score in scores:
if int(score['song']) != songid:
continue
if int(score['chart']) != songchart:
continue
newscore = self.__format_score(game, version, songid, songchart,
score)
if topscore is None:
# No merging needed
topscore = newscore
continue
topscore = self.__merge_score(game, version, topscore, newscore)
return topscore
def get_all_profiles(self, game: str, version: int) -> List[Tuple[UserID, ValidatedDict]]:
# Fetch local and remote profiles, and then merge by adding remote profiles to local
# profiles when we don't have a profile for that user ID yet.
local_cards, local_profiles, remote_profiles = Parallel.execute([
self.user.get_all_cards,
lambda: self.user.get_all_profiles(game, version),
lambda: Parallel.flatten(Parallel.call(
[client.get_profiles for client in self.clients],
game,
version,
APIConstants.ID_TYPE_SERVER,
[],
)),
])
card_to_id = {cardid: userid for (cardid, userid) in local_cards}
id_to_profile = {userid: profile for (userid, profile) in local_profiles}
for profile in remote_profiles:
cardids = sorted([card.upper() for card in profile.get('cards', [])])
if len(cardids) == 0:
# We don't care about anonymous profiles
continue
local_cards = [cardid for cardid in cardids if cardid in card_to_id]
if len(local_cards) > 0:
# We have a local version of this profile!
continue
# Create a fake user with this profile
del profile['cards']
exact_match = profile.get('match', 'partial') == 'exact'
if not exact_match:
continue
userid = RemoteUser.card_to_userid(cardids[0])
refid = self.user.get_refid(game, version, userid)
extid = self.user.get_extid(game, version, userid)
# Add in our defaults we always provide
profile['game'] = game
profile['version'] = version
profile['refid'] = refid
profile['extid'] = extid
id_to_profile[userid] = self.__format_profile(ValidatedDict(profile))
return [(userid, id_to_profile[userid]) for userid in id_to_profile]
def test_basic(self) -> None:
results = Parallel.execute([
lambda: 1,
lambda: 2,
lambda: 3,
lambda: 4,
lambda: 5,
])
self.assertEqual(results, [1, 2, 3, 4, 5])
def test_function(self) -> None:
def fun(x: int) -> int:
return -x
results = Parallel.execute([
lambda: fun(1),
lambda: fun(2),
lambda: fun(3),
lambda: fun(4),
lambda: fun(5),
])
self.assertEqual(results, [-1, -2, -3, -4, -5])
def get_all_songs(
self,
game: str,
version: Optional[int] = None,
) -> List[Song]:
"""
Given a game and a version, look up all song/chart combos associated with that game.
Parameters:
game - String representing a game series.
version - Integer representing which version of the game.
Returns:
A list of Song objects detailing the song information for each song.
"""
if version is None:
# We could do a ton of work to support this by iterating over all versions
# and combining, but this isn't going to be used in that manner, so lets
# skip that for now.
return []
catalogs: List[Dict[str, List[Dict[str, Any]]]] = Parallel.call(
[client.get_catalog for client in self.clients], game, version)
retval: List[Song] = []
seen: Set[str] = set()
for catalog in catalogs:
for entry in catalog.get('songs', []):
song = self.__format_song(
game,
version,
int(entry['song']),
int(entry['chart']),
str(entry['title'] if entry['title'] is not None else "")
or None,
str(entry['artist'] if entry['artist'] is not None else "")
or None,
str(entry['genre'] if entry['genre'] is not None else "")
or None,
entry,
)
if song is None:
continue
key = f"{song.id}_{song.chart}"
if key in seen:
continue
retval.append(song)
seen.add(key)
return retval
def get_items(self, game: str, version: int) -> List[Item]:
"""
Given a game/userid, find all items in the catalog.
Parameters:
game - String identifier of the game looking up the catalog.
version - Integer identifier of the version looking up this catalog.
Returns:
A list of item objects.
"""
catalogs: List[Dict[str, List[Dict[str, Any]]]] = Parallel.call(
[client.get_catalog for client in self.clients], game, version)
retval: List[Item] = []
seen: Set[str] = set()
for catalog in catalogs:
for catalogtype in catalog:
# Simple LUT for now, might need to be complicated later
if game == GameConstants.SDVX:
translation = {
"purchases": self.__translate_sdvx_song_unlock,
"appealcards": self.__translate_sdvx_appealcard,
}.get(catalogtype, None)
elif game == GameConstants.JUBEAT:
translation = {
"emblems": self.__translate_jubeat_emblems,
}.get(catalogtype, None)
elif game == GameConstants.IIDX:
translation = {
"qpros": self.__translate_iidx_qpros,
}.get(catalogtype, None)
else:
translation = None
# If we don't have a mapping for this, ignore it
if translation is None:
continue
for entry in catalog[catalogtype]:
# Translate the entry
item = translation(entry)
# Now, see if it is unique, and if so, remember it
key = f"{item.type}_{item.id}"
if key in seen:
continue
retval.append(item)
seen.add(key)
return retval
def __profile_request(self, game: str, version: int, userid: UserID, exact: bool) -> Optional[ValidatedDict]:
# First, get or create the extid/refid for this virtual user
cardid = RemoteUser.userid_to_card(userid)
refid = self.user.get_refid(game, version, userid)
extid = self.user.get_extid(game, version, userid)
profiles = Parallel.flatten(Parallel.call(
[client.get_profiles for client in self.clients],
game,
version,
APIConstants.ID_TYPE_CARD,
[cardid],
))
for profile in profiles:
cards = [card.upper() for card in profile.get('cards', [])]
if cardid in cards:
# Sanitize the returned data
profile = copy.deepcopy(profile)
del profile['cards']
exact_match = profile.get('match', 'partial') == 'exact'
if exact and (not exact_match):
# This is a partial match, not for this game/version
continue
if 'match' in profile:
del profile['match']
# Add in our defaults we always provide
profile['game'] = game
profile['version'] = version if exact_match else 0
profile['refid'] = refid
profile['extid'] = extid
return self.__format_profile(ValidatedDict(profile))
return None
def test_call(self) -> None:
def fun1(x: int) -> int:
return x * 10
def fun2(x: int) -> int:
return -x * 10
def fun3(x: int) -> int:
return x * 2
def fun4(x: int) -> int:
return -x * 2
def fun5(x: int) -> int:
return x
results = Parallel.call([fun1, fun2, fun3, fun4, fun5], 2)
self.assertEqual(results, [20, -20, 4, -4, 2])
def test_class(self) -> None:
class A:
def fun(self, x: int) -> int:
return x * 10
class B:
def fun(self, x: int) -> int:
return x * 20
class C:
def fun(self, x: int) -> int:
return x * 30
class D:
def fun(self, x: int) -> int:
return x * 40
class E:
def fun(self, x: int) -> int:
return x * 50
classes = [A(), B(), C(), D(), E()]
results = Parallel.call([c.fun for c in classes], 2)
self.assertEqual(results, [20, 40, 60, 80, 100])
def get_any_profiles(self, game: str, version: int, userids: List[UserID]) -> List[Tuple[UserID, Optional[ValidatedDict]]]:
if len(userids) == 0:
return []
remote_ids = [
userid for userid in userids
if RemoteUser.is_remote(userid)
]
local_ids = [
userid for userid in userids
if not RemoteUser.is_remote(userid)
]
if len(remote_ids) == 0:
# We only have local profiles here, just pass on to the underlying layer
return self.user.get_any_profiles(game, version, local_ids)
else:
# We have to fetch some local profiles and some remote profiles, and then
# merge them together
card_to_userid = {
RemoteUser.userid_to_card(userid): userid
for userid in remote_ids
}
local_profiles, remote_profiles = Parallel.execute([
lambda: self.user.get_any_profiles(game, version, local_ids),
lambda: Parallel.flatten(Parallel.call(
[client.get_profiles for client in self.clients],
game,
version,
APIConstants.ID_TYPE_CARD,
[RemoteUser.userid_to_card(userid) for userid in remote_ids],
))
])
for profile in remote_profiles:
cards = [card.upper() for card in profile.get('cards', [])]
for card in cards:
# Map it back to the requested user
userid = card_to_userid.get(card)
if userid is None:
continue
# Sanitize the returned data
profile = copy.deepcopy(profile)
del profile['cards']
exact_match = profile.get('match', 'partial') == 'exact'
if 'match' in profile:
del profile['match']
refid = self.user.get_refid(game, version, userid)
extid = self.user.get_extid(game, version, userid)
# Add in our defaults we always provide
profile['game'] = game
profile['version'] = version if exact_match else 0
profile['refid'] = refid
profile['extid'] = extid
local_profiles.append(
(userid, self.__format_profile(ValidatedDict(profile))),
)
# Mark that we saw this card/user
del card_to_userid[card]
# Finally, mark all missing remote profiles as None
for card in card_to_userid:
local_profiles.append((card_to_userid[card], None))
return local_profiles
def get_clear_rates(self) -> Dict[int, Dict[int, Dict[str, int]]]:
"""
Returns a dictionary similar to the following:
{
musicid: {
chart: {
total: total plays,
clears: total clears,
average: average score,
},
},
}
"""
all_attempts, remote_attempts = Parallel.execute([
lambda: self.data.local.music.get_all_attempts(
game=self.game,
version=self.version,
), lambda: self.data.remote.music.get_clear_rates(
game=self.game,
version=self.version,
)
])
attempts: Dict[int, Dict[int, Dict[str, int]]] = {}
for (_, attempt) in all_attempts:
# Terrible temporary structure is terrible.
if attempt.id not in attempts:
attempts[attempt.id] = {}
if attempt.chart not in attempts[attempt.id]:
attempts[attempt.id][attempt.chart] = {
'total': 0,
'clears': 0,
'average': 0,
}
# We saw an attempt, keep the total attempts in sync.
attempts[attempt.id][attempt.chart]['average'] = int(
((attempts[attempt.id][attempt.chart]['average'] *
attempts[attempt.id][attempt.chart]['total']) +
attempt.points) /
(attempts[attempt.id][attempt.chart]['total'] + 1))
attempts[attempt.id][attempt.chart]['total'] += 1
if attempt.data.get_int('clear_type', self.CLEAR_TYPE_NO_PLAY) in [
self.CLEAR_TYPE_NO_PLAY, self.CLEAR_TYPE_FAILED
]:
# This attempt was a failure, so don't count it against clears of full combos
continue
# It was at least a clear
attempts[attempt.id][attempt.chart]['clears'] += 1
# Merge in remote attempts
for songid in remote_attempts:
if songid not in attempts:
attempts[songid] = {}
for songchart in remote_attempts[songid]:
if songchart not in attempts[songid]:
attempts[songid][songchart] = {
'total': 0,
'clears': 0,
'average': 0,
}
attempts[songid][songchart]['total'] += remote_attempts[
songid][songchart]['plays']
attempts[songid][songchart]['clears'] += remote_attempts[
songid][songchart]['clears']
return attempts
def test_flatten(self) -> None:
results = Parallel.flatten([[1, 2, 3], [4, 5, 6], [7, 8, 9], []])
self.assertEqual(results, [1, 2, 3, 4, 5, 6, 7, 8, 9])
def test_map(self) -> None:
def fun(x: int) -> int:
return x * 2
results = Parallel.map(fun, [1, 2, 3, 4, 5])
self.assertEqual(results, [2, 4, 6, 8, 10])