def start(self):
while (True):
teamsList = self.loadTeams()
team_id_list = self.genereateTeamIDList(teamsList)
try:
currentRelevantGames = self.getRelevantLiveLeagueGames(
team_id_list)
except ApiException:
currentRelevantGames = self.oldRelevantGames
new_finished_games = self.findFinishedGames(
self.oldRelevantGames, currentRelevantGames)
if len(new_finished_games) > 0:
self.finishedGames.extend(new_finished_games)
# process matches
match_processor = MatchProcessor()
processed_matches = []
to_remove_finished_games = [
] # games not up after 2hrs, can remove
for obj in self.finishedGames:
if obj["countdown"] <= 0:
match_details_obj = self.api_wrapper.getMatchDetails(
obj["match_id"])
if match_details_obj.isEmpty():
# match data is not available yet
obj["age"] = (obj["age"] + 1)
if obj["age"] > 60:
to_remove_finished_games.append(obj)
obj["countdown"] = int(
self.configMap["match_parser_countdown"])
continue
match_processor.process(match_details_obj)
processed_matches.append(obj)
# delete finished games older than 2hrs
for obj in to_remove_finished_games:
HelperTools.log("gave up on match " + obj["match_id"])
self.finishedGames.remove(obj)
# delete all parsed matches from finishedGames list
for game in processed_matches:
self.finishedGames.remove(game)
# iterate cooldowns of finished gamess
for obj in self.finishedGames:
countdown = obj["countdown"]
obj["countdown"] = countdown - int(
self.configMap["crawler_sleep_time"])
for hook in self.hooks:
hook.update()
if self.stopBool:
HelperTools.log("stop message received")
break
time.sleep(int(self.configMap["crawler_sleep_time"]))
def start(self):
while(True):
teamsList = self.loadTeams()
team_id_list = self.genereateTeamIDList(teamsList)
try:
currentRelevantGames = self.getRelevantLiveLeagueGames(team_id_list)
except ApiException:
currentRelevantGames = self.oldRelevantGames
new_finished_games = self.findFinishedGames(self.oldRelevantGames, currentRelevantGames)
if len(new_finished_games) > 0:
self.finishedGames.extend(new_finished_games)
# process matches
match_processor = MatchProcessor()
processed_matches = []
to_remove_finished_games = [] # games not up after 2hrs, can remove
for obj in self.finishedGames:
if obj["countdown"] <= 0:
match_details_obj = self.api_wrapper.getMatchDetails(obj["match_id"])
if match_details_obj.isEmpty():
# match data is not available yet
obj["age"] = (obj["age"] + 1)
if obj["age"] > 60:
to_remove_finished_games.append(obj)
obj["countdown"] = int(self.configMap["match_parser_countdown"])
continue
match_processor.process(match_details_obj)
processed_matches.append(obj)
# delete finished games older than 2hrs
for obj in to_remove_finished_games:
HelperTools.log("gave up on match " + obj["match_id"])
self.finishedGames.remove(obj)
# delete all parsed matches from finishedGames list
for game in processed_matches:
self.finishedGames.remove(game)
# iterate cooldowns of finished gamess
for obj in self.finishedGames:
countdown = obj["countdown"]
obj["countdown"] = countdown - int(self.configMap["crawler_sleep_time"])
for hook in self.hooks:
hook.update()
if self.stopBool:
HelperTools.log("stop message received")
break
time.sleep(int(self.configMap["crawler_sleep_time"]))
def update(self):
players_dir = HelperTools.getPlayersDir()
player_file = os.path.join(HelperTools.getWebDir(), "live", "player_list.json")
json_obj = {}
player_ids = []
for f in os.listdir(players_dir):
player_ids.append(int(f.split(".")[0]))
json_obj["all_players"] = player_ids
saveJsonToFile(json_obj, player_file)
def processPlayer(self, match_details_player, local_player):
if local_player is None:
return
player_info = local_player.getTeamName() + "." + local_player.getPlayerName()
player_info += " (" + local_player.getPlayerPosition() + ")"
HelperTools.log(" processing player: " + player_info)
avg_values = AverageValues(local_player.getPlayerPosition())
local_player.applyMatchDetails(match_details_player, avg_values)
avg_values.applyMatchDetails(match_details_player)
local_player.save()
avg_values.save()
def update(self):
players_dir = HelperTools.getPlayersDir()
player_file = os.path.join(HelperTools.getWebDir(), "live",
"player_list.json")
json_obj = {}
player_ids = []
for f in os.listdir(players_dir):
player_ids.append(int(f.split(".")[0]))
json_obj["all_players"] = player_ids
saveJsonToFile(json_obj, player_file)
def processPlayer(self, match_details_player, local_player):
if local_player is None:
return
player_info = local_player.getTeamName() + "." + local_player.getPlayerName()
player_info += " (" + local_player.getPlayerPosition() + ")"
HelperTools.log(" processing player: " + player_info)
avg_values = AverageValues(local_player.getPlayerPosition())
local_player.applyMatchDetails(match_details_player, avg_values)
avg_values.applyMatchDetails(match_details_player)
local_player.save()
avg_values.save()
def test_getMethodsWithNonTestedCodeString(self):
sample_file = os.path.join(HelperTools.getParentDir(__file__), "res",
"TestSample.test")
test_list = [4, 13, 14]
result = CoverageData.getMethodsWithNonTestedCodeString(
self, sample_file, test_list)
self.assertEqual(result, "test1() test3()")
def __init__(self, player_id):
self.local_player_filename = os.path.join(HelperTools.getWebDir(), "live", "players", str(player_id) + ".json")
self.json_player_data = loadJsonFromFile(self.local_player_filename)
self.isAnonymous = False
if self.isEmpty():
# new player, needs to be created
player_pos_and_name = TeamsFile().getPlayerPositionAndName(player_id)
player_pos = player_pos_and_name["player_position"]
player_name = player_pos_and_name["player_name"]
team_name = player_pos_and_name["team_name"]
self.json_player_data["player_id"] = player_id
self.json_player_data["player_name"] = player_name
self.json_player_data["player_position"] = player_pos
self.json_player_data["team_name"] = team_name
self.json_player_data["average_kda"] = 0
self.json_player_data["average_gpm"] = 0
self.json_player_data["average_lh_per_min"] = 0
self.json_player_data["average_xpm"] = 0
self.json_player_data["average_tower_damage"] = 0
self.json_player_data["average_hero_damage"] = 0
self.json_player_data["average_kda_rating"] = 50
self.json_player_data["average_gpm_rating"] = 50
self.json_player_data["average_lhpm_rating"] = 50
self.json_player_data["average_xpm_rating"] = 50
self.json_player_data["average_td_rating"] = 50
self.json_player_data["average_hd_rating"] = 50
self.json_player_data["average_fight_rating"] = 50
self.json_player_data["average_push_rating"] = 50
self.json_player_data["average_farm_rating"] = 50
self.json_player_data["average_total_rating"] = 50
self.json_player_data["counter"] = 0
self.json_player_data["10_last_matches"] = []
def loadJsonFromTestFile(self, filename):
sample = open(os.path.join(HelperTools.getParentDir(__file__), "res",
filename + ".json"),
encoding="utf8")
content = sample.read().strip()
json_obj = json.loads(content)
return json_obj
def findFinishedGames(self, old_relevant_games, current_relevant_games):
finished_games = []
for match_id in current_relevant_games:
if match_id not in old_relevant_games:
# relevant game just went live
HelperTools.log("a relevant game just went live: " + str(match_id))
for match_id in old_relevant_games:
if match_id not in current_relevant_games:
# relevant game is finished, can be parsed
obj = {"countdown": int(self.configMap["match_parser_countdown"]), "match_id": match_id, "age": 0}
finished_games.append(obj)
HelperTools.log("a relevant game just finished: " + str(match_id))
self.oldRelevantGames = current_relevant_games
return finished_games
def loadConfig(self):
configMap = {}
config_file = open(HelperTools.getConfigFile())
for line in config_file:
key = line.split(':')[0].strip()
value = line.split(':')[1].strip()
configMap[str(key)] = str(value)
return configMap
def test_loadLocalPlayer(self, MockHelperTools):
players_test_dir = os.path.join(HelperTools.getParentDir(__file__),
"res", "players_test")
MockHelperTools.getPlayersDir = MagicMock(
return_value=players_test_dir)
mp = MatchProcessor()
local_player_obj = mp.loadLocalPlayer("21232132")
self.assertEquals(local_player_obj.getPlayerID(), 21232132)
def loadConfig(self):
configMap = {}
config_file = open(HelperTools.getConfigFile())
for line in config_file:
key = line.split(':')[0].strip()
value = line.split(':')[1].strip()
configMap[str(key)] = str(value)
return configMap
def __init__(self):
self.hooks = []
self.hooks.append(PlayerListHook())
self.configMap = self.loadConfig()
api_key = self.loadApiKey()
self.api_wrapper = Dota2ApiWrapper(api_key)
self.stopBool = False
# list of match IDs
self.oldRelevantGames = []
# contains list of {"countdown": <cd>, "match_id": <match_id>, "age": <age>s}
self.finishedGames = []
HelperTools.log("crawler started")
def __init__(self):
self.hooks = []
self.hooks.append(PlayerListHook())
self.configMap = self.loadConfig()
api_key = self.loadApiKey()
self.api_wrapper = Dota2ApiWrapper(api_key)
self.stopBool = False
# list of match IDs
self.oldRelevantGames = []
# contains list of {"countdown": <cd>, "match_id": <match_id>, "age": <age>s}
self.finishedGames = []
HelperTools.log("crawler started")
def __init__(self, json_object):
ApiObject.__init__(self, json_object["result"])
self.players = []
for player in self.json_object["players"]:
self.players.append(MatchDetailsPlayer(player, self.getMatchLength()))
del player["ability_upgrades"]
# stuff that is not needed
del self.json_object["picks_bans"]
self.match_file = HelperTools.getMatchFile(self.getMatchID())
def __init__(self, json_object):
ApiObject.__init__(self, json_object["result"])
self.players = []
for player in self.json_object["players"]:
self.players.append(
MatchDetailsPlayer(player, self.getMatchLength()))
del player["ability_upgrades"]
# stuff that is not needed
del self.json_object["picks_bans"]
self.match_file = HelperTools.getMatchFile(self.getMatchID())
def __init__(self, player_position):
self.player_pos = player_position
self.avg_values_file = os.path.join(HelperTools.getWebDir(), "avg", player_position, "average_values.json")
self.json_avg_data = loadJsonFromFile(self.avg_values_file)
if self.isEmpty():
# instantiate avg file
self.json_avg_data["average_kda"] = 2.5
self.json_avg_data["average_gpm"] = 400
self.json_avg_data["average_lh_per_min"] = 5
self.json_avg_data["average_xpm"] = 400
self.json_avg_data["average_tower_damage"] = 25
self.json_avg_data["average_hero_damage"] = 250
self.json_avg_data["counter"] = 0
def findFinishedGames(self, old_relevant_games, current_relevant_games):
finished_games = []
for match_id in current_relevant_games:
if match_id not in old_relevant_games:
# relevant game just went live
HelperTools.log("a relevant game just went live: " +
str(match_id))
for match_id in old_relevant_games:
if match_id not in current_relevant_games:
# relevant game is finished, can be parsed
obj = {
"countdown": int(self.configMap["match_parser_countdown"]),
"match_id": match_id,
"age": 0
}
finished_games.append(obj)
HelperTools.log("a relevant game just finished: " +
str(match_id))
self.oldRelevantGames = current_relevant_games
return finished_games
def process(self, match_details_instance):
teams = match_details_instance.getRadiantTeamName() + " vs " + match_details_instance.getDireTeamName()
HelperTools.log(" processing match: " + str(match_details_instance.getMatchID()) + ": " + teams)
for match_details_player in match_details_instance.getPlayers():
# process player and average data
self.processPlayer(match_details_player, self.loadLocalPlayer(str(match_details_player.getAccountID())))
self.processMatch(match_details_instance)
HelperTools.log(" match processed.")
HelperTools.log("")
def process(self, match_details_instance):
teams = match_details_instance.getRadiantTeamName() + " vs " + match_details_instance.getDireTeamName()
HelperTools.log(" processing match: " + str(match_details_instance.getMatchID()) + ": " + teams)
for match_details_player in match_details_instance.getPlayers():
# process player and average data
self.processPlayer(match_details_player, self.loadLocalPlayer(str(match_details_player.getAccountID())))
self.processMatch(match_details_instance)
HelperTools.log(" match processed.")
HelperTools.log("")
def test_Local_Player(self):
par_dir = HelperTools.getParentDir(__file__)
path = os.path.join(par_dir, "res", "players_test", "47434686.json")
path2 = os.path.join(par_dir, "res", "players_test", "21232132.json")
player_obj = LocalPlayer(path)
player_obj2 = LocalPlayer(path2)
self.assertEqual(player_obj.getAverageKDA(), 0)
self.assertFalse(player_obj2.isEmpty())
self.assertEqual(player_obj2.getAverageGPM(), 630)
self.assertEqual(player_obj2.getAverageXPM(), 200)
self.assertEqual(player_obj2.getTeamName(), "TestTeam")
self.assertEqual(player_obj2.getPlayerName(), "h0r5t")
self.assertEqual(player_obj2.getAverageLHPM(), 8.42)
self.assertEqual(player_obj2.getAverageTowerDamage(), 2333)
self.assertEqual(player_obj2.getAverageHeroDamage(), 21000)
self.assertEqual(player_obj2.getCounter(), 23)
def test_Local_Player(self):
par_dir = HelperTools.getParentDir(__file__)
path = os.path.join(par_dir, "res", "players_test", "47434686.json")
path2 = os.path.join(par_dir, "res", "players_test", "21232132.json")
player_obj = LocalPlayer(path)
player_obj2 = LocalPlayer(path2)
self.assertEqual(player_obj.getAverageKDA(), 0)
self.assertFalse(player_obj2.isEmpty())
self.assertEqual(player_obj2.getAverageGPM(), 630)
self.assertEqual(player_obj2.getAverageXPM(), 200)
self.assertEqual(player_obj2.getTeamName(), "TestTeam")
self.assertEqual(player_obj2.getPlayerName(), "h0r5t")
self.assertEqual(player_obj2.getAverageLHPM(), 8.42)
self.assertEqual(player_obj2.getAverageTowerDamage(), 2333)
self.assertEqual(player_obj2.getAverageHeroDamage(), 21000)
self.assertEqual(player_obj2.getCounter(), 23)
def test_loadLocalPlayer(self, MockHelperTools):
players_test_dir = os.path.join(HelperTools.getParentDir(__file__), "res", "players_test")
MockHelperTools.getPlayersDir = MagicMock(return_value=players_test_dir)
mp = MatchProcessor()
local_player_obj = mp.loadLocalPlayer("21232132")
self.assertEquals(local_player_obj.getPlayerID(), 21232132)
def loadApiKey(self):
key_file = open(HelperTools.getApiKeyFile())
key = key_file.read().strip()
return key
def loadTeams(self):
teamsFile = HelperTools.getTeamsFile()
teams_json = loadJsonFromFile(teamsFile)
return teams_json
def loadTeams(self):
teamsFile = HelperTools.getTeamsFile()
teams_json = loadJsonFromFile(teamsFile)
return teams_json
def test_getParentDir(self):
path = os.path.join("a", "b", "c")
self.assertEqual(HelperTools.getParentDir(path), os.path.abspath(os.path.join("a", "b")))
def test_getParentDir(self):
path = os.path.join("a", "b", "c")
self.assertEqual(HelperTools.getParentDir(path),
os.path.abspath(os.path.join("a", "b")))
def __init__(self):
self.teams_data = loadJsonFromFile(HelperTools.getTeamsFile())
def loadApiKey():
key_file = open(HelperTools.getApiKeyFile())
key = key_file.read().strip()
return key
def loadJsonFromTestFile(self, filename):
sample = open(os.path.join(HelperTools.getParentDir(__file__), "res", filename + ".json"), encoding="utf8")
content = sample.read().strip()
json_obj = json.loads(content)
return json_obj
def test_getPlayersDir(self):
atrap_dir = HelperTools.getParentDir(HelperTools.getParentDir(HelperTools.getParentDir(__file__)))
path = os.path.join(atrap_dir, "web", "live", "players")
self.assertEqual(HelperTools.getPlayersDir(), path)
def applyRating(self, rating):
rating_data = rating.getRatings()
self.updateAverageWithName("average_kda_rating", rating_data["kda_rating"])
HelperTools.log(" kda_rating: " + str(rating_data["kda_rating"]))
self.updateAverageWithName("average_gpm_rating", rating_data["gpm_rating"])
HelperTools.log(" gpm_rating: " + str(rating_data["gpm_rating"]))
self.updateAverageWithName("average_lhpm_rating", rating_data["lhpm_rating"])
HelperTools.log(" lhpm_rating: " + str(rating_data["lhpm_rating"]))
self.updateAverageWithName("average_xpm_rating", rating_data["xpm_rating"])
HelperTools.log(" xpm_rating: " + str(rating_data["xpm_rating"]))
self.updateAverageWithName("average_hd_rating", rating_data["hd_rating"])
HelperTools.log(" hd_rating: " + str(rating_data["hd_rating"]))
self.updateAverageWithName("average_td_rating", rating_data["td_rating"])
HelperTools.log(" td_rating: " + str(rating_data["td_rating"]))
self.updateAverageWithName("average_fight_rating", rating_data["fight_rating"])
HelperTools.log(" fight_rating: " + str(rating_data["fight_rating"]))
self.updateAverageWithName("average_push_rating", rating_data["push_rating"])
HelperTools.log(" push_rating: " + str(rating_data["push_rating"]))
self.updateAverageWithName("average_farm_rating", rating_data["farm_rating"])
HelperTools.log(" farm_rating: " + str(rating_data["farm_rating"]))
self.updateAverageWithName("average_total_rating", rating_data["total_rating"])
HelperTools.log(" total_rating: " + str(rating_data["total_rating"]))
HelperTools.log("")
def applyMatchDetails(self, match_details_player, avg_values_instance):
self.updateAverageWithName("average_kda", match_details_player.getKDA())
HelperTools.log(" kda: " + str(match_details_player.getKDA()))
self.updateAverageWithName("average_gpm", match_details_player.getGPM())
HelperTools.log(" gpm: " + str(match_details_player.getGPM()))
self.updateAverageWithName("average_lh_per_min", match_details_player.getLastHitsPerMinute())
HelperTools.log(" lhpm: " + str(match_details_player.getLastHitsPerMinute()))
self.updateAverageWithName("average_xpm", match_details_player.getXPM())
HelperTools.log(" xpm: " + str(match_details_player.getXPM()))
self.updateAverageWithName("average_hero_damage", match_details_player.getHeroDamage())
HelperTools.log(" hd: " + str(match_details_player.getHeroDamage()))
self.updateAverageWithName("average_tower_damage", match_details_player.getTowerDamage())
HelperTools.log(" td: " + str(match_details_player.getTowerDamage()))
rating = MatchDetailsPlayerRating(match_details_player, avg_values_instance)
self.applyRating(rating)
self.incrementCounter()
def test_getMethodsWithNonTestedCodeString(self):
sample_file = os.path.join(HelperTools.getParentDir(__file__), "res", "TestSample.test")
test_list = [4, 13, 14]
result = CoverageData.getMethodsWithNonTestedCodeString(self, sample_file, test_list)
self.assertEqual(result, "test1() test3()")
def test_getPlayersDir(self):
atrap_dir = HelperTools.getParentDir(
HelperTools.getParentDir(HelperTools.getParentDir(__file__)))
path = os.path.join(atrap_dir, "web", "live", "players")
self.assertEqual(HelperTools.getPlayersDir(), path)
