def parse_matches(path):
matches = []
index = 0
for dirpath, _, names in os.walk(path):
for name in filter( lambda n: 'json' in n, names):
index += 1
with open(os.path.join(dirpath, name)) as f:
match_dicts = json.load(f)
matches.extend(list(map(parse_match, match_dicts)))
print("Parsed file No. " + str(index))
nonempty = list(filter( lambda d: d != {}, matches))
write_dicts_to_file(nonempty, os.path.join(path, 'matches.csv'))
def parse_competitions(path):
competitions = []
index = 0
for dirpath, _, names in os.walk(path):
for name in filter(lambda n: 'json' in n, names):
index += 1
with open(os.path.join(dirpath, name)) as f:
match_dicts = json.load(f)
for competition in map(parse_competition, match_dicts):
if competition not in competitions:
competitions.append(competition)
print("Parsed file No. " + str(index))
nonempty = list(filter(lambda d: d != {}, competitions))
write_dicts_to_file(nonempty, os.path.join(path, 'competitions.csv'))
def scrape_pages(path):
teams = []
statistics = []
players = []
index = 0
for dirpath, _, names in os.walk(path):
for name in filter( lambda n: 'json' in n, names):
with open(os.path.join(dirpath, name)) as f:
d = scrape_page(f.read())
statistics.extend(d['stats'])
if d['team_a'] not in teams:
teams.append(d['team_a'])
if d['team_b'] not in teams:
teams.append(d['team_b'])
for p in d['players']:
if p not in players:
players.append(p)
print("Parsed file No. " + str(index))
index += 1
write_dicts_to_file(statistics, os.path.join(path, 'stats.csv'))
write_dicts_to_file(teams, os.path.join(path, 'teams.csv'))
write_dicts_to_file(players, os.path.join(path, 'players.csv'))