from hltv_scraper import helper, scraper, get_match_ids, get_finished_events
threads = 32
statsids = helper.get_existing_data("statsIDs", 1)
existing_statsids = helper.get_existing_data('rounds', 0)
unscraped_stats = helper.remove_existing_data(existing_statsids, statsids,
'matches')
chunks = (unscraped_stats[i:i + 250]
for i in range(0, len(unscraped_stats), 250))
for group in chunks:
statsToCheck = group
# Step 1: add new matches to the event join table
stats = helper.scrape(statsToCheck, scraper.get_match_stats, threads)
new_stats = helper.fix_rounds(stats)
# Step 2: Tabulate
helper.tabulate("rounds", new_stats)
from hltv_scraper import helper, scraper, get_match_ids, get_finished_events
processes = 32
players = list(range(0, 16050))
for i in range(0, len(players)):
players[i] = str(players[i])
existing_players = helper.get_existing_data('players', 0)
unscraped_players = helper.remove_existing_data(existing_players, players,
'players')
chunks = (unscraped_players[i:i + 250]
for i in range(0, len(unscraped_players), 250))
for group in chunks:
players_to_check = group
# Step 1: Add new events to eventIDs.csv
new_players = helper.scrape(players_to_check, scraper.get_players,
processes)
# Step 2: Tabulate
helper.tabulate("players", new_players)
new_players = scraper.get_new_iterable_items("player",
helper.find_max("players", 0))
new_players = helper.scrape(new_players, scraper.get_players, processes)
# Step 11: Check event prize data for eventPrizes.csv and eventWinners.csv
completed_events_to_check = helper.remove_existing_data(
existing_completed_events, helper.un_dimension(new_completed_events,
0), 'completed events')
event_rewards = helper.scrape(completed_events_to_check,
scraper.get_event_rewards, processes)
event_winners = helper.scrape(completed_events_to_check,
scraper.get_event_winners, processes)
# Step 12: Tabulate
if tab:
helper.tabulate("matchIDs", new_match_ids)
helper.tabulate("joinMatchEvent", new_events)
helper.tabulate("matchResults", new_match_info)
helper.tabulate("playerStats", new_player_stats)
helper.tabulate("picksAndBans", picks_and_bans)
helper.tabulate("statsIDs", new_stats_ids)
helper.tabulate("rounds", round_stats_clean)
helper.tabulate("eventIDs", new_event_ids)
helper.tabulate("teams", new_teams)
helper.tabulate("players", new_players)
helper.tabulate("completedEvents", new_completed_events)
helper.tabulate("eventPrizes", event_rewards)
helper.tabulate("eventWinners", event_winners)
# Step 13: Debug
if helper.check_args('debug', sys.argv):
unscraped_matches[i] = match_lookup[unscraped_matches[i]]
chunks = (unscraped_matches[i:i+250] for i in range(0, len(unscraped_matches), 250))
for group in chunks:
matchesToCheck = group
# Step 1: add new matches to the event join table
newEvents = helper.scrape(matchesToCheck, scraper.get_match_events, threads)
# Step 2: Update matchResults.csv
newMatchInfo = helper.scrape(matchesToCheck, scraper.get_match_info, threads)
# Sometimes this returns a multi-dimensional array, so we remove it
newMatchInfo = helper.fix_match_results(newMatchInfo, 16)
# Step 3: Update playerStats.csv
newPlayerStats = helper.scrape(matchesToCheck, scraper.get_player_stats, threads)
# This returns a single array for each match with all of the player stats, so we un-array it
newPlayerStats = helper.fix_player_stats(newPlayerStats)
# Step 4: Update picksAndBans.csv
raw_picks_and_bans = helper.scrape(matchesToCheck, scraper.get_match_vetos, threads)
picks_and_bans = helper.fix_player_stats(raw_picks_and_bans)
# Step 5: Tabulate
helper.tabulate("joinMatchEvent", newEvents)
helper.tabulate("matchResults", newMatchInfo)
helper.tabulate("playerStats", newPlayerStats)
helper.tabulate("picksAndBans", picks_and_bans)
from hltv_scraper import helper, scraper, get_match_ids, get_finished_events
threads = 32
matches = helper.get_existing_data("matchIDs", 0)
match_lookup = helper.csv_lookup('matchIDs', 0, 1)
existing_matches = helper.get_existing_data('joinMatchEvent', 0)
unscraped_matches = helper.remove_existing_data(existing_matches, matches,
'matches')
for i in range(0, len(unscraped_matches)):
unscraped_matches[i] = match_lookup[unscraped_matches[i]]
chunks = (unscraped_matches[i:i + 250]
for i in range(0, len(unscraped_matches), 250))
for group in chunks:
matchesToCheck = group
# Step 1: add new matches to the event join table
newEvents = helper.scrape(matchesToCheck, scraper.get_match_events,
threads)
# Step 2: Tabulate
helper.tabulate("joinMatchEvent", newEvents)
from hltv_scraper import helper, scraper, get_match_ids, get_finished_events
processes = 32
completed_events = get_finished_events.get_finished_events('0')
helper.tabulate("completedEvents", completed_events)
events = helper.un_dimension(completed_events, 0)
chunks = (events[i:i+250] for i in range(0, len(events), 250))
for group in chunks:
events_to_check = group
# Step 1: Add new events
event_rewards = helper.scrape(events_to_check, scraper.get_event_rewards, processes)
event_winners = helper.scrape(events_to_check, scraper.get_event_winners, processes)
# Step 2: Tabulate
helper.tabulate("eventPrizes", event_rewards)
helper.tabulate("eventWinners", event_winners)
import scraper
from hltv_scraper import helper, scraper, get_match_ids, get_finished_events
processes = 32
events = get_finished_events.get_finished_events()
print(len(events))
existing_events = helper.get_existing_data('eventIDs', 3)
print(len(existing_events))
unscraped_events = helper.remove_existing_data(existing_events, events,
'events')
chunks = (unscraped_events[i:i + 250]
for i in range(0, len(unscraped_events), 250))
for group in chunks:
events_to_check = group
# Step 1: Add new events to eventIDs.csv
new_event_ids = helper.scrape(events_to_check, scraper.get_event_names,
processes)
# Step 2: Tabulate
helper.tabulate("eventIDs", new_event_ids)
from hltv_scraper import helper, scraper, get_match_ids, get_finished_events
processes = 32
players = list(range(0, 16050))
ids = get_match_ids.get_match_ids(0)
helper.tabulate("matchIDs", ids)
from hltv_scraper import helper, scraper
threads = 32
matches = helper.get_existing_data("matchIDs", 1)
existing_matches = helper.get_existing_data('playerStats', 9)
unscraped_matches = helper.remove_existing_data(existing_matches, matches,
'matches')
chunks = (unscraped_matches[i:i + 250]
for i in range(0, len(unscraped_matches), 250))
for group in chunks:
matches_to_check = group
# Step 3: Update playerStats.csv
new_player_stats = helper.scrape(matches_to_check,
scraper.get_player_stats, threads)
# This returns a single array for each match with all of the player stats, so we un-array it
new_player_stats = helper.fix_player_stats(new_player_stats)
# Step 5: Tabulate
helper.tabulate("playerStats", new_player_stats)