from hltv_scraper import helper, scraper, get_match_ids, get_finished_events threads = 32 statsids = helper.get_existing_data("statsIDs", 1) existing_statsids = helper.get_existing_data('rounds', 0) unscraped_stats = helper.remove_existing_data(existing_statsids, statsids, 'matches') chunks = (unscraped_stats[i:i + 250] for i in range(0, len(unscraped_stats), 250)) for group in chunks: statsToCheck = group # Step 1: add new matches to the event join table stats = helper.scrape(statsToCheck, scraper.get_match_stats, threads) new_stats = helper.fix_rounds(stats) # Step 2: Tabulate helper.tabulate("rounds", new_stats)
from hltv_scraper import helper, scraper, get_match_ids, get_finished_events processes = 32 players = list(range(0, 16050)) for i in range(0, len(players)): players[i] = str(players[i]) existing_players = helper.get_existing_data('players', 0) unscraped_players = helper.remove_existing_data(existing_players, players, 'players') chunks = (unscraped_players[i:i + 250] for i in range(0, len(unscraped_players), 250)) for group in chunks: players_to_check = group # Step 1: Add new events to eventIDs.csv new_players = helper.scrape(players_to_check, scraper.get_players, processes) # Step 2: Tabulate helper.tabulate("players", new_players)
from hltv_scraper import helper, scraper, get_match_ids, get_finished_events threads = 32 matches = helper.get_existing_data("matchIDs", 0) match_lookup = helper.csv_lookup('matchIDs', 0, 1) existing_matches = helper.get_existing_data('matchResults', 15) unscraped_matches = helper.remove_existing_data(existing_matches, matches, 'matches') for i in range(0, len(unscraped_matches)): unscraped_matches[i] = match_lookup[unscraped_matches[i]] chunks = (unscraped_matches[i:i+250] for i in range(0, len(unscraped_matches), 250)) for group in chunks: matchesToCheck = group # Step 1: add new matches to the event join table newEvents = helper.scrape(matchesToCheck, scraper.get_match_events, threads) # Step 2: Update matchResults.csv newMatchInfo = helper.scrape(matchesToCheck, scraper.get_match_info, threads) # Sometimes this returns a multi-dimensional array, so we remove it newMatchInfo = helper.fix_match_results(newMatchInfo, 16) # Step 3: Update playerStats.csv newPlayerStats = helper.scrape(matchesToCheck, scraper.get_player_stats, threads) # This returns a single array for each match with all of the player stats, so we un-array it newPlayerStats = helper.fix_player_stats(newPlayerStats) # Step 4: Update picksAndBans.csv
# Handle player stats stats = helper.fix_player_stats( helper.scrape(match_id, scraper.get_player_stats, processes)) for i in range(0, len(stats)): stats[i][1] = team_id_to_name[stats[i][1]] stats[i][2] = player_id_to_name[stats[i][2]] # Handle printing print(f"\nTest scrape results for {match_id[0]}:\n") print(f"Event: {eventID[0][1]}\n") helper.print_array("Map results", matchInfo, 1) helper.print_array("Player stats", stats, 1) sys.exit("Test complete.") # Make an array of existing Match and Event IDs existing_match_ids = helper.get_existing_data("matchIDs", 1) existing_event_ids = helper.get_existing_data("eventIDs", 3) existing_completed_events = helper.get_existing_data("completedEvents", 0) # Get the last ID so we know when to stop looking new_match_ids = get_match_ids.get_match_ids(existing_match_ids[-1]) new_completed_events = get_finished_events.get_finished_events( existing_completed_events[-1]) # Exit if there are no new matches if len(new_match_ids) < 1 and len(new_completed_events) < 1: print("Nothing to scrape!") # Just check for new matches and break out of the loop elif helper.check_args('check', sys.argv): print(
from hltv_scraper import helper, scraper, get_match_ids, get_finished_events threads = 32 matches = helper.get_existing_data("matchIDs", 0) match_lookup = helper.csv_lookup('matchIDs', 0, 1) existing_matches = helper.get_existing_data('joinMatchEvent', 0) unscraped_matches = helper.remove_existing_data(existing_matches, matches, 'matches') for i in range(0, len(unscraped_matches)): unscraped_matches[i] = match_lookup[unscraped_matches[i]] chunks = (unscraped_matches[i:i + 250] for i in range(0, len(unscraped_matches), 250)) for group in chunks: matchesToCheck = group # Step 1: add new matches to the event join table newEvents = helper.scrape(matchesToCheck, scraper.get_match_events, threads) # Step 2: Tabulate helper.tabulate("joinMatchEvent", newEvents)
import scraper from hltv_scraper import helper, scraper, get_match_ids, get_finished_events processes = 32 events = get_finished_events.get_finished_events() print(len(events)) existing_events = helper.get_existing_data('eventIDs', 3) print(len(existing_events)) unscraped_events = helper.remove_existing_data(existing_events, events, 'events') chunks = (unscraped_events[i:i + 250] for i in range(0, len(unscraped_events), 250)) for group in chunks: events_to_check = group # Step 1: Add new events to eventIDs.csv new_event_ids = helper.scrape(events_to_check, scraper.get_event_names, processes) # Step 2: Tabulate helper.tabulate("eventIDs", new_event_ids)
from hltv_scraper import helper, scraper threads = 32 matches = helper.get_existing_data("matchIDs", 1) existing_matches = helper.get_existing_data('playerStats', 9) unscraped_matches = helper.remove_existing_data(existing_matches, matches, 'matches') chunks = (unscraped_matches[i:i + 250] for i in range(0, len(unscraped_matches), 250)) for group in chunks: matches_to_check = group # Step 3: Update playerStats.csv new_player_stats = helper.scrape(matches_to_check, scraper.get_player_stats, threads) # This returns a single array for each match with all of the player stats, so we un-array it new_player_stats = helper.fix_player_stats(new_player_stats) # Step 5: Tabulate helper.tabulate("playerStats", new_player_stats)