import sys from functions import tournaments, array2csv # Command line input start_year = str(sys.argv[1]) end_year = str(sys.argv[2]) # Iterate through the years and scrape tourney data print('') print('Year Tournaments') print('---- -----------') tourney_data = [] for h in range(int(start_year), int(end_year) + 1): year = str(h) tourney_data += tournaments(year) # Output to CSV filename = 'tournaments_' + start_year + '-' + end_year array2csv(tourney_data, filename)
from functions import html_parse_tree, xpath_parse, regex_strip_array, array2csv weeks_url = "http://www.atpworldtour.com/en/rankings/singles" weeks_tree = html_parse_tree(weeks_url) weeks_xpath = "//ul[@data-value = 'rankDate']/li/@data-value" weeks_parsed = xpath_parse(weeks_tree, weeks_xpath) weeks_cleaned = regex_strip_array(weeks_parsed) weeks_list = [[week] for week in weeks_cleaned] # Output to CSV filename = 'weeks' array2csv(weeks_list, filename)
# Match stats URL XPath match_stats_url_xpath = tourney_match_count_xpath = "//table[contains(@class, 'day-table')]/tbody[*]/tr[*]/td[contains(@class, 'day-table-score')]/a/@href" match_stats_url_cleaned = xpath_parse(tourney_tree, match_stats_url_xpath) # Filter problematic URL's match_stats_url_suffixes = [] for foo in match_stats_url_cleaned: if foo.find('//') == -1: match_stats_url_suffixes.append(foo) # STEP 2: Parse match stats if len(match_stats_url_suffixes) > 0: # Parse match stats asynchronously match_stats_data_scrape += asynchronous(match_stats_url_suffixes, scrape_match_stats, tourney_index, tourney_slug) # Parse match stats synchronously #match_stats_data_scrape += synchronous(match_stats_url_suffixes, scrape_match_stats, tourney_index, tourney_slug) else: spacing1 = format_spacing(5, tourney_index) spacing2 = format_spacing(15, tourney_slug) print tourney_index + spacing1 + ' ' + tourney_slug + spacing2 + ' Match stats URL problems' # STEP 3: Output to CSV filename = "match_stats_" + year + "_" + start_index array2csv(match_stats_data_scrape, filename)
tourney_url_suffix_split = tourney_url_suffixes[i].split('/') tourney_slug = tourney_url_suffix_split[4] # Match stats URL XPath match_stats_url_xpath = tourney_match_count_xpath = "//table[contains(@class, 'day-table')]/tbody[*]/tr[*]/td[contains(@class, 'day-table-score')]/a/@href" match_stats_url_cleaned = xpath_parse(tourney_tree, match_stats_url_xpath) # Filter problematic URL's match_stats_url_suffixes = [] for foo in match_stats_url_cleaned: if foo.find('//') == -1: match_stats_url_suffixes.append(foo) # STEP 2: Parse match stats if len(match_stats_url_suffixes) > 0: # Parse match stats asynchronously match_stats_data_scrape += asynchronous(match_stats_url_suffixes, scrape_match_stats, tourney_index, tourney_slug) # Parse match stats synchronously #match_stats_data_scrape += synchronous(match_stats_url_suffixes, scrape_match_stats, tourney_index, tourney_slug) else: spacing1 = format_spacing(5, tourney_index) spacing2 = format_spacing(15, tourney_slug) print tourney_index + spacing1 + ' ' + tourney_slug + spacing2 + ' Match stats URL problems' # STEP 3: Output to CSV filename = "match_stats_" + year + "_" + start_index array2csv(match_stats_data_scrape, filename)
elif len(move_down_parsed) > 0: move_direction = 'down' else: move_direction = '' age = age_cleaned[i] points = int(points_parsed[i].replace(',', '')) tourneys = tourneys_parsed[i] try: name_text.encode('ascii') except UnicodeEncodeError: name_text = unidecode.unidecode(name_text) data = [ week_title, week_year, week_month, week_day, rank_text, rank_number, move, move_direction, age, points, tourneys, player_url, player_slug, name_text, country_text, player_id ] rankings.append(data) filename = 'rankings_' + str(h) + '_' + week """" with codecs.open(filename + '2.csv', 'w', encoding='utf8') as f: writer = csv.writer(f, delimiter=',') for row in rankings: writer.writerow(row) """ array2csv(rankings, filename) print(str(h) + " " + week)
for i in xrange(0 , len(tourney_urls_scrape)): if len(tourney_urls_scrape[i]) > 0: # STEP 2: Scrape tournament page match_data_scrape = [] match_urls_scrape = [] scrape_tourney_output = scrape_tourney(tourney_urls_scrape[i]) match_data_scrape = scrape_tourney_output[0] match_urls_scrape = scrape_tourney_output[1] #match_counter += len(match_data_scrape) # STEP 3: tourney_data + match_data for match in match_data_scrape: foo = tourney_data_scrape[i] + match tourney_match.append(foo) spacing_count1 = len('Order') - len(str(tourney_data_scrape[i][1])) spacing1 = '' for j in xrange(0, spacing_count1): spacing1 += ' ' spacing_count2 = 41 - len(tourney_data_scrape[i][2]) spacing2 = '' for j in xrange(0, spacing_count2): spacing2 += ' ' print year + ' ' + str(tourney_data_scrape[i][1]) + spacing1 + ' ' + tourney_data_scrape[i][2] + spacing2 + ' ' + str(len(match_data_scrape)) filename = "match_scores_" + start_year + "-" + end_year array2csv(tourney_match, filename)
checkIfEmptyReturnFirst(first_name_cleaned), checkIfEmptyReturnFirst(last_name_cleaned), rank, player_url, profile_picture, flag_code, checkIfEmptyReturnFirst(residence_cleaned), checkIfEmptyReturnFirst(birthplace_cleaned), checkIfEmptyReturnFirst(birthdate_cleaned), birth_year, birth_month, birth_day, checkIfEmptyReturnFirst(turned_pro_cleaned), checkIfEmptyReturnFirst(weight_lbs_cleaned), weight_kg_cleaned_extracted, height_ft_cleaned_extracted, height_inches, height_cm_cleaned_extracted, handedness_cleaned_extracted, backhand, checkIfEmptyReturnFirst(coach_cleaned), checkIfEmptyReturnFirst(career_high_cleaned), career_high_date_cleaned_extracted, checkIfEmptyReturnFirst(prize_money_year_cleaned), checkIfEmptyReturnFirst(prize_money_cleaned), checkIfEmptyReturnFirst(titles_year_cleaned), checkIfEmptyReturnFirst(titles_cleaned), checkIfEmptyReturnFirst(win_loss_year_cleaned), checkIfEmptyReturnFirst(win_loss_cleaned) ]) array2csv(new_rows, 'rankings_0_2019-07-01') # array2csv(profiles, 'profiles') array2csv8utf(profiles, 'profiles') end = time.time() print(end - start)
import sys from functions import tournaments, array2csv # Command line input start_year = str(sys.argv[1]) end_year = str(sys.argv[2]) # Iterate through the years and scrape tourney data print '' print 'Year Tournaments' print '---- -----------' tourney_data = [] for h in xrange(int(start_year), int(end_year) + 1): year = str(h) tourney_data += tournaments(year) # Output to CSV filename = 'tournaments_' + start_year + '-' + end_year array2csv(tourney_data, filename)
week_split = week.split('-') week_year = int(week_split[0]) week_month = int(week_split[1]) week_day = int(week_split[2]) week_title = week.replace('-','.') move = move_cleaned[i] move_up_xpath = "//table[@class='mega-table']/tbody/tr[" + str(i + 1) + "]/td[@class='move-cell']/div[@class='move-up']" move_up_parsed = xpath_parse(week_tree, move_up_xpath) move_down_xpath = "//table[@class='mega-table']/tbody/tr[" + str(i + 1) + "]/td[@class='move-cell']/div[@class='move-down']" move_down_parsed = xpath_parse(week_tree, move_down_xpath) if len(move_up_parsed) > 0: move_direction = 'up' elif len(move_down_parsed) > 0: move_direction = 'down' else: move_direction = '' age = age_cleaned[i] points = int(points_parsed[i].replace(',', '')) tourneys = tourneys_parsed[i] data = [week_title, week_year, week_month, week_day, rank_text, rank_number, move, move_direction, age, points, tourneys, player_url, player_slug, player_id] rankings.append(data) filename = 'rankings_' + str(h) + '_' + week array2csv(rankings, filename) print str(h) + " " + week
for i in xrange(0 , len(tourney_urls_scrape)): if len(tourney_urls_scrape[i]) > 0: # STEP 2: Scrape tournament page match_data_scrape = [] match_urls_scrape = [] scrape_tourney_output = scrape_tourney(tourney_urls_scrape[i]) match_data_scrape = scrape_tourney_output[0] match_urls_scrape = scrape_tourney_output[1] #match_counter += len(match_data_scrape) # STEP 3: tourney_data + match_data for match in match_data_scrape: foo = tourney_data_scrape[i] + match tourney_match.append(foo) spacing_count1 = len('Order') - len(str(tourney_data_scrape[i][1])) spacing1 = '' for j in xrange(0, spacing_count1): spacing1 += ' ' spacing_count2 = 41 - len(tourney_data_scrape[i][2]) spacing2 = '' for j in xrange(0, spacing_count2): spacing2 += ' ' print year + ' ' + str(tourney_data_scrape[i][1]) + spacing1 + ' ' + tourney_data_scrape[i][2] + spacing2 + ' ' + str(len(match_data_scrape)) filename = "match_scores_" + start_year + "-" + end_year array2csv(tourney_match, filename)