# This thing gets all the players from all the teams. import sqlite3 import scrape from bs4 import BeautifulSoup conn = sqlite3.connect('pool.sqlite') c = conn.cursor() insert = [] for row in c.execute('SELECT * FROM teams'): print row[1] + '/tab/players/season/2012' soup = scrape.get_link(row[1] + '/tab/players/season/2012') table = soup.find('tbody') rows = table.findAll('tr') for rows in table.findAll('tr'): cols = rows.findAll('td') insert.append((cols[1].a.get_text(), cols[1].a.get('href'), cols[2].get_text(), row[0])) c.executemany("""INSERT INTO players VALUES (?, ?, ?, ?)""", insert) conn.commit()
def dint(dint): if isinstance( dint, int ): return dint elif dint == "": return 0 else: return int(dint) c.execute('DELETE FROM "main"."player_data"') c.execute('SELECT * FROM players') result = c.fetchall() for row in result: print row[1] try: soup = scrape.get_link(row[1]) table = soup.find('tbody') rows = table.findAll('tr') # [0link, 1apps, 2subs, 3goals, 4Y, 5R, 6clean, 7conceded, 8position, 9team] score = [row[1], 0, 0, 0, 0, 0, 0, 0, '', ''] for rows in table.findAll('tr'): cols = rows.findAll('td') # Apps score[1] += dint(cols[3].get_text()) # Subs score[2] += dint(cols[4].get_text())