def scrape_cur_standings():
table_url = 'http://thensbl.com/orgstand.htm'
tables = get_tables(table_url)
standings_changed = False
for table in tables:
titles = table.find_all('tr', class_=re.compile('dmrptsecttitle'))
for title in titles:
element = []
tit = title.get_text()
if tit == 'Divisional':
sql_table = 'team_standings'
rows = table.find_all('tr', class_=re.compile('dmrptbody'))
for row in rows:
element = []
for data in row:
if data.get_text() == ' ':
element.append(None)
else:
#strip takes white space away from the front and end of a text string
element.append(data.get_text().strip())
year = element[0]
team_location_name = element[1]
wins = element[2]
losses = element[3]
if team_location_name is not None:
full_name = helper.get_team_name(
team_location_name, year)
qry = """SELECT ts.year
, ts.team_name
, MAX(ts.games_played) AS gp
FROM team_standings ts
WHERE 1
AND ts.team_name = '%s'
AND ts.year = %s
GROUP BY ts.team_name, ts.year"""
prev_gp = db.query(qry % (full_name, year))
if prev_gp == ():
print "\n\nNEW SEASON!!!!!\n\n"
prev_gp == 0
else:
prev_gp = prev_gp[0][2]
if int(wins) + int(losses) != prev_gp:
standings_changed = True
# print full_name, int(wins)+int(losses), prev_gp, standings_changed
return standings_changed
def input_data(ratings, sql_table, cats, year):
print '\t' + sql_table
entries = []
for player in ratings:
entry = {}
entry['year'] = year
for cat, val in zip(cats, player):
# any category we aren't interested in recording, we mark as foo
if cat != 'foo':
# entry[cat] = val #####
if cat == 'player_name' and val is not None:
entry[cat] = val.replace('*', '').replace('#', '')
else:
entry[cat] = val
if (entry.get("player_name") not in ('Total', None, '', 'Other')
and entry.get("team_abb") not in ('Total', None, '', 'Other')):
entries.append(entry)
elif entry.get("team_name") not in ('Total', None, '', 'Other'):
full_name = helper.get_team_name(entry.get("team_name"), year)
entry['team_name'] = full_name
if sql_table == 'team_standings':
entry['games_played'] = int(entry.get('w')) + int(
entry.get('l'))
entries.append(entry)
if 'player_name' in entry:
helper.input_name(entry.get('player_name'))
# used for debugging
# if entries != []:
# for entry in entries[0:30]:
# print '\t\t',
# print entry
# raw_input("")
if entries != []:
db.insertRowDict(entries,
sql_table,
insertMany=True,
rid=0,
replace=True)
db.conn.commit()
