def create_player_table(self):
mysql_db.execute(self.mysql, """delete from players where True;""")
self.mysql.flush()
self.mysql.commit()
all_players = []
for letter in list(string.ascii_lowercase):
if letter == 'x':
continue
url = PLAYERS_URL + letter + '/'
df = bbalref_scraper.PLAYERS.fetch_and_clean(url)
time.sleep(WAIT_BETWEEN_REQUESTS)
df_links = bbalref_scraper.scrape_hrefs_from_player_table(url, bbalref_scraper.PLAYERS.dom_id)
df_links = df_links.drop('player', axis=1)
# df = pd.merge(df, df_links, on='player', how='outer')
df = df.join(df_links)
all_players.append(df)
time.sleep(WAIT_BETWEEN_REQUESTS)
df = pd.concat(all_players, ignore_index=True)
df['scraped'] = False
df2 = df.astype(object).where(pd.notnull(df), None)
df2.to_sql('players', con=mysql_db.engine, if_exists='append', index=False)
self.mysql.flush()
self.mysql.commit()
def table_exists(self, table):
results = mysql_db.execute(self.mysql, """SHOW TABLES LIKE '%s';""" % table, df=False)
return True if results else False
def ensure_data_not_already_in_table(self, table, player_key):
mysql_db.execute(self.mysql, """DELETE FROM %s where player_key = '%s';""" % (table, player_key))
self.mysql.flush()
self.mysql.commit()
