def get_time(time_string):
cal = Calendar()
now = datetime.now(timezone("UTC"))
if time_string == "":
time = cal.parseDT("1 day", now)
else:
time = cal.parseDT(time_string, now)
return time[0]
async def poll(self):
cal = Calendar()
now = datetime.now(timezone("UTC"))
then = cal.parseDT("1 minute", now)
print("polling...")
for r in Reminder.select().where(Reminder.time.between(now, then[0])):
await self.bot.get_channel(r.channel_id).send(r.reminder_text)
def _guess_game_date(datetime_str):
# Try to turn a datetime string from FD into an actual datetime
datetime_str.replace('Sun', 'Sunday')
datetime_str.replace('Mon', 'Monday')
datetime_str.replace('Tues', 'Tuesday')
datetime_str.replace('Wed', 'Wednesday')
datetime_str.replace('Thurs', 'Thursday')
datetime_str.replace('Fri', 'Friday')
datetime_str.replace('Sat', 'Saturday')
cal = Calendar()
dt, ret_code = cal.parseDT(datetime_str)
return dt
def _guess_game_date(datetime_str):
# Try to turn a datetime string from FD into an actual datetime
datetime_str.replace('Sun', 'Sunday')
datetime_str.replace('Mon', 'Monday')
datetime_str.replace('Tues', 'Tuesday')
datetime_str.replace('Wed', 'Wednesday')
datetime_str.replace('Thurs', 'Thursday')
datetime_str.replace('Fri', 'Friday')
datetime_str.replace('Sat', 'Saturday')
cal = Calendar()
dt, ret_code = cal.parseDT(datetime_str)
return dt
def parse_mtime_single(value):
"""Convert a human readable time description to a """
if not HAS_PARSEDATETIME:
return int(value)
calendar = Calendar()
guess, rc = calendar.parseDT(value)
if rc is 0:
LOGGER.warning("Could not parse date: %s", value)
return int(value)
return guess.timestamp()
def parse_mtime_single(value):
"""Convert a human readable time description to a """
if not HAS_PARSEDATETIME:
return int(value)
calendar = Calendar()
guess, rc = calendar.parseDT(value)
if rc == 0:
LOGGER.warning('Could not parse date: %s', value)
return int(value)
return guess.timestamp()
async def convert(self, ctx: commands.Context, argument: str):
cal = Calendar()
utc = timezone("UTC")
utcnow = datetime.utcnow().replace(microsecond=0, tzinfo=utc)
parsed = cal.parseDT(argument, sourceTime=utcnow, tzinfo=utc)[0]
if parsed == utcnow:
raise commands.BadArgument(f'"{argument}" is not a valid time.')
elif parsed < utcnow:
raise commands.BadArgument(f'"{argument}" is in the past.')
return parsed
def parse_contest_element(browser_elem):
"""
Parse contest element for entry button, FanDuel game ID, FD table ID, game start as datetime, game name, and entry cost
:param selenium.webdriver.remote.webelement.WebElement browser_elem: contest div element
:return tuple[selenium.webdriver.remote.webelement.WebElement, int, int, datetime.datetime, str, int]: see description
"""
match_groups = contest_element_id_regex.match(browser_elem.get_attribute("id"))
fd_game_id = int(match_groups.group("gameid"))
fd_table_id = int(match_groups.group("tableid"))
game_title = browser_elem.find_element_by_xpath("div[1]/a/span[1]").text
game_fee = browser_elem.find_element_by_xpath("div[2]/span").text.replace('$','').replace(',','')
game_time = browser_elem.find_element_by_xpath("div[2]/time").text
cal = Calendar()
fd_game_date, ret_code = cal.parseDT(game_time)
entry_button_element = browser_elem.find_element_by_xpath("a")
return entry_button_element, fd_game_id, fd_table_id, fd_game_date, game_title, game_fee
def parse_contest_element(browser_elem):
"""
Parse contest element for entry button, FanDuel game ID, FD table ID, game start as datetime, game name, and entry cost
:param selenium.webdriver.remote.webelement.WebElement browser_elem: contest div element
:return tuple[selenium.webdriver.remote.webelement.WebElement, int, int, datetime.datetime, str, int]: see description
"""
match_groups = contest_element_id_regex.match(
browser_elem.get_attribute("id"))
fd_game_id = int(match_groups.group("gameid"))
fd_table_id = int(match_groups.group("tableid"))
game_title = browser_elem.find_element_by_xpath("div[1]/a/span[1]").text
game_fee = browser_elem.find_element_by_xpath("div[2]/span").text.replace(
'$', '').replace(',', '')
game_time = browser_elem.find_element_by_xpath("div[2]/time").text
cal = Calendar()
fd_game_date, ret_code = cal.parseDT(game_time)
entry_button_element = browser_elem.find_element_by_xpath("a")
return entry_button_element, fd_game_id, fd_table_id, fd_game_date, game_title, game_fee
def scrape_single_game(sport, browser, debug=False):
# Since we are separating contests by their start time now, we don't currently try to choose a
# best contest to scrape. We should certainly upgrade this in the future!
contest_element_id_regex = re.compile(r"contest_(?P<gameid>\d+)-(?P<tableid>\d+)")
browser_elem = browser.find_element_by_class_name('contest-list-item')
match_groups = contest_element_id_regex.match(browser_elem.get_attribute("id"))
fd_game_id = int(match_groups.group("gameid"))
fd_table_id = int(match_groups.group("tableid"))
fd_game_title = browser_elem.find_element_by_class_name("contest-name-text").text
entry_fee = browser_elem.find_element_by_class_name("entry-fee-cell").text.replace('$','').replace(',','')
game_time = browser_elem.find_element_by_class_name("startdate-cell").text
cal = Calendar()
new_parsed_dt, ret_code = cal.parseDT(game_time)
# If it is between 9 PM and midnight locally (Pacific), we'll have an issue:
# FD shows us the game time as e.g. "7 PM" but we'll interpret that as today.
# Workaround: if the game time is in the past, its probably tomorrow. Add 1 day.
# Break if it's not between 9PM and midnight so we know this happened
if new_parsed_dt < datetime.datetime.now():
if datetime.datetime.now().hour < 21:
print 'Parsed time of FanDuel game start is in the past, but this isnt a TZ issue!'
IPython.embed()
else:
new_parsed_dt += datetime.timedelta(days=1)
assert new_parsed_dt > datetime.datetime.now()
browser_elem.click()
print 'Scraping this game:'
print ' ', fd_game_title
print ' on', new_parsed_dt.isoformat()
game_entry_url = 'https://www.fanduel.com/games/{game_id}/contests/{game_id}-{table_id}/enter'.format(game_id=fd_game_id, table_id=fd_table_id)
# Go to the details for the game, to find list of eligible players for this game.
browser.get(game_entry_url)
time.sleep(1)
# Get the salary cap for the game directly
salary_text = browser.find_element_by_xpath('//*[@id="ui-skeleton"]/div/section/div[2]/div[4]/div[2]/section/header/'
'remaining-salary/div/div[1]/figure').text
cap = int(salary_text.replace('$','').replace(',',''))
# Download the master player / salary list
player_list_link = browser.find_element_by_link_text("Download players list")
# Sometimes this doesn't work the first time...
download_filename = get_csv_file(sport, new_parsed_dt, fd_game_id)
attempts = 1
while not os.path.exists(download_filename) and attempts <= 10:
print '...trying to download file (attempt %d)' % attempts
time.sleep(3)
player_list_link.click()
time.sleep(3)
attempts += 1
if attempts > 5:
print "Problem downloading player list -- not saving anything"
return False
else:
print '...success!'
# Get player lineups
if sport == 'mlb':
print "Accessing lineups..."
lineups_link = browser.find_element_by_link_text("Lineup Info")
# You can't just click it, it opens in a new tab :( So we visit it with get() and then go back()
lineup_url = lineups_link.get_attribute('href')
browser.get(lineup_url)
lineup_info = parse_lineup_page(browser.page_source)
browser.back()
print " %d / %d lineups submitted (%d players)" % (len(lineup_info.loaded_teams),
len(lineup_info.loaded_teams) + len(lineup_info.unloaded_teams),
len(lineup_info.parsed_players))
else:
# Should consider grabbing injury / GTD status here
lineup_info = None
# Except that doesn't work too well
# parse_nba_player_list(browser.page_source)
add_game_info_to_db(sport,
fd_game_id,
cap,
entry_fee,
new_parsed_dt,
fd_table_id,
game_title=fd_game_title,
lineup=lineup_info)
# Return to the main lobby page with back()
browser.back()
return True
