def parse_competition(competition, sport="football", *sites):
"""
Retourne les cotes d'une competition donnée pour un ou plusieurs sites de
paris. Si aucun site n'est choisi, le parsing se fait sur l'ensemble des
bookmakers reconnus par l'ARJEL
"""
if sportsbetting.ABORT:
raise sportsbetting.AbortException
try:
_id, formatted_name = get_id_formatted_competition_name(
competition, sport)
except TypeError:
print("Competition inconnue")
return
print(formatted_name, *sites)
if not sites:
sites = [
'betclic', 'betstars', 'bwin', 'france_pari', 'joa', 'netbet',
'parionssport', 'pasinobet', 'pmu', 'unibet', 'winamax', 'zebet'
]
res_parsing = {}
for site in sites:
if len(sites) > 1:
print(site)
url = get_competition_by_id(_id, site)
try:
if url:
try:
res_parsing[site] = parse(site, url)
except urllib3.exceptions.MaxRetryError:
selenium_init.DRIVER[site].quit()
print("Redémarrage de selenium")
selenium_init.start_selenium(site, timeout=20)
res_parsing[site] = parse(site, url)
except sqlite3.OperationalError:
print(
"Erreur dans la base de données, redémarrage en cours")
res_parsing[site] = parse(site, url)
except urllib.error.URLError:
print("{} non accessible sur {} (délai écoulé)".format(
competition, site))
except KeyboardInterrupt:
res_parsing[site] = {}
except selenium.common.exceptions.TimeoutException:
print("Element non trouvé par selenium ({} sur {})".format(
competition, site))
except sportsbetting.UnavailableCompetitionException:
print("{} non disponible sur {}".format(competition, site))
except socket.timeout:
print("{} non accessible sur {} (timeout socket)".format(
competition, site))
except selenium.common.exceptions.StaleElementReferenceException:
print("StaleElement non trouvé par selenium ({} sur {})".format(
competition, site))
except selenium.common.exceptions.WebDriverException:
print("Connection closed ({} sur {})".format(competition, site))
res = format_team_names(res_parsing, sport, competition)
out = valid_odds(merge_dict_odds(res), sport)
if inspect.currentframe().f_back.f_code.co_name != "<module>":
return out
def parse_competitions2(competitions, sport="football", *sites):
if not sites:
sites = ['betclic', 'betstars', 'bwin', 'france_pari', 'joa', 'netbet',
'parionssport', 'pasinobet', 'pmu', 'unibet', 'winamax', 'zebet']
sportsbetting.EXPECTED_TIME = 28 + len(competitions) * 12.5
selenium_sites = {"betstars", "bwin", "joa", "parionssport", "pasinobet", "unibet"}
selenium_required = ((inspect.currentframe().f_back.f_code.co_name
in ["<module>", "parse_thread"]
or 'test' in inspect.currentframe().f_back.f_code.co_name)
and (selenium_sites.intersection(sites) or not sites))
sportsbetting.SELENIUM_REQUIRED = selenium_required
if selenium_required:
ThreadPool(6).map(lambda x: selenium_init.start_selenium(x),
selenium_sites.intersection(sites))
sportsbetting.PROGRESS = 0
sportsbetting.SUB_PROGRESS_LIMIT = len(sites)
for competition in competitions:
if competition == sport or "Tout le" in competition:
import_teams_by_sport(sport)
else:
id_competition = get_id_formatted_competition_name(competition, sport)[0]
if id_competition < 0:
import_teams_by_competition_id_thesportsdb(id_competition)
else:
import_teams_by_url("http://www.comparateur-de-cotes.fr/comparateur/" + sport
+ "/a-ed" + str(id_competition))
sportsbetting.IS_PARSING = True
list_odds = ThreadPool(6).map(lambda x: parse_competitions_site(competitions, sport, x), sites)
sportsbetting.IS_PARSING = False
if selenium_required:
ThreadPool(6).map(lambda x: selenium_init.DRIVER[x].quit(),
selenium_sites.intersection(sites))
sportsbetting.ODDS[sport] = merge_dict_odds(list_odds)
def parse_competitions(competitions, sport="football", *sites):
sites_order = [
'bwin', 'parionssport', 'betstars', 'pasinobet', 'joa', 'unibet',
'betclic', 'pmu', 'france_pari', 'netbet', 'winamax', 'zebet'
]
if not sites:
sites = sites_order
sportsbetting.EXPECTED_TIME = 28 + len(competitions) * 12.5
selenium_sites = sportsbetting.SELENIUM_SITES.intersection(sites)
selenium_required = ((inspect.currentframe().f_back.f_code.co_name in [
"<module>", "parse_thread"
] or 'test' in inspect.currentframe().f_back.f_code.co_name)
and (selenium_sites or not sites))
sportsbetting.SELENIUM_REQUIRED = selenium_required
sites = [site for site in sites_order if site in sites]
sportsbetting.PROGRESS = 0
if selenium_required:
for site in selenium_sites:
while True:
headless = sport != "handball" or site != "bwin"
if sportsbetting.ABORT or selenium_init.start_selenium(
site, headless, timeout=15):
break
colorama.init()
print(termcolor.colored('Restarting', 'yellow'))
colorama.Style.RESET_ALL
colorama.deinit()
sportsbetting.PROGRESS += 100 / len(selenium_sites)
sportsbetting.PROGRESS = 0
sportsbetting.SUB_PROGRESS_LIMIT = len(sites)
for competition in competitions:
if competition == sport or "Tout le" in competition:
import_teams_by_sport(sport)
else:
id_competition = get_id_formatted_competition_name(
competition, sport)[0]
if id_competition < 0:
import_teams_by_competition_id_thesportsdb(id_competition)
else:
import_teams_by_url(
"http://www.comparateur-de-cotes.fr/comparateur/" + sport +
"/a-ed" + str(id_competition))
list_odds = []
try:
sportsbetting.IS_PARSING = True
list_odds = ThreadPool(7).map(
lambda x: parse_competitions_site(competitions, sport, x), sites)
sportsbetting.ODDS[sport] = merge_dict_odds(list_odds)
except Exception:
print(traceback.format_exc(), file=sys.stderr)
sportsbetting.IS_PARSING = False
if selenium_required:
colorama.init()
print(termcolor.colored('Drivers closed', 'green'))
colorama.Style.RESET_ALL
colorama.deinit()
sportsbetting.ABORT = False
def merge_dicts_nba(match, id_betclic, id_parionssport, id_pinnacle, id_pmu,
id_unibet, id_winamax, id_zebet):
odds = [
betclic.get_sub_markets_players_basketball_betclic(id_betclic),
parionssport.get_sub_markets_players_basketball_parionssport(
id_parionssport),
pinnacle.get_sub_markets_players_basketball_pinnacle(id_pinnacle),
pmu.get_sub_markets_players_basketball_pmu(id_pmu),
unibet.get_sub_markets_players_basketball_unibet(id_unibet),
winamax.get_sub_markets_players_basketball_winamax(id_winamax),
zebet.get_sub_markets_players_basketball_zebet(id_zebet)
]
bookmakers = [
"betclic", "pasionssport", "pinnacle", "pmu", "unibet", "winamax",
"zebet"
]
sub_markets = [
'Points + passes + rebonds', 'Passes', 'Rebonds', 'Points + passes',
'Points + rebonds', 'Passes + rebonds', 'Points'
]
best = {}
best_books = {}
middles = {}
surebets = {}
for sub_market in sub_markets:
odds_sub_market = valid_odds(
merge_dict_odds([x.get(sub_market, {}) for x in odds], False),
"basketball")
players_limits = odds_sub_market.keys()
previous_player = ""
previous_limit = 0
players = []
for player_limit in players_limits:
player, limit = player_limit.split("_")
players.append([player, float(limit)])
for player, limit in sorted(list(players)):
same_player = previous_player == player
player_dict = player + "_" + str(limit)
previous_player_dict = previous_player + "_" + str(previous_limit)
surebets[player + " / " + str(limit) + " " + sub_market] = {
"match": match,
"odds": odds_sub_market[player_dict]["odds"]
}
if same_player:
odds_middle = get_middle_odds(
odds_sub_market[previous_player_dict]["odds"],
odds_sub_market[player_dict]["odds"])
if odds_middle:
middles[player + " / " + str(previous_limit) + " - " +
str(limit) + " " + sub_market] = {
"odds": odds_middle,
"match": match
}
previous_player, previous_limit = player, limit
return surebets, middles
def parse_competition(competition, sport="football", *sites):
"""
Retourne les cotes d'une competition donnée pour un ou plusieurs sites de
paris. Si aucun site n'est choisi, le parsing se fait sur l'ensemble des
bookmakers reconnus par l'ARJEL
"""
try:
_id, formatted_name = get_id_formatted_competition_name(competition, sport)
except TypeError:
print("Competition inconnue")
return
print(formatted_name, *sites)
if not sites:
sites = ['betclic', 'betstars', 'bwin', 'france_pari', 'joa', 'netbet',
'parionssport', 'pasinobet', 'pmu', 'unibet', 'winamax', 'zebet']
selenium_sites = {"betstars", "bwin", "joa", "parionssport", "pasinobet", "unibet"}
selenium_required = (inspect.currentframe().f_back.f_code.co_name == "<module>"
and (selenium_sites.intersection(sites) or not sites))
if selenium_required:
selenium_init.start_selenium()
res_parsing = {}
for site in sites:
if len(sites) > 1:
print(site)
url = get_competition_by_id(_id, site)
try:
if url:
try:
res_parsing[site] = parse(site, url)
except urllib3.exceptions.MaxRetryError:
selenium_init.DRIVER.quit()
print("Redémarrage de selenium")
selenium_init.start_selenium()
res_parsing[site] = parse(site, url)
except urllib.error.URLError:
print("Site non accessible (délai écoulé)")
except KeyboardInterrupt:
res_parsing[site] = {}
except selenium.common.exceptions.TimeoutException:
print("Element non trouvé par selenium")
except sportsbetting.UnavailableCompetitionException:
print("Compétition non disponible")
# sportsbetting.PROGRESS += 100/(len(sites)*sportsbetting.SUBPROGRESS_LIMIT)
if selenium_required:
selenium_init.DRIVER.quit()
# if len(sites) > 1:
res = format_team_names(res_parsing, sport)
out = valid_odds(merge_dict_odds(res), sport)
# else:
# out = valid_odds(res_parsing[sites[0]], sport)
if inspect.currentframe().f_back.f_code.co_name != "<module>":
return out
sportsbetting.ODDS[sport] = out
def parse_competition(competition, sport, *sites):
"""
Retourne les cotes d'une competition donnée pour un ou plusieurs sites de
paris. Si aucun site n'est choisi, le parsing se fait sur l'ensemble des
bookmakers reconnus par l'ARJEL
"""
if sb.ABORT:
raise sb.AbortException
try:
_id = get_id_from_competition_name(competition, sport)
except TypeError:
print("Competition inconnue")
return
print(competition, *sites)
if not sites:
sites = [
'betclic', 'betstars', 'bwin', 'france_pari', 'joa', 'netbet',
'parionssport', 'pasinobet', 'pmu', 'unibet', 'winamax', 'zebet'
]
res_parsing = {}
for site in sites:
if len(sites) > 1:
print(site)
url = get_competition_by_id(_id, site)
try:
if url:
res_parsing[site] = parse(site, url)
else:
print("Pas d'url en base pour {} sur {}".format(
competition, site))
except urllib.error.URLError:
print("{} non accessible sur {} (délai écoulé)".format(
competition, site))
except KeyboardInterrupt:
res_parsing[site] = {}
except selenium.common.exceptions.TimeoutException:
print("Element non trouvé par selenium ({} sur {})".format(
competition, site))
except sb.UnavailableCompetitionException:
print("{} non disponible sur {}".format(competition, site))
except socket.timeout:
print("{} non accessible sur {} (timeout socket)".format(
competition, site))
except selenium.common.exceptions.StaleElementReferenceException:
print("StaleElement non trouvé par selenium ({} sur {})".format(
competition, site))
except selenium.common.exceptions.WebDriverException:
print("Connection closed ({} sur {})".format(competition, site))
res = format_team_names(res_parsing, sport, competition)
out = valid_odds(merge_dict_odds(res), sport)
return out
def parse_competitions_site(competitions, sport, site):
selenium_sites = {"betstars", "bwin", "joa", "parionssport", "pasinobet", "unibet"}
selenium_required = (
inspect.currentframe().f_back.f_code.co_name in ["<module>", "parse_thread"]
and site in selenium_sites)
if selenium_required:
selenium_init.start_selenium()
list_odds = []
for competition in competitions:
list_odds.append(parse_competition(competition, sport, site))
sportsbetting.PROGRESS += 100 / (len(competitions) * sportsbetting.SUB_PROGRESS_LIMIT)
if selenium_required:
selenium_init.DRIVER.quit()
return merge_dict_odds(list_odds)
def parse_competitions_site(competitions, sport, site):
list_odds = []
if len(competitions
) > 40 and site == "winamax": # to avoid being blocked by winamax
competitions = competitions[:40]
sb.SITE_PROGRESS[site] = 0
try:
for competition in competitions:
list_odds.append(parse_competition(competition, sport, site))
sb.PROGRESS += 100 / (len(competitions) * sb.SUB_PROGRESS_LIMIT)
sb.SITE_PROGRESS[site] += 100 / len(competitions)
except sb.UnavailableSiteException:
print("{} non accessible".format(site))
sb.SITE_PROGRESS[site] = 100
except sb.AbortException:
print("Interruption", site)
return merge_dict_odds(list_odds)
def parse_competitions(competitions, sport, *sites):
sites_order = [
'betfair', 'joa', 'pmu', 'barrierebet', 'pasinobet', 'france_pari',
'netbet', 'zebet', 'winamax', 'pinnacle', 'betclic', 'pokerstars',
'unibet', 'bwin', 'parionssport'
]
if not sites:
sites = sites_order
sb.EXPECTED_TIME = 28 + len(competitions) * 12.5
sites = [site for site in sites_order if site in sites]
sb.PROGRESS = 0
selenium_sites = sb.SELENIUM_SITES.intersection(sites)
for site in selenium_sites:
selenium_init.start_driver(site)
sb.PROGRESS += 100 / len(selenium_sites)
sb.PROGRESS = 0
sb.SUB_PROGRESS_LIMIT = len(sites)
if sb.DB_MANAGEMENT:
for competition in competitions:
if competition == sport or "Tout le" in competition:
import_teams_by_sport(sport)
else:
id_competition = get_id_from_competition_name(
competition, sport)
if id_competition < 0:
import_teams_by_competition_id_thesportsdb(id_competition)
else:
import_teams_by_url(
"http://www.comparateur-de-cotes.fr/comparateur/" +
sport + "/a-ed" + str(id_competition))
list_odds = []
try:
sb.IS_PARSING = True
list_odds = ThreadPool(3).map(
lambda x: parse_competitions_site(competitions, sport, x), sites)
sb.ODDS[sport] = merge_dict_odds(list_odds)
except Exception:
print(traceback.format_exc(), file=sys.stderr)
sb.IS_PARSING = False
sb.ABORT = False
sb.SEEN_SUREBET[sport] = False
def parse_competitions_site(competitions, sport, site):
list_odds = []
if len(competitions
) > 40 and site == "winamax": # to avoid being blocked by winamax
competitions = competitions[:40]
sportsbetting.SITE_PROGRESS[site] = 0
try:
for competition in competitions:
list_odds.append(parse_competition(competition, sport, site))
sportsbetting.PROGRESS += 100 / (len(competitions) *
sportsbetting.SUB_PROGRESS_LIMIT)
sportsbetting.SITE_PROGRESS[site] += 100 / len(competitions)
except sportsbetting.UnavailableSiteException:
print("{} non accessible".format(site))
sportsbetting.SITE_PROGRESS[site] = 100
except sportsbetting.AbortException:
print("Interruption", site)
except sqlite3.OperationalError:
print("Database is locked", site)
if site in sportsbetting.SELENIUM_SITES:
selenium_init.DRIVER[site].quit()
return merge_dict_odds(list_odds)
