def update_all_database(start="", *sites):
"""
Ajoute les noms manquants dans toute la base de données
"""
conn = sqlite3.connect("sportsbetting/resources/teams.db")
c = conn.cursor()
c.execute("""
SELECT sport, competition FROM competitions
""")
if start:
start_found = False
else:
start_found = True
selenium_init.start_selenium()
for line in c.fetchall():
try:
if start in line[1]:
start_found = True
if start_found:
add_names_to_db(line[1], line[0], *sites)
except KeyboardInterrupt:
pass
except sportsbetting.UnavailableCompetitionException:
pass
selenium_init.DRIVER.quit()
def parse_football(*sites):
"""
Stocke les cotes des principaux championnats de football en global
"""
selenium_sites = {
"betstars", "bwin", "joa", "parionssport", "pasinobet", "unibet"
}
selenium_required = (inspect.currentframe().f_back.f_code.co_name
== "<module>"
and (selenium_sites.intersection(sites) or not sites))
if selenium_required:
selenium_init.start_selenium()
competitions = [
"france ligue 1", "angleterre premier league", "espagne liga",
"italie serie", "allemagne bundesliga"
] #,
#"ligue des champions"]
sportsbetting.ODDS["football"] = parse_competitions(
competitions, "football", *sites)
if selenium_required:
selenium_init.DRIVER.quit()
if inspect.currentframe().f_back.f_code.co_name == "<module>":
try:
toaster = ToastNotifier()
toaster.show_toast("Sports-betting", "Fin du parsing")
except NameError:
subprocess.Popen(['notify-send', "Fin du parsing"])
def parse_competition(competition, sport="football", *sites):
"""
Retourne les cotes d'une competition donnée pour un ou plusieurs sites de
paris. Si aucun site n'est choisi, le parsing se fait sur l'ensemble des
bookmakers reconnus par l'ARJEL
"""
if sportsbetting.ABORT:
raise sportsbetting.AbortException
try:
_id, formatted_name = get_id_formatted_competition_name(
competition, sport)
except TypeError:
print("Competition inconnue")
return
print(formatted_name, *sites)
if not sites:
sites = [
'betclic', 'betstars', 'bwin', 'france_pari', 'joa', 'netbet',
'parionssport', 'pasinobet', 'pmu', 'unibet', 'winamax', 'zebet'
]
res_parsing = {}
for site in sites:
if len(sites) > 1:
print(site)
url = get_competition_by_id(_id, site)
try:
if url:
try:
res_parsing[site] = parse(site, url)
except urllib3.exceptions.MaxRetryError:
selenium_init.DRIVER[site].quit()
print("Redémarrage de selenium")
selenium_init.start_selenium(site, timeout=20)
res_parsing[site] = parse(site, url)
except sqlite3.OperationalError:
print(
"Erreur dans la base de données, redémarrage en cours")
res_parsing[site] = parse(site, url)
except urllib.error.URLError:
print("{} non accessible sur {} (délai écoulé)".format(
competition, site))
except KeyboardInterrupt:
res_parsing[site] = {}
except selenium.common.exceptions.TimeoutException:
print("Element non trouvé par selenium ({} sur {})".format(
competition, site))
except sportsbetting.UnavailableCompetitionException:
print("{} non disponible sur {}".format(competition, site))
except socket.timeout:
print("{} non accessible sur {} (timeout socket)".format(
competition, site))
except selenium.common.exceptions.StaleElementReferenceException:
print("StaleElement non trouvé par selenium ({} sur {})".format(
competition, site))
except selenium.common.exceptions.WebDriverException:
print("Connection closed ({} sur {})".format(competition, site))
res = format_team_names(res_parsing, sport, competition)
out = valid_odds(merge_dict_odds(res), sport)
if inspect.currentframe().f_back.f_code.co_name != "<module>":
return out
def parse_competitions(competitions, sport="football", *sites):
"""
Retourne les cotes de plusieurs competitions
"""
selenium_sites = {
"betstars", "bwin", "joa", "parionssport", "pasinobet", "unibet"
}
selenium_required = (inspect.currentframe().f_back.f_code.co_name
== "<module>"
and (selenium_sites.intersection(sites) or not sites))
if selenium_required:
selenium_init.start_selenium()
list_odds = []
for competition in competitions:
list_odds.append(parse_competition(competition, sport, *sites))
print()
if selenium_required:
selenium_init.DRIVER.quit()
if inspect.currentframe().f_back.f_code.co_name == "<module>":
try:
toaster = ToastNotifier()
toaster.show_toast("Sports-betting", "Fin du parsing")
except NameError:
subprocess.Popen(['notify-send', "Fin du parsing"])
if inspect.currentframe().f_back.f_code.co_name != "<module>":
return merge_dicts(list_odds)
sportsbetting.ODDS[sport] = merge_dicts(list_odds)
def add_names_to_db(competition, sport="football", *sites):
"""
Ajoute à la base de données les noms d'équipe/joueur pour une competition donnée sur tous les
sites
"""
try:
id_competition, formated_name = get_id_formated_competition_name(
competition, sport)
except TypeError:
print("Competition inconnue")
return {}
print(formated_name)
if competition == sport or "Tout le" in competition:
import_teams_by_sport(sport)
else:
import_teams_by_url("http://www.comparateur-de-cotes.fr/comparateur/" +
sport + "/a-ed" + str(id_competition))
if not sites:
sites = [
'betclic', 'betstars', 'bwin', 'france_pari', 'joa', 'netbet',
'parionssport', 'pasinobet', 'pmu', 'unibet', 'winamax', 'zebet'
]
selenium_sites = {
"betstars", "bwin", "joa", "parionssport", "pasinobet", "unibet"
}
selenium_required = (inspect.currentframe().f_back.f_code.co_name
== "<module>"
and (selenium_sites.intersection(sites) or not sites))
if selenium_required:
selenium_init.start_selenium()
for site in sites:
print(site)
url = get_competition_url(competition, sport, site)
if url:
try:
teams = parse_and_add_to_db(site, sport, url)
if teams:
sportsbetting.TEAMS_NOT_FOUND.append(teams)
except KeyboardInterrupt:
start = time.time()
print("Recommencez pour arrêter le parsing")
while time.time() - start < 0.5:
time.sleep(0.5)
except urllib3.exceptions.MaxRetryError:
selenium_init.DRIVER.quit()
print("Redémarrage de selenium")
selenium_init.start_selenium()
teams = parse_and_add_to_db(site, sport, url)
if teams:
sportsbetting.TEAMS_NOT_FOUND.append(teams)
except selenium.common.exceptions.TimeoutException:
pass
except urllib.error.HTTPError:
pass
if selenium_required:
selenium_init.DRIVER.quit()
def parse_competitions(competitions, sport="football", *sites):
sites_order = [
'bwin', 'parionssport', 'betstars', 'pasinobet', 'joa', 'unibet',
'pmu', 'betclic', 'france_pari', 'netbet', 'winamax', 'zebet'
]
if not sites:
sites = sites_order
sportsbetting.EXPECTED_TIME = 28 + len(competitions) * 12.5
selenium_sites = {
"betstars", "bwin", "joa", "parionssport", "pasinobet", "unibet"
}
selenium_required = ((inspect.currentframe().f_back.f_code.co_name in [
"<module>", "parse_thread"
] or 'test' in inspect.currentframe().f_back.f_code.co_name)
and (selenium_sites.intersection(sites) or not sites))
sportsbetting.SELENIUM_REQUIRED = selenium_required
sites = [site for site in sites_order if site in sites]
sportsbetting.PROGRESS = 0
if selenium_required:
for site in selenium_sites.intersection(sites):
selenium_init.start_selenium(site)
sportsbetting.PROGRESS += 100 / len(
selenium_sites.intersection(sites))
sportsbetting.PROGRESS = 0
sportsbetting.SUB_PROGRESS_LIMIT = len(sites)
for competition in competitions:
if competition == sport or "Tout le" in competition:
import_teams_by_sport(sport)
else:
id_competition = get_id_formatted_competition_name(
competition, sport)[0]
if id_competition < 0:
import_teams_by_competition_id_thesportsdb(id_competition)
else:
import_teams_by_url(
"http://www.comparateur-de-cotes.fr/comparateur/" + sport +
"/a-ed" + str(id_competition))
list_odds = []
try:
sportsbetting.IS_PARSING = True
list_odds = ThreadPool(6).map(
lambda x: parse_competitions_site(competitions, sport, x), sites)
sportsbetting.ODDS[sport] = merge_dict_odds(list_odds)
except Exception:
print(traceback.format_exc(), file=sys.stderr)
sportsbetting.IS_PARSING = False
if selenium_required:
ThreadPool(6).map(lambda x: selenium_init.DRIVER[x].quit(),
selenium_sites.intersection(sites))
colorama.init()
print(termcolor.colored('Drivers closed', 'green'))
colorama.Style.RESET_ALL
colorama.deinit()
sportsbetting.ABORT = False
def parse_competition(competition, sport="football", *sites):
"""
Retourne les cotes d'une competition donnée pour un ou plusieurs sites de
paris. Si aucun site n'est choisi, le parsing se fait sur l'ensemble des
bookmakers reconnus par l'ARJEL
"""
try:
_id, formatted_name = get_id_formatted_competition_name(competition, sport)
except TypeError:
print("Competition inconnue")
return
print(formatted_name, *sites)
if not sites:
sites = ['betclic', 'betstars', 'bwin', 'france_pari', 'joa', 'netbet',
'parionssport', 'pasinobet', 'pmu', 'unibet', 'winamax', 'zebet']
selenium_sites = {"betstars", "bwin", "joa", "parionssport", "pasinobet", "unibet"}
selenium_required = (inspect.currentframe().f_back.f_code.co_name == "<module>"
and (selenium_sites.intersection(sites) or not sites))
if selenium_required:
selenium_init.start_selenium()
res_parsing = {}
for site in sites:
if len(sites) > 1:
print(site)
url = get_competition_by_id(_id, site)
try:
if url:
try:
res_parsing[site] = parse(site, url)
except urllib3.exceptions.MaxRetryError:
selenium_init.DRIVER.quit()
print("Redémarrage de selenium")
selenium_init.start_selenium()
res_parsing[site] = parse(site, url)
except urllib.error.URLError:
print("Site non accessible (délai écoulé)")
except KeyboardInterrupt:
res_parsing[site] = {}
except selenium.common.exceptions.TimeoutException:
print("Element non trouvé par selenium")
except sportsbetting.UnavailableCompetitionException:
print("Compétition non disponible")
# sportsbetting.PROGRESS += 100/(len(sites)*sportsbetting.SUBPROGRESS_LIMIT)
if selenium_required:
selenium_init.DRIVER.quit()
# if len(sites) > 1:
res = format_team_names(res_parsing, sport)
out = valid_odds(merge_dict_odds(res), sport)
# else:
# out = valid_odds(res_parsing[sites[0]], sport)
if inspect.currentframe().f_back.f_code.co_name != "<module>":
return out
sportsbetting.ODDS[sport] = out
def parse_handball(*sites):
"""
Stocke les cotes de handball en global
"""
selenium_sites = {"betstars", "bwin", "joa", "parionssport", "pasinobet", "unibet"}
selenium_required = (inspect.currentframe().f_back.f_code.co_name == "<module>"
and (selenium_sites.intersection(sites) or not sites))
if selenium_required:
selenium_init.start_selenium()
sportsbetting.ODDS["handball"] = parse_competition("champions", "handball", *sites)
if selenium_required:
selenium_init.DRIVER.quit()
def parse_competitions_site(competitions, sport, site):
selenium_sites = {"betstars", "bwin", "joa", "parionssport", "pasinobet", "unibet"}
selenium_required = (
inspect.currentframe().f_back.f_code.co_name in ["<module>", "parse_thread"]
and site in selenium_sites)
if selenium_required:
selenium_init.start_selenium()
list_odds = []
for competition in competitions:
list_odds.append(parse_competition(competition, sport, site))
sportsbetting.PROGRESS += 100 / (len(competitions) * sportsbetting.SUB_PROGRESS_LIMIT)
if selenium_required:
selenium_init.DRIVER.quit()
return merge_dict_odds(list_odds)
def parse_football(*sites):
"""
Stocke les cotes des principaux championnats de football en global
"""
selenium_sites = {"betstars", "bwin", "joa", "parionssport", "pasinobet", "unibet"}
selenium_required = (inspect.currentframe().f_back.f_code.co_name == "<module>"
and (selenium_sites.intersection(sites) or not sites))
if selenium_required:
selenium_init.start_selenium()
competitions = ["france ligue 1", "angleterre premier league",
"espagne liga", "italie serie", "allemagne bundesliga",
"ligue des champions"]
parse_competitions(competitions, "football", *sites)
if selenium_required:
selenium_init.DRIVER.quit()
def parse_pasinobet_sport(sport):
selenium_init.start_selenium("pasinobet", False)
selenium_init.DRIVER["pasinobet"].maximize_window()
selenium_init.DRIVER["pasinobet"].get(
"https://www.pasinobet.fr/#/sport/?type=0&sport=1®ion=-1")
sport_items = WebDriverWait(selenium_init.DRIVER["pasinobet"], 15).until(
EC.presence_of_all_elements_located((By.CLASS_NAME, "sport-header")))
for sport_item in sport_items:
if sport_item.text.split() and sport_item.text.split()[0] == sport:
sport_item.find_element_by_xpath("..").click()
break
regions = WebDriverWait(selenium_init.DRIVER["pasinobet"], 15).until(
EC.presence_of_all_elements_located((By.CLASS_NAME, "region")))
input()
selenium_init.DRIVER["pasinobet"].quit()
def parse_competitions2(competitions, sport="football", *sites):
if not sites:
sites = ['betclic', 'betstars', 'bwin', 'france_pari', 'joa', 'netbet',
'parionssport', 'pasinobet', 'pmu', 'unibet', 'winamax', 'zebet']
sportsbetting.EXPECTED_TIME = 28 + len(competitions) * 12.5
selenium_sites = {"betstars", "bwin", "joa", "parionssport", "pasinobet", "unibet"}
selenium_required = ((inspect.currentframe().f_back.f_code.co_name
in ["<module>", "parse_thread"]
or 'test' in inspect.currentframe().f_back.f_code.co_name)
and (selenium_sites.intersection(sites) or not sites))
sportsbetting.SELENIUM_REQUIRED = selenium_required
if selenium_required:
ThreadPool(6).map(lambda x: selenium_init.start_selenium(x),
selenium_sites.intersection(sites))
sportsbetting.PROGRESS = 0
sportsbetting.SUB_PROGRESS_LIMIT = len(sites)
for competition in competitions:
if competition == sport or "Tout le" in competition:
import_teams_by_sport(sport)
else:
id_competition = get_id_formatted_competition_name(competition, sport)[0]
if id_competition < 0:
import_teams_by_competition_id_thesportsdb(id_competition)
else:
import_teams_by_url("http://www.comparateur-de-cotes.fr/comparateur/" + sport
+ "/a-ed" + str(id_competition))
sportsbetting.IS_PARSING = True
list_odds = ThreadPool(6).map(lambda x: parse_competitions_site(competitions, sport, x), sites)
sportsbetting.IS_PARSING = False
if selenium_required:
ThreadPool(6).map(lambda x: selenium_init.DRIVER[x].quit(),
selenium_sites.intersection(sites))
sportsbetting.ODDS[sport] = merge_dict_odds(list_odds)
def get_promotions_unibet():
selenium_init.start_selenium()
selenium_init.DRIVER.get("https://www.unibet.fr/promotions.do")
WebDriverWait(selenium_init.DRIVER, 15).until(
EC.presence_of_all_elements_located((By.CLASS_NAME, "headline-item")))
inner_html = selenium_init.DRIVER.execute_script(
"return document.body.innerHTML")
soup = BeautifulSoup(inner_html, features="lxml")
urls = []
for line in soup.find_all("li", {"data-headline-display": "HAD"}):
child = line.findChildren("a", recursive=False)[0]
if "par-ami" in child["href"]:
break
urls.append("https://www.unibet.fr" + child["href"])
for url in urls:
parse_promotion_unibet(url)
if not urls:
print("Pas de promotion en cours")
selenium_init.DRIVER.quit()
def parse_competitions(competitions, sport="football", *sites):
"""
Retourne les cotes de plusieurs competitions
"""
selenium_sites = {"betstars", "bwin", "joa", "parionssport", "pasinobet", "unibet"}
selenium_required = (
inspect.currentframe().f_back.f_code.co_name in ["<module>", "parse_thread"]
and (selenium_sites.intersection(sites) or not sites))
if selenium_required:
selenium_init.start_selenium()
list_odds = []
sportsbetting.PROGRESS = 0
sportsbetting.SUB_PROGRESS_LIMIT = len(competitions)
for competition in competitions:
list_odds.append(parse_competition(competition, sport, *sites))
print()
if selenium_required:
selenium_init.DRIVER.quit()
# if inspect.currentframe().f_back.f_code.co_name != "<module>":
# return merge_dicts(list_odds)
sportsbetting.ODDS[sport] = merge_dicts(list_odds)
def parse_handball(*sites):
"""
Stocke les cotes de handball en global
"""
selenium_sites = {
"betstars", "bwin", "joa", "parionssport", "pasinobet", "unibet"
}
selenium_required = (inspect.currentframe().f_back.f_code.co_name
== "<module>"
and (selenium_sites.intersection(sites) or not sites))
if selenium_required:
selenium_init.start_selenium()
sportsbetting.ODDS["handball"] = parse_competition("champions", "handball",
*sites)
if selenium_required:
selenium_init.DRIVER.quit()
if inspect.currentframe().f_back.f_code.co_name == "<module>":
try:
toaster = ToastNotifier()
toaster.show_toast("Sports-betting", "Fin du parsing")
except NameError:
subprocess.Popen(['notify-send', "Fin du parsing"])