def parse_competitions2(competitions, sport="football", *sites):
if not sites:
sites = ['betclic', 'betstars', 'bwin', 'france_pari', 'joa', 'netbet',
'parionssport', 'pasinobet', 'pmu', 'unibet', 'winamax', 'zebet']
sportsbetting.EXPECTED_TIME = 28 + len(competitions) * 12.5
selenium_sites = {"betstars", "bwin", "joa", "parionssport", "pasinobet", "unibet"}
selenium_required = ((inspect.currentframe().f_back.f_code.co_name
in ["<module>", "parse_thread"]
or 'test' in inspect.currentframe().f_back.f_code.co_name)
and (selenium_sites.intersection(sites) or not sites))
sportsbetting.SELENIUM_REQUIRED = selenium_required
if selenium_required:
ThreadPool(6).map(lambda x: selenium_init.start_selenium(x),
selenium_sites.intersection(sites))
sportsbetting.PROGRESS = 0
sportsbetting.SUB_PROGRESS_LIMIT = len(sites)
for competition in competitions:
if competition == sport or "Tout le" in competition:
import_teams_by_sport(sport)
else:
id_competition = get_id_formatted_competition_name(competition, sport)[0]
if id_competition < 0:
import_teams_by_competition_id_thesportsdb(id_competition)
else:
import_teams_by_url("http://www.comparateur-de-cotes.fr/comparateur/" + sport
+ "/a-ed" + str(id_competition))
sportsbetting.IS_PARSING = True
list_odds = ThreadPool(6).map(lambda x: parse_competitions_site(competitions, sport, x), sites)
sportsbetting.IS_PARSING = False
if selenium_required:
ThreadPool(6).map(lambda x: selenium_init.DRIVER[x].quit(),
selenium_sites.intersection(sites))
sportsbetting.ODDS[sport] = merge_dict_odds(list_odds)
def parse_competitions(competitions, sport="football", *sites):
sites_order = [
'bwin', 'parionssport', 'betstars', 'pasinobet', 'joa', 'unibet',
'betclic', 'pmu', 'france_pari', 'netbet', 'winamax', 'zebet'
]
if not sites:
sites = sites_order
sportsbetting.EXPECTED_TIME = 28 + len(competitions) * 12.5
selenium_sites = sportsbetting.SELENIUM_SITES.intersection(sites)
selenium_required = ((inspect.currentframe().f_back.f_code.co_name in [
"<module>", "parse_thread"
] or 'test' in inspect.currentframe().f_back.f_code.co_name)
and (selenium_sites or not sites))
sportsbetting.SELENIUM_REQUIRED = selenium_required
sites = [site for site in sites_order if site in sites]
sportsbetting.PROGRESS = 0
if selenium_required:
for site in selenium_sites:
while True:
headless = sport != "handball" or site != "bwin"
if sportsbetting.ABORT or selenium_init.start_selenium(
site, headless, timeout=15):
break
colorama.init()
print(termcolor.colored('Restarting', 'yellow'))
colorama.Style.RESET_ALL
colorama.deinit()
sportsbetting.PROGRESS += 100 / len(selenium_sites)
sportsbetting.PROGRESS = 0
sportsbetting.SUB_PROGRESS_LIMIT = len(sites)
for competition in competitions:
if competition == sport or "Tout le" in competition:
import_teams_by_sport(sport)
else:
id_competition = get_id_formatted_competition_name(
competition, sport)[0]
if id_competition < 0:
import_teams_by_competition_id_thesportsdb(id_competition)
else:
import_teams_by_url(
"http://www.comparateur-de-cotes.fr/comparateur/" + sport +
"/a-ed" + str(id_competition))
list_odds = []
try:
sportsbetting.IS_PARSING = True
list_odds = ThreadPool(7).map(
lambda x: parse_competitions_site(competitions, sport, x), sites)
sportsbetting.ODDS[sport] = merge_dict_odds(list_odds)
except Exception:
print(traceback.format_exc(), file=sys.stderr)
sportsbetting.IS_PARSING = False
if selenium_required:
colorama.init()
print(termcolor.colored('Drivers closed', 'green'))
colorama.Style.RESET_ALL
colorama.deinit()
sportsbetting.ABORT = False
def add_names_to_db(competition, sport="football", *sites):
"""
Ajoute à la base de données les noms d'équipe/joueur pour une competition donnée sur tous les
sites
"""
try:
id_competition, formated_name = get_id_formated_competition_name(
competition, sport)
except TypeError:
print("Competition inconnue")
return {}
print(formated_name)
if competition == sport or "Tout le" in competition:
import_teams_by_sport(sport)
else:
import_teams_by_url("http://www.comparateur-de-cotes.fr/comparateur/" +
sport + "/a-ed" + str(id_competition))
if not sites:
sites = [
'betclic', 'betstars', 'bwin', 'france_pari', 'joa', 'netbet',
'parionssport', 'pasinobet', 'pmu', 'unibet', 'winamax', 'zebet'
]
selenium_sites = {
"betstars", "bwin", "joa", "parionssport", "pasinobet", "unibet"
}
selenium_required = (inspect.currentframe().f_back.f_code.co_name
== "<module>"
and (selenium_sites.intersection(sites) or not sites))
if selenium_required:
selenium_init.start_selenium()
for site in sites:
print(site)
url = get_competition_url(competition, sport, site)
if url:
try:
teams = parse_and_add_to_db(site, sport, url)
if teams:
sportsbetting.TEAMS_NOT_FOUND.append(teams)
except KeyboardInterrupt:
start = time.time()
print("Recommencez pour arrêter le parsing")
while time.time() - start < 0.5:
time.sleep(0.5)
except urllib3.exceptions.MaxRetryError:
selenium_init.DRIVER.quit()
print("Redémarrage de selenium")
selenium_init.start_selenium()
teams = parse_and_add_to_db(site, sport, url)
if teams:
sportsbetting.TEAMS_NOT_FOUND.append(teams)
except selenium.common.exceptions.TimeoutException:
pass
except urllib.error.HTTPError:
pass
if selenium_required:
selenium_init.DRIVER.quit()
def parse_competitions(competitions, sport, *sites):
sites_order = [
'betfair', 'joa', 'pmu', 'barrierebet', 'pasinobet', 'france_pari',
'netbet', 'zebet', 'winamax', 'pinnacle', 'betclic', 'pokerstars',
'unibet', 'bwin', 'parionssport'
]
if not sites:
sites = sites_order
sb.EXPECTED_TIME = 28 + len(competitions) * 12.5
sites = [site for site in sites_order if site in sites]
sb.PROGRESS = 0
selenium_sites = sb.SELENIUM_SITES.intersection(sites)
for site in selenium_sites:
selenium_init.start_driver(site)
sb.PROGRESS += 100 / len(selenium_sites)
sb.PROGRESS = 0
sb.SUB_PROGRESS_LIMIT = len(sites)
if sb.DB_MANAGEMENT:
for competition in competitions:
if competition == sport or "Tout le" in competition:
import_teams_by_sport(sport)
else:
id_competition = get_id_from_competition_name(
competition, sport)
if id_competition < 0:
import_teams_by_competition_id_thesportsdb(id_competition)
else:
import_teams_by_url(
"http://www.comparateur-de-cotes.fr/comparateur/" +
sport + "/a-ed" + str(id_competition))
list_odds = []
try:
sb.IS_PARSING = True
list_odds = ThreadPool(3).map(
lambda x: parse_competitions_site(competitions, sport, x), sites)
sb.ODDS[sport] = merge_dict_odds(list_odds)
except Exception:
print(traceback.format_exc(), file=sys.stderr)
sb.IS_PARSING = False
sb.ABORT = False
sb.SEEN_SUREBET[sport] = False
