def parse_competitions2(competitions, sport="football", *sites): if not sites: sites = ['betclic', 'betstars', 'bwin', 'france_pari', 'joa', 'netbet', 'parionssport', 'pasinobet', 'pmu', 'unibet', 'winamax', 'zebet'] sportsbetting.EXPECTED_TIME = 28 + len(competitions) * 12.5 selenium_sites = {"betstars", "bwin", "joa", "parionssport", "pasinobet", "unibet"} selenium_required = ((inspect.currentframe().f_back.f_code.co_name in ["<module>", "parse_thread"] or 'test' in inspect.currentframe().f_back.f_code.co_name) and (selenium_sites.intersection(sites) or not sites)) sportsbetting.SELENIUM_REQUIRED = selenium_required if selenium_required: ThreadPool(6).map(lambda x: selenium_init.start_selenium(x), selenium_sites.intersection(sites)) sportsbetting.PROGRESS = 0 sportsbetting.SUB_PROGRESS_LIMIT = len(sites) for competition in competitions: if competition == sport or "Tout le" in competition: import_teams_by_sport(sport) else: id_competition = get_id_formatted_competition_name(competition, sport)[0] if id_competition < 0: import_teams_by_competition_id_thesportsdb(id_competition) else: import_teams_by_url("http://www.comparateur-de-cotes.fr/comparateur/" + sport + "/a-ed" + str(id_competition)) sportsbetting.IS_PARSING = True list_odds = ThreadPool(6).map(lambda x: parse_competitions_site(competitions, sport, x), sites) sportsbetting.IS_PARSING = False if selenium_required: ThreadPool(6).map(lambda x: selenium_init.DRIVER[x].quit(), selenium_sites.intersection(sites)) sportsbetting.ODDS[sport] = merge_dict_odds(list_odds)
def parse_competitions(competitions, sport="football", *sites): sites_order = [ 'bwin', 'parionssport', 'betstars', 'pasinobet', 'joa', 'unibet', 'betclic', 'pmu', 'france_pari', 'netbet', 'winamax', 'zebet' ] if not sites: sites = sites_order sportsbetting.EXPECTED_TIME = 28 + len(competitions) * 12.5 selenium_sites = sportsbetting.SELENIUM_SITES.intersection(sites) selenium_required = ((inspect.currentframe().f_back.f_code.co_name in [ "<module>", "parse_thread" ] or 'test' in inspect.currentframe().f_back.f_code.co_name) and (selenium_sites or not sites)) sportsbetting.SELENIUM_REQUIRED = selenium_required sites = [site for site in sites_order if site in sites] sportsbetting.PROGRESS = 0 if selenium_required: for site in selenium_sites: while True: headless = sport != "handball" or site != "bwin" if sportsbetting.ABORT or selenium_init.start_selenium( site, headless, timeout=15): break colorama.init() print(termcolor.colored('Restarting', 'yellow')) colorama.Style.RESET_ALL colorama.deinit() sportsbetting.PROGRESS += 100 / len(selenium_sites) sportsbetting.PROGRESS = 0 sportsbetting.SUB_PROGRESS_LIMIT = len(sites) for competition in competitions: if competition == sport or "Tout le" in competition: import_teams_by_sport(sport) else: id_competition = get_id_formatted_competition_name( competition, sport)[0] if id_competition < 0: import_teams_by_competition_id_thesportsdb(id_competition) else: import_teams_by_url( "http://www.comparateur-de-cotes.fr/comparateur/" + sport + "/a-ed" + str(id_competition)) list_odds = [] try: sportsbetting.IS_PARSING = True list_odds = ThreadPool(7).map( lambda x: parse_competitions_site(competitions, sport, x), sites) sportsbetting.ODDS[sport] = merge_dict_odds(list_odds) except Exception: print(traceback.format_exc(), file=sys.stderr) sportsbetting.IS_PARSING = False if selenium_required: colorama.init() print(termcolor.colored('Drivers closed', 'green')) colorama.Style.RESET_ALL colorama.deinit() sportsbetting.ABORT = False
def add_names_to_db(competition, sport="football", *sites): """ Ajoute à la base de données les noms d'équipe/joueur pour une competition donnée sur tous les sites """ try: id_competition, formated_name = get_id_formated_competition_name( competition, sport) except TypeError: print("Competition inconnue") return {} print(formated_name) if competition == sport or "Tout le" in competition: import_teams_by_sport(sport) else: import_teams_by_url("http://www.comparateur-de-cotes.fr/comparateur/" + sport + "/a-ed" + str(id_competition)) if not sites: sites = [ 'betclic', 'betstars', 'bwin', 'france_pari', 'joa', 'netbet', 'parionssport', 'pasinobet', 'pmu', 'unibet', 'winamax', 'zebet' ] selenium_sites = { "betstars", "bwin", "joa", "parionssport", "pasinobet", "unibet" } selenium_required = (inspect.currentframe().f_back.f_code.co_name == "<module>" and (selenium_sites.intersection(sites) or not sites)) if selenium_required: selenium_init.start_selenium() for site in sites: print(site) url = get_competition_url(competition, sport, site) if url: try: teams = parse_and_add_to_db(site, sport, url) if teams: sportsbetting.TEAMS_NOT_FOUND.append(teams) except KeyboardInterrupt: start = time.time() print("Recommencez pour arrêter le parsing") while time.time() - start < 0.5: time.sleep(0.5) except urllib3.exceptions.MaxRetryError: selenium_init.DRIVER.quit() print("Redémarrage de selenium") selenium_init.start_selenium() teams = parse_and_add_to_db(site, sport, url) if teams: sportsbetting.TEAMS_NOT_FOUND.append(teams) except selenium.common.exceptions.TimeoutException: pass except urllib.error.HTTPError: pass if selenium_required: selenium_init.DRIVER.quit()
def parse_competitions(competitions, sport, *sites): sites_order = [ 'betfair', 'joa', 'pmu', 'barrierebet', 'pasinobet', 'france_pari', 'netbet', 'zebet', 'winamax', 'pinnacle', 'betclic', 'pokerstars', 'unibet', 'bwin', 'parionssport' ] if not sites: sites = sites_order sb.EXPECTED_TIME = 28 + len(competitions) * 12.5 sites = [site for site in sites_order if site in sites] sb.PROGRESS = 0 selenium_sites = sb.SELENIUM_SITES.intersection(sites) for site in selenium_sites: selenium_init.start_driver(site) sb.PROGRESS += 100 / len(selenium_sites) sb.PROGRESS = 0 sb.SUB_PROGRESS_LIMIT = len(sites) if sb.DB_MANAGEMENT: for competition in competitions: if competition == sport or "Tout le" in competition: import_teams_by_sport(sport) else: id_competition = get_id_from_competition_name( competition, sport) if id_competition < 0: import_teams_by_competition_id_thesportsdb(id_competition) else: import_teams_by_url( "http://www.comparateur-de-cotes.fr/comparateur/" + sport + "/a-ed" + str(id_competition)) list_odds = [] try: sb.IS_PARSING = True list_odds = ThreadPool(3).map( lambda x: parse_competitions_site(competitions, sport, x), sites) sb.ODDS[sport] = merge_dict_odds(list_odds) except Exception: print(traceback.format_exc(), file=sys.stderr) sb.IS_PARSING = False sb.ABORT = False sb.SEEN_SUREBET[sport] = False