def get_leagues(): url = settings.FOOTBALL_API_BASE_URL + settings.FOOTBALL_API_URLS[ "competitions"] request = make_request(url, settings.REQUEST_HEADERS["footballApi"]) try: data = request["data"] for league in data["competitions"]: if league["plan"] == "TIER_ONE": try: new_league = League.objects.get( Q(api_id=league["id"]) | Q(name=league["name"])) if new_league and new_league.api_id is None: new_league.api_id = league["id"] new_league.save() except League.DoesNotExist: print(league["name"] + " does not exists. Creating...") new_league = League() new_league.api_id = league["id"] new_league.name = league["name"] new_league.code = league["code"] new_league.country = league["area"]["name"] new_league.save() print("Successfully created " + league["name"]) except KeyError: print(request["message"]) raise print("Leagues created/updated successfully!") return True
def get_lineups(): for league in LEAGUES: if league in settings.ROTOWIRE_LEAGUE_URLS: url = settings.ROTOWIRE_BASE_URL + settings.ROTOWIRE_LEAGUE_URLS[ league] else: continue rotowire_html = make_request(url, settings.REQUEST_HEADERS['rotowire'], 'text') lineup_html = BeautifulSoup(rotowire_html['data'], parse_only=SoupStrainer(class_="lineups"), features="html.parser") # ONLY GET NEW LINEUP IF IT CHANGED current_html = RotowireRequestLog.objects.get_html(league) if current_html: current_html = current_html.html else: current_html = '' if current_html != str(lineup_html): rr = RotowireRequestLog() rr.html = str(lineup_html) rr.league = League.objects.get(code=league) rr.save() return True
def scrape_games(): matches = [] url = settings.REDDIT_BASE_URL + settings.REDDIT_API_URL["soccerstreams"] request = make_request(url, settings.REQUEST_HEADERS["reddit"]) data = '' try: if request['data']: data = request['data'] except Exception as e: print('An error occurred when scraping for games: ' + request['message']) raise reg_ex = "(\[\d\d\:\d\d\s\D\D\D\])" try: for child in data['data']['children']: game_title_check = re.search( reg_ex, child['data']['title']) # Title with GMT Time ([00:00 GMT]) if game_title_check: soccer_match = { "GameTitle": child['data']['title'].replace(game_title_check.group(), '').strip(), "Time": game_title_check.group()[1:10], "PostLink": child['data']['url'] } matches.append(soccer_match) except Exception as e: print('An error occurred: ' + repr(e)) return matches
def scrape_ace_links(arr): # print('Scraping ace links ' + arr); links = [] for j in arr: url_string = j["postUrl"] request_url = str(url_string) + ".json" request = make_request(request_url, settings.REQUEST_HEADERS["reddit"]) data = '' try: if request['data']: data = request['data'] except Exception as e: print('An error occurred when scraping for links ' + request['message']) raise acestream_reg_ex = 'acestream://\w+' try: for dict in data: for child in dict['data']['children']: if child['kind'] == 't1': regex_result = re.finditer(acestream_reg_ex, child['data']['body']) for i in regex_result: link_obj = { "link": i.group(), "score": child['data']['score'], "author": child['data']['author'], "gameID": j["id"] } links.append(link_obj) else: continue except Exception as e: print('An error occurred: ' + repr(e)) return links
def update_match_day(): leagues = League.objects.filter(tracked=True) for league in leagues: url = settings.FOOTBALL_API_BASE_URL + settings.FOOTBALL_API_URLS[ "competitionSeasons"] % str(league.api_id) request = make_request(url, settings.REQUEST_HEADERS["footballApi"]) try: data = request["data"] try: league.current_match_day = data["currentSeason"][ "currentMatchday"] league.save() except KeyError as e: print(e) except KeyError as e: print("{0} KeyError - {1}".format(str(request["message"]), e.__str__())) return True
def get_teams(): # Clear previous set leagues. # For example, a team might no longer be in the Champions league or the First division teams = Team.objects.all() for i in teams: i.leagues.clear() # Get new teams and update leagues on current teams only for the supported leagues tracked_leagues = League.objects.filter(tracked=True) for tracked_league in tracked_leagues: url = settings.FOOTBALL_API_BASE_URL + settings.FOOTBALL_API_URLS[ "teams"] % str(tracked_league.api_id) request = make_request(url, settings.REQUEST_HEADERS["footballApi"]) try: data = request["data"] for team in data["teams"]: try: new_team = Team.objects.get(api_id=team["id"]) if new_team: new_team.leagues.add( League.objects.get(id=tracked_league.id)) except Team.DoesNotExist: print(team["name"] + " does not exist. Creating...") new_team = Team() new_team.api_id = team["id"] new_team.club_colors = team["clubColors"] new_team.name = team["name"] new_team.short_name = team["shortName"] new_team.venue = team["venue"] new_team.tla = team["tla"] new_team.save() # New team needs to exist before adding values to Many-to-Many field new_team.leagues.add( League.objects.get(id=tracked_league.id)) print("Successfully created " + team["name"]) except KeyError: print(request["message"]) raise print("Teams created/updated successfully!") return True
def get_matches(start_date=None, end_date=None): today = datetime.today() last_day_of_month = monthrange(today.year, today.month)[1] # Returns tuple Ex. (3, 31) if start_date is None: start_date = datetime(today.year, today.month, 1) if end_date is None: end_date = datetime(today.year, today.month, last_day_of_month) try: start_date = datetime.strftime(start_date, '%Y-%m-%d') end_date = datetime.strftime(end_date, '%Y-%m-%d') except ValueError: raise ValueError("Invalid Date Format!") query = "?dateFrom=%s&dateTo=%s" % (start_date, end_date) tracked_leagues = League.objects.filter(tracked=True) match_status_choices = dict( (key, value) for (value, key) in Match.STATUS_CHOICES) match_winner_choices = dict( (key, value) for (value, key) in Match.WINNER_CHOICES) for tracked_league in tracked_leagues: url = settings.FOOTBALL_API_BASE_URL + settings.FOOTBALL_API_URLS[ "matches"] % str(tracked_league.api_id) + query request = make_request(url, settings.REQUEST_HEADERS["footballApi"]) try: data = request["data"] for match in data["matches"]: try: home_team = Team.objects.get( api_id=match["homeTeam"]["id"]) away_team = Team.objects.get( api_id=match["awayTeam"]["id"]) match_status = sanitize_string( match["status"], delimiter="_") # Ex. IN_PLAY -> In Play match_winner = sanitize_string(match["score"]["winner"], delimiter="_") match_datetime = datetime.strptime(match["utcDate"], "%Y-%m-%dT%H:%M:%SZ") match_datetime = match_datetime.replace(tzinfo=pytz.utc) new_match = Match() try: new_match = Match.objects.get(api_match_id=match["id"]) except Match.DoesNotExist as e: print(match["homeTeam"]["name"] + " vs " + match["awayTeam"]["name"] + " does not exist. Creating...") new_match.api_match_id = match["id"] new_match.match_day = match["matchday"] new_match.match_date_time = match_datetime new_match.home_team = home_team new_match.away_team = away_team new_match.league = League.objects.get( id=tracked_league.id) finally: new_match.status = match_status_choices[match_status] new_match.goals_scored_home_team = match["score"][ "fullTime"]["homeTeam"] new_match.goals_scored_away_team = match["score"][ "fullTime"]["awayTeam"] new_match.penalty_goals_home_team = match["score"][ "penalties"]["homeTeam"] new_match.penalty_goals_away_team = match["score"][ "penalties"]["awayTeam"] if match_winner is not None and match_winner != "": new_match.winner = match_winner_choices[ match_winner] new_match.save() except Team.DoesNotExist as e: print("Home or Away does not exists...") raise except KeyError: raise except ValueError: raise ValueError("Invalid Date Format!") except KeyError: print(request["message"]) raise print("Matches created/updated successfully!") return True