def handle(self, *args, **options): # Footyroom + Hoofoot highlights fetching highlights = fetcher_footyroom.fetch_highlights(num_pagelet=10, max_days_ago=1000) \ + fetcher_hoofoot.fetch_highlights(num_pagelet=10, max_days_ago=1000)\ + fetcher_footyroom_videos.fetch_highlights(num_pagelet=10, max_days_ago=1000) # Add new highlights for highlight in highlights: # Parse the date before inserting it (date needs to be a string) highlight.time_since_added = str( dateparser.parse(highlight.time_since_added)) if not latest_highlight_manager.has_highlight(highlight): latest_highlight_manager.add_highlight(highlight, sent=True)
def check_scrapping_status(): # Define scrapping exception class ScrappingException(Exception): pass scrapping_problems = [] highlights_footyroom = fetcher_footyroom.fetch_highlights( num_pagelet=1, max_days_ago=1000) highlights_footyroom_video = [ h for h in highlights_footyroom if isinstance(h, FootyroomVideoHighlight) ] highlights_footyroom = [ h for h in highlights_footyroom if isinstance(h, FootyroomHighlight) ] if not highlights_footyroom: scrapping_problems.append('FOOTYROOM') if not highlights_footyroom_video: scrapping_problems.append('FOOTYROOM VIDEOS') highlights_hoofoot = fetcher_hoofoot.fetch_highlights(num_pagelet=1, max_days_ago=1000) if not highlights_hoofoot: scrapping_problems.append('HOOFOOT') highlights_highlightsfootball = fetcher_highlightsfootball.fetch_highlights( num_pagelet=1, max_days_ago=1000) if not highlights_highlightsfootball: scrapping_problems.append('HIGHLIGHTS FOOTBALL') highlights_sportyhl = fetcher_sportyhl.fetch_highlights(num_pagelet=1, max_days_ago=1000) if not highlights_sportyhl: scrapping_problems.append('SPORTYHL') if scrapping_problems: raise ScrappingException("Failed to scrape " + ', '.join(scrapping_problems))
def handle(self, *args, **options): try: highlights_footyroom = fetcher_footyroom.fetch_highlights(num_pagelet=1, max_days_ago=1000) if not highlights_footyroom: raise ScrappingException("Failed to scrape FOOTYROOM") highlights_hoofoot = fetcher_hoofoot.fetch_highlights(num_pagelet=1, max_days_ago=1000) if not highlights_hoofoot: raise ScrappingException("Failed to scrape HOOFOOT") except Exception as error: if not settings.DEBUG: # Report to sentry if problem detected client.captureException() else: raise error
def handle(self, *args, **options): highlights = fetcher_footyroom.fetch_highlights(100, 1000) for highlight in highlights: football_team_manager.add_football_team(highlight.team1.lower()) football_team_manager.add_football_team(highlight.team2.lower())
def send_most_recent_highlights(): # Footyroom + Hoofoot highlights fetching highlights = fetcher_footyroom.fetch_highlights(num_pagelet=1, max_days_ago=2) \ + fetcher_hoofoot.fetch_highlights(num_pagelet=4, max_days_ago=7) \ + fetcher_footyroom_videos.fetch_highlights(num_pagelet=3, max_days_ago=7) # Add new highlights for highlight in highlights: # Parse the date before inserting it (date needs to be a string) highlight.time_since_added = str( dateparser.parse(highlight.time_since_added)) if latest_highlight_manager.has_highlight(highlight): # Skip if highlight already in database continue sent = False # Mark as sent if a similar highlight (same match, different provider) is in database and has already been sent if latest_highlight_manager.get_similar_sent_highlights(highlight): sent = True latest_highlight_manager.add_highlight(highlight, sent=sent) # Set Footyroom images for hoofoot highlights for hoofoot_highlight in latest_highlight_manager.get_all_highlights_from_source( source='hoofoot'): img_link = latest_highlight_manager.get_highlight_img_link_from_footyroom( hoofoot_highlight) if not img_link: continue latest_highlight_manager.set_img_link(hoofoot_highlight, img_link) # Send highlights not already sent not_sent_highlights = latest_highlight_manager.get_not_sent_highlights() today = datetime.today() for highlight in not_sent_highlights: time_since_added = highlight.get_parsed_time_since_added() # Add time to make sure video is good if timedelta(minutes=30) < abs(today - time_since_added): if highlight.sent: # highlight has already been sent continue # Log highlights sent logger.log("Highlight sent: " + highlight.get_match_name()) # Send highlight for team1 team1 = highlight.team1.name.lower() send_highlight_to_users(highlight, team1) # Send highlight for team2 team2 = highlight.team2.name.lower() send_highlight_to_users(highlight, team2) # Set highlights for same match to sent similar_highlights = latest_highlight_manager.get_similar_highlights( highlight, not_sent_highlights) for h in similar_highlights: latest_highlight_manager.set_sent(h) # Delete old highlights all_highlights = latest_highlight_manager.get_all_highlights() for highlight in all_highlights: time_since_added = highlight.get_parsed_time_since_added() # Old highlight, delete if (today - time_since_added) > timedelta(days=60): latest_highlight_manager.delete_highlight(highlight)
def send_most_recent_highlights(footyroom_pagelet=3, hoofoot_pagelet=4, highlightsfootball_pagelet=3, sportyhl_pagelet=3): # Fetch highlights from multiple sources highlights = fetcher_footyroom.fetch_highlights(num_pagelet=footyroom_pagelet, max_days_ago=7) \ + fetcher_hoofoot.fetch_highlights(num_pagelet=hoofoot_pagelet, max_days_ago=7) \ + fetcher_highlightsfootball.fetch_highlights(num_pagelet=highlightsfootball_pagelet, max_days_ago=7) \ + fetcher_sportyhl.fetch_highlights(num_pagelet=sportyhl_pagelet, max_days_ago=7) # Add new highlights for highlight in highlights: # Parse the date before inserting it (date needs to be a string) highlight.time_since_added = str( dateparser.parse(highlight.time_since_added)) if latest_highlight_manager.has_highlight(highlight): # Skip if highlight already in database continue sent = False # Mark as sent if a similar highlight (same match, different provider) is in database and has already been sent if latest_highlight_manager.get_similar_sent_highlights(highlight): sent = True latest_highlight_manager.add_highlight(highlight, sent=sent) # Set Footyroom infos for h in latest_highlight_manager.get_all_highlights_from_source( sources=['hoofoot', 'sportyhl', 'highlightsfootball']): footyroom_highlight = latest_highlight_manager.get_same_highlight_footyroom( h) if not footyroom_highlight: continue latest_highlight_manager.set_img_link(h, footyroom_highlight.img_link) latest_highlight_manager.set_goal_data(h, footyroom_highlight.goal_data) # Also add game score for specific sources if h.source in ['sportyhl', 'highlightsfootball']: latest_highlight_manager.set_score(h, footyroom_highlight.score1, footyroom_highlight.score2) # Send highlights not already sent not_sent_highlights = latest_highlight_manager.get_not_sent_highlights( AVAILABLE_SOURCES) today = datetime.today() for highlight in not_sent_highlights: time_since_added = highlight.get_parsed_time_since_added() # Add time to make sure video is good if timedelta(minutes=30) < abs( today - time_since_added) or highlight.priority_short > 0: if highlight.sent: # highlight has already been sent continue if highlight.score1 < 0 or highlight.score2 < 0: # score was not set as no similar video - invalid latest_highlight_manager.set_invalid(highlight) continue # Log highlights sent logger.log("Highlight sent: " + highlight.get_match_name(), forward=True) # Set highlights for same match to sent similar_highlights = latest_highlight_manager.get_similar_highlights( highlight, not_sent_highlights) for h in similar_highlights: latest_highlight_manager.set_sent(h) # Send highlight to users _send_highlight_to_users(highlight) # Delete old highlights # FIXME: try to find a way to keep old highlights all_highlights = latest_highlight_manager.get_all_highlights() for highlight in all_highlights: time_since_added = highlight.get_parsed_time_since_added() # Old highlight, delete if (today - time_since_added) > timedelta(days=60): latest_highlight_manager.delete_highlight(highlight)