def handle(self, *args, **options):
# Footyroom + Hoofoot highlights fetching
highlights = fetcher_footyroom.fetch_highlights(num_pagelet=10, max_days_ago=1000) \
+ fetcher_hoofoot.fetch_highlights(num_pagelet=10, max_days_ago=1000)\
+ fetcher_footyroom_videos.fetch_highlights(num_pagelet=10, max_days_ago=1000)
# Add new highlights
for highlight in highlights:
# Parse the date before inserting it (date needs to be a string)
highlight.time_since_added = str(
dateparser.parse(highlight.time_since_added))
if not latest_highlight_manager.has_highlight(highlight):
latest_highlight_manager.add_highlight(highlight, sent=True)
def check_scrapping_status():
# Define scrapping exception
class ScrappingException(Exception):
pass
scrapping_problems = []
highlights_footyroom = fetcher_footyroom.fetch_highlights(
num_pagelet=1, max_days_ago=1000)
highlights_footyroom_video = [
h for h in highlights_footyroom
if isinstance(h, FootyroomVideoHighlight)
]
highlights_footyroom = [
h for h in highlights_footyroom if isinstance(h, FootyroomHighlight)
]
if not highlights_footyroom:
scrapping_problems.append('FOOTYROOM')
if not highlights_footyroom_video:
scrapping_problems.append('FOOTYROOM VIDEOS')
highlights_hoofoot = fetcher_hoofoot.fetch_highlights(num_pagelet=1,
max_days_ago=1000)
if not highlights_hoofoot:
scrapping_problems.append('HOOFOOT')
highlights_highlightsfootball = fetcher_highlightsfootball.fetch_highlights(
num_pagelet=1, max_days_ago=1000)
if not highlights_highlightsfootball:
scrapping_problems.append('HIGHLIGHTS FOOTBALL')
highlights_sportyhl = fetcher_sportyhl.fetch_highlights(num_pagelet=1,
max_days_ago=1000)
if not highlights_sportyhl:
scrapping_problems.append('SPORTYHL')
if scrapping_problems:
raise ScrappingException("Failed to scrape " +
', '.join(scrapping_problems))
def handle(self, *args, **options):
try:
highlights_footyroom = fetcher_footyroom.fetch_highlights(num_pagelet=1, max_days_ago=1000)
if not highlights_footyroom:
raise ScrappingException("Failed to scrape FOOTYROOM")
highlights_hoofoot = fetcher_hoofoot.fetch_highlights(num_pagelet=1, max_days_ago=1000)
if not highlights_hoofoot:
raise ScrappingException("Failed to scrape HOOFOOT")
except Exception as error:
if not settings.DEBUG:
# Report to sentry if problem detected
client.captureException()
else:
raise error
def handle(self, *args, **options):
highlights = fetcher_footyroom.fetch_highlights(100, 1000)
for highlight in highlights:
football_team_manager.add_football_team(highlight.team1.lower())
football_team_manager.add_football_team(highlight.team2.lower())
def send_most_recent_highlights():
# Footyroom + Hoofoot highlights fetching
highlights = fetcher_footyroom.fetch_highlights(num_pagelet=1, max_days_ago=2) \
+ fetcher_hoofoot.fetch_highlights(num_pagelet=4, max_days_ago=7) \
+ fetcher_footyroom_videos.fetch_highlights(num_pagelet=3, max_days_ago=7)
# Add new highlights
for highlight in highlights:
# Parse the date before inserting it (date needs to be a string)
highlight.time_since_added = str(
dateparser.parse(highlight.time_since_added))
if latest_highlight_manager.has_highlight(highlight):
# Skip if highlight already in database
continue
sent = False
# Mark as sent if a similar highlight (same match, different provider) is in database and has already been sent
if latest_highlight_manager.get_similar_sent_highlights(highlight):
sent = True
latest_highlight_manager.add_highlight(highlight, sent=sent)
# Set Footyroom images for hoofoot highlights
for hoofoot_highlight in latest_highlight_manager.get_all_highlights_from_source(
source='hoofoot'):
img_link = latest_highlight_manager.get_highlight_img_link_from_footyroom(
hoofoot_highlight)
if not img_link:
continue
latest_highlight_manager.set_img_link(hoofoot_highlight, img_link)
# Send highlights not already sent
not_sent_highlights = latest_highlight_manager.get_not_sent_highlights()
today = datetime.today()
for highlight in not_sent_highlights:
time_since_added = highlight.get_parsed_time_since_added()
# Add time to make sure video is good
if timedelta(minutes=30) < abs(today - time_since_added):
if highlight.sent:
# highlight has already been sent
continue
# Log highlights sent
logger.log("Highlight sent: " + highlight.get_match_name())
# Send highlight for team1
team1 = highlight.team1.name.lower()
send_highlight_to_users(highlight, team1)
# Send highlight for team2
team2 = highlight.team2.name.lower()
send_highlight_to_users(highlight, team2)
# Set highlights for same match to sent
similar_highlights = latest_highlight_manager.get_similar_highlights(
highlight, not_sent_highlights)
for h in similar_highlights:
latest_highlight_manager.set_sent(h)
# Delete old highlights
all_highlights = latest_highlight_manager.get_all_highlights()
for highlight in all_highlights:
time_since_added = highlight.get_parsed_time_since_added()
# Old highlight, delete
if (today - time_since_added) > timedelta(days=60):
latest_highlight_manager.delete_highlight(highlight)
def send_most_recent_highlights(footyroom_pagelet=3,
hoofoot_pagelet=4,
highlightsfootball_pagelet=3,
sportyhl_pagelet=3):
# Fetch highlights from multiple sources
highlights = fetcher_footyroom.fetch_highlights(num_pagelet=footyroom_pagelet, max_days_ago=7) \
+ fetcher_hoofoot.fetch_highlights(num_pagelet=hoofoot_pagelet, max_days_ago=7) \
+ fetcher_highlightsfootball.fetch_highlights(num_pagelet=highlightsfootball_pagelet, max_days_ago=7) \
+ fetcher_sportyhl.fetch_highlights(num_pagelet=sportyhl_pagelet, max_days_ago=7)
# Add new highlights
for highlight in highlights:
# Parse the date before inserting it (date needs to be a string)
highlight.time_since_added = str(
dateparser.parse(highlight.time_since_added))
if latest_highlight_manager.has_highlight(highlight):
# Skip if highlight already in database
continue
sent = False
# Mark as sent if a similar highlight (same match, different provider) is in database and has already been sent
if latest_highlight_manager.get_similar_sent_highlights(highlight):
sent = True
latest_highlight_manager.add_highlight(highlight, sent=sent)
# Set Footyroom infos
for h in latest_highlight_manager.get_all_highlights_from_source(
sources=['hoofoot', 'sportyhl', 'highlightsfootball']):
footyroom_highlight = latest_highlight_manager.get_same_highlight_footyroom(
h)
if not footyroom_highlight:
continue
latest_highlight_manager.set_img_link(h, footyroom_highlight.img_link)
latest_highlight_manager.set_goal_data(h,
footyroom_highlight.goal_data)
# Also add game score for specific sources
if h.source in ['sportyhl', 'highlightsfootball']:
latest_highlight_manager.set_score(h, footyroom_highlight.score1,
footyroom_highlight.score2)
# Send highlights not already sent
not_sent_highlights = latest_highlight_manager.get_not_sent_highlights(
AVAILABLE_SOURCES)
today = datetime.today()
for highlight in not_sent_highlights:
time_since_added = highlight.get_parsed_time_since_added()
# Add time to make sure video is good
if timedelta(minutes=30) < abs(
today - time_since_added) or highlight.priority_short > 0:
if highlight.sent:
# highlight has already been sent
continue
if highlight.score1 < 0 or highlight.score2 < 0:
# score was not set as no similar video - invalid
latest_highlight_manager.set_invalid(highlight)
continue
# Log highlights sent
logger.log("Highlight sent: " + highlight.get_match_name(),
forward=True)
# Set highlights for same match to sent
similar_highlights = latest_highlight_manager.get_similar_highlights(
highlight, not_sent_highlights)
for h in similar_highlights:
latest_highlight_manager.set_sent(h)
# Send highlight to users
_send_highlight_to_users(highlight)
# Delete old highlights
# FIXME: try to find a way to keep old highlights
all_highlights = latest_highlight_manager.get_all_highlights()
for highlight in all_highlights:
time_since_added = highlight.get_parsed_time_since_added()
# Old highlight, delete
if (today - time_since_added) > timedelta(days=60):
latest_highlight_manager.delete_highlight(highlight)