def get_league_on_page(self):
"""
get information of the league
:return:
"""
try:
return Cache.get_element(self.league_data_stage, "CRAWL_LEAGUE_PAGE")
except KeyError:
pass
if not self.soup:
page = requests.get(self.link_league_to_check).text
self.soup = BeautifulSoup(page, "html.parser")
div_league = self.soup.find('div',
{'class': 'mx-dropdown-container mx-flexbox mx-float-left mx-template-dropdown'})
league_name = str(div_league.span.string).strip()
leagues = League.read_by_name(league_name, like=True)
league = None
if len(leagues) == 0:
# No league found, also with the name in the web page
log.warning("No league found, also with the name in the web page ["+self.league_data_stage+"]")
elif len(leagues) == 1:
league = leagues[0]
else:
# too many leagues found
log.warning("Too many leagues found [" + self.league_data_stage + "]")
Cache.add_element(self.league_data_stage, league, "CRAWL_LEAGUE_PAGE")
return league
def get_league(self):
"""
Return the league of this match
:return:
"""
import src.application.Domain.League as League
return League.read_by_id(self.league_id)
def start_crawling():
"""
:return:
"""
c = Crawler()
link_league_found = c.look_for_leagues()
find_thesaurus_legues(link_league_found.values())
for new_league_name in find_new_league_to_manage(
link_league_found.values()):
log.info("New league that can be managed [" + new_league_name + "]")
for league_link, league_name in link_league_found.items():
leagues = League.read_by_name(league_name, like=True)
if len(leagues) == 0:
log.debug("League by name not found [" + league_name + "]")
elif len(leagues) == 1:
league = leagues[0]
print("Elaborating " + league.name + "...")
cl = CrawlerLeague(league, league_link)
cl.start_crawling()
else:
log.warning("Too many leagues by name found [" + league_name + "]")
def doTest():
league = League.read_by_name("Italy", like=True)[0]
exp = Experiment(0)
exp.run(league,
complete=True,
**{
"type_evaluation": 4,
"ml_train_input_id": 5
})
def search_by_name():
user_input = input("Insert league name: ")
leagues = League.read_by_name(user_input, like=True)
if len(leagues) == 0:
GuiUtil.print_att("No Leagues found", user_input)
else:
GuiUtil.show_list_answer(
[get_league_str(league) for league in leagues],
print_index=True,
label="League by name",
label_value=user_input)
def search_by_league():
user_input = input("Insert league name: ")
GuiUtil.print_info("Looking for", user_input)
leagues = League.read_by_name(user_input, like=True)
if len(leagues) == 0:
GuiUtil.print_att("No leagues found with name", user_input)
elif len(leagues) == 1:
league = leagues[0]
matches = league.get_matches(season=util.get_current_season(),
ordered=True)
for i, match in enumerate(matches):
match_out = get_printable_match(match)
GuiUtil.print_indent_answer(i + 1, match_out, True)
def find_thesaurus_legues(leagues_found):
"""
try to match leagues in the DB with those one found on the web page
:param leagues_found:
:return:
"""
leagues = League.read_all()
for league in leagues:
att = True
for league_name in league.name.split("|"):
if league_name.strip() in leagues_found:
att = False
if att:
print("ATT: league name not found, a thesarus should be there [ " +
league_name + " ]")
def search_by_league():
user_input = input("Insert league name: ")
leagues = League.read_by_name(user_input, like=True)
if len(leagues) == 0:
GuiUtil.print_att("No leagues found", user_input)
elif len(leagues) == 1:
teams = sorted(leagues[0].get_teams(),
key=lambda team: team.team_long_name)
teams_to_print = [
t.team_long_name + ": http://sofifa.com/team/" +
str(t.team_fifa_api_id) for t in teams
]
GuiUtil.show_list_answer(teams_to_print, print_index=True)
else:
GuiUtil.print_att("Too many leagues found", user_input)
def find_new_league_to_manage(leagues_found):
"""
Return leagues that are not managed right now, but they can be
:param leagues_found:
:return:
"""
new_league_names = []
leagues = League.read_all()
for league_found in leagues_found:
new = True
for league in leagues:
if league_found in league.name:
new = False
if new:
new_league_names.append(league_found)
return new_league_names
def get_printable_match(match, show_event_link=False):
league = League.read_by_id(match.league_id)
match_str = league.name + " - Stage: " + str(
match.stage) + " - " + match.date + "\n"
home_team = match.get_home_team()
away_team = match.get_away_team()
# HOME TEAM INFO
if home_team:
match_str += home_team.team_long_name
else:
match_str += str(match.home_team_api_id)
match_str += " " + str(match.home_team_goal)
match_str += " vs "
# AWAY TEAM INFO
if away_team:
match_str += away_team.team_long_name
else:
match_str += str(match.away_team_api_id)
match_str += " " + str(match.away_team_goal)
if not match.is_finished():
match_str += "\n* TO BE CRAWLED *"
# team formation home
match_str += get_formation(match, home_team, home=True)
# team formation away
match_str += get_formation(match, away_team, home=False)
if show_event_link:
match_str += "\n(http://json.mx-api.enetscores.com/live_data/event/" + str(
match.match_api_id) + "/0)"
return match_str
def get_leagues(self):
"""
Return the league of this country
:return:
"""
return League.read_by_country(self.id)
def do_test_4():
league = League.read_by_name("ita", like=True)[0]
#for league in League.read_all():
print(league.name)
exp = Experiment(5)
exp.run(league, complete=True, **{"type_evaluation": 1})