class RestClient(RestClientInterface):
'''Define concrete Adapter
'''
_name = 'provider1'
_abiospool = Provider1Pool()
def __init__(self, *args, **kwargs):
cherrypy.thread_data.client_obj = self
if 'log' in kwargs:
self._log = kwargs['log']
super().__init__(*args, **kwargs)
def _logging(self, message: str, url: str = None, log_type: str = None, log_data: dict = {}) -> None:
if self._log:
api_config[self._name]['log'] = True
if api_config[self._name]['log'] is False:
return
if not hasattr(self, 'logger'):
self.logger = LibPool().logger
logdata = log_data
logdata = {
'connection_details': api_config[self._name]
}
if url:
logdata['url'] = url
if log_type is not None and log_type == 'error':
self.logger.error(message, logdata)
else:
self.logger.info(message, logdata)
def send_request(self, api_endpoint: str, data: dict = {}) -> dict:
auth_token = self._abiospool.auth_token
if auth_token is None:
# Auth token does not exists, get new one
self.authenticate()
auth_token = self._abiospool.auth_token
if auth_token is None:
error_message = 'Cannot connect to Provider 2'
self._logging(error_message, log_type='error')
raise cherrypy.HTTPError(401, error_message)
ac = api_config[self._name]
url_start = '{}{}?'.format(ac['url'], api_endpoint)
if '?' in api_endpoint:
url_start = '{}&'.format(url_start[0:-1])
url = '{}access_token={}'.format(url_start, auth_token)
if self._abiospool.request_permission:
response = requests.get(url, data=to_json(data))
if response.status_code == 401 and response.json()['error_description'] == 'Access token is not valid.':
# Unauthorized -> authenticate & repeat request
error_message = 'Provider 2 Unauthorized: {} {}'.format(response.json()['error_description'], auth_token)
self._logging(error_message, url, 'error')
info_message = 'Reconnect to Provider 2'
self._logging(info_message)
self.authenticate()
response = requests.get(url, data=to_json(data))
tools.check_request_timeout() # Application hook to return 408 if time is over
return response.headers, response.json()
def authenticate(self) -> None:
ac = api_config[self._name]
auth_data = {
'grant_type': 'client_credentials',
'client_id': ac['client_id'],
'client_secret': ac['client_secret']
}
headers = {'Content-Type': 'application/x-www-form-urlencoded'}
url = '{}{}'.format(ac['url'], 'oauth/access_token')
try: # Authenticate
response = requests.post(url, data=auth_data, headers=headers)
if 'error' in response.json():
raise AuthenticationError(response.json()['error_description'])
self._abiospool.auth_token = response.json()['access_token']
except Exception as e:
error_message = 'Authenticate to Provider 2 was not succesfull: {}'.format(e)
self._logging(error_message, url, 'error')
raise cherrypy.HTTPError(401, error_message)
def monitor(self) -> float:
# Try to authenticate at REST endpoint
start_time = time.perf_counter()
if self.authenticate() is False:
return False
return time.perf_counter() - start_time
class Watcher(WatcherInterface):
'''Define concrete Adapter
'''
_name = 'provider1'
lib_pool = LibPool()
def __init__(self, *args, **kwargs):
self.sql = self.lib_pool.libsql
self.send_request = cherrypy.thread_data.client_obj.send_request
if 'game_name' in kwargs:
self._game_name = kwargs['game_name']
if 'log' in kwargs:
self._log = kwargs['log']
self.validator = get_validator_for_api(self._name, self._game_name,
self._log)
self.transformer = get_transformer_for_api(self._name, self._game_name,
self._log)
self.time_limit = int(get_time_limit())
super().__init__(*args, **kwargs)
def _log_msg(self, type: str, msg: str, **kwargs) -> None:
if self._log:
api_config[self._name]['log'] = True
if api_config[self._name]['log'] is False:
return
if not hasattr(self, 'logger'):
self.logger = LibPool().logger
logdata = {'connection_details': api_config[self._name]}
for name, value in kwargs.items():
logdata[name] = value
if type == 'warning':
self.logger.warning(msg, logdata)
elif type == 'error':
self.logger.error(msg, logdata)
def watch_current_games(self) -> None:
self.sql.start_trans_mode()
cur = self.sql.cursor()
# If there are some new just finished games mentioned in specification founded,
# save them in database table `current_game_watch`
valid_tournament_ids = list(
toolz.itertoolz.unique([
y[self._name]['tournament_id'] for x, y in api_config[
'{}_tournaments'.format(self._game_name)].items()
if y[self._name] and y[self._name]['tournament_id']
]))
if not valid_tournament_ids:
msg = 'No valid tournaments were found for Provider 2'
self._log_msg('error', msg)
return
for tournament_id in valid_tournament_ids:
series = {'last_page': 1} # Fake for loop start
series_current_page = 0
while series['last_page'] > series_current_page: # Pagination
series_current_page += 1
# NOTE: `&tiers[]=1` is not necessary, when we filter tournament ids,
# its open to lower levels of tournaments for testing
url_s = 'series?games[]={}&with[]=matches&is_over=true&tournaments[]={}&page={}'.format(
api_config[self._game_name]['provider1_id'], tournament_id,
series_current_page)
headers, series = self.send_request(api_endpoint=url_s)
if not series or 'data' not in series:
msg = 'No data were found for Provider 2'
self._log_msg('error', msg, url=url_s)
return
limit_datetime = datetime.datetime.utcnow(
) - datetime.timedelta(minutes=self.time_limit)
for serie in series['data']:
if (serie['end'] is None or datetime.datetime.strptime(
serie['end'], '%Y-%m-%d %H:%M:%S') <
limit_datetime):
continue
url_m = 'series/{}?with[]=matches'.format(serie['id'])
headers, matches = self.send_request(api_endpoint=url_m)
if not self.validator.validate_match(
url_m, matches, serie, 'current_game_invalid'):
continue
for match in matches['matches']:
existing = cur.qfo('''
SELECT * FROM "current_game_watch"
WHERE "data_src_game_id" = %(src_game_id)s
AND "is_deleted" = false
''',
params={'src_game_id': match['id']})
if existing:
continue
common_data = {
'data_src': self._name,
'game_name': self._game_name,
'data_src_url': url_s,
'data_src_game_id': match['id'],
'data_src_game_title': matches['title'],
'data_src_start_datetime': matches['start'],
'data_src_finish_datetime': matches['end'],
'data_src_tournament_id': serie['tournament_id'],
'data_src_tournament_title': None,
'insert_datetime': datetime.datetime.utcnow(),
'is_watching': True
}
cur.insert('current_game_watch', common_data)
watch_game_id = cur.get_last_id('current_game_watch')
url_g = 'matches/{}?with[]=summary'.format(match['id'])
headers, data = self.send_request(api_endpoint=url_g)
if not self.validator.validate_game(
url_g, watch_game_id, data, match,
'current_game_invalid'):
cur.update('current_game_watch', {
'is_watching': False,
'is_deleted': True
}, {'id': watch_game_id})
self.sql.finish_trans_mode()
def collect_current_data(self) -> None:
# Look for finished games (not longer than hour ago) mentioned in database table `current_game_watch`
# and collect data for them into tables with team and player stats
self.sql.start_trans_mode()
cur = self.sql.cursor()
matches = cur.qfa(
'''
SELECT * FROM "current_game_watch"
WHERE "is_watching" = true
AND "is_deleted" = false
AND "data_src" = %s
AND "game_name" = %s
''', [self._name, self._game_name])
for match in matches:
url_m = 'matches/{}?with[]=summary'.format(
match['data_src_game_id'])
headers, data = self.send_request(api_endpoint=url_m)
# Save game stats connection object
stats_game_id, unchanged_game_id = self._save_game_stats_connection_objects(
match['id'])
# Need to find last stats id
prev_stats_game_id = self._get_previous_game_id(match['id'])
# Save team and player game stats
changes_count = {}
changes_count['team'] = self._save_team_stats(
url_m, stats_game_id, prev_stats_game_id, unchanged_game_id,
data)
changes_count['player'] = self._save_player_stats(
url_m, stats_game_id, prev_stats_game_id, unchanged_game_id,
data)
self._solve_changes_count_aftermath(changes_count, stats_game_id,
unchanged_game_id)
# Set watching false if its over limit now
self._check_watching_limit(match['id'], match['insert_datetime'])
self.sql.finish_trans_mode()
def _save_team_stats(self, url: str, stats_game_id: int,
prev_stats_game_id: int, unchanged_game_id: int,
data: dict) -> DictNone:
# Load last stats to have a data for comparison
last_data = self._get_last_data(prev_stats_game_id, 'team')
prepared_data = self.transformer.prepare_teams_data(data)
common_data = {'data_src_url': url}
teams_data = self.transformer.get_teams_data(prepared_data, data)
return self._save_team_stats_common(stats_game_id, unchanged_game_id,
common_data, teams_data, last_data)
class Validator(ValidatorInterface):
'''Define concrete Adapter
'''
_name = 'provider2'
lib_pool = LibPool()
def __init__(self, *args, **kwargs):
self.sql = self.lib_pool.libsql
if 'game_name' in kwargs:
self._game_name = kwargs['game_name']
if 'log' in kwargs:
self._log = kwargs['log']
def _log_msg(self, type: str, msg: str, **kwargs) -> None:
if self._log:
api_config[self._name]['log'] = True
if api_config[self._name]['log'] is False:
return
if not hasattr(self, 'logger'):
self.logger = LibPool().logger
logdata = {'connection_details': api_config[self._name]}
for name, value in kwargs.items():
logdata[name] = value
if type == 'warning':
self.logger.warning(msg, logdata)
elif type == 'error':
self.logger.error(msg, logdata)
def validate_match(self, url: str, matches_data: dict, table: str) -> bool:
msg = None
if 'games' not in matches_data or not matches_data['games']:
msg = '{} - "games" missing in data or empty'.format(
self._game_name.upper())
# Save it as invalid and log it
insert_data = {
'data_src_url': url,
'insert_datetime': datetime.datetime.utcnow(),
'problem_msg': msg
}
self.sql.cursor().insert(table, insert_data)
self._log_msg('error', msg, url=url)
return False
return True
def validate_game(self, url: str, parent_game_id: int, data: dict,
match_data: dict, table: str) -> bool:
teams = [x['opponent']['id'] for x in data['match']['opponents']]
msg = None
if not teams or len(teams) != 2:
msg = '{} - "opponents" empty in data or invalid for match {}'.format(
self._game_name.upper(), match_data['id'])
elif 'players' not in data or not data['players']:
msg = '{} - "players" missing in data or empty for match {}'.format(
self._game_name.upper(), match_data['id'])
if msg:
# Save it as invalid and log it
insert_data = {
'data_src_url': url,
'insert_datetime': datetime.datetime.utcnow(),
'problem_msg': msg
}
if table == 'current_game_invalid':
insert_data['watch_game_id'] = parent_game_id
elif table == 'past_game_invalid':
insert_data['stats_game_id'] = parent_game_id
self.sql.cursor().insert(table, insert_data)
self._log_msg('error', msg, url=url)
return False
return True
class Watcher(WatcherInterface):
'''Define concrete Adapter
'''
_name = 'provider2'
lib_pool = LibPool()
def __init__(self, *args, **kwargs):
self.sql = self.lib_pool.libsql
self.send_request = cherrypy.thread_data.client_obj.send_request
if 'game_name' in kwargs:
self._game_name = kwargs['game_name']
if 'log' in kwargs:
self._log = kwargs['log']
self.validator = get_validator_for_api(self._name, self._game_name,
self._log)
self.time_limit = int(get_time_limit())
super().__init__(*args, **kwargs)
def _log_msg(self, type: str, msg: str, **kwargs) -> None:
if self._log:
api_config[self._name]['log'] = True
if api_config[self._name]['log'] is False:
return
if not hasattr(self, 'logger'):
self.logger = LibPool().logger
logdata = {'connection_details': api_config[self._name]}
for name, value in kwargs.items():
logdata[name] = value
if type == 'warning':
self.logger.warning(msg, logdata)
elif type == 'error':
self.logger.error(msg, logdata)
def watch_current_games(self) -> None:
self.sql.start_trans_mode()
cur = self.sql.cursor()
# If there are some new just finished games mentioned in specification founded,
# save them in database table `current_game_watch`
valid_league_ids = list(
toolz.itertoolz.unique([
y[self._name]['league_id'] for x, y in api_config[
'{}_tournaments'.format(self._game_name)].items()
if y[self._name] and y[self._name]['league_id']
]))
if not valid_league_ids:
msg = 'No valid leagues were found for Provider 1'
self._log_msg('error', msg)
return
matches_last_page = 1 # Fake for loop start
matches_current_page = 0
while matches_last_page > matches_current_page: # Pagination
matches_current_page += 1
url_m = '{}/matches/past?filter[status]=finished&league_id={}&page[size]=100&page[number]={}'.format(
api_config[self._game_name]['provider2_slug'],
str(valid_league_ids), matches_current_page)
matches_headers, matches = self.send_request(api_endpoint=url_m)
matches_last_page = self._round_up(
int(matches_headers['X-Total']) /
int(matches_headers['X-Per-Page']))
if not matches:
return {'return_msg': 'No data were found for Provider 1.'}
msg = 'No data were found for Provider 1'
self._log_msg('error', msg, url=url_m)
return
limit_datetime = datetime.datetime.utcnow() - datetime.timedelta(
minutes=self.time_limit)
for match in matches:
if match['league_id'] not in valid_league_ids:
continue
elif (match['end_at'] is None or datetime.datetime.strptime(
match['end_at'], '%Y-%m-%dT%H:%M:%SZ') <
limit_datetime):
continue
if not self.validator.validate_match(url_m, match,
'current_game_invalid'):
continue
for game in match['games']:
existing = cur.qfo('''
SELECT * FROM "current_game_watch"
WHERE "data_src_game_id" = %(src_game_id)s
AND "is_deleted" = false
''',
params={'src_game_id': game['id']})
if existing:
continue
common_data = {
'data_src': self._name,
'game_name': self._game_name,
'data_src_url': url_m,
'data_src_game_id': game['id'],
'data_src_game_title': match['name'],
'data_src_start_datetime': match['begin_at'],
'data_src_finish_datetime': match['end_at'],
'data_src_tournament_id': match['serie_id'],
'data_src_tournament_title':
match['serie']['full_name'],
'insert_datetime': datetime.datetime.utcnow(),
'is_watching': True
}
cur.insert('current_game_watch', common_data)
watch_game_id = cur.get_last_id('current_game_watch')
url_g = '{}/games/{}'.format(
api_config[self._game_name]['provider2_slug'],
game['id'])
headers, data = self.send_request(api_endpoint=url_g)
if not self.validator.validate_game(
url_m, watch_game_id, data, match,
'current_game_invalid'):
cur.update('current_game_watch', {
'is_watching': False,
'is_deleted': True
}, {'id': watch_game_id})
self.sql.finish_trans_mode()
def collect_current_data(self):
# Look for finished games (not longer than hour ago) mentioned in database table `current_game_watch`
# and collect data for them into tables with team and player stats
self.sql.start_trans_mode()
cur = self.sql.cursor()
games = cur.qfa(
'''
SELECT * FROM "current_game_watch"
WHERE "is_watching" = true
AND "is_deleted" = false
AND "data_src" = %s
AND "game_name" = %s
''', [self._name, self._game_name])
for game in games:
url_g = '{}/games/{}'.format(
api_config[self._game_name]['provider2_slug'],
game['data_src_game_id'])
headers, data = self.send_request(api_endpoint=url_g)
# Save game stats connection object
stats_game_id, unchanged_game_id = self._save_game_stats_connection_objects(
game['id'])
# Need to find last stats id
prev_stats_game_id = self._get_previous_game_id(game['id'])
# Save team and player game stats
changes_count = {}
changes_count['team'] = self._save_team_stats(
url_g, stats_game_id, prev_stats_game_id, unchanged_game_id,
game['data_src_game_id'], data, data['match']['games'])
changes_count['player'] = self._save_player_stats(
url_g, stats_game_id, prev_stats_game_id, unchanged_game_id,
data)
self._solve_changes_count_aftermath(changes_count, stats_game_id,
unchanged_game_id)
# Set watching false if its over limit now
self._check_watching_limit(game['id'], game['insert_datetime'])
self.sql.finish_trans_mode()
def _save_team_stats(self, url: str, stats_game_id: int,
prev_stats_game_id: int, unchanged_game_id: int,
game_id: int, data: dict,
games_data: dict) -> DictNone:
# Load last stats to have a data for comparison
last_data = self._get_last_data(prev_stats_game_id, 'team')
teams = [x['opponent']['id'] for x in data['match']['opponents']]
prepared_data = self.transformer.prepare_teams_data(
teams, data, game_id, games_data)
common_data = {'data_src_url': url}
teams_data = self.transformer.get_teams_data(prepared_data, data,
teams)
return self._save_team_stats_common(stats_game_id, unchanged_game_id,
common_data, teams_data, last_data)
def _round_up(self, n, decimals=0):
multiplier = 10**decimals
return math.ceil(n * multiplier) / multiplier
class Validator(ValidatorInterface):
'''Define concrete Adapter
'''
_name = 'provider1'
lib_pool = LibPool()
def __init__(self, *args, **kwargs):
self.sql = self.lib_pool.libsql
if 'game_name' in kwargs:
self._game_name = kwargs['game_name']
if 'log' in kwargs:
self._log = kwargs['log']
def _log_msg(self, type: str, msg: str, **kwargs) -> None:
if self._log:
api_config[self._name]['log'] = True
if api_config[self._name]['log'] is False:
return
if not hasattr(self, 'logger'):
self.logger = LibPool().logger
logdata = {'connection_details': api_config[self._name]}
for name, value in kwargs.items():
logdata[name] = value
if type == 'warning':
self.logger.warning(msg, logdata)
elif type == 'error':
self.logger.error(msg, logdata)
def validate_match(self, url: str, matches_data: dict,
tournament_data: dict, table: str) -> bool:
msg = None
if 'matches' not in matches_data or not matches_data['matches']:
msg = '{} - "matches" missing in data or empty'.format(
self._game_name.upper())
# Save it as invalid and log it
insert_data = {
'data_src_url': url,
'insert_datetime': datetime.datetime.utcnow(),
'problem_msg': msg
}
self.sql.cursor().insert(table, insert_data)
self._log_msg('error', msg, url=url)
return False
return True
def validate_game(self, url: str, parent_game_id: int, data: dict,
match_data: dict, table: str) -> bool:
side1 = api_config[self._game_name]['provider1_sides'][0]
side2 = api_config[self._game_name]['provider1_sides'][1]
msg = None
if 'rosters' not in match_data or not match_data['rosters']:
msg = '{} - "rosters" missing in data or empty for match {}'.format(
self._game_name.upper(), match_data['id'])
else:
if 'match_summary' not in data or not data['match_summary']:
msg = '{} - "match_summary" missing in data or empty'.format(
self._game_name.upper())
elif self._game_name == 'csgo':
if side1 not in data['match_summary'] and side2 not in data[
'match_summary']:
msg = '{} - "{}" and "{}" missing in summary data'.format(
self._game_name.upper(), side1, side2)
elif side1 not in data['match_summary']:
msg = '{} - "{}" missing in summary data'.format(
self._game_name.upper(), side1)
elif side2 not in data['match_summary']:
msg = '{} - "{}" missing in summary data'.format(
self._game_name.upper(), side2)
elif (side1 not in data['match_summary']['scoreboard']
and side2 not in data['match_summary']['scoreboard']):
msg = '{} - "{}" and "{}" missing in scoreboard data'.format(
self._game_name.upper(), side1, side2)
elif side1 not in data['match_summary']['scoreboard']:
msg = '{} - "{}" missing in scoreboard data'.format(
self._game_name.upper(), side1, side2)
elif side2 not in data['match_summary']['scoreboard']:
msg = '{} - "{"} missing in scoreboard data'.format(
self._game_name.upper(), side1, side2)
elif self._game_name == 'dota2':
if 'rosters' not in data or not data['rosters']:
msg = '{} - "rosters" missing in data or empty for match {}'.format(
self._game_name.upper(), match_data['id'])
elif 'winner' not in data or data['winner'] is None:
msg = '{} - "winner" missing in summary data or empty'.format(
self._game_name.upper())
elif 'players_stats' not in data['match_summary']:
msg = '{} - "players_stats" missing in data[match_summary]'.format(
self._game_name.upper(), side1, side2)
else:
if (data['match_summary'][side1] !=
data['rosters'][0]['id']
or data['match_summary'][side2] !=
data['rosters'][1]['id']):
msg = '{} - home or away team does not correspond with rosters'.format(
self._game_name.upper())
elif self._game_name == 'lol':
if 'rosters' not in data or not data['rosters']:
msg = '{} - missing "rosters" in data or empty for match {}'.format(
self._game_name.upper(), match_data['id'])
elif side1 not in data['match_summary'] and side2 not in data[
'match_summary']:
msg = '{} - "{}" and "{}" missing in data[match_summary]'.format(
self._game_name.upper(), side1, side2)
elif side1 not in data['match_summary']:
msg = '{} - "{}" missing in data[match_summary]'.format(
self._game_name.upper(), side1)
elif side2 not in data['match_summary']:
msg = '{} - "{}" missing in data[match_summary]'.format(
self._game_name.upper(), side2)
elif 'players' not in data['match_summary'][
side1] and 'players' not in data['match_summary'][
side2]:
msg = '{} - "players" missing in data[match_summary][{}] and data[match_summary][{}]'.format(
self._game_name.upper(), side1, side2)
elif 'players' not in data['match_summary'][side1]:
msg = '{} - "players" missing in data[match_summary][{}]'.format(
self._game_name.upper(), side1)
elif 'players' not in data['match_summary'][side2]:
msg = '{} - "players" missing in data[match_summary][{}]'.format(
self._game_name.upper(), side2)
else:
if (data['match_summary'][side1]['id'] !=
data['rosters'][0]['id']
or data['match_summary'][side2]['id'] !=
data['rosters'][1]['id']):
msg = '{} - home or away team does not correspond with rosters'.format(
self._game_name.upper())
if msg:
# Save it as invalid and log it
insert_data = {
'data_src_url': url,
'insert_datetime': datetime.datetime.utcnow(),
'problem_msg': msg
}
if table == 'current_game_invalid':
insert_data['watch_game_id'] = parent_game_id
elif table == 'past_game_invalid':
insert_data['stats_game_id'] = parent_game_id
self.sql.cursor().insert(table, insert_data)
self._log_msg('error', msg, url=url, match_id=match_data['id'])
return False
return True
class RestClient(RestClientInterface):
'''Define concrete Adapter
'''
_name = 'provider2'
_pandapool = Provider2Pool()
def __init__(self, *args, **kwargs):
cherrypy.thread_data.client_obj = self
if 'log' in kwargs:
self._log = kwargs['log']
super().__init__(*args, **kwargs)
def _logging(self,
message: str,
url: str = None,
log_type: str = None) -> None:
if self._log:
api_config[self._name]['log'] = True
if api_config[self._name]['log'] is False:
return
if not hasattr(self, 'logger'):
self.logger = LibPool().logger
logdata = {'connection_details': api_config[self._name]}
if url:
logdata['url'] = url
if log_type is not None and log_type == 'error':
self.logger.error(message, logdata)
else:
self.logger.info(message, logdata)
def send_request(self, api_endpoint: str, data: dict = {}) -> dict:
ac = api_config[self._name]
url_start = '{}{}?'.format(ac['url'], api_endpoint)
if '?' in api_endpoint:
url_start = '{}&'.format(url_start[0:-1])
url = '{}token={}'.format(url_start, ac['auth_token'])
try:
if self._pandapool.request_permission:
response = requests.get(url, data=to_json(data))
if response.status_code == 401:
raise AuthenticationError(response.json()['error'])
elif 'error' in response.json():
raise Exception(response.json()['error'])
except AuthenticationError as e:
error_message = 'Authentication error on Provider 1: {}'.format(e)
self._logging(error_message, url, 'error')
raise cherrypy.HTTPError(401, error_message)
except Exception as e:
error_message = 'Request to Provider 1 was not succesfull: {}'.format(
e)
self._logging(error_message, url, 'error')
raise cherrypy.HTTPError(500, error_message)
tools.check_request_timeout(
) # Application hook to return 408 if time is over
return response.headers, response.json()
def authenticate(self) -> None:
# Provider 1 is always authenticated, hit lives matches to check it
url = '{}{}'.format(api_config[self._name]['url'], 'lives')
response = requests.get(url)
if response.status_code == 401:
return False
def monitor(self) -> float:
# Try to authenticate at REST endpoint
start_time = time.perf_counter()
if self.authenticate() is False:
return False
return time.perf_counter() - start_time
class Grabber(GrabberInterface):
'''Define concrete Adapter
'''
_name = 'provider1'
lib_pool = LibPool()
def __init__(self, *args, **kwargs):
self.sql = self.lib_pool.libsql
self.send_request = cherrypy.thread_data.client_obj.send_request
if 'game_name' in kwargs:
self._game_name = kwargs['game_name']
if 'log' in kwargs:
self._log = kwargs['log']
self.validator = get_validator_for_api(self._name, self._game_name, self._log)
self.transformer = get_transformer_for_api(self._name, self._game_name, self._log)
super().__init__(*args, **kwargs)
def _log_msg(self, type: str, msg: str, **kwargs) -> None:
if self._log:
api_config[self._name]['log'] = True
if api_config[self._name]['log'] is False:
return
if not hasattr(self, 'logger'):
self.logger = LibPool().logger
logdata = {'connection_details': api_config[self._name]}
for name, value in kwargs.items():
logdata[name] = value
if type == 'warning':
self.logger.warning(msg, logdata)
elif type == 'error':
self.logger.error(msg, logdata)
def _find_and_save_past_games(
self, date_from: datetime.date = None, date_to: datetime.date = None, stats: dict = {}) -> dict:
self.sql.start_trans_mode()
cur = self.sql.cursor()
# Find games mentioned in configuration, save them in database table `past_game_stats`
valid_tournament_ids = list(toolz.itertoolz.unique([
y[self._name]['tournament_id'] for x, y in api_config['{}_tournaments'.format(self._game_name)].items()
if y[self._name] and y[self._name]['tournament_id']]))
series = {'last_page': 1} # Fake for loop start
series_current_page = 0
while series['last_page'] > series_current_page: # Pagination
series_current_page += 1
# NOTE: `&tiers[]=1` is not necessary, when we filter tournament ids,
# its open to lower levels of tournaments for testing
# NOTE: &tournaments[]=123,345 does not work as expected, should be only one ID -> DO NOT USE IT!!!
url_s = 'series?games[]={}&with[]=matches&page={}'.format(
api_config[self._game_name]['provider1_id'],
series_current_page)
headers, series = self.send_request(api_endpoint=url_s)
if not series or 'data' not in series:
return {'return_msg': 'No data were found for Provider 2.'}
limit_from = None
if date_from:
limit_from = datetime.datetime.strptime('{} 00:00:00'.format(date_from), '%Y-%m-%d %H:%M:%S')
limit_to = None
if date_to:
limit_to = datetime.datetime.strptime('{} 23:59:59'.format(date_to), '%Y-%m-%d %H:%M:%S')
for serie in series['data']:
if serie['tournament_id'] not in valid_tournament_ids:
continue
elif ((limit_from and limit_to
and serie['start'] and serie['end']
and (datetime.datetime.strptime(serie['start'], '%Y-%m-%d %H:%M:%S') < limit_from
or datetime.datetime.strptime(serie['end'], '%Y-%m-%d %H:%M:%S') > limit_to))
or (limit_from and serie['start']
and datetime.datetime.strptime(serie['start'], '%Y-%m-%d %H:%M:%S')
< limit_from)
or (limit_to and serie['end']
and datetime.datetime.strptime(serie['end'], '%Y-%m-%d %H:%M:%S')
> limit_to)):
continue
stats['matches_total_count'] += 1
url_m = 'series/{}?with[]=matches'.format(serie['id'])
headers, matches = self.send_request(api_endpoint=url_m)
if not self.validator.validate_match(url_m, matches, serie, 'past_game_invalid'):
stats['matches_invalid_count'] += 1
continue
for match in matches['matches']:
stats['games_total_count'] += 1
existing = cur.qfo('''
SELECT * FROM "past_game_stats" WHERE "data_src_game_id" = %(src_game_id)s
''', params={'src_game_id': match['id']})
common_data = {
'data_src': self._name,
'game_name': self._game_name,
'data_src_url': url_s,
'data_src_game_id': match['id'],
'data_src_game_title': serie['title'],
'data_src_start_datetime': matches['start'],
'data_src_finish_datetime': matches['end'],
'data_src_tournament_id': serie['tournament_id'],
'data_src_tournament_title': None
}
if not existing:
common_data['insert_datetime'] = datetime.datetime.utcnow()
cur.insert('past_game_stats', common_data)
stats_game_id = cur.get_last_id('past_game_stats')
else:
stats_game_id = existing['id']
del(existing['id'], existing['insert_datetime'], existing['update_datetime'])
diff_data = dict(toolz.itertoolz.diff(common_data, existing))
if diff_data:
common_data['update_datetime'] = datetime.datetime.utcnow()
cur.update('past_game_stats', diff_data, conditions={'data_src_game_id': match['id']})
url_g = 'matches/{}?with[]=summary'.format(match['id'])
headers, data = self.send_request(api_endpoint=url_g)
if not self.validator.validate_game(url_g, stats_game_id, data, match, 'past_game_invalid'):
print(url_g)
stats['games_invalid_count'] += 1
continue
# Save team and player game stats
stats['datapoints_missing_count'] += self._save_team_stats(url_m, stats_game_id, data)
stats['datapoints_missing_count'] += self._save_player_stats(url_m, stats_game_id, data)
# Save statistics into "past_game_analysis" table
final_stats = self._transform_stats_and_save_analysis(self._name, stats)
self.sql.finish_trans_mode()
return final_stats
def _save_team_stats(
self, url: str, stats_game_id: int, data: dict) -> int:
prepared_data = self.transformer.prepare_teams_data(data)
common_data = {
'stats_game_id': stats_game_id,
'data_src_url': url
}
teams_data = self.transformer.get_teams_data(prepared_data, data)
return self._save_team_stats_common(common_data, teams_data)
