def new_message():
def format_text(project_name, message):
cases = {
"error": "\u26D4New error from %s:\n"
"%s",
"warning": "⚠️New warning from %s:\n"
"%s",
"log": "\U0001F4ACNew log from %s:\n"
"%s",
}
return cases[message["type"]] % (project_name,
log["message"]["content"])
log = request.json
session = Session()
project = session.query(Project).filter_by(token=log["token"]).first()
if not project:
return {
"ok": False,
"error": "Project not found! Please check your token"
}
text = format_text(project.name, log["message"])
bot.sendMessage(chat_id=project.user_id, text=text)
return jsonify({"ok": True})
def __init__(self, url: str):
"""
make sure site is in db
check if site is up
if it is, update last_up in db
"""
# this should be statically defined in child implementations of the class
assert self.actor != ""
self.current_victims = []
self.new_victims = []
self.url = url
self.session = Session()
q = self.session.query(Site).filter_by(actor=self.actor)
if q.count() == 0:
# site is new, init obj
self.site = Site(actor=self.actor, url=self.url)
self.session.add(self.site)
self.session.commit()
self.first_run = True
else:
# site exists, set obj
self.site = q.first()
# if we haven't successfully scraped the site before, consider this the first run
if self.site.last_scraped is None:
self.first_run = True
# check if site is up
self.is_up = self.is_site_up()
def is_password_correct(password: str) -> bool:
"""Returns whether a user email exists in the database.
Args:
email: An email that may belong to a user.
Returns:
Returns true if the email exists, false otherwise.
"""
session = Session()
return session.query(User).filter_by(password=password).first() != None
def get_role_groups(role_id: int) -> List[Group]:
"""Returns a list of groups with the given role.
Args:
role_id: An role's role_id.
Returns:
Returns a list of groups if an role has the role_id, otherwise returns None.
"""
session = Session()
return session.query(Group).join(RoleToGroup).filter(
RoleToGroup.role_id == role_id).all()
def get_employee_roles(email: str) -> List[EmployeeToRole]:
"""Returns a list of roles with the given email.
Args:
email: An employee's email.
Returns:
Returns a list of roles if an employee has the email, otherwise returns None.
"""
session = Session()
return session.query(EmployeeToRole).filter(
EmployeeToRole.email == email).all()
def get_slack_groups() -> List[Group]:
"""Returns a list of groups that belong to Slack.
Returns:
Returns a list containing all the groups from Slack.
"""
groups = None
session = Session()
try:
groups = session.query(Group).filter(Group.app_id == 1).all()
except Exception as e:
print(e)
return groups
async def upload_file(file_path: str, league_name: str, stat_type: str,
last_5: bool) -> bool:
session: SQLSession = Session()
try:
with open(file_path, 'rb') as file:
msg = await bot.send_photo(admins[0],
file,
disable_notification=True)
file_id = msg.photo[-1].file_id
session.query(MediaIds).filter(
and_(MediaIds.league == league_name,
MediaIds.stat_type == stat_type,
MediaIds.last_5 == last_5)).delete()
session.add(
MediaIds(league_name, stat_type, last_5, file_id, file_path))
return True
except Exception as ex:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logging.warning(f"Ex={ex} in file={fname} line={exc_tb.tb_lineno}")
return False
finally:
session.commit()
session.close()
def __update_xg(self, league_name: str) -> bool:
logging.info(f"Start update xg for league={league_name}")
session: SQLSession = Session()
try:
for player_stat in self.fbref.get_xg_stats(league_name):
cur_player: PlayerStats = self.get_player(player_stat)
if cur_player:
last3_xg_per_game, last3_npxg_per_game, last3_xa_per_game, last5_xg_per_game, \
last5_npxg_per_game, last5_xa_per_game = self.__compute_xg(player_stat, cur_player)
session.query(PlayerStats).filter(
PlayerStats.id == cur_player.id).update({
'r5_xg':
player_stat["xg"],
'r5_npxg':
player_stat["npxg"],
'r5_xa':
player_stat["xa"],
'last3_xg_per_game':
last3_xg_per_game,
'last3_npxg_per_game':
last3_npxg_per_game,
'last3_xa_per_game':
last3_xa_per_game,
'last5_xg_per_game':
last5_xg_per_game,
'last5_npxg_per_game':
last5_npxg_per_game,
'last5_xa_per_game':
last5_xa_per_game
})
session.commit()
else: # if new player
session.add(
PlayerStats(player_stat['name'], league_name,
player_stat['team'],
player_stat['position']))
session.commit()
cur_player = self.get_player(player_stat)
if cur_player:
session.query(PlayerStats).filter(
PlayerStats.id == cur_player.id).update({
'r5_xg':
player_stat["xg"],
'r5_npxg':
player_stat["npxg"],
'r5_xa':
player_stat["xa"]
})
session.commit()
return True
except Exception as ex:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logging.warning(f"Ex={ex} in file={fname} line={exc_tb.tb_lineno}")
return False
finally:
session.commit()
session.close()
def get_player(self, player_info: dict) -> PlayerStats:
session: SQLSession = Session()
result = session.query(PlayerStats).filter(
and_(PlayerStats.name == player_info['name'],
PlayerStats.league == player_info['league'],
PlayerStats.team == player_info['team'])).first()
session.close()
return result
def get_coeffs(self, league_name: str, cur_round: bool) -> str:
session: SQLSession = Session()
coeffs = session.query(Coeff).filter(
and_(Coeff.league == league_name, Coeff.is_cur_round == cur_round))
if not coeffs:
return ""
coeffs_list: List[Coeff] = [cf for cf in coeffs]
return self.__transform_coeffs(coeffs_list, cur_round)
async def update_coeffs(self, league_name: str) -> bool:
# delete last round
db_session: SQLSession = Session()
db_session.query(Coeff).filter(Coeff.league == league_name).delete()
db_session.commit()
db_session.close()
return await self.xbet.update_league(league_name)
def is_new_round(self, league_name: str) -> bool:
session: SQLSession = Session()
league_info = session.query(League_info).filter(
League_info.league == league_name).first()
session.close()
if not league_info:
return False
return datetime.now() > league_info.deadline
def get_employees_with_ids(ids: List[int]) -> List[Employee]:
"""Returns a list of employees that match the specified ids.
Args:
ids: The ids being used to query employees.
Returns:
Returns a list of employess.
"""
session = Session()
employees = []
try:
employees = session.query(Employee).filter(Employee.id.in_(ids)).all()
except Exception as e:
print(e)
return employees
def get_roles_with_ids(ids: List[int]) -> List[Role]:
"""Returns a list of roles that match the specified ids.
Args:
ids: The ids being used to query roles.
Returns:
Returns a list of roles.
"""
session = Session()
roles = []
try:
roles = session.query(Role).filter(Role.id.in_(ids)).all()
except Exception as e:
print(e)
return roles
def get_employees_by_role(role: Role) -> List[Employee]:
"""Returns a list of employees with the given role.
Args:
role: An instance of Role.
Returns:
Returns a list of employees.
"""
employees = []
session = Session()
try:
employees = session.query(Employee).join(EmployeeToRole).filter(
EmployeeToRole.role_id == role.id).all()
except Exception as e:
print(e)
return employees
def show_projects(update, context):
chat_id = update.callback_query.message.chat.id
try:
session = Session()
projects = session.query(Project).filter_by(user_id=chat_id)
text = "Your projects"
buttons = []
count = projects.count()
context.user_data["projects"] = projects
for i in range(0, count, 2):
row = [
InlineKeyboardButton(text=projects[i].name,
callback_data=str(EDITING) +
str(projects[i].name)),
]
if i + 1 < count:
row.append(
InlineKeyboardButton(text=projects[i + 1].name,
callback_data=str(EDITING) +
str(projects[i + 1].name)))
buttons.append(row)
buttons.append([
InlineKeyboardButton(text='Return to main',
callback_data=str(MAIN))
])
keyboard = InlineKeyboardMarkup(buttons)
update.callback_query.answer()
update.callback_query.edit_message_text(text=text,
reply_markup=keyboard)
return SELECTING_ACTION
except Exception as e:
print(str(e))
def get_sources(self, league_name: str) -> List[Source]:
session: SQLSession = Session()
try:
sources = session.query(Source).filter(
Source.league == league_name).all()
return sources
except Exception as ex:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logging.warning(f"Ex={ex} in file={fname} line={exc_tb.tb_lineno}")
return []
finally:
session.close()
def add_source(self, name: str, league_name: str, url: str,
description: str) -> bool:
logging.info(f"Add source {name} for league={league_name}")
session: SQLSession = Session()
try:
session.add(Source(name, league_name, url, description))
return True
except Exception as ex:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logging.warning(f"Ex={ex} in file={fname} line={exc_tb.tb_lineno}")
return False
finally:
session.commit()
session.close()
def delete_source(self, name: str, league_name: str, url: str) -> bool:
logging.info(f"Delete source {name} for league={league_name}")
session: SQLSession = Session()
try:
session.query(Source).filter(
and_(Source.league == league_name, Source.name == name,
Source.url == url)).delete()
return True
except Exception as ex:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logging.warning(f"Ex={ex} in file={fname} line={exc_tb.tb_lineno}")
return False
finally:
session.commit()
session.close()
def get_players(self, league_name: str) -> str:
xbet = XBet()
if league_name not in xbet.leagues:
return "Ошибка в названии лиги"
session: SQLSession = Session()
best_players = session.query(Player).filter(Player.league == league_name) \
.order_by(Player.dif_popularity.desc(), Player.old_popularity.desc()).limit(10).all()
worst_players = session.query(Player).filter(Player.league == league_name) \
.order_by(Player.dif_popularity, Player.old_popularity.desc()).limit(10).all()
session.close()
result = ["\U0001F4C8 Популярные игроки:\n"]
result += self.__transform_players(best_players)
result += ["\n\U0001F4C9 Непопулярные игроки:\n"]
result += self.__transform_players(worst_players)
return '\n'.join(result)
def delete_source_by_id(self, source_id: int) -> str:
logging.info(f"Delete source with id={source_id}")
session: SQLSession = Session()
try:
league_name = session.query(Source).filter(
and_(Source.id == source_id)).first().league
session.query(Source).filter(and_(Source.id == source_id)).delete()
return league_name
except Exception as ex:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logging.warning(f"Ex={ex} in file={fname} line={exc_tb.tb_lineno}")
return ""
finally:
session.commit()
session.close()
def save(update, context):
session = Session()
new_name = context.user_data["new_name"]
project = context.user_data["current_project"]
session.query(Project).filter(Project.id == project.id).update(
{"name": new_name})
session.commit()
project.name = new_name
context.user_data["current_project"] = project
return edit(update, context)
async def __update_shoots_files(self, league_name: str) -> bool:
try:
logging.info(f"Start update shoots files for league={league_name}")
# get best players
session: SQLSession = Session()
best_players_last_3 = session.query(PlayerStats).filter(PlayerStats.league == league_name) \
.order_by(PlayerStats.last3_shoots_per_game.desc(), PlayerStats.last3_on_target_per_shoot.desc()) \
.limit(20).all()
best_players_last_5 = session.query(PlayerStats).filter(PlayerStats.league == league_name) \
.order_by(PlayerStats.last5_shoots_per_game.desc(), PlayerStats.last5_on_target_per_shoot.desc()) \
.limit(20).all()
# create pd pataframes
last_3_info = [[
round(player.last3_shoots_per_game, 2),
int(player.last3_on_target_per_shoot * 100), player.name,
player.team
] for player in best_players_last_3]
last_5_info = [[
round(player.last5_shoots_per_game, 2),
int(player.last5_on_target_per_shoot * 100), player.name,
player.team
] for player in best_players_last_5]
last_3_df = pd.DataFrame(
last_3_info, columns=['Уд/И', 'УдС/Уд(%)', 'Игрок', 'Команда'])
last_3_df_styled = last_3_df.style.background_gradient()
last_5_df = pd.DataFrame(
last_5_info, columns=['Уд/И', 'УдС/Уд(%)', 'Игрок', 'Команда'])
last_5_df_styled = last_5_df.style.background_gradient()
result = self.__update_files(league_name, 'shoots',
last_3_df_styled, last_5_df_styled)
result *= await upload_files(league_name, 'shoots')
return result
except Exception as ex:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logging.warning(f"Ex={ex} in file={fname} line={exc_tb.tb_lineno}")
return False
finally:
session.close()
def update_deadline(self, league_name: str) -> bool:
"""
Update deadline for league
"""
logging.info(f"Update deadline for league={league_name}")
new_deadline = self.sports.get_deadline(league_name)
if not new_deadline:
return False
session: SQLSession = Session()
league = session.query(League_info).filter(
League_info.league == league_name).first()
if league:
session.query(League_info).filter(League_info.id == league.id) \
.update({League_info.deadline: new_deadline})
else:
session.add(League_info(league_name, new_deadline))
session.commit()
session.close()
return True
def get_gca_id(self, league_name: str, last_5: bool = False) -> str:
if league_name not in self.fbref.shoots_creation_leagues:
return "Для данной лиги нет данных"
try:
session: SQLSession = Session()
image = session.query(MediaIds).filter(
and_(MediaIds.league == league_name,
MediaIds.stat_type == 'gca',
MediaIds.last_5 == last_5)).first()
if image is not None:
return image.file_id
else:
return ""
except Exception as ex:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logging.warning(f"Ex={ex} in file={fname} line={exc_tb.tb_lineno}")
return ""
finally:
session.close()
def update_league(self, league_name: str, new_round: bool = False) -> bool:
"""
Updates players popularity for league
"""
logging.info(
f"Start update players for league={league_name}, new_round={new_round}"
)
session: SQLSession = Session()
try:
if new_round:
session.query(Player).filter(
Player.league == league_name).update({'dif_popularity': 0})
for player in self.sports.get_league_players(league_name):
cur_player = self.get_player(player)
if cur_player:
if new_round:
session.query(Player).filter(
Player.id == cur_player.id).update(
{'old_popularity': player['popularity']})
else:
session.query(Player).filter(
Player.id == cur_player.id).update({
'dif_popularity':
player['popularity'] - Player.old_popularity
})
else: # if new player
session.add(
Player(player['name'], player['league'],
player['team'], player['amplua'],
player['popularity'], 0))
return True
except Exception as ex:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
logging.warning(f"Ex={ex} in file={fname} line={exc_tb.tb_lineno}")
return False
finally:
session.commit()
session.close()
def typing_name(update, context):
try:
session = Session()
token = uuid.uuid1()
project = Project(token=token,
name=update.message.text,
user_id=update.message.chat.id)
session.add(project)
session.commit()
update.message.reply_text(text=str(token))
return main(update, context)
except Exception as e:
print(str(e))
def start(update, context):
session = Session()
try:
u = User(chat_id=update.message.chat.id)
session.add(u)
session.commit()
except IntegrityError:
print("User already exists")
buttons = [[
InlineKeyboardButton(text='To main menu', callback_data=str(MAIN)),
]]
keyboard = InlineKeyboardMarkup(buttons)
update.message.reply_text("Choose the menu", reply_markup=keyboard)
return SELECTING_ACTION
def shutdown_session(exception=None):
Session.remove()
class SiteCrawler:
# threat actor associated with the leak site
actor: str = ""
# url for the leak site
url: str = ""
# list of victims on the leak site from current scrape
current_victims: List[Victim] = []
# new victims on the leak site from current scrape
new_victims: List[Victim] = []
# is the site up? set by is_site_up()
is_up: bool = False
# db session, set in __init__()
session: SessionType
# site object from db, set in __init__()
site: Site
# is this the first ingest of the site? set in __init__()
# if the first run, don't notify on new victims (b/c they are all "new")
first_run: bool = False
# headers to be used for requests
headers: Dict[str, str]
def __init__(self, url: str):
"""
make sure site is in db
check if site is up
if it is, update last_up in db
"""
# this should be statically defined in child implementations of the class
assert self.actor != ""
self.current_victims = []
self.new_victims = []
self.url = url
self.session = Session()
q = self.session.query(Site).filter_by(actor=self.actor)
self.headers: Dict[str, str] = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; rv:78.0) Gecko/20100101 Firefox/78.0"
}
if q.count() == 0:
# site is new, init obj
self.site = Site(actor=self.actor, url=self.url)
self.session.add(self.site)
self.session.commit()
self.first_run = True
else:
# site exists, set obj
self.site = q.first()
# if we haven't successfully scraped the site before, consider this the first run
if self.site.last_scraped is None:
self.first_run = True
# check if site is up
self.is_up = self.is_site_up()
def is_site_up(self) -> bool:
"""
check if the site is up
this might have specific criteria for some sites
"""
with Proxy() as p:
try:
r = p.get(self.url,
headers=self.headers,
timeout=Config["timeout"])
if r.status_code >= 400:
return False
except Exception as e:
return False
self.site.last_up = datetime.utcnow()
return True
def scrape_victims(self):
"""
pull each listing on the site
check if its already in the db
if it is, update the last seen
if it isn't, add it to the db
store each org name in a list (self.current_victims)
this also sets self.new_victims, which has new victims to notify with
"""
raise Exception("Function implementation not found")
def identify_removed_victims(self) -> List[Victim]:
"""
check org name list against db
if something is in the db, not already removed, and not in the list, alert
"""
# get the current victims from the last scrape
victims = self.session.query(Victim).filter_by(site=self.site,
removed=False).all()
# remove anything from the last scrape that was also in this scrape
# the remaining set is things that were present last time, but not this time
for v in self.current_victims:
try:
victims.remove(v)
except ValueError:
# i think there is an edge case here that can be caused by
# running the scrape while a victim is removed
# it _should_ get picked up on the next run though, so we
# can safely ignore this
pass
# mark the victims as removed, since they are no longer on the leak site
for v in victims:
v.removed = True
self.session.commit()
return victims
