def new_message(): def format_text(project_name, message): cases = { "error": "\u26D4New error from %s:\n" "%s", "warning": "⚠️New warning from %s:\n" "%s", "log": "\U0001F4ACNew log from %s:\n" "%s", } return cases[message["type"]] % (project_name, log["message"]["content"]) log = request.json session = Session() project = session.query(Project).filter_by(token=log["token"]).first() if not project: return { "ok": False, "error": "Project not found! Please check your token" } text = format_text(project.name, log["message"]) bot.sendMessage(chat_id=project.user_id, text=text) return jsonify({"ok": True})
def __init__(self, url: str): """ make sure site is in db check if site is up if it is, update last_up in db """ # this should be statically defined in child implementations of the class assert self.actor != "" self.current_victims = [] self.new_victims = [] self.url = url self.session = Session() q = self.session.query(Site).filter_by(actor=self.actor) if q.count() == 0: # site is new, init obj self.site = Site(actor=self.actor, url=self.url) self.session.add(self.site) self.session.commit() self.first_run = True else: # site exists, set obj self.site = q.first() # if we haven't successfully scraped the site before, consider this the first run if self.site.last_scraped is None: self.first_run = True # check if site is up self.is_up = self.is_site_up()
def is_password_correct(password: str) -> bool: """Returns whether a user email exists in the database. Args: email: An email that may belong to a user. Returns: Returns true if the email exists, false otherwise. """ session = Session() return session.query(User).filter_by(password=password).first() != None
def get_role_groups(role_id: int) -> List[Group]: """Returns a list of groups with the given role. Args: role_id: An role's role_id. Returns: Returns a list of groups if an role has the role_id, otherwise returns None. """ session = Session() return session.query(Group).join(RoleToGroup).filter( RoleToGroup.role_id == role_id).all()
def get_employee_roles(email: str) -> List[EmployeeToRole]: """Returns a list of roles with the given email. Args: email: An employee's email. Returns: Returns a list of roles if an employee has the email, otherwise returns None. """ session = Session() return session.query(EmployeeToRole).filter( EmployeeToRole.email == email).all()
def get_slack_groups() -> List[Group]: """Returns a list of groups that belong to Slack. Returns: Returns a list containing all the groups from Slack. """ groups = None session = Session() try: groups = session.query(Group).filter(Group.app_id == 1).all() except Exception as e: print(e) return groups
async def upload_file(file_path: str, league_name: str, stat_type: str, last_5: bool) -> bool: session: SQLSession = Session() try: with open(file_path, 'rb') as file: msg = await bot.send_photo(admins[0], file, disable_notification=True) file_id = msg.photo[-1].file_id session.query(MediaIds).filter( and_(MediaIds.league == league_name, MediaIds.stat_type == stat_type, MediaIds.last_5 == last_5)).delete() session.add( MediaIds(league_name, stat_type, last_5, file_id, file_path)) return True except Exception as ex: exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] logging.warning(f"Ex={ex} in file={fname} line={exc_tb.tb_lineno}") return False finally: session.commit() session.close()
def __update_xg(self, league_name: str) -> bool: logging.info(f"Start update xg for league={league_name}") session: SQLSession = Session() try: for player_stat in self.fbref.get_xg_stats(league_name): cur_player: PlayerStats = self.get_player(player_stat) if cur_player: last3_xg_per_game, last3_npxg_per_game, last3_xa_per_game, last5_xg_per_game, \ last5_npxg_per_game, last5_xa_per_game = self.__compute_xg(player_stat, cur_player) session.query(PlayerStats).filter( PlayerStats.id == cur_player.id).update({ 'r5_xg': player_stat["xg"], 'r5_npxg': player_stat["npxg"], 'r5_xa': player_stat["xa"], 'last3_xg_per_game': last3_xg_per_game, 'last3_npxg_per_game': last3_npxg_per_game, 'last3_xa_per_game': last3_xa_per_game, 'last5_xg_per_game': last5_xg_per_game, 'last5_npxg_per_game': last5_npxg_per_game, 'last5_xa_per_game': last5_xa_per_game }) session.commit() else: # if new player session.add( PlayerStats(player_stat['name'], league_name, player_stat['team'], player_stat['position'])) session.commit() cur_player = self.get_player(player_stat) if cur_player: session.query(PlayerStats).filter( PlayerStats.id == cur_player.id).update({ 'r5_xg': player_stat["xg"], 'r5_npxg': player_stat["npxg"], 'r5_xa': player_stat["xa"] }) session.commit() return True except Exception as ex: exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] logging.warning(f"Ex={ex} in file={fname} line={exc_tb.tb_lineno}") return False finally: session.commit() session.close()
def get_player(self, player_info: dict) -> PlayerStats: session: SQLSession = Session() result = session.query(PlayerStats).filter( and_(PlayerStats.name == player_info['name'], PlayerStats.league == player_info['league'], PlayerStats.team == player_info['team'])).first() session.close() return result
def get_coeffs(self, league_name: str, cur_round: bool) -> str: session: SQLSession = Session() coeffs = session.query(Coeff).filter( and_(Coeff.league == league_name, Coeff.is_cur_round == cur_round)) if not coeffs: return "" coeffs_list: List[Coeff] = [cf for cf in coeffs] return self.__transform_coeffs(coeffs_list, cur_round)
async def update_coeffs(self, league_name: str) -> bool: # delete last round db_session: SQLSession = Session() db_session.query(Coeff).filter(Coeff.league == league_name).delete() db_session.commit() db_session.close() return await self.xbet.update_league(league_name)
def is_new_round(self, league_name: str) -> bool: session: SQLSession = Session() league_info = session.query(League_info).filter( League_info.league == league_name).first() session.close() if not league_info: return False return datetime.now() > league_info.deadline
def get_employees_with_ids(ids: List[int]) -> List[Employee]: """Returns a list of employees that match the specified ids. Args: ids: The ids being used to query employees. Returns: Returns a list of employess. """ session = Session() employees = [] try: employees = session.query(Employee).filter(Employee.id.in_(ids)).all() except Exception as e: print(e) return employees
def get_roles_with_ids(ids: List[int]) -> List[Role]: """Returns a list of roles that match the specified ids. Args: ids: The ids being used to query roles. Returns: Returns a list of roles. """ session = Session() roles = [] try: roles = session.query(Role).filter(Role.id.in_(ids)).all() except Exception as e: print(e) return roles
def get_employees_by_role(role: Role) -> List[Employee]: """Returns a list of employees with the given role. Args: role: An instance of Role. Returns: Returns a list of employees. """ employees = [] session = Session() try: employees = session.query(Employee).join(EmployeeToRole).filter( EmployeeToRole.role_id == role.id).all() except Exception as e: print(e) return employees
def show_projects(update, context): chat_id = update.callback_query.message.chat.id try: session = Session() projects = session.query(Project).filter_by(user_id=chat_id) text = "Your projects" buttons = [] count = projects.count() context.user_data["projects"] = projects for i in range(0, count, 2): row = [ InlineKeyboardButton(text=projects[i].name, callback_data=str(EDITING) + str(projects[i].name)), ] if i + 1 < count: row.append( InlineKeyboardButton(text=projects[i + 1].name, callback_data=str(EDITING) + str(projects[i + 1].name))) buttons.append(row) buttons.append([ InlineKeyboardButton(text='Return to main', callback_data=str(MAIN)) ]) keyboard = InlineKeyboardMarkup(buttons) update.callback_query.answer() update.callback_query.edit_message_text(text=text, reply_markup=keyboard) return SELECTING_ACTION except Exception as e: print(str(e))
def get_sources(self, league_name: str) -> List[Source]: session: SQLSession = Session() try: sources = session.query(Source).filter( Source.league == league_name).all() return sources except Exception as ex: exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] logging.warning(f"Ex={ex} in file={fname} line={exc_tb.tb_lineno}") return [] finally: session.close()
def add_source(self, name: str, league_name: str, url: str, description: str) -> bool: logging.info(f"Add source {name} for league={league_name}") session: SQLSession = Session() try: session.add(Source(name, league_name, url, description)) return True except Exception as ex: exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] logging.warning(f"Ex={ex} in file={fname} line={exc_tb.tb_lineno}") return False finally: session.commit() session.close()
def delete_source(self, name: str, league_name: str, url: str) -> bool: logging.info(f"Delete source {name} for league={league_name}") session: SQLSession = Session() try: session.query(Source).filter( and_(Source.league == league_name, Source.name == name, Source.url == url)).delete() return True except Exception as ex: exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] logging.warning(f"Ex={ex} in file={fname} line={exc_tb.tb_lineno}") return False finally: session.commit() session.close()
def get_players(self, league_name: str) -> str: xbet = XBet() if league_name not in xbet.leagues: return "Ошибка в названии лиги" session: SQLSession = Session() best_players = session.query(Player).filter(Player.league == league_name) \ .order_by(Player.dif_popularity.desc(), Player.old_popularity.desc()).limit(10).all() worst_players = session.query(Player).filter(Player.league == league_name) \ .order_by(Player.dif_popularity, Player.old_popularity.desc()).limit(10).all() session.close() result = ["\U0001F4C8 Популярные игроки:\n"] result += self.__transform_players(best_players) result += ["\n\U0001F4C9 Непопулярные игроки:\n"] result += self.__transform_players(worst_players) return '\n'.join(result)
def delete_source_by_id(self, source_id: int) -> str: logging.info(f"Delete source with id={source_id}") session: SQLSession = Session() try: league_name = session.query(Source).filter( and_(Source.id == source_id)).first().league session.query(Source).filter(and_(Source.id == source_id)).delete() return league_name except Exception as ex: exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] logging.warning(f"Ex={ex} in file={fname} line={exc_tb.tb_lineno}") return "" finally: session.commit() session.close()
def save(update, context): session = Session() new_name = context.user_data["new_name"] project = context.user_data["current_project"] session.query(Project).filter(Project.id == project.id).update( {"name": new_name}) session.commit() project.name = new_name context.user_data["current_project"] = project return edit(update, context)
async def __update_shoots_files(self, league_name: str) -> bool: try: logging.info(f"Start update shoots files for league={league_name}") # get best players session: SQLSession = Session() best_players_last_3 = session.query(PlayerStats).filter(PlayerStats.league == league_name) \ .order_by(PlayerStats.last3_shoots_per_game.desc(), PlayerStats.last3_on_target_per_shoot.desc()) \ .limit(20).all() best_players_last_5 = session.query(PlayerStats).filter(PlayerStats.league == league_name) \ .order_by(PlayerStats.last5_shoots_per_game.desc(), PlayerStats.last5_on_target_per_shoot.desc()) \ .limit(20).all() # create pd pataframes last_3_info = [[ round(player.last3_shoots_per_game, 2), int(player.last3_on_target_per_shoot * 100), player.name, player.team ] for player in best_players_last_3] last_5_info = [[ round(player.last5_shoots_per_game, 2), int(player.last5_on_target_per_shoot * 100), player.name, player.team ] for player in best_players_last_5] last_3_df = pd.DataFrame( last_3_info, columns=['Уд/И', 'УдС/Уд(%)', 'Игрок', 'Команда']) last_3_df_styled = last_3_df.style.background_gradient() last_5_df = pd.DataFrame( last_5_info, columns=['Уд/И', 'УдС/Уд(%)', 'Игрок', 'Команда']) last_5_df_styled = last_5_df.style.background_gradient() result = self.__update_files(league_name, 'shoots', last_3_df_styled, last_5_df_styled) result *= await upload_files(league_name, 'shoots') return result except Exception as ex: exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] logging.warning(f"Ex={ex} in file={fname} line={exc_tb.tb_lineno}") return False finally: session.close()
def update_deadline(self, league_name: str) -> bool: """ Update deadline for league """ logging.info(f"Update deadline for league={league_name}") new_deadline = self.sports.get_deadline(league_name) if not new_deadline: return False session: SQLSession = Session() league = session.query(League_info).filter( League_info.league == league_name).first() if league: session.query(League_info).filter(League_info.id == league.id) \ .update({League_info.deadline: new_deadline}) else: session.add(League_info(league_name, new_deadline)) session.commit() session.close() return True
def get_gca_id(self, league_name: str, last_5: bool = False) -> str: if league_name not in self.fbref.shoots_creation_leagues: return "Для данной лиги нет данных" try: session: SQLSession = Session() image = session.query(MediaIds).filter( and_(MediaIds.league == league_name, MediaIds.stat_type == 'gca', MediaIds.last_5 == last_5)).first() if image is not None: return image.file_id else: return "" except Exception as ex: exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] logging.warning(f"Ex={ex} in file={fname} line={exc_tb.tb_lineno}") return "" finally: session.close()
def update_league(self, league_name: str, new_round: bool = False) -> bool: """ Updates players popularity for league """ logging.info( f"Start update players for league={league_name}, new_round={new_round}" ) session: SQLSession = Session() try: if new_round: session.query(Player).filter( Player.league == league_name).update({'dif_popularity': 0}) for player in self.sports.get_league_players(league_name): cur_player = self.get_player(player) if cur_player: if new_round: session.query(Player).filter( Player.id == cur_player.id).update( {'old_popularity': player['popularity']}) else: session.query(Player).filter( Player.id == cur_player.id).update({ 'dif_popularity': player['popularity'] - Player.old_popularity }) else: # if new player session.add( Player(player['name'], player['league'], player['team'], player['amplua'], player['popularity'], 0)) return True except Exception as ex: exc_type, exc_obj, exc_tb = sys.exc_info() fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] logging.warning(f"Ex={ex} in file={fname} line={exc_tb.tb_lineno}") return False finally: session.commit() session.close()
def typing_name(update, context): try: session = Session() token = uuid.uuid1() project = Project(token=token, name=update.message.text, user_id=update.message.chat.id) session.add(project) session.commit() update.message.reply_text(text=str(token)) return main(update, context) except Exception as e: print(str(e))
def start(update, context): session = Session() try: u = User(chat_id=update.message.chat.id) session.add(u) session.commit() except IntegrityError: print("User already exists") buttons = [[ InlineKeyboardButton(text='To main menu', callback_data=str(MAIN)), ]] keyboard = InlineKeyboardMarkup(buttons) update.message.reply_text("Choose the menu", reply_markup=keyboard) return SELECTING_ACTION
def shutdown_session(exception=None): Session.remove()
class SiteCrawler: # threat actor associated with the leak site actor: str = "" # url for the leak site url: str = "" # list of victims on the leak site from current scrape current_victims: List[Victim] = [] # new victims on the leak site from current scrape new_victims: List[Victim] = [] # is the site up? set by is_site_up() is_up: bool = False # db session, set in __init__() session: SessionType # site object from db, set in __init__() site: Site # is this the first ingest of the site? set in __init__() # if the first run, don't notify on new victims (b/c they are all "new") first_run: bool = False # headers to be used for requests headers: Dict[str, str] def __init__(self, url: str): """ make sure site is in db check if site is up if it is, update last_up in db """ # this should be statically defined in child implementations of the class assert self.actor != "" self.current_victims = [] self.new_victims = [] self.url = url self.session = Session() q = self.session.query(Site).filter_by(actor=self.actor) self.headers: Dict[str, str] = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; rv:78.0) Gecko/20100101 Firefox/78.0" } if q.count() == 0: # site is new, init obj self.site = Site(actor=self.actor, url=self.url) self.session.add(self.site) self.session.commit() self.first_run = True else: # site exists, set obj self.site = q.first() # if we haven't successfully scraped the site before, consider this the first run if self.site.last_scraped is None: self.first_run = True # check if site is up self.is_up = self.is_site_up() def is_site_up(self) -> bool: """ check if the site is up this might have specific criteria for some sites """ with Proxy() as p: try: r = p.get(self.url, headers=self.headers, timeout=Config["timeout"]) if r.status_code >= 400: return False except Exception as e: return False self.site.last_up = datetime.utcnow() return True def scrape_victims(self): """ pull each listing on the site check if its already in the db if it is, update the last seen if it isn't, add it to the db store each org name in a list (self.current_victims) this also sets self.new_victims, which has new victims to notify with """ raise Exception("Function implementation not found") def identify_removed_victims(self) -> List[Victim]: """ check org name list against db if something is in the db, not already removed, and not in the list, alert """ # get the current victims from the last scrape victims = self.session.query(Victim).filter_by(site=self.site, removed=False).all() # remove anything from the last scrape that was also in this scrape # the remaining set is things that were present last time, but not this time for v in self.current_victims: try: victims.remove(v) except ValueError: # i think there is an edge case here that can be caused by # running the scrape while a victim is removed # it _should_ get picked up on the next run though, so we # can safely ignore this pass # mark the victims as removed, since they are no longer on the leak site for v in victims: v.removed = True self.session.commit() return victims