def backup_db(prefix='', postfix=''): log.warning('Backuping DB...') term(f'mongodump --uri {MONGO_URI} --out {BACKUP_DIR}/temp_dir') date_str = datetime.now().strftime("%d-%m-%Y=%H:%M:%S") term( f'cd {BACKUP_DIR} && 7z a "bac_{prefix}{date_str}{postfix}.zip" temp_dir/* -p{get_str_key("BACKUP_PASS")}' ) term(f'rm -rf {BACKUP_DIR}/temp_dir') log.info('Backup done!')
async def startup(): log.debug("Starting before serving task for all modules...") loop.create_task(before_srv_task(loop)) if not get_bool_key("DEBUG_MODE"): log.debug("Waiting 2 seconds...") await asyncio.sleep(2) log.info("Aiogram: Using polling method") loop.create_task(dp.start_polling()) log.info("Bot is alive!")
async def update_user(chat_id, new_user): old_user = await db.user_list.find_one({'user_id': new_user.id}) new_chat = [chat_id] if old_user and 'chats' in old_user: if old_user['chats']: new_chat = old_user['chats'] if not new_chat or chat_id not in new_chat: new_chat.append(chat_id) if old_user and 'first_detected_date' in old_user: first_detected_date = old_user['first_detected_date'] else: first_detected_date = datetime.datetime.now() if new_user.username: username = new_user.username.lower() else: username = None if hasattr(new_user, 'last_name') and new_user.last_name: last_name = html.escape(new_user.last_name) else: last_name = None first_name = html.escape(new_user.first_name) user_new = { 'user_id': new_user.id, 'first_name': first_name, 'last_name': last_name, 'username': username, 'user_lang': new_user.language_code, 'chats': new_chat, 'first_detected_date': first_detected_date } # Check on old user in DB with same username find_old_user = { 'username': user_new['username'], 'user_id': { '$ne': user_new['user_id'] } } if user_new['username'] and (check := await db.user_list.find_one(find_old_user)): await db.user_list.delete_one({'_id': check['_id']}) log.info( f"Found user ({check['user_id']}) with same username as ({user_new['user_id']}), old user was deleted." )
async def update_users_handler(message): chat_id = message.chat.id # Update chat new_chat = message.chat if not new_chat.type == 'private': old_chat = await db.chat_list.find_one({'chat_id': chat_id}) if not hasattr(new_chat, 'username'): chatnick = None else: chatnick = new_chat.username if old_chat and 'first_detected_date' in old_chat: first_detected_date = old_chat['first_detected_date'] else: first_detected_date = datetime.datetime.now() chat_new = { "chat_id": chat_id, "chat_title": html.escape(new_chat.title), "chat_nick": chatnick, "type": new_chat.type, "first_detected_date": first_detected_date } # Check on old chat in DB with same username find_old_chat = { 'chat_nick': chat_new['chat_nick'], 'chat_id': { '$ne': chat_new['chat_id'] } } if chat_new['chat_nick'] and (check := await db.chat_list.find_one(find_old_chat)): await db.chat_list.delete_one({'_id': check['_id']}) log.info( f"Found chat ({check['chat_id']}) with same username as ({chat_new['chat_id']}), old chat was deleted." ) await db.chat_list.update_one({'chat_id': chat_id}, {"$set": chat_new}, upsert=True) log.debug(f"Users: Chat {chat_id} updated")
async def cleanup_chats_and_users(): await channel_log('Starting cleanup process...', info_log=True) log.info("Cleaning old chats...") deleted_chats = 0 async for item in db.chat_list.find(): await asyncio.sleep(0.1) chat_id = item['chat_id'] try: await bot.get_chat(chat_id) except Unauthorized: # Cleanup for collection in await db.list_collection_names(): async for item in db[collection].find({'chat_id': chat_id}): await db[collection].delete_many({'_id': item['_id']}) deleted_chats += 1 log.info("Cleaning old users...") deleted_users = 0 async for item in db.user_list.find(): await asyncio.sleep(0.1) user_id = item['user_id'] try: await bot.get_chat(user_id) except Unauthorized: # Cleanup for collection in await db.list_collection_names(): async for item in db[collection].find( {'$or': [{ 'user_id': user_id }, { 'chat_id': user_id }]}): await db[collection].delete_many({'_id': item['_id']}) deleted_users += 1 await channel_log( f'Cleanup done! Cleared {deleted_chats} chats and {deleted_users} users!', info_log=True) await channel_log('<b>Current stats:</b>\n' + await __stats__())
from apscheduler.executors.asyncio import AsyncIOExecutor from apscheduler.jobstores.redis import RedisJobStore from apscheduler.schedulers.asyncio import AsyncIOScheduler from pytz import utc from sophie_bot.config import get_str_key, get_int_key from sophie_bot.utils.logger import log DEFAULT = "default" jobstores = { DEFAULT: RedisJobStore( host=get_str_key("REDIS_HOST"), port=get_str_key("REDIS_PORT"), db=get_int_key("REDIS_DB_FSM") ) } executors = {DEFAULT: AsyncIOExecutor()} job_defaults = {"coalesce": False, "max_instances": 3} scheduler = AsyncIOScheduler( jobstores=jobstores, executors=executors, job_defaults=job_defaults, timezone=utc ) log.info("Starting apscheduller...") scheduler.start()
# # Licensed under the Raphielscape Public License, Version 1.c (the "License"); # you may not use this file except in compliance with the License. import os import yaml from babel.core import Locale from sophie_bot.services.mongo import db from sophie_bot.services.redis import redis from sophie_bot.utils.logger import log LANGUAGES = {} log.info("Loading localizations...") for filename in os.listdir('sophie_bot/localization'): log.debug('Loading language file ' + filename) f = open('sophie_bot/localization/' + filename, "r", encoding='utf8') lang = yaml.load(f, Loader=yaml.CLoader) lang_code = lang['language_info']['code'] lang['language_info']['babel'] = Locale(lang_code) LANGUAGES[lang_code] = lang log.info("Languages loaded: {}".format( [l['language_info']['name'] for l in LANGUAGES.values()]))
# Copyright (C) 2018 - 2020 MrYacha. All rights reserved. Source code available under the AGPL. # Copyright (C) 2019 Aiogram # # This file is part of SophieBot. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as # published by the Free Software Foundation, either version 3 of the # License, or (at your option) any later version. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. import sentry_sdk from sentry_sdk.integrations.redis import RedisIntegration from sophie_bot.config import get_str_key from sophie_bot.utils.logger import log log.info("Starting sentry.io integraion...") sentry_sdk.init(get_str_key('SENTRY_API_KEY'), integrations=[RedisIntegration()])
if get_bool_key("DEBUG_MODE"): log.debug("Enabling logging middleware.") dp.middleware.setup(LoggingMiddleware()) LOAD = get_list_key("LOAD") DONT_LOAD = get_list_key("DONT_LOAD") if get_bool_key('LOAD_MODULES'): if len(LOAD) > 0: modules = LOAD else: modules = ALL_MODULES modules = [x for x in modules if x not in DONT_LOAD] log.info("Modules to load: %s", str(modules)) for module_name in modules: log.debug(f"Importing <d><n>{module_name}</></>") imported_module = import_module("sophie_bot.modules." + module_name) LOADED_MODULES.append(imported_module) log.info("Modules loaded!") else: log.warning("Not importing modules!") # asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) loop = asyncio.get_event_loop() # Import misc stuff import_module("sophie_bot.utils.exit_gracefully") if not get_bool_key('DEBUG_MODE'): import_module("sophie_bot.utils.sentry")
# # Licensed under the Raphielscape Public License, Version 1.c (the "License"); # you may not use this file except in compliance with the License. import os import yaml from babel.core import Locale from sophie_bot.services.mongo import db from sophie_bot.services.redis import redis from sophie_bot.utils.logger import log LANGUAGES = {} log.info("Loading localizations...") for filename in os.listdir('sophie_bot/localization'): log.debug('Loading language file ' + filename) f = open('sophie_bot/localization/' + filename, "r", encoding='utf8') lang = yaml.load(f, Loader=yaml.CLoader) lang_code = lang['language_info']['code'] lang['language_info']['babel'] = Locale(lang_code) LANGUAGES[lang_code] = lang log.info("Languages loaded: {}".format( [l['language_info']['babel'].display_name for l in LANGUAGES.values()]))
# it under the terms of the GNU Affero General Public License as # published by the Free Software Foundation, either version 3 of the # License, or (at your option) any later version. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. from sophie_bot.services.mongo import mongodb from sophie_bot.utils.logger import log log.info('Sophie Database v4') log.info("Filters: move 'note' to 'note_name'") log.info('Starting updating all filters...') all_filters = mongodb.filters.find({}) all_filters_count = all_filters.count() counter = 0 changed_filters = 0 for item in all_filters: counter += 1 log.info(f'Updating {counter} of {all_filters_count}...') if 'note' in item: changed_filters += 1 item['note_name'] = item['note'] del item['note']
from sentry_sdk import configure_scope from sophie_bot import BOT_USERNAME, dp from sophie_bot.config import get_bool_key from sophie_bot.utils.filters import ALL_FILTERS from sophie_bot.utils.logger import log DEBUG_MODE = get_bool_key('DEBUG_MODE') ALLOW_F_COMMANDS = get_bool_key("allow_forwards_commands") ALLOW_COMMANDS_FROM_EXC = get_bool_key("allow_commands_with_!") REGISTRED_COMMANDS = [] COMMANDS_ALIASES = {} # Import filters log.info("Filters to load: %s", str(ALL_FILTERS)) for module_name in ALL_FILTERS: log.debug("Importing " + module_name) imported_module = import_module("sophie_bot.utils.filters." + module_name) log.info("Filters loaded!") def register(*args, cmds=None, f=None, allow_edited=True, allow_kwargs=False, **kwargs): if cmds and type(cmds) == str: cmds = [cmds]
from sophie_bot import dp from sophie_bot.config import get_bool_key from sophie_bot.modules import ALL_MODULES, LOADED_MODULES from sophie_bot.services.quart import quart from sophie_bot.utils.db_backup import backup_db from sophie_bot.utils.logger import log # import uvloop if get_bool_key("DEBUG_MODE"): log.debug("Enabling logging middleware.") dp.middleware.setup(LoggingMiddleware()) if get_bool_key('LOAD_MODULES'): # Import modules log.info("Modules to load: %s", str(ALL_MODULES)) for module_name in ALL_MODULES: log.debug(f"Importing <d><n>{module_name}</></>") imported_module = import_module("sophie_bot.modules." + module_name) LOADED_MODULES.append(imported_module) log.info("Modules loaded!") else: log.warning("Not importing modules!") # asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) loop = asyncio.get_event_loop() # Import misc stuff import_module("sophie_bot.utils.exit_gracefully") if not get_bool_key('DEBUG_MODE'): import_module("sophie_bot.utils.sentry")
# License, or (at your option) any later version. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. import os import signal from sophie_bot.services.redis import redis from sophie_bot.utils.logger import log def exit_gracefully(signum, frame): log.warning("Bye!") try: redis.save() except Exception: log.error("Exiting immediately!") os.kill(os.getpid(), signal.SIGUSR1) # Signal exit log.info("Setting exit_gracefully task...") signal.signal(signal.SIGINT, exit_gracefully)
# it under the terms of the GNU Affero General Public License as # published by the Free Software Foundation, either version 3 of the # License, or (at your option) any later version. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. from sophie_bot.services.mongo import mongodb from sophie_bot.utils.logger import log log.info('Sophie Database v3') log.info('Support notes aliases') log.info('Starting updating all notes...') all_notes = mongodb.notes_v2.find({}) all_notes_count = all_notes.count() counter = 0 changed_notes = 0 for note in all_notes: counter += 1 log.info(f'Updating {counter} of {all_notes_count}...') if 'name' in note: changed_notes += 1 names = [note['name']] del note['name']
# License, or (at your option) any later version. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. from pymongo import InsertOne from sophie_bot.services.mongo import mongodb from sophie_bot.utils.logger import log log.info('Sophie Database v5') log.info("Feds: migrate to old feds database structure") log.info('Starting updating all feds...') all_feds = mongodb.feds.find({}) all_feds_count = all_feds.count() counter = 0 changed_feds = 0 for fed in all_feds: counter += 1 log.info(f'Updating {counter} of {all_feds_count}...') queue = [] if 'banned' not in fed: continue
from sophie_bot.versions import DB_STRUCTURE_VER async def notify_bot_owner(old_ver, new_ver): await bot.send_message( OWNER_ID, f"Sophie database structure was updated from <code>{old_ver}</code> to <code>{new_ver}</code>" ) # TODO: Logs channel log.debug("Checking on database structure update...") if not (data := mongodb.db_structure.find_one({'db_ver': {"$exists": True}})): log.info("Your database is empty! Creating database structure key...") mongodb.db_structure.insert_one({'db_ver': DB_STRUCTURE_VER}) log.info("Database structure version is: " + str(DB_STRUCTURE_VER)) else: curr_ver = data['db_ver'] log.info("Your database structure version is: " + str(curr_ver)) if DB_STRUCTURE_VER > curr_ver: log.error("Your database is old! Waiting 20 seconds till update...") log.info("Press CTRL + C to cancel!") time.sleep(20) log.debug("Trying to update database structure...") log.info("--------------------------------") log.info("Your current database structure version: " + str(curr_ver)) log.info("New database structure version: " + str(DB_STRUCTURE_VER)) log.info("--------------------------------") old_ver = curr_ver
async def channel_log(msg, info_log=True): chat_id = get_int_key('LOGS_CHANNEL_ID') if info_log: log.info(msg) await bot.send_message(chat_id, html.escape(msg))
'MONGODB_URI': 'localhost', 'MONGO_DB': 'sophie', 'BACKUP_PASS': "******", 'JOIN_CONFIRM_DURATION': '30m', } CONFIG_PATH = 'data/bot_conf.yaml' if os.name == 'nt': log.debug("Detected Windows, changing config path...") CONFIG_PATH = os.getcwd() + "\\data\\bot_conf.yaml" if os.path.isfile(CONFIG_PATH): log.info(CONFIG_PATH) for item in (data := yaml.load(open('data/bot_conf.yaml', "r"), Loader=yaml.CLoader)): DEFAULTS[item.upper()] = data[item] else: log.info("Using env vars") def get_str_key(name, required=False): if name in DEFAULTS: default = DEFAULTS[name] else: default = None if not (data := env.str(name, default=default)) and not required: log.warn('No str key: ' + name) return None elif not data:
# GNU Affero General Public License for more details. # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. import asyncio import logging from aiogram import Bot, Dispatcher, types from aiogram.contrib.fsm_storage.redis import RedisStorage2 from sophie_bot.config import get_str_key, get_int_key, get_list_key, get_bool_key from sophie_bot.utils.logger import log from sophie_bot.versions import SOPHIE_VERSION log.info("----------------------") log.info("| SophieBot |") log.info("----------------------") log.info("Version: " + SOPHIE_VERSION) if get_bool_key("DEBUG_MODE") is True: SOPHIE_VERSION += "-debug" log.setLevel(logging.DEBUG) log.warn( "! Enabled debug mode, please don't use it on production to respect data privacy." ) TOKEN = get_str_key("TOKEN", required=True) OWNER_ID = get_int_key("OWNER_ID", required=True) OPERATORS = list(get_list_key("OPERATORS"))
# License, or (at your option) any later version. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Affero General Public License for more details. # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. from sophie_bot.services.mongo import mongodb from sophie_bot.utils.logger import log from pymongo import DeleteOne log.info('Sophie Database v6') log.info("Feds: fix str user_id and fix duplications") log.info('Starting updating all feds...') queue = [] all_bans = mongodb.fed_bans.find({'user_id': {'$type': 'string'}}) all_bans_count = all_bans.count() counter = 0 changed_feds = 0 for ban in all_bans: counter += 1 changed_feds += 1 queue.append(DeleteOne({'_id': ban['_id']}))