def backup_db(prefix='', postfix=''):
log.warning('Backuping DB...')
term(f'mongodump --uri {MONGO_URI} --out {BACKUP_DIR}/temp_dir')
date_str = datetime.now().strftime("%d-%m-%Y=%H:%M:%S")
term(
f'cd {BACKUP_DIR} && 7z a "bac_{prefix}{date_str}{postfix}.zip" temp_dir/* -p{get_str_key("BACKUP_PASS")}'
)
term(f'rm -rf {BACKUP_DIR}/temp_dir')
log.info('Backup done!')
async def startup():
log.debug("Starting before serving task for all modules...")
loop.create_task(before_srv_task(loop))
if not get_bool_key("DEBUG_MODE"):
log.debug("Waiting 2 seconds...")
await asyncio.sleep(2)
log.info("Aiogram: Using polling method")
loop.create_task(dp.start_polling())
log.info("Bot is alive!")
async def update_user(chat_id, new_user):
old_user = await db.user_list.find_one({'user_id': new_user.id})
new_chat = [chat_id]
if old_user and 'chats' in old_user:
if old_user['chats']:
new_chat = old_user['chats']
if not new_chat or chat_id not in new_chat:
new_chat.append(chat_id)
if old_user and 'first_detected_date' in old_user:
first_detected_date = old_user['first_detected_date']
else:
first_detected_date = datetime.datetime.now()
if new_user.username:
username = new_user.username.lower()
else:
username = None
if hasattr(new_user, 'last_name') and new_user.last_name:
last_name = html.escape(new_user.last_name)
else:
last_name = None
first_name = html.escape(new_user.first_name)
user_new = {
'user_id': new_user.id,
'first_name': first_name,
'last_name': last_name,
'username': username,
'user_lang': new_user.language_code,
'chats': new_chat,
'first_detected_date': first_detected_date
}
# Check on old user in DB with same username
find_old_user = {
'username': user_new['username'],
'user_id': {
'$ne': user_new['user_id']
}
}
if user_new['username'] and (check := await
db.user_list.find_one(find_old_user)):
await db.user_list.delete_one({'_id': check['_id']})
log.info(
f"Found user ({check['user_id']}) with same username as ({user_new['user_id']}), old user was deleted."
)
async def update_users_handler(message):
chat_id = message.chat.id
# Update chat
new_chat = message.chat
if not new_chat.type == 'private':
old_chat = await db.chat_list.find_one({'chat_id': chat_id})
if not hasattr(new_chat, 'username'):
chatnick = None
else:
chatnick = new_chat.username
if old_chat and 'first_detected_date' in old_chat:
first_detected_date = old_chat['first_detected_date']
else:
first_detected_date = datetime.datetime.now()
chat_new = {
"chat_id": chat_id,
"chat_title": html.escape(new_chat.title),
"chat_nick": chatnick,
"type": new_chat.type,
"first_detected_date": first_detected_date
}
# Check on old chat in DB with same username
find_old_chat = {
'chat_nick': chat_new['chat_nick'],
'chat_id': {
'$ne': chat_new['chat_id']
}
}
if chat_new['chat_nick'] and (check := await
db.chat_list.find_one(find_old_chat)):
await db.chat_list.delete_one({'_id': check['_id']})
log.info(
f"Found chat ({check['chat_id']}) with same username as ({chat_new['chat_id']}), old chat was deleted."
)
await db.chat_list.update_one({'chat_id': chat_id}, {"$set": chat_new},
upsert=True)
log.debug(f"Users: Chat {chat_id} updated")
async def cleanup_chats_and_users():
await channel_log('Starting cleanup process...', info_log=True)
log.info("Cleaning old chats...")
deleted_chats = 0
async for item in db.chat_list.find():
await asyncio.sleep(0.1)
chat_id = item['chat_id']
try:
await bot.get_chat(chat_id)
except Unauthorized:
# Cleanup
for collection in await db.list_collection_names():
async for item in db[collection].find({'chat_id': chat_id}):
await db[collection].delete_many({'_id': item['_id']})
deleted_chats += 1
log.info("Cleaning old users...")
deleted_users = 0
async for item in db.user_list.find():
await asyncio.sleep(0.1)
user_id = item['user_id']
try:
await bot.get_chat(user_id)
except Unauthorized:
# Cleanup
for collection in await db.list_collection_names():
async for item in db[collection].find(
{'$or': [{
'user_id': user_id
}, {
'chat_id': user_id
}]}):
await db[collection].delete_many({'_id': item['_id']})
deleted_users += 1
await channel_log(
f'Cleanup done! Cleared {deleted_chats} chats and {deleted_users} users!',
info_log=True)
await channel_log('<b>Current stats:</b>\n' + await __stats__())
from apscheduler.executors.asyncio import AsyncIOExecutor
from apscheduler.jobstores.redis import RedisJobStore
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from pytz import utc
from sophie_bot.config import get_str_key, get_int_key
from sophie_bot.utils.logger import log
DEFAULT = "default"
jobstores = {
DEFAULT: RedisJobStore(
host=get_str_key("REDIS_HOST"),
port=get_str_key("REDIS_PORT"),
db=get_int_key("REDIS_DB_FSM")
)
}
executors = {DEFAULT: AsyncIOExecutor()}
job_defaults = {"coalesce": False, "max_instances": 3}
scheduler = AsyncIOScheduler(
jobstores=jobstores, executors=executors, job_defaults=job_defaults, timezone=utc
)
log.info("Starting apscheduller...")
scheduler.start()
#
# Licensed under the Raphielscape Public License, Version 1.c (the "License");
# you may not use this file except in compliance with the License.
import os
import yaml
from babel.core import Locale
from sophie_bot.services.mongo import db
from sophie_bot.services.redis import redis
from sophie_bot.utils.logger import log
LANGUAGES = {}
log.info("Loading localizations...")
for filename in os.listdir('sophie_bot/localization'):
log.debug('Loading language file ' + filename)
f = open('sophie_bot/localization/' + filename, "r", encoding='utf8')
lang = yaml.load(f, Loader=yaml.CLoader)
lang_code = lang['language_info']['code']
lang['language_info']['babel'] = Locale(lang_code)
LANGUAGES[lang_code] = lang
log.info("Languages loaded: {}".format(
[l['language_info']['name'] for l in LANGUAGES.values()]))
# Copyright (C) 2018 - 2020 MrYacha. All rights reserved. Source code available under the AGPL.
# Copyright (C) 2019 Aiogram
#
# This file is part of SophieBot.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import sentry_sdk
from sentry_sdk.integrations.redis import RedisIntegration
from sophie_bot.config import get_str_key
from sophie_bot.utils.logger import log
log.info("Starting sentry.io integraion...")
sentry_sdk.init(get_str_key('SENTRY_API_KEY'),
integrations=[RedisIntegration()])
if get_bool_key("DEBUG_MODE"):
log.debug("Enabling logging middleware.")
dp.middleware.setup(LoggingMiddleware())
LOAD = get_list_key("LOAD")
DONT_LOAD = get_list_key("DONT_LOAD")
if get_bool_key('LOAD_MODULES'):
if len(LOAD) > 0:
modules = LOAD
else:
modules = ALL_MODULES
modules = [x for x in modules if x not in DONT_LOAD]
log.info("Modules to load: %s", str(modules))
for module_name in modules:
log.debug(f"Importing <d><n>{module_name}</></>")
imported_module = import_module("sophie_bot.modules." + module_name)
LOADED_MODULES.append(imported_module)
log.info("Modules loaded!")
else:
log.warning("Not importing modules!")
# asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
loop = asyncio.get_event_loop()
# Import misc stuff
import_module("sophie_bot.utils.exit_gracefully")
if not get_bool_key('DEBUG_MODE'):
import_module("sophie_bot.utils.sentry")
#
# Licensed under the Raphielscape Public License, Version 1.c (the "License");
# you may not use this file except in compliance with the License.
import os
import yaml
from babel.core import Locale
from sophie_bot.services.mongo import db
from sophie_bot.services.redis import redis
from sophie_bot.utils.logger import log
LANGUAGES = {}
log.info("Loading localizations...")
for filename in os.listdir('sophie_bot/localization'):
log.debug('Loading language file ' + filename)
f = open('sophie_bot/localization/' + filename, "r", encoding='utf8')
lang = yaml.load(f, Loader=yaml.CLoader)
lang_code = lang['language_info']['code']
lang['language_info']['babel'] = Locale(lang_code)
LANGUAGES[lang_code] = lang
log.info("Languages loaded: {}".format(
[l['language_info']['babel'].display_name for l in LANGUAGES.values()]))
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from sophie_bot.services.mongo import mongodb
from sophie_bot.utils.logger import log
log.info('Sophie Database v4')
log.info("Filters: move 'note' to 'note_name'")
log.info('Starting updating all filters...')
all_filters = mongodb.filters.find({})
all_filters_count = all_filters.count()
counter = 0
changed_filters = 0
for item in all_filters:
counter += 1
log.info(f'Updating {counter} of {all_filters_count}...')
if 'note' in item:
changed_filters += 1
item['note_name'] = item['note']
del item['note']
from sentry_sdk import configure_scope
from sophie_bot import BOT_USERNAME, dp
from sophie_bot.config import get_bool_key
from sophie_bot.utils.filters import ALL_FILTERS
from sophie_bot.utils.logger import log
DEBUG_MODE = get_bool_key('DEBUG_MODE')
ALLOW_F_COMMANDS = get_bool_key("allow_forwards_commands")
ALLOW_COMMANDS_FROM_EXC = get_bool_key("allow_commands_with_!")
REGISTRED_COMMANDS = []
COMMANDS_ALIASES = {}
# Import filters
log.info("Filters to load: %s", str(ALL_FILTERS))
for module_name in ALL_FILTERS:
log.debug("Importing " + module_name)
imported_module = import_module("sophie_bot.utils.filters." + module_name)
log.info("Filters loaded!")
def register(*args,
cmds=None,
f=None,
allow_edited=True,
allow_kwargs=False,
**kwargs):
if cmds and type(cmds) == str:
cmds = [cmds]
from sophie_bot import dp
from sophie_bot.config import get_bool_key
from sophie_bot.modules import ALL_MODULES, LOADED_MODULES
from sophie_bot.services.quart import quart
from sophie_bot.utils.db_backup import backup_db
from sophie_bot.utils.logger import log
# import uvloop
if get_bool_key("DEBUG_MODE"):
log.debug("Enabling logging middleware.")
dp.middleware.setup(LoggingMiddleware())
if get_bool_key('LOAD_MODULES'):
# Import modules
log.info("Modules to load: %s", str(ALL_MODULES))
for module_name in ALL_MODULES:
log.debug(f"Importing <d><n>{module_name}</></>")
imported_module = import_module("sophie_bot.modules." + module_name)
LOADED_MODULES.append(imported_module)
log.info("Modules loaded!")
else:
log.warning("Not importing modules!")
# asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
loop = asyncio.get_event_loop()
# Import misc stuff
import_module("sophie_bot.utils.exit_gracefully")
if not get_bool_key('DEBUG_MODE'):
import_module("sophie_bot.utils.sentry")
# License, or (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os
import signal
from sophie_bot.services.redis import redis
from sophie_bot.utils.logger import log
def exit_gracefully(signum, frame):
log.warning("Bye!")
try:
redis.save()
except Exception:
log.error("Exiting immediately!")
os.kill(os.getpid(), signal.SIGUSR1)
# Signal exit
log.info("Setting exit_gracefully task...")
signal.signal(signal.SIGINT, exit_gracefully)
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from sophie_bot.services.mongo import mongodb
from sophie_bot.utils.logger import log
log.info('Sophie Database v3')
log.info('Support notes aliases')
log.info('Starting updating all notes...')
all_notes = mongodb.notes_v2.find({})
all_notes_count = all_notes.count()
counter = 0
changed_notes = 0
for note in all_notes:
counter += 1
log.info(f'Updating {counter} of {all_notes_count}...')
if 'name' in note:
changed_notes += 1
names = [note['name']]
del note['name']
# License, or (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from pymongo import InsertOne
from sophie_bot.services.mongo import mongodb
from sophie_bot.utils.logger import log
log.info('Sophie Database v5')
log.info("Feds: migrate to old feds database structure")
log.info('Starting updating all feds...')
all_feds = mongodb.feds.find({})
all_feds_count = all_feds.count()
counter = 0
changed_feds = 0
for fed in all_feds:
counter += 1
log.info(f'Updating {counter} of {all_feds_count}...')
queue = []
if 'banned' not in fed:
continue
from sophie_bot.versions import DB_STRUCTURE_VER
async def notify_bot_owner(old_ver, new_ver):
await bot.send_message(
OWNER_ID,
f"Sophie database structure was updated from <code>{old_ver}</code> to <code>{new_ver}</code>"
)
# TODO: Logs channel
log.debug("Checking on database structure update...")
if not (data := mongodb.db_structure.find_one({'db_ver': {"$exists": True}})):
log.info("Your database is empty! Creating database structure key...")
mongodb.db_structure.insert_one({'db_ver': DB_STRUCTURE_VER})
log.info("Database structure version is: " + str(DB_STRUCTURE_VER))
else:
curr_ver = data['db_ver']
log.info("Your database structure version is: " + str(curr_ver))
if DB_STRUCTURE_VER > curr_ver:
log.error("Your database is old! Waiting 20 seconds till update...")
log.info("Press CTRL + C to cancel!")
time.sleep(20)
log.debug("Trying to update database structure...")
log.info("--------------------------------")
log.info("Your current database structure version: " + str(curr_ver))
log.info("New database structure version: " + str(DB_STRUCTURE_VER))
log.info("--------------------------------")
old_ver = curr_ver
async def channel_log(msg, info_log=True):
chat_id = get_int_key('LOGS_CHANNEL_ID')
if info_log:
log.info(msg)
await bot.send_message(chat_id, html.escape(msg))
'MONGODB_URI': 'localhost',
'MONGO_DB': 'sophie',
'BACKUP_PASS': "******",
'JOIN_CONFIRM_DURATION': '30m',
}
CONFIG_PATH = 'data/bot_conf.yaml'
if os.name == 'nt':
log.debug("Detected Windows, changing config path...")
CONFIG_PATH = os.getcwd() + "\\data\\bot_conf.yaml"
if os.path.isfile(CONFIG_PATH):
log.info(CONFIG_PATH)
for item in (data := yaml.load(open('data/bot_conf.yaml', "r"), Loader=yaml.CLoader)):
DEFAULTS[item.upper()] = data[item]
else:
log.info("Using env vars")
def get_str_key(name, required=False):
if name in DEFAULTS:
default = DEFAULTS[name]
else:
default = None
if not (data := env.str(name, default=default)) and not required:
log.warn('No str key: ' + name)
return None
elif not data:
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import asyncio
import logging
from aiogram import Bot, Dispatcher, types
from aiogram.contrib.fsm_storage.redis import RedisStorage2
from sophie_bot.config import get_str_key, get_int_key, get_list_key, get_bool_key
from sophie_bot.utils.logger import log
from sophie_bot.versions import SOPHIE_VERSION
log.info("----------------------")
log.info("| SophieBot |")
log.info("----------------------")
log.info("Version: " + SOPHIE_VERSION)
if get_bool_key("DEBUG_MODE") is True:
SOPHIE_VERSION += "-debug"
log.setLevel(logging.DEBUG)
log.warn(
"! Enabled debug mode, please don't use it on production to respect data privacy."
)
TOKEN = get_str_key("TOKEN", required=True)
OWNER_ID = get_int_key("OWNER_ID", required=True)
OPERATORS = list(get_list_key("OPERATORS"))
# License, or (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from sophie_bot.services.mongo import mongodb
from sophie_bot.utils.logger import log
from pymongo import DeleteOne
log.info('Sophie Database v6')
log.info("Feds: fix str user_id and fix duplications")
log.info('Starting updating all feds...')
queue = []
all_bans = mongodb.fed_bans.find({'user_id': {'$type': 'string'}})
all_bans_count = all_bans.count()
counter = 0
changed_feds = 0
for ban in all_bans:
counter += 1
changed_feds += 1
queue.append(DeleteOne({'_id': ban['_id']}))
