def check_for_new_titles(bot):
old_data = {}
new_data = {}
json_path = join(ROOT_PATH, 'telegram_bot', 'data', 'forums.json')
if exists(json_path):
with open(json_path, encoding='utf-8') as f:
old_data = json.load(f)
for forum in forums:
content = load_page(forum)
titles = re.findall('^==(.*)==\s*$', content, re.MULTILINE)
titles = list(map(str.strip, titles))
new_data[forum] = titles
old_titles = old_data.get(forum, [])
new_titles = list(set(titles) - set(old_titles))
if new_titles:
forum_name = forum[len('Викисловарь:'):]
message_text = f'📝 <b>Новая тема на форуме!</b>\n' \
f'«{forum_name}»\n'
for title in new_titles:
title = title.replace('<', '<')
title = title.replace('>', '>')
title = title.strip('=').strip()
link = f'→ <a href="https://ru.wiktionary.org/wiki/{forum}">' \
f'{title}</a>\n'
message_text += link
# chat_id = DEV_CHAT_ID
chat_id = MAIN_GROUP_CHAT_ID
send(bot, chat_id, message_text)
with open(json_path, 'w', encoding='utf-8') as f:
json.dump(new_data, f, indent=4)
def __init__(self):
super().__init__()
self.base_path = 'Отчёты/ru/Глаголы/Без созданных деепричастий'
self.base_class = derive(KeyTitle, CustomDetails, ReversedIndex)
self.report_keys = [
'-ать',
'-аться',
'-ереть',
'-ереться',
'-еть',
'-еться',
'-ить',
'-иться',
'-оть',
'-оться',
'-уть',
'-уться',
'-ыть',
'-ыться',
'-ять',
'-яться',
'-ресть',
'-ресться',
'-честь',
'-честься',
'-сть',
'-сться',
'-сти',
'-стись',
'-чь',
'-чься',
'-зти',
'-зтись',
'-зть',
'-зться',
'-йти',
'-йтись',
'-???',
'(безличные)',
'(через дефис)',
]
self.create_reports()
self.aspects = {}
for i in range(11):
if i == 7:
continue
lines = load_page(f'User:Vesailok/verb{i}').split('\n')
for line in lines:
verb, aspect, verb_stressed = \
re.match('# \[\[([^]]+)\]\] \((сов|несов)\) (.+)$', line).\
groups()
# if verb != self.remove_stress(verb_stressed).strip() \
# or '́' not in verb_stressed and 'ё' not in verb_stressed:
# if verb != verb_stressed:
# raise Exception()
# print('-', verb)
# # raise ImpossibleError(f"Ошибка в данных: {i}/{verb}")
self.aspects[verb] = aspect
def get_new_data(title):
content = load_page(title, skip_absent=True)
if not content:
return {}
data = {}
for day, title in re.findall('\|\s*(\d+)\s*=\s*\[\[([^]]+)]]', content):
print('*', day, '-', title)
data[day] = title
return data
def prepare_template(title, day, month, year):
try:
content = load_page(title)
except NoPage:
link = get_title_link(title)
send(f'⛔️ Страница не найдена: {link}')
return None, None
template = '{{слово дня|' + f'{day}|{month}|{year}' + '}}'
return content, template
def download_module(title, path):
print(f'- {title}', end='')
content = load_page(title) + '\n'
content = content.replace("\n-- dev_prefix = 'User:Vitalik/'",
"\ndev_prefix = 'User:Vitalik/'")
outs = ['', '.out'] if not debug else [
'',
]
for out in outs:
write(path.replace('[.out]', out), content)
print(' - OK')
def upload_changes(self, desc, minor=True, debug=False):
server_content = load_page(self.title)
if server_content != self._old_content:
raise Exception('Content on the server was suddenly changed.')
new_content = self.new_content
if new_content != self._old_content:
if debug:
print('=' * 10)
print(self.title)
print('-' * 10)
print(new_content)
print('-' * 10)
return
save_page(self.title, new_content, desc, minor, check_changes=False)
def __init__(self):
super().__init__()
self.aspects = {}
for i in range(11):
if i == 7:
continue
lines = load_page(f'User:Vesailok/verb{i}').split('\n')
for line in lines:
verb, aspect, verb_stressed = \
re.match('# \[\[([^]]+)\]\] \((сов|несов)\) (.+)$', line).\
groups()
# if verb != self.remove_stress(verb_stressed).strip() \
# or '́' not in verb_stressed and 'ё' not in verb_stressed:
# if verb != verb_stressed:
# raise Exception()
# print('-', verb)
# # raise ImpossibleError(f"Ошибка в данных: {i}/{verb}")
self.aspects[verb] = aspect
def download_page(title, path):
print(f'- {title}', end='')
try:
content = load_page(title) + '\n'
except NoPage:
print(' - No page')
return
content = content.replace("\n-- dev_prefix = 'User:Vitalik/'",
"\ndev_prefix = 'User:Vitalik/'")
if exists(path):
old_content = read(path)
if old_content != content:
print(' - OK')
else:
write(path, content)
print(' - Not changed')
else:
write(path, content)
print(' - NEW')
def process_words_of_days():
current_year = datetime.now().year
content = load_page('Викисловарь:Слово_дня')
content = \
re.search('== Кандидаты ==\n(.*?)\n== ', content, re.DOTALL).group(1)
processed = False
for title in re.findall('\[\[(Викисловарь:Слово дня/[^]]+)]]', content):
title = title.replace('{{#expr: - 1 + {{CURRENTYEAR}} }}',
f'{current_year - 1}')
title = title.replace('{{#expr: + 1 + {{CURRENTYEAR}} }}',
f'{current_year + 1}')
title = title.replace('{{CURRENTYEAR}}', f'{current_year}')
print('@', title)
if process_list(title):
processed = True
if processed:
send('✔️ Бот завершил работу')
def check_for_new_titles(bot):
old_data = {}
new_data = {}
json_path = join(conf.data_path, 'forum_news', 'forums.json')
if exists(json_path):
with open(json_path, encoding='utf-8') as f:
old_data = json.load(f)
for forum in forums:
old_titles = old_data.get(forum, [])
content = load_page(forum)
titles = re.findall('^==(.*)==\s*$', content, re.MULTILINE)
titles = list(map(str.strip, titles))
new_data[forum] = titles
new_titles = []
for title in titles:
if title not in old_titles:
new_titles.append(title)
if new_titles:
forum_name = forum[len('Викисловарь:'):]
message_text = f'📝 <b>Новая тема на форуме!</b>\n' \
f'«{forum_name}»\n'
for title in new_titles:
title = title.replace('<', '<')
title = title.replace('>', '>')
title = title.strip('=').strip()
forum = forum.replace(' ', '_')
link = f'→ <a href="https://ru.wiktionary.org/wiki/{forum}">' \
f'{title}</a>\n'
message_text += link
# chat_id = DEV_CHAT_ID
chat_id = conf.main_group_id
send(bot, chat_id, message_text)
with open(json_path, 'w', encoding='utf-8') as f:
json.dump(new_data, f, indent=4)
def start(self):
name = re.search(u':Cinemantique/(.+)', self.data_page).group(1)
if f'* [[{self.data_page}|{name}]] = on' \
not in load_page(u'User:Cinemantique/bot'):
print(f'Bot `{self.data_page}` is offline.')
return
entries = load_page(self.data_page).strip()
if not entries:
print(f'Page `{self.data_page}` is empty.')
return
self.report.status('Бот запущен')
processed = []
for i, entry in enumerate(entries.split('\n\n')):
title, *values = entry.split('\n ')
if not values:
continue
if not self.check_entries(title, values):
continue
if title in processed:
self.report.error(title, 'дублируется в списке')
continue
processed.append(title)
try:
if self.debug:
page = StoragePage(title, silent=True)
else:
page = OnlinePage(title, silent=True)
except PageNotFound:
self.report.error(title, 'в ВС нет статьи')
continue
if page.is_redirect:
self.report.error(title, 'статья-редирект')
continue
if not page.ru:
self.report.error(title, 'нет русского раздела в статье')
continue
if len(page.ru.homonyms.all()) > 1:
self.report.error(title, 'омонимы в статье')
continue
if not self.process_page(page, title, values):
continue
try:
if not self.debug:
page.upload_changes(self.desc)
except Exception:
self.report.error(title, 'что-то пошло не так')
self.report.save('unexpected error')
raise
self.report.success(title)
self.report.status('Бот завершён')
if not self.debug:
desc = f'Готово: [[{self.data_page}/report|отчёт]]'
save_page(self.data_page, '', desc)
def append(self, report_title, lines, desc):
title = f"{self.base.data_page}/{report_title}"
old_content = load_page(title, skip_absent=True)
if not self.base.debug:
save_page(title, f'{old_content}\n{lines}', desc)
def process_message(update, context):
bot = context.bot
message = update.message
if not message:
slack_error('❔ Событие без сообщения?')
return
chat = message.chat
title = message.text.strip()
user = message.from_user
is_main_group = chat.id == conf.main_group_id
is_new_group = chat.id == conf.new_group_id
system_message = user.id == 777000
if is_new_group and message.text_html.startswith(new_foreign_header):
sub_messages = new_foreign_messages.load()
for sub_message in sub_messages:
send(bot, conf.new_group_id, sub_message,
reply_to=message.message_id, pause=2)
time.sleep(1)
return
if is_new_group and message.text_html.startswith(daily_stats_header):
sub_messages = daily_stats_messages.load()
for sub_message in sub_messages:
send(bot, conf.new_group_id, sub_message,
reply_to=message.message_id, pause=2)
time.sleep(1)
return
if '\n' in title or 'https://' in title or 'http://' in title:
if is_main_group or is_new_group and system_message:
pass # do nothing
else:
slack_message(message, 'ignore (multiline or link)')
return
update_command = title in ["=", "update", "upd", "refresh",
"обновить", "обнови", "обновись"]
reply_bot = None
reply = message.reply_to_message
if reply and reply.from_user.id in [conf.bot_chat_id, 777000]: # fixme: 777000 doesn't work (and perhaps we don't need it?)
reply_bot = reply
if message.chat_id < 0: # if we are in a group chat
if title.endswith(('=', '~', ' ?')):
pass # okay
elif update_command and reply_bot:
pass # okay
else:
if not is_main_group \
and chat.id not in [-1001369947386, -1001163650570]:
slack_message(message, 'ignore (chat messages)')
return # do nothing
if user.id in conf.admin_ids and title.lower() == 'update!':
content = load_page('User:VitalikBot/conf/telegram/tpls.json')
data = json.loads(content)
json_dump(tpls_filename, data)
send(bot, chat.id, 'Done')
if not is_main_group:
slack_message(message, 'Done')
return
if update_command and reply_bot:
msg = reply_bot
m = re.search('<b>(?P<title>.*?)</b>', reply_bot.text_html)
if not m:
send(bot, chat.id, '🤷🏻♂️ Внутренняя ошибка '
'(не получилось распарсить сообщение)')
return
title = remove_stress(m.group('title'))
else:
bot.send_chat_action(chat.id, telegram.ChatAction.TYPING) # typing...
msg = send(bot, chat.id, '🔎 <i>Делаю запрос...</i>')
skip_content = title.endswith('==')
title = re.sub(r'\s*=+$', '', title)
homonym, title = get_homonym(title)
lang, title = get_lang(title)
title = re.sub(r'#(\w+)', '', title)
title = title.strip()
save_log(message, title)
def edit_message(reply, suffix=''):
buttons = reply.buttons
if message.chat.id == conf.new_channel_id:
buttons = None
edit(bot, chat.id, msg.message_id, reply.text + suffix, buttons)
replies.append(reply)
replies = []
if skip_content:
edit_message(ShortReply(title))
else:
try:
edit_message(StorageReply(title, lang, homonym))
except StorageError as e:
edit_message(ErrorReply(title))
slack_exception('storage', e)
try:
suffix = ''
if message.chat.id == conf.new_channel_id:
suffix = get_author(title)
edit_message(Reply(title, lang, homonym), suffix)
except Exception:
edit_message(ErrorReply(title))
raise
slack_message_raw(':large_blue_circle: ' * 3)
for reply in replies:
slack_message(message, reply.text)
def __init__(self, title, site_lang: str, silent=False):
content = load_page(title)
# todo: process `is_redirect`
super().__init__(title, content, site_lang, silent=silent)
def read_lines(title):
lines = load_page(title).split('\n')
for line in lines:
if not line:
continue
yield line
def save_new_page(title, content, desc='Заливка деепричастий'):
try:
load_page(title)
return
except NoPage:
save_page(title, content, desc)