def check_for_new_titles(bot): old_data = {} new_data = {} json_path = join(ROOT_PATH, 'telegram_bot', 'data', 'forums.json') if exists(json_path): with open(json_path, encoding='utf-8') as f: old_data = json.load(f) for forum in forums: content = load_page(forum) titles = re.findall('^==(.*)==\s*$', content, re.MULTILINE) titles = list(map(str.strip, titles)) new_data[forum] = titles old_titles = old_data.get(forum, []) new_titles = list(set(titles) - set(old_titles)) if new_titles: forum_name = forum[len('Викисловарь:'):] message_text = f'📝 <b>Новая тема на форуме!</b>\n' \ f'«{forum_name}»\n' for title in new_titles: title = title.replace('<', '<') title = title.replace('>', '>') title = title.strip('=').strip() link = f'→ <a href="https://ru.wiktionary.org/wiki/{forum}">' \ f'{title}</a>\n' message_text += link # chat_id = DEV_CHAT_ID chat_id = MAIN_GROUP_CHAT_ID send(bot, chat_id, message_text) with open(json_path, 'w', encoding='utf-8') as f: json.dump(new_data, f, indent=4)
def __init__(self): super().__init__() self.base_path = 'Отчёты/ru/Глаголы/Без созданных деепричастий' self.base_class = derive(KeyTitle, CustomDetails, ReversedIndex) self.report_keys = [ '-ать', '-аться', '-ереть', '-ереться', '-еть', '-еться', '-ить', '-иться', '-оть', '-оться', '-уть', '-уться', '-ыть', '-ыться', '-ять', '-яться', '-ресть', '-ресться', '-честь', '-честься', '-сть', '-сться', '-сти', '-стись', '-чь', '-чься', '-зти', '-зтись', '-зть', '-зться', '-йти', '-йтись', '-???', '(безличные)', '(через дефис)', ] self.create_reports() self.aspects = {} for i in range(11): if i == 7: continue lines = load_page(f'User:Vesailok/verb{i}').split('\n') for line in lines: verb, aspect, verb_stressed = \ re.match('# \[\[([^]]+)\]\] \((сов|несов)\) (.+)$', line).\ groups() # if verb != self.remove_stress(verb_stressed).strip() \ # or '́' not in verb_stressed and 'ё' not in verb_stressed: # if verb != verb_stressed: # raise Exception() # print('-', verb) # # raise ImpossibleError(f"Ошибка в данных: {i}/{verb}") self.aspects[verb] = aspect
def get_new_data(title): content = load_page(title, skip_absent=True) if not content: return {} data = {} for day, title in re.findall('\|\s*(\d+)\s*=\s*\[\[([^]]+)]]', content): print('*', day, '-', title) data[day] = title return data
def prepare_template(title, day, month, year): try: content = load_page(title) except NoPage: link = get_title_link(title) send(f'⛔️ Страница не найдена: {link}') return None, None template = '{{слово дня|' + f'{day}|{month}|{year}' + '}}' return content, template
def download_module(title, path): print(f'- {title}', end='') content = load_page(title) + '\n' content = content.replace("\n-- dev_prefix = 'User:Vitalik/'", "\ndev_prefix = 'User:Vitalik/'") outs = ['', '.out'] if not debug else [ '', ] for out in outs: write(path.replace('[.out]', out), content) print(' - OK')
def upload_changes(self, desc, minor=True, debug=False): server_content = load_page(self.title) if server_content != self._old_content: raise Exception('Content on the server was suddenly changed.') new_content = self.new_content if new_content != self._old_content: if debug: print('=' * 10) print(self.title) print('-' * 10) print(new_content) print('-' * 10) return save_page(self.title, new_content, desc, minor, check_changes=False)
def __init__(self): super().__init__() self.aspects = {} for i in range(11): if i == 7: continue lines = load_page(f'User:Vesailok/verb{i}').split('\n') for line in lines: verb, aspect, verb_stressed = \ re.match('# \[\[([^]]+)\]\] \((сов|несов)\) (.+)$', line).\ groups() # if verb != self.remove_stress(verb_stressed).strip() \ # or '́' not in verb_stressed and 'ё' not in verb_stressed: # if verb != verb_stressed: # raise Exception() # print('-', verb) # # raise ImpossibleError(f"Ошибка в данных: {i}/{verb}") self.aspects[verb] = aspect
def download_page(title, path): print(f'- {title}', end='') try: content = load_page(title) + '\n' except NoPage: print(' - No page') return content = content.replace("\n-- dev_prefix = 'User:Vitalik/'", "\ndev_prefix = 'User:Vitalik/'") if exists(path): old_content = read(path) if old_content != content: print(' - OK') else: write(path, content) print(' - Not changed') else: write(path, content) print(' - NEW')
def process_words_of_days(): current_year = datetime.now().year content = load_page('Викисловарь:Слово_дня') content = \ re.search('== Кандидаты ==\n(.*?)\n== ', content, re.DOTALL).group(1) processed = False for title in re.findall('\[\[(Викисловарь:Слово дня/[^]]+)]]', content): title = title.replace('{{#expr: - 1 + {{CURRENTYEAR}} }}', f'{current_year - 1}') title = title.replace('{{#expr: + 1 + {{CURRENTYEAR}} }}', f'{current_year + 1}') title = title.replace('{{CURRENTYEAR}}', f'{current_year}') print('@', title) if process_list(title): processed = True if processed: send('✔️ Бот завершил работу')
def check_for_new_titles(bot): old_data = {} new_data = {} json_path = join(conf.data_path, 'forum_news', 'forums.json') if exists(json_path): with open(json_path, encoding='utf-8') as f: old_data = json.load(f) for forum in forums: old_titles = old_data.get(forum, []) content = load_page(forum) titles = re.findall('^==(.*)==\s*$', content, re.MULTILINE) titles = list(map(str.strip, titles)) new_data[forum] = titles new_titles = [] for title in titles: if title not in old_titles: new_titles.append(title) if new_titles: forum_name = forum[len('Викисловарь:'):] message_text = f'📝 <b>Новая тема на форуме!</b>\n' \ f'«{forum_name}»\n' for title in new_titles: title = title.replace('<', '<') title = title.replace('>', '>') title = title.strip('=').strip() forum = forum.replace(' ', '_') link = f'→ <a href="https://ru.wiktionary.org/wiki/{forum}">' \ f'{title}</a>\n' message_text += link # chat_id = DEV_CHAT_ID chat_id = conf.main_group_id send(bot, chat_id, message_text) with open(json_path, 'w', encoding='utf-8') as f: json.dump(new_data, f, indent=4)
def start(self): name = re.search(u':Cinemantique/(.+)', self.data_page).group(1) if f'* [[{self.data_page}|{name}]] = on' \ not in load_page(u'User:Cinemantique/bot'): print(f'Bot `{self.data_page}` is offline.') return entries = load_page(self.data_page).strip() if not entries: print(f'Page `{self.data_page}` is empty.') return self.report.status('Бот запущен') processed = [] for i, entry in enumerate(entries.split('\n\n')): title, *values = entry.split('\n ') if not values: continue if not self.check_entries(title, values): continue if title in processed: self.report.error(title, 'дублируется в списке') continue processed.append(title) try: if self.debug: page = StoragePage(title, silent=True) else: page = OnlinePage(title, silent=True) except PageNotFound: self.report.error(title, 'в ВС нет статьи') continue if page.is_redirect: self.report.error(title, 'статья-редирект') continue if not page.ru: self.report.error(title, 'нет русского раздела в статье') continue if len(page.ru.homonyms.all()) > 1: self.report.error(title, 'омонимы в статье') continue if not self.process_page(page, title, values): continue try: if not self.debug: page.upload_changes(self.desc) except Exception: self.report.error(title, 'что-то пошло не так') self.report.save('unexpected error') raise self.report.success(title) self.report.status('Бот завершён') if not self.debug: desc = f'Готово: [[{self.data_page}/report|отчёт]]' save_page(self.data_page, '', desc)
def append(self, report_title, lines, desc): title = f"{self.base.data_page}/{report_title}" old_content = load_page(title, skip_absent=True) if not self.base.debug: save_page(title, f'{old_content}\n{lines}', desc)
def process_message(update, context): bot = context.bot message = update.message if not message: slack_error('❔ Событие без сообщения?') return chat = message.chat title = message.text.strip() user = message.from_user is_main_group = chat.id == conf.main_group_id is_new_group = chat.id == conf.new_group_id system_message = user.id == 777000 if is_new_group and message.text_html.startswith(new_foreign_header): sub_messages = new_foreign_messages.load() for sub_message in sub_messages: send(bot, conf.new_group_id, sub_message, reply_to=message.message_id, pause=2) time.sleep(1) return if is_new_group and message.text_html.startswith(daily_stats_header): sub_messages = daily_stats_messages.load() for sub_message in sub_messages: send(bot, conf.new_group_id, sub_message, reply_to=message.message_id, pause=2) time.sleep(1) return if '\n' in title or 'https://' in title or 'http://' in title: if is_main_group or is_new_group and system_message: pass # do nothing else: slack_message(message, 'ignore (multiline or link)') return update_command = title in ["=", "update", "upd", "refresh", "обновить", "обнови", "обновись"] reply_bot = None reply = message.reply_to_message if reply and reply.from_user.id in [conf.bot_chat_id, 777000]: # fixme: 777000 doesn't work (and perhaps we don't need it?) reply_bot = reply if message.chat_id < 0: # if we are in a group chat if title.endswith(('=', '~', ' ?')): pass # okay elif update_command and reply_bot: pass # okay else: if not is_main_group \ and chat.id not in [-1001369947386, -1001163650570]: slack_message(message, 'ignore (chat messages)') return # do nothing if user.id in conf.admin_ids and title.lower() == 'update!': content = load_page('User:VitalikBot/conf/telegram/tpls.json') data = json.loads(content) json_dump(tpls_filename, data) send(bot, chat.id, 'Done') if not is_main_group: slack_message(message, 'Done') return if update_command and reply_bot: msg = reply_bot m = re.search('<b>(?P<title>.*?)</b>', reply_bot.text_html) if not m: send(bot, chat.id, '🤷🏻♂️ Внутренняя ошибка ' '(не получилось распарсить сообщение)') return title = remove_stress(m.group('title')) else: bot.send_chat_action(chat.id, telegram.ChatAction.TYPING) # typing... msg = send(bot, chat.id, '🔎 <i>Делаю запрос...</i>') skip_content = title.endswith('==') title = re.sub(r'\s*=+$', '', title) homonym, title = get_homonym(title) lang, title = get_lang(title) title = re.sub(r'#(\w+)', '', title) title = title.strip() save_log(message, title) def edit_message(reply, suffix=''): buttons = reply.buttons if message.chat.id == conf.new_channel_id: buttons = None edit(bot, chat.id, msg.message_id, reply.text + suffix, buttons) replies.append(reply) replies = [] if skip_content: edit_message(ShortReply(title)) else: try: edit_message(StorageReply(title, lang, homonym)) except StorageError as e: edit_message(ErrorReply(title)) slack_exception('storage', e) try: suffix = '' if message.chat.id == conf.new_channel_id: suffix = get_author(title) edit_message(Reply(title, lang, homonym), suffix) except Exception: edit_message(ErrorReply(title)) raise slack_message_raw(':large_blue_circle: ' * 3) for reply in replies: slack_message(message, reply.text)
def __init__(self, title, site_lang: str, silent=False): content = load_page(title) # todo: process `is_redirect` super().__init__(title, content, site_lang, silent=silent)
def read_lines(title): lines = load_page(title).split('\n') for line in lines: if not line: continue yield line
def save_new_page(title, content, desc='Заливка деепричастий'): try: load_page(title) return except NoPage: save_page(title, content, desc)