def analyze_missing_data(df, logger=None): ''' Find variables with missing data and return those in a list. Input: - df: (pandas dataframe) dataframe of the dataset Output: - nan_vars: (list) list of the name of the variables with missing values ''' msg = "\n# Attempting to analyze the missing data..." log_msg(logger, msg) try: log_msg(logger, "SUCCESS") nan = df.isna().sum() nan_perc = round(100 * nan / len(df.index), 2) nan_df = pd.concat([nan, nan_perc], axis=1) nan_df = nan_df.rename(columns={0: 'NaN', 1: 'Percent of NaN'}) nan_df = nan_df.sort_values(by=['Percent of NaN'], ascending=False) only_nan_df = nan_df[nan_df['NaN'] > 0] nan_vars = only_nan_df.index.tolist() #logging msg1 = "\nThe following features have missing values:\n >>>{}\n" log_msg(logger, msg1.format(nan_vars)) msg2 = "- {} has {} missing values, which are {}% of the data" for var in nan_vars: num = only_nan_df.loc[var][0] perc = only_nan_df.loc[var][1] log_msg(logger, msg2.format(var, int(num), perc)) return nan_vars except: log_msg(logger, "FAILED")
def generate_dummy(train_df, test_df, features, logger=None): ''' Given the dataset and the variable, generate dummy variables for it. - df: (pandas dataframe) dataframe of the dataset - variable: (str) the name of the variable Output: - Nothing ''' msg = "\n# For the following variables, dummy variables have been created." log_msg(logger, msg) log_msg(logger, " >>>{}".format(features)) try: for feature in features: train_dummy = pd.get_dummies(train_df[feature], prefix=feature) train_df = pd.concat([train_df, train_dummy], axis=1) train_df = train_df.drop(columns=[feature]) test_dummy = pd.get_dummies(test_df[feature], prefix=feature) test_df = pd.concat([test_df, test_dummy], axis=1) test_df = test_df.drop(columns=[feature]) log_msg(logger, "SUCCESS") return train_df, test_df except: log_msg(logger, "FAILED") return None
def impute_missing_data(train_df, test_df, columns, logger=None): ''' Given the dataset, impute missing data for the given variables. - df: (pandas dataframe) dataframe of the dataset - columns: (list) the name of columns Output: - Nothing ''' msg = "\n# Attempting to IMPUTE the missing data...\n" log_msg(logger, msg) try: for col in columns: if train_df[col].dtype == 'object': estimate = train_df[col].value_counts().idxmax() msg = "- For '{}', the most FREQUENT value is selected." else: if abs(train_df[col].kurt()) > 3: cond = train_df[col].median() msg = "- For '{}', the MEDIAN is selected." else: cond = train_df[col].mean() msg = "- For '{}', the MEAN is selected." estimate = round(cond) log_msg(logger, msg.format(col)) train_df[col] = train_df[col].fillna(estimate) test_df[col] = test_df[col].fillna(estimate) log_msg(logger, "SUCCESS") except: log_msg(logger, "FAILED")
def read_data(filename, convert=False, logger=None): ''' Read a dataset and print a short summary of the data. Return a dataframe of the dataset. Input: - filename: (string) the directory of the dataset Output: - df: (pandas dataframe) dataframe of the dataset ''' log_msg(logger, "\n# LOADING the data...") try: _, ext = os.path.splitext(filename) if ext == '.csv': df = pd.read_csv(filename, index_col=0) log_msg(logger, "SUCCESS") elif ext in ['.xls', '.xlsx']: df = pd.read_excel(filename, index_col=None) log_msg(logger, "SUCCESS") if convert is True: convert_dtypes(df, logger) return df except: log_msg(logger, "FAILED")
def show_section(bot, update, some_section=False): """Here need some documentation""" log.info(log_msg(update)) reply_markup = InlineKeyboardMarkup( [[InlineKeyboardButton('Меню', callback_data='_menu')]]) if not some_section: user_query = chosen_owns(update) else: user_query = Show.get(user_id=update.effective_user.id) query = User.select().where(User.house == user_query.house, User.section == user_query.section).order_by( User.floor) neighbors = [str(user) + '\n' for user in query] show_list = ('<b>Мешканці під\'їзду № ' + str(user_query.section) + ' Будинку № ' + str(user_query.house) + '</b>:\n' + '{}' * len(neighbors)).format(*neighbors) bot.sendMessage(chat_id=update.effective_user.id, parse_mode=ParseMode.HTML, disable_web_page_preview=True, text=show_list, reply_markup=reply_markup) update.callback_query.answer()
def save_user_data(bot, update): """callbackQuery from reject. save user data""" log.info(log_msg(update)) user = chosen_owns(update) if not user.updated: text = 'В базу ДОДАНО сусіда:\n' else: text = 'В базі ОНОВЛЕНО сусіда:\n' if update.callback_query.data == '_apart_reject': user_mode = Show.get(user_id=update.effective_user.id) user_mode.msg_apart_mode = False user_mode.save() user.apartment = None user.updated = datetime.now().strftime('%y.%m.%d %H:%M:%S.%f')[:-4] user.save() update.callback_query.answer() user_created_report(bot, update, created_user=user, text=text) new_neighbor_report(bot, update, created_user=user) bot.sendMessage( chat_id=update.effective_user.id, parse_mode=ParseMode.HTML, text='<b>Дякую, Ваші дані збережені</b>. Бажаєте подивитись сусідів?') start_command(bot, update)
def menu_kbd(bot, update): """show keyboard to chose: show neighbors or edit own info""" log.info(log_msg(update)) if User.get(user_id=update.effective_user.id).house and User.get( user_id=update.effective_user.id).section: keyboard = [[ InlineKeyboardButton('Дивитись сусідів 👫', callback_data='show') ], [InlineKeyboardButton('Змінити свої дані ✏', callback_data='edit')], [ InlineKeyboardButton('Важлива інфа ℹ', callback_data='building') ], [ InlineKeyboardButton('Статистика бота 📊️', callback_data='statistics') ], [ InlineKeyboardButton('Мій будинок 🏠', callback_data='house_neighbors'), InlineKeyboardButton('Мій під\'їзд 🔢', callback_data='section_neighbors') ], [ InlineKeyboardButton('Сповіщення 🔔', callback_data='notifications') ]] else: keyboard = [[ InlineKeyboardButton('Додати свої дані 📝', callback_data='edit') ]] reply_markup = InlineKeyboardMarkup(keyboard) bot.sendMessage(chat_id=update.effective_user.id, text='Меню:', reply_markup=reply_markup, parse_mode=ParseMode.HTML)
def jubilee(bot, update, created_user): """Check if new added user is 'hero of the day' i.e some round number in db""" log.info(log_msg(update)) celebration_count = [i for i in range(0, 2000, 50)] query = Own.select().where(Own.house, Own.section) check_list = [query.where(Own.house == i).count() for i in range(1, 5)] total = query.count() text = f'сусідів 🎇 🎈 🎉 🎆 🍹\nВітаємо\n{created_user.joined_str}' for count, house in enumerate(check_list, start=1): if house in celebration_count: x, created = Jubilee.get_or_create(house=count, count=house) if created: text = f'В будинку № {count} Вже зареєстровано {house} ' + text try: bot.sendMessage(chat_id=-1001076439601, text=text, parse_mode=ParseMode.HTML) # test chat except BadRequest: bot.sendMessage(chat_id=-1001307649156, text=text, parse_mode=ParseMode.HTML) return if total in celebration_count: text = f'Вже зареэстровано {total} сусідів 🎇 🎈 🎉 🎆 🍹\nВітаємо\n{created_user.joined_str}' x, created = Jubilee.get_or_create(house=0, count=total) if created: try: bot.sendMessage(chat_id=-1001076439601, text=text, parse_mode=ParseMode.HTML) # test chat except BadRequest: bot.sendMessage(chat_id=-1001307649156, text=text, parse_mode=ParseMode.HTML)
def save_user_data(update, context): """callbackQuery from reject. save user data""" log.info(log_msg(update)) user = chosen_owns(update) if not user.updated: text = 'В базу ДОБАВЛЕН:\n' else: text = 'В базе ОБНОВЛЕН:\n' if update.callback_query.data == '_apart_reject': user_mode = Show.get(user_id=update.effective_user.id) user_mode.msg_apart_mode = False user.apartment = None user_mode.save() update.callback_query.message.reply_text( text='Квартиру не сохраняем пока что.') user.updated = datetime.now().strftime('%y.%m.%d %H:%M:%S.%f')[:-4] user.save() # TODO: properly do user_created_report # user_created_report(update, context, created_user=user, text=text) new_neighbor_report(update, context, created_user=user) reply_markup = InlineKeyboardMarkup( [[InlineKeyboardButton('Меню', callback_data='_menu')]]) update.callback_query.message.reply_text(text="Данные сохранены.", parse_mode=ParseMode.HTML, reply_markup=reply_markup)
def set_floor_kbd(update, _): """callbackQuery from set_section_kbd(). Show keyboard to chose its own floor""" log.info(log_msg(update)) user = chosen_owns(update) user.section = int(update.callback_query.data[2]) user.save() floors = houses_arr['house_' + str(user.house)]['section_' + str(user.section)] # TODO: add proper floor count keyboard = [] count_ = len(floors) while count_ > 0: floor = [] for i in range(3): if count_ == 0: break floor.append( InlineKeyboardButton(str(floors[-count_]), callback_data='_f' + str(floors[-count_]))) count_ -= 1 keyboard.append(floor) reply_markup = InlineKeyboardMarkup(keyboard) update.callback_query.message.reply_text('На каком этаже?', reply_markup=reply_markup) update.callback_query.answer()
def start_command(update, context): """handle /start command""" log.info(log_msg(update)) is_changed(update) if update.callback_query: update.callback_query.answer() menu_kbd(update, context)
def notifications_kbd(bot, update): """callbackQuery handler. pattern:^notifications$. Show notifications keyboard settings""" log.info(log_msg(update)) keyboard = [[ InlineKeyboardButton('В моєму будинку 🏠', callback_data='_notify_house') ], [ InlineKeyboardButton('В моєму під\'їзді 🔢', callback_data='_notify_section') ], [ InlineKeyboardButton('Вимкнути сповіщення 🔕', callback_data='_notify_OFF') ], [InlineKeyboardButton('Меню', callback_data='_menu')]] reply_markup = InlineKeyboardMarkup(keyboard) user = Show.get(user_id=update.effective_user.id) _dict = { None: 'Вимкнено', '_notify_OFF': 'Вимкнено', '_notify_section': 'В моєму під\'їзді 🔢', '_notify_house': 'В моєму будинку 🏠' } text = f'Зараз сповіщення встановленні в режим\n' \ f'<b>{_dict[user.notification_mode]}</b>\nОтримувати сповіщення коли з\'явиться новий сусід:' bot.editMessageText(chat_id=update.effective_user.id, parse_mode=ParseMode.HTML, text=text, reply_markup=reply_markup, message_id=update.effective_message.message_id) update.callback_query.answer()
def show_house(bot, update): """ """ log.info(log_msg(update)) update.callback_query.answer() reply_markup = InlineKeyboardMarkup([[InlineKeyboardButton('Меню', callback_data='_menu')]]) if update.callback_query.data == 'show_this_house': # if user want see selected house user_query = Show.get(user_id=update.effective_user.id) else: # if user want see own house and have one user_query = chosen_owns(update) neighbors = [] sections = Own.select(Own.section).where(Own.house == user_query.house, Own.section).distinct().order_by( Own.section) query = Own.select().where(Own.house == user_query.house, Own.section).order_by(Own.floor) for i in sections: neighbors.append(f'\n{"📭 <b>Секція".rjust(30, " ")} {i.section}</b>\n') for user in query.where(Own.section == i.section): neighbors.append(f'{user.user} {user}\n') show_list = (f'<b>Мешканці будинку № {user_query.house}</b>:\n' + '{}' * len(neighbors)).format(*neighbors) if len(show_list) < 6200: bot.sendMessage(chat_id=update.effective_user.id, parse_mode=ParseMode.HTML, text=show_list, reply_markup=reply_markup) else: part_1, part_2, part_3 = show_list.partition('📭 <b>Секція 4'.rjust(30, ' ') + '</b>' + '\n') bot.sendMessage(chat_id=update.effective_user.id, parse_mode=ParseMode.HTML, text=part_1[:-2]) # to do: remove "." from 2nd msg. Without that dot, rjust not works bot.sendMessage(chat_id=update.effective_user.id, parse_mode=ParseMode.HTML, text='.' + part_2 + part_3, reply_markup=reply_markup)
def building(bot, update): """CallbackQueryHandler. pattern ^building$""" log.info(log_msg(update)) update.callback_query.answer() reply_markup = InlineKeyboardMarkup([[InlineKeyboardButton('Меню', callback_data='_menu')]]) bot.sendMessage(chat_id=update.effective_user.id, text=building_msg, parse_mode=ParseMode.HTML, disable_web_page_preview=True, reply_markup=reply_markup)
def new_neighbor_report(bot, update, created_user): """Send message for users who enabled notifications""" log.info(log_msg(update)) # query for users who set notifications as _notify_house query_params = Show.select(Show.user_id).where(Show.notification_mode == '_notify_house') query_users = Own.select(Own.user).where(Own.house == created_user.house) query = query_params & query_users # prevent telegram blocking spam for i, user in enumerate(query): if i % 29 == 0: time.sleep(1) try: bot.sendMessage(chat_id=user.user_id, parse_mode=ParseMode.HTML, text=f'Новий сусід\n{created_user.joined_str}') except BadRequest as err: bot.sendMessage(chat_id=ADMIN_ID, text=f'failed to send notification for user {user.user_id} {err}', parse_mode=ParseMode.HTML) # query for users who set notifications as _notify_section query_params = Show.select(Show.user_id).where(Show.notification_mode == '_notify_section') query_users = query_users.where(Own.section == created_user.section) query = query_params & query_users for i, user in enumerate(query): if i % 29 == 0: time.sleep(1) try: bot.sendMessage(chat_id=user.user_id, parse_mode=ParseMode.HTML, text=f'Новий сусід\n{created_user.joined_str}') except BadRequest as err: bot.sendMessage(chat_id=ADMIN_ID, text=f'failed to send notification for user {user.user_id} {err}', parse_mode=ParseMode.HTML)
def save_user_data(bot, update): """callbackQuery handler. pattern: ^_apart_reject$|^_section_reject$ AND integer text handler""" log.info(log_msg(update)) if update.callback_query: update.callback_query.answer() user = chosen_owns(update) user_mode = Show.get(user_id=update.effective_user.id) text = f'В базе {"ОБНОВЛЕН" if user.updated else "СОЗДАН"} пользователь:\n' if user_mode.msg_apart_mode and update.message: apartment = int(update.message.text) user.apartment = apartment else: if update.callback_query.data == '_apart_reject': user.apartment = None user.updated = datetime.now().strftime('%y.%m.%d %H:%M:%S.%f')[:-4] user.save() user_mode.msg_apart_mode = False user_mode.save() user_created_report(bot, update, created_user=user, text=text) new_neighbor_report(bot, update, created_user=user) text_success = '<b>Дякую, Ваші дані збережені</b>. Бажаєте подивитись сусідів?' bot.sendMessage(text=text_success, chat_id=update.effective_user.id, parse_mode=ParseMode.HTML) menu_kbd(bot, update) prepared_data = prepare_data() make_pie(prepared_data) make_bars(prepared_data)
def notifications_kbd(update, _): """callbackQuery handler. pattern:^notifications$. Show notifications keyboard settings""" log.info(log_msg(update)) keyboard = [[ InlineKeyboardButton('В моем доме 🏠', callback_data='_notify_house') ], [ InlineKeyboardButton('В моем подъезде 🔢', callback_data='_notify_section') ], [ InlineKeyboardButton('Выключить оповещения 🔕', callback_data='_notify_OFF') ], [InlineKeyboardButton('Меню', callback_data='_menu')]] reply_markup = InlineKeyboardMarkup(keyboard) user = Show.get(user_id=update.effective_user.id) _dict = { None: 'Выключено', '_notify_OFF': 'Выключено', '_notify_section': 'В моем подъезде 🔢', '_notify_house': 'В моєм доме 🏠' } text = f'Сейчас оповещения установлены в режим\n' \ f'<b>{_dict[user.notification_mode]}</b>\nПолучать оповещения ' \ f'когда появится новый сосед:' update.message.reply_text(parse_mode=ParseMode.HTML, text=text, reply_markup=reply_markup, message_id=update.effective_message.message_id) update.callback_query.answer()
def show_section(update, _, some_section=False): """Here need some documentation""" # TODO: rework that func completely log.info(log_msg(update)) reply_markup = InlineKeyboardMarkup( [[InlineKeyboardButton('Меню', callback_data='_menu')]]) if not some_section: user_query = chosen_owns(update) else: user_query = Show.get(user_id=update.effective_user.id) query = User.select().where(User.house == user_query.house, User.section == user_query.section).order_by( User.floor) neighbors = [str(user) + '\n' for user in query] show_list = ('<b>Жители подъезда № ' + str(user_query.section) + ' Дома № ' + str(user_query.house) + '</b>:\n' + '{}' * len(neighbors)).format(*neighbors) update.message.reply_text(parse_mode=ParseMode.HTML, disable_web_page_preview=True, text=show_list, reply_markup=reply_markup) update.callback_query.answer()
def greeting(bot, update): """handle new chat members, and sent greeting message. Delete after delay. Running async""" log.info(log_msg(update)) new_member_name = update.message.new_chat_members[0].full_name text = greeting_msg.format(new_member_name) update.message.reply_text(text=text, parse_mode=ParseMode.HTML, disable_web_page_preview=True)
def help_command(update, _): """handle /help command""" log.info(log_msg(update)) reply_markup = InlineKeyboardMarkup( [[InlineKeyboardButton('Меню', callback_data='_menu')]]) update.message.reply_text(text=help_msg, parse_mode=ParseMode.HTML, reply_markup=reply_markup)
def __init__(self, bot, update): log.info(log_msg(update)) update.callback_query.answer() keyboard = [[ InlineKeyboardButton('Назад', callback_data='statistics'), InlineKeyboardButton('Меню', callback_data='_menu') ]] self.reply_markup = InlineKeyboardMarkup(keyboard)
def save_params(update, context): """callbackQuery from section_kbd(). save params to db table""" log.info(log_msg(update)) user_query = Show.get(user_id=update.effective_user.id) user_query.section = int(update.callback_query.data[3]) user_query.save() some_section = True show_section(update, context, some_section) update.callback_query.answer()
def msg_handler(bot, update): """handle all text msg except other filters do""" msg = update.message.text reply_markup = InlineKeyboardMarkup([[InlineKeyboardButton('Меню', callback_data='_menu')]]) bot.sendPhoto(chat_id=update.effective_user.id, photo=open(os.path.join('img', 'maybe.jpg'), 'rb'), reply_markup=reply_markup, caption=f'Я ще не розумію людської мови, але вчусь, і скоро буду розуміть деякі слова і фрази\n' f'Краще скористайтесь меню') log.info(log_msg(update) + f' text: {msg}')
def talkative(update, _): """Statistics for messaging in group chat. Show top 10 by msgs and by chars""" log.info(log_msg(update)) reply_markup = InlineKeyboardMarkup( [[InlineKeyboardButton('Меню', callback_data='_menu')]]) log_files_list = [ f for f in os.listdir('logfiles') if not f.startswith('.') ] data = {} id_pattern = r' ([0-9]{6,10}) ' pattern = r' ([0-9]{6,10}) name: (.*) usrnm: ' for log_file in log_files_list: with open(os.path.join('logfiles', log_file), mode='r', encoding='utf-8') as file: text = file.read() match = list(set(re.findall(pattern, text))) data = {i[0]: [0, 0, i[1]] for i in match} for log_file in log_files_list: with open(os.path.join('logfiles', log_file), mode='r', encoding='utf-8') as file: for line in file.readlines(): try: id_ = re.search(id_pattern, line).group().strip() data[id_][0] += len(line.split('msg: ')[1].strip()) data[id_][1] += 1 except (KeyError, AttributeError): pass by_chars = sorted(data.items(), key=lambda x: x[1][0], reverse=True) by_msgs = sorted(data.items(), key=lambda x: x[1][1], reverse=True) template = '<a href="tg://user?id={}">{}</a> {}' talkatives_chars = [ template.format(user[0], user[1][2], user[1][0]) + '\n' for user in by_chars[:10] ] talkatives_msgs = [ template.format(user[0], user[1][2], user[1][1]) + '\n' for user in by_msgs[:10] ] show_list = ('<b>Лидеры по количеству знаков</b>\n' + '{}' * len(talkatives_chars)).format(*talkatives_chars) + '\n' + \ ('<b>Лидеры по количеству сообщений</b>\n' + '{}' * len(talkatives_msgs)).format( *talkatives_msgs) update.message.reply_text(parse_mode=ParseMode.HTML, disable_web_page_preview=True, text=show_list, reply_markup=reply_markup)
def help_command(bot, update): """handle /help command""" log.info(log_msg(update)) reply_markup = InlineKeyboardMarkup( [[InlineKeyboardButton('Меню', callback_data='_menu')]]) bot.sendMessage(chat_id=update.effective_user.id, text=help_msg, parse_mode=ParseMode.HTML, reply_markup=reply_markup)
def about_command(update, _): """handle /about command""" log.info(log_msg(update)) reply_markup = InlineKeyboardMarkup( [[InlineKeyboardButton('Меню', callback_data='_menu')]]) update.message.reply_text(text=about_msg, parse_mode=ParseMode.HTML, disable_web_page_preview=True, reply_markup=reply_markup)
def building(update, _): """CallbackQueryHandler. pattern ^building$""" log.info(log_msg(update)) reply_markup = InlineKeyboardMarkup( [[InlineKeyboardButton('Меню', callback_data='_menu')]]) update.message.reply_text(text=building_msg, parse_mode=ParseMode.HTML, disable_web_page_preview=True, reply_markup=reply_markup) update.callback_query.answer()
def group_chat_logging(bot, update): """handle text msgs in group chat. MessageHandler((Filters.text & Filters.group)""" log.info(log_msg(update)) user_id = update.effective_user.id full_name = update.effective_user.full_name msg_len = len(update.message.text) msg = update.message.text Chat.create(user_id=user_id, full_name=full_name, msg_len=msg_len, msg=msg)
def about_command(bot, update): """handle /about command""" log.info(log_msg(update)) reply_markup = InlineKeyboardMarkup( [[InlineKeyboardButton('Меню', callback_data='_menu')]]) bot.sendMessage(chat_id=update.effective_user.id, text=about_msg, parse_mode=ParseMode.HTML, disable_web_page_preview=True, reply_markup=reply_markup)
def msg_handler(update, _): """handle all text msg except other filters do""" msg = update.message.text reply_markup = InlineKeyboardMarkup( [[InlineKeyboardButton('Меню', callback_data='_menu')]]) update.message.reply_text( photo=open(os.path.join('img', 'maybe.jpg'), 'rb'), reply_markup=reply_markup, caption=f'Я еще не понимать человеческий. Лучше воспользуйтесь меню.') log.info(log_msg(update) + f' text: {msg}')