def analyze_missing_data(df, logger=None):
'''
Find variables with missing data and return those in a list.
Input:
- df: (pandas dataframe) dataframe of the dataset
Output:
- nan_vars: (list) list of the name of the variables with missing values
'''
msg = "\n# Attempting to analyze the missing data..."
log_msg(logger, msg)
try:
log_msg(logger, "SUCCESS")
nan = df.isna().sum()
nan_perc = round(100 * nan / len(df.index), 2)
nan_df = pd.concat([nan, nan_perc], axis=1)
nan_df = nan_df.rename(columns={0: 'NaN', 1: 'Percent of NaN'})
nan_df = nan_df.sort_values(by=['Percent of NaN'], ascending=False)
only_nan_df = nan_df[nan_df['NaN'] > 0]
nan_vars = only_nan_df.index.tolist()
#logging
msg1 = "\nThe following features have missing values:\n >>>{}\n"
log_msg(logger, msg1.format(nan_vars))
msg2 = "- {} has {} missing values, which are {}% of the data"
for var in nan_vars:
num = only_nan_df.loc[var][0]
perc = only_nan_df.loc[var][1]
log_msg(logger, msg2.format(var, int(num), perc))
return nan_vars
except:
log_msg(logger, "FAILED")
def generate_dummy(train_df, test_df, features, logger=None):
'''
Given the dataset and the variable, generate dummy variables for it.
- df: (pandas dataframe) dataframe of the dataset
- variable: (str) the name of the variable
Output:
- Nothing
'''
msg = "\n# For the following variables, dummy variables have been created."
log_msg(logger, msg)
log_msg(logger, " >>>{}".format(features))
try:
for feature in features:
train_dummy = pd.get_dummies(train_df[feature], prefix=feature)
train_df = pd.concat([train_df, train_dummy], axis=1)
train_df = train_df.drop(columns=[feature])
test_dummy = pd.get_dummies(test_df[feature], prefix=feature)
test_df = pd.concat([test_df, test_dummy], axis=1)
test_df = test_df.drop(columns=[feature])
log_msg(logger, "SUCCESS")
return train_df, test_df
except:
log_msg(logger, "FAILED")
return None
def impute_missing_data(train_df, test_df, columns, logger=None):
'''
Given the dataset, impute missing data for the given variables.
- df: (pandas dataframe) dataframe of the dataset
- columns: (list) the name of columns
Output:
- Nothing
'''
msg = "\n# Attempting to IMPUTE the missing data...\n"
log_msg(logger, msg)
try:
for col in columns:
if train_df[col].dtype == 'object':
estimate = train_df[col].value_counts().idxmax()
msg = "- For '{}', the most FREQUENT value is selected."
else:
if abs(train_df[col].kurt()) > 3:
cond = train_df[col].median()
msg = "- For '{}', the MEDIAN is selected."
else:
cond = train_df[col].mean()
msg = "- For '{}', the MEAN is selected."
estimate = round(cond)
log_msg(logger, msg.format(col))
train_df[col] = train_df[col].fillna(estimate)
test_df[col] = test_df[col].fillna(estimate)
log_msg(logger, "SUCCESS")
except:
log_msg(logger, "FAILED")
def read_data(filename, convert=False, logger=None):
'''
Read a dataset and print a short summary of the data.
Return a dataframe of the dataset.
Input:
- filename: (string) the directory of the dataset
Output:
- df: (pandas dataframe) dataframe of the dataset
'''
log_msg(logger, "\n# LOADING the data...")
try:
_, ext = os.path.splitext(filename)
if ext == '.csv':
df = pd.read_csv(filename, index_col=0)
log_msg(logger, "SUCCESS")
elif ext in ['.xls', '.xlsx']:
df = pd.read_excel(filename, index_col=None)
log_msg(logger, "SUCCESS")
if convert is True:
convert_dtypes(df, logger)
return df
except:
log_msg(logger, "FAILED")
def show_section(bot, update, some_section=False):
"""Here need some documentation"""
log.info(log_msg(update))
reply_markup = InlineKeyboardMarkup(
[[InlineKeyboardButton('Меню', callback_data='_menu')]])
if not some_section:
user_query = chosen_owns(update)
else:
user_query = Show.get(user_id=update.effective_user.id)
query = User.select().where(User.house == user_query.house,
User.section == user_query.section).order_by(
User.floor)
neighbors = [str(user) + '\n' for user in query]
show_list = ('<b>Мешканці під\'їзду № ' + str(user_query.section) +
' Будинку № ' + str(user_query.house) + '</b>:\n' +
'{}' * len(neighbors)).format(*neighbors)
bot.sendMessage(chat_id=update.effective_user.id,
parse_mode=ParseMode.HTML,
disable_web_page_preview=True,
text=show_list,
reply_markup=reply_markup)
update.callback_query.answer()
def save_user_data(bot, update):
"""callbackQuery from reject. save user data"""
log.info(log_msg(update))
user = chosen_owns(update)
if not user.updated:
text = 'В базу ДОДАНО сусіда:\n'
else:
text = 'В базі ОНОВЛЕНО сусіда:\n'
if update.callback_query.data == '_apart_reject':
user_mode = Show.get(user_id=update.effective_user.id)
user_mode.msg_apart_mode = False
user_mode.save()
user.apartment = None
user.updated = datetime.now().strftime('%y.%m.%d %H:%M:%S.%f')[:-4]
user.save()
update.callback_query.answer()
user_created_report(bot, update, created_user=user, text=text)
new_neighbor_report(bot, update, created_user=user)
bot.sendMessage(
chat_id=update.effective_user.id,
parse_mode=ParseMode.HTML,
text='<b>Дякую, Ваші дані збережені</b>. Бажаєте подивитись сусідів?')
start_command(bot, update)
def menu_kbd(bot, update):
"""show keyboard to chose: show neighbors or edit own info"""
log.info(log_msg(update))
if User.get(user_id=update.effective_user.id).house and User.get(
user_id=update.effective_user.id).section:
keyboard = [[
InlineKeyboardButton('Дивитись сусідів 👫', callback_data='show')
], [InlineKeyboardButton('Змінити свої дані ✏', callback_data='edit')],
[
InlineKeyboardButton('Важлива інфа ℹ',
callback_data='building')
],
[
InlineKeyboardButton('Статистика бота 📊️',
callback_data='statistics')
],
[
InlineKeyboardButton('Мій будинок 🏠',
callback_data='house_neighbors'),
InlineKeyboardButton('Мій під\'їзд 🔢',
callback_data='section_neighbors')
],
[
InlineKeyboardButton('Сповіщення 🔔',
callback_data='notifications')
]]
else:
keyboard = [[
InlineKeyboardButton('Додати свої дані 📝', callback_data='edit')
]]
reply_markup = InlineKeyboardMarkup(keyboard)
bot.sendMessage(chat_id=update.effective_user.id,
text='Меню:',
reply_markup=reply_markup,
parse_mode=ParseMode.HTML)
def jubilee(bot, update, created_user):
"""Check if new added user is 'hero of the day' i.e some round number in db"""
log.info(log_msg(update))
celebration_count = [i for i in range(0, 2000, 50)]
query = Own.select().where(Own.house, Own.section)
check_list = [query.where(Own.house == i).count() for i in range(1, 5)]
total = query.count()
text = f'сусідів 🎇 🎈 🎉 🎆 🍹\nВітаємо\n{created_user.joined_str}'
for count, house in enumerate(check_list, start=1):
if house in celebration_count:
x, created = Jubilee.get_or_create(house=count, count=house)
if created:
text = f'В будинку № {count} Вже зареєстровано {house} ' + text
try:
bot.sendMessage(chat_id=-1001076439601, text=text, parse_mode=ParseMode.HTML) # test chat
except BadRequest:
bot.sendMessage(chat_id=-1001307649156, text=text, parse_mode=ParseMode.HTML)
return
if total in celebration_count:
text = f'Вже зареэстровано {total} сусідів 🎇 🎈 🎉 🎆 🍹\nВітаємо\n{created_user.joined_str}'
x, created = Jubilee.get_or_create(house=0, count=total)
if created:
try:
bot.sendMessage(chat_id=-1001076439601, text=text, parse_mode=ParseMode.HTML) # test chat
except BadRequest:
bot.sendMessage(chat_id=-1001307649156, text=text, parse_mode=ParseMode.HTML)
def save_user_data(update, context):
"""callbackQuery from reject. save user data"""
log.info(log_msg(update))
user = chosen_owns(update)
if not user.updated:
text = 'В базу ДОБАВЛЕН:\n'
else:
text = 'В базе ОБНОВЛЕН:\n'
if update.callback_query.data == '_apart_reject':
user_mode = Show.get(user_id=update.effective_user.id)
user_mode.msg_apart_mode = False
user.apartment = None
user_mode.save()
update.callback_query.message.reply_text(
text='Квартиру не сохраняем пока что.')
user.updated = datetime.now().strftime('%y.%m.%d %H:%M:%S.%f')[:-4]
user.save()
# TODO: properly do user_created_report
# user_created_report(update, context, created_user=user, text=text)
new_neighbor_report(update, context, created_user=user)
reply_markup = InlineKeyboardMarkup(
[[InlineKeyboardButton('Меню', callback_data='_menu')]])
update.callback_query.message.reply_text(text="Данные сохранены.",
parse_mode=ParseMode.HTML,
reply_markup=reply_markup)
def set_floor_kbd(update, _):
"""callbackQuery from set_section_kbd().
Show keyboard to chose its own floor"""
log.info(log_msg(update))
user = chosen_owns(update)
user.section = int(update.callback_query.data[2])
user.save()
floors = houses_arr['house_' + str(user.house)]['section_' +
str(user.section)]
# TODO: add proper floor count
keyboard = []
count_ = len(floors)
while count_ > 0:
floor = []
for i in range(3):
if count_ == 0:
break
floor.append(
InlineKeyboardButton(str(floors[-count_]),
callback_data='_f' +
str(floors[-count_])))
count_ -= 1
keyboard.append(floor)
reply_markup = InlineKeyboardMarkup(keyboard)
update.callback_query.message.reply_text('На каком этаже?',
reply_markup=reply_markup)
update.callback_query.answer()
def start_command(update, context):
"""handle /start command"""
log.info(log_msg(update))
is_changed(update)
if update.callback_query:
update.callback_query.answer()
menu_kbd(update, context)
def notifications_kbd(bot, update):
"""callbackQuery handler. pattern:^notifications$. Show notifications keyboard settings"""
log.info(log_msg(update))
keyboard = [[
InlineKeyboardButton('В моєму будинку 🏠',
callback_data='_notify_house')
],
[
InlineKeyboardButton('В моєму під\'їзді 🔢',
callback_data='_notify_section')
],
[
InlineKeyboardButton('Вимкнути сповіщення 🔕',
callback_data='_notify_OFF')
], [InlineKeyboardButton('Меню', callback_data='_menu')]]
reply_markup = InlineKeyboardMarkup(keyboard)
user = Show.get(user_id=update.effective_user.id)
_dict = {
None: 'Вимкнено',
'_notify_OFF': 'Вимкнено',
'_notify_section': 'В моєму під\'їзді 🔢',
'_notify_house': 'В моєму будинку 🏠'
}
text = f'Зараз сповіщення встановленні в режим\n' \
f'<b>{_dict[user.notification_mode]}</b>\nОтримувати сповіщення коли з\'явиться новий сусід:'
bot.editMessageText(chat_id=update.effective_user.id,
parse_mode=ParseMode.HTML,
text=text,
reply_markup=reply_markup,
message_id=update.effective_message.message_id)
update.callback_query.answer()
def show_house(bot, update):
""" """
log.info(log_msg(update))
update.callback_query.answer()
reply_markup = InlineKeyboardMarkup([[InlineKeyboardButton('Меню', callback_data='_menu')]])
if update.callback_query.data == 'show_this_house':
# if user want see selected house
user_query = Show.get(user_id=update.effective_user.id)
else:
# if user want see own house and have one
user_query = chosen_owns(update)
neighbors = []
sections = Own.select(Own.section).where(Own.house == user_query.house, Own.section).distinct().order_by(
Own.section)
query = Own.select().where(Own.house == user_query.house, Own.section).order_by(Own.floor)
for i in sections:
neighbors.append(f'\n{"📭 <b>Секція".rjust(30, " ")} {i.section}</b>\n')
for user in query.where(Own.section == i.section):
neighbors.append(f'{user.user} {user}\n')
show_list = (f'<b>Мешканці будинку № {user_query.house}</b>:\n'
+ '{}' * len(neighbors)).format(*neighbors)
if len(show_list) < 6200:
bot.sendMessage(chat_id=update.effective_user.id, parse_mode=ParseMode.HTML, text=show_list,
reply_markup=reply_markup)
else:
part_1, part_2, part_3 = show_list.partition('📭 <b>Секція 4'.rjust(30, ' ') + '</b>' + '\n')
bot.sendMessage(chat_id=update.effective_user.id, parse_mode=ParseMode.HTML, text=part_1[:-2])
# to do: remove "." from 2nd msg. Without that dot, rjust not works
bot.sendMessage(chat_id=update.effective_user.id, parse_mode=ParseMode.HTML, text='.' + part_2 + part_3,
reply_markup=reply_markup)
def building(bot, update):
"""CallbackQueryHandler. pattern ^building$"""
log.info(log_msg(update))
update.callback_query.answer()
reply_markup = InlineKeyboardMarkup([[InlineKeyboardButton('Меню', callback_data='_menu')]])
bot.sendMessage(chat_id=update.effective_user.id, text=building_msg,
parse_mode=ParseMode.HTML, disable_web_page_preview=True, reply_markup=reply_markup)
def new_neighbor_report(bot, update, created_user):
"""Send message for users who enabled notifications"""
log.info(log_msg(update))
# query for users who set notifications as _notify_house
query_params = Show.select(Show.user_id).where(Show.notification_mode == '_notify_house')
query_users = Own.select(Own.user).where(Own.house == created_user.house)
query = query_params & query_users
# prevent telegram blocking spam
for i, user in enumerate(query):
if i % 29 == 0:
time.sleep(1)
try:
bot.sendMessage(chat_id=user.user_id, parse_mode=ParseMode.HTML,
text=f'Новий сусід\n{created_user.joined_str}')
except BadRequest as err:
bot.sendMessage(chat_id=ADMIN_ID, text=f'failed to send notification for user {user.user_id} {err}',
parse_mode=ParseMode.HTML)
# query for users who set notifications as _notify_section
query_params = Show.select(Show.user_id).where(Show.notification_mode == '_notify_section')
query_users = query_users.where(Own.section == created_user.section)
query = query_params & query_users
for i, user in enumerate(query):
if i % 29 == 0:
time.sleep(1)
try:
bot.sendMessage(chat_id=user.user_id, parse_mode=ParseMode.HTML,
text=f'Новий сусід\n{created_user.joined_str}')
except BadRequest as err:
bot.sendMessage(chat_id=ADMIN_ID, text=f'failed to send notification for user {user.user_id} {err}',
parse_mode=ParseMode.HTML)
def save_user_data(bot, update):
"""callbackQuery handler. pattern: ^_apart_reject$|^_section_reject$ AND integer text handler"""
log.info(log_msg(update))
if update.callback_query:
update.callback_query.answer()
user = chosen_owns(update)
user_mode = Show.get(user_id=update.effective_user.id)
text = f'В базе {"ОБНОВЛЕН" if user.updated else "СОЗДАН"} пользователь:\n'
if user_mode.msg_apart_mode and update.message:
apartment = int(update.message.text)
user.apartment = apartment
else:
if update.callback_query.data == '_apart_reject':
user.apartment = None
user.updated = datetime.now().strftime('%y.%m.%d %H:%M:%S.%f')[:-4]
user.save()
user_mode.msg_apart_mode = False
user_mode.save()
user_created_report(bot, update, created_user=user, text=text)
new_neighbor_report(bot, update, created_user=user)
text_success = '<b>Дякую, Ваші дані збережені</b>. Бажаєте подивитись сусідів?'
bot.sendMessage(text=text_success, chat_id=update.effective_user.id, parse_mode=ParseMode.HTML)
menu_kbd(bot, update)
prepared_data = prepare_data()
make_pie(prepared_data)
make_bars(prepared_data)
def notifications_kbd(update, _):
"""callbackQuery handler. pattern:^notifications$. Show notifications keyboard settings"""
log.info(log_msg(update))
keyboard = [[
InlineKeyboardButton('В моем доме 🏠', callback_data='_notify_house')
],
[
InlineKeyboardButton('В моем подъезде 🔢',
callback_data='_notify_section')
],
[
InlineKeyboardButton('Выключить оповещения 🔕',
callback_data='_notify_OFF')
], [InlineKeyboardButton('Меню', callback_data='_menu')]]
reply_markup = InlineKeyboardMarkup(keyboard)
user = Show.get(user_id=update.effective_user.id)
_dict = {
None: 'Выключено',
'_notify_OFF': 'Выключено',
'_notify_section': 'В моем подъезде 🔢',
'_notify_house': 'В моєм доме 🏠'
}
text = f'Сейчас оповещения установлены в режим\n' \
f'<b>{_dict[user.notification_mode]}</b>\nПолучать оповещения ' \
f'когда появится новый сосед:'
update.message.reply_text(parse_mode=ParseMode.HTML,
text=text,
reply_markup=reply_markup,
message_id=update.effective_message.message_id)
update.callback_query.answer()
def show_section(update, _, some_section=False):
"""Here need some documentation"""
# TODO: rework that func completely
log.info(log_msg(update))
reply_markup = InlineKeyboardMarkup(
[[InlineKeyboardButton('Меню', callback_data='_menu')]])
if not some_section:
user_query = chosen_owns(update)
else:
user_query = Show.get(user_id=update.effective_user.id)
query = User.select().where(User.house == user_query.house,
User.section == user_query.section).order_by(
User.floor)
neighbors = [str(user) + '\n' for user in query]
show_list = ('<b>Жители подъезда № ' + str(user_query.section) +
' Дома № ' + str(user_query.house) + '</b>:\n' +
'{}' * len(neighbors)).format(*neighbors)
update.message.reply_text(parse_mode=ParseMode.HTML,
disable_web_page_preview=True,
text=show_list,
reply_markup=reply_markup)
update.callback_query.answer()
def greeting(bot, update):
"""handle new chat members, and sent greeting message. Delete after delay. Running async"""
log.info(log_msg(update))
new_member_name = update.message.new_chat_members[0].full_name
text = greeting_msg.format(new_member_name)
update.message.reply_text(text=text,
parse_mode=ParseMode.HTML,
disable_web_page_preview=True)
def help_command(update, _):
"""handle /help command"""
log.info(log_msg(update))
reply_markup = InlineKeyboardMarkup(
[[InlineKeyboardButton('Меню', callback_data='_menu')]])
update.message.reply_text(text=help_msg,
parse_mode=ParseMode.HTML,
reply_markup=reply_markup)
def __init__(self, bot, update):
log.info(log_msg(update))
update.callback_query.answer()
keyboard = [[
InlineKeyboardButton('Назад', callback_data='statistics'),
InlineKeyboardButton('Меню', callback_data='_menu')
]]
self.reply_markup = InlineKeyboardMarkup(keyboard)
def save_params(update, context):
"""callbackQuery from section_kbd(). save params to db table"""
log.info(log_msg(update))
user_query = Show.get(user_id=update.effective_user.id)
user_query.section = int(update.callback_query.data[3])
user_query.save()
some_section = True
show_section(update, context, some_section)
update.callback_query.answer()
def msg_handler(bot, update):
"""handle all text msg except other filters do"""
msg = update.message.text
reply_markup = InlineKeyboardMarkup([[InlineKeyboardButton('Меню', callback_data='_menu')]])
bot.sendPhoto(chat_id=update.effective_user.id, photo=open(os.path.join('img', 'maybe.jpg'), 'rb'),
reply_markup=reply_markup,
caption=f'Я ще не розумію людської мови, але вчусь, і скоро буду розуміть деякі слова і фрази\n'
f'Краще скористайтесь меню')
log.info(log_msg(update) + f' text: {msg}')
def talkative(update, _):
"""Statistics for messaging in group chat.
Show top 10 by msgs and by chars"""
log.info(log_msg(update))
reply_markup = InlineKeyboardMarkup(
[[InlineKeyboardButton('Меню', callback_data='_menu')]])
log_files_list = [
f for f in os.listdir('logfiles') if not f.startswith('.')
]
data = {}
id_pattern = r' ([0-9]{6,10}) '
pattern = r' ([0-9]{6,10}) name: (.*) usrnm: '
for log_file in log_files_list:
with open(os.path.join('logfiles', log_file),
mode='r',
encoding='utf-8') as file:
text = file.read()
match = list(set(re.findall(pattern, text)))
data = {i[0]: [0, 0, i[1]] for i in match}
for log_file in log_files_list:
with open(os.path.join('logfiles', log_file),
mode='r',
encoding='utf-8') as file:
for line in file.readlines():
try:
id_ = re.search(id_pattern, line).group().strip()
data[id_][0] += len(line.split('msg: ')[1].strip())
data[id_][1] += 1
except (KeyError, AttributeError):
pass
by_chars = sorted(data.items(), key=lambda x: x[1][0], reverse=True)
by_msgs = sorted(data.items(), key=lambda x: x[1][1], reverse=True)
template = '<a href="tg://user?id={}">{}</a> {}'
talkatives_chars = [
template.format(user[0], user[1][2], user[1][0]) + '\n'
for user in by_chars[:10]
]
talkatives_msgs = [
template.format(user[0], user[1][2], user[1][1]) + '\n'
for user in by_msgs[:10]
]
show_list = ('<b>Лидеры по количеству знаков</b>\n' + '{}'
* len(talkatives_chars)).format(*talkatives_chars) + '\n' + \
('<b>Лидеры по количеству сообщений</b>\n' + '{}' * len(talkatives_msgs)).format(
*talkatives_msgs)
update.message.reply_text(parse_mode=ParseMode.HTML,
disable_web_page_preview=True,
text=show_list,
reply_markup=reply_markup)
def help_command(bot, update):
"""handle /help command"""
log.info(log_msg(update))
reply_markup = InlineKeyboardMarkup(
[[InlineKeyboardButton('Меню', callback_data='_menu')]])
bot.sendMessage(chat_id=update.effective_user.id,
text=help_msg,
parse_mode=ParseMode.HTML,
reply_markup=reply_markup)
def about_command(update, _):
"""handle /about command"""
log.info(log_msg(update))
reply_markup = InlineKeyboardMarkup(
[[InlineKeyboardButton('Меню', callback_data='_menu')]])
update.message.reply_text(text=about_msg,
parse_mode=ParseMode.HTML,
disable_web_page_preview=True,
reply_markup=reply_markup)
def building(update, _):
"""CallbackQueryHandler. pattern ^building$"""
log.info(log_msg(update))
reply_markup = InlineKeyboardMarkup(
[[InlineKeyboardButton('Меню', callback_data='_menu')]])
update.message.reply_text(text=building_msg,
parse_mode=ParseMode.HTML,
disable_web_page_preview=True,
reply_markup=reply_markup)
update.callback_query.answer()
def group_chat_logging(bot, update):
"""handle text msgs in group chat. MessageHandler((Filters.text & Filters.group)"""
log.info(log_msg(update))
user_id = update.effective_user.id
full_name = update.effective_user.full_name
msg_len = len(update.message.text)
msg = update.message.text
Chat.create(user_id=user_id, full_name=full_name, msg_len=msg_len, msg=msg)
def about_command(bot, update):
"""handle /about command"""
log.info(log_msg(update))
reply_markup = InlineKeyboardMarkup(
[[InlineKeyboardButton('Меню', callback_data='_menu')]])
bot.sendMessage(chat_id=update.effective_user.id,
text=about_msg,
parse_mode=ParseMode.HTML,
disable_web_page_preview=True,
reply_markup=reply_markup)
def msg_handler(update, _):
"""handle all text msg except other filters do"""
msg = update.message.text
reply_markup = InlineKeyboardMarkup(
[[InlineKeyboardButton('Меню', callback_data='_menu')]])
update.message.reply_text(
photo=open(os.path.join('img', 'maybe.jpg'), 'rb'),
reply_markup=reply_markup,
caption=f'Я еще не понимать человеческий. Лучше воспользуйтесь меню.')
log.info(log_msg(update) + f' text: {msg}')
