def build_histogram(message, launched_by_tests=False):
"""
Answer to user on /histogram command
:param message: Message by decorator
:param launched_by_tests: Is it launched by tests
:return: Code 1 if everything was ok, Code 2 if profile was closed/not exists, Code 3 - wrong format
"""
by = f"{message.chat.first_name} {message.chat.last_name} ({message.chat.id})"
try:
target = (message.text.split(' '))[1]
except IndexError:
if settings.log_needed:
logging.info(f"histogram_response^Wrong format. Requested by {by}")
if not launched_by_tests:
bot.send_message(message.chat.id, "Введите по формату\n"
"/histogram {id}")
return 3
if settings.log_needed:
logging.info(f"histogram_request^{by} wants to build histogram {target}")
ages = age_analyzer.get_friends_ages(target)
if age_analyzer.is_profile_closed(target) or ages == "PC":
if settings.log_needed:
logging.info(f"histogram_response^{target} - no profile. Requested by {by}")
if not launched_by_tests:
bot.send_message(message.chat.id, "Страница закрыта или не существует. Попробуйте еще раз.")
return 2
else:
if not launched_by_tests:
bot.send_message(message.chat.id, f"Мы начали анализировать {target}")
if settings.log_needed:
logging.info(f"histogram_start^Started analyze {target} to build histogram. Requested by {by}")
target_name = age_analyzer.get_name(target)
ages = age_analyzer.get_friends_ages(target=target)
y = counts_by_arr(ages)
x = range(len(y))
plt.figure(figsize=(15, 5), dpi=80)
plt.bar(x=x, height=y)
plt.xlim(min(ages) - 5, max(ages) + 5)
plt.ylim(0, max(y) + 5)
plt.yticks(np.arange(0, max(y) + 5, 5))
plt.xticks(np.arange(min(ages) - (int(min(ages)) % 5), max(ages) + 5, 5))
plt.title(f"{target_name['first_name']} {target_name['last_name']}", fontsize=24)
plt.ylabel("Count", fontsize=16)
plt.xlabel("Age", fontsize=16)
if not launched_by_tests:
plt.savefig(f"{settings.project_folder}/graph/{target}.png")
if settings.log_needed:
logging.info(
f"histogram_response^Histogram saved to {settings.project_folder}/graph/{target}.png. Sending it back.")
if not launched_by_tests:
photo = open(f"{settings.project_folder}/graph/{target}.png", 'rb')
bot.send_message(message.chat.id,
f"Мы построили гистограмму возрастов друзей"
f" пользователя {target_name['first_name']} {target_name['last_name']}.")
bot.send_chat_action(message.chat.id, 'upload_photo')
bot.send_photo(message.chat.id, photo)
photo.close()
plt.close()
return 1
def analyze(message, launched_by_tests=False, model=neural_network):
by = f"{message.chat.first_name} {message.chat.last_name} ({message.chat.id})"
try:
target = (message.text.split(' '))[1]
except IndexError:
if settings.log_needed:
logging.info(f"analyze_response^Wrong format. Requested by {by}")
if not launched_by_tests:
bot.send_message(message.chat.id, "Введите по формату\n"
"/analyze {id}")
return 3
if settings.log_needed:
logging.info(f"analyze_request^{by} wants to analyze {target}".encode(
"ascii", errors='xmlcharrefreplace'))
ages = age_analyzer.get_friends_ages(target)
if age_analyzer.is_profile_closed(target) or ages == "PC":
if settings.log_needed:
logging.info(
f"analyze_response^{target} - no profile. Requested by {by}")
if not launched_by_tests:
bot.send_message(
message.chat.id,
"Страница закрыта или не существует. Попробуйте еще раз.")
return 2
else:
if not launched_by_tests:
bot.send_message(message.chat.id,
f"Мы начали анализировать {target}")
if settings.log_needed:
logging.info(
f"analyze_analyzing^Started analyze {target}. Requested by {by}."
)
target_name = age_analyzer.get_name(target)
target_age = age_analyzer.get_age(target)
if target_age == -1:
target_age = "не указан"
friends_ages = age_analyzer.get_friends_ages(target)
predicted = (model.query(friends_ages))
mode = find_max_mode(friends_ages)
response = f"Мы проанализировали {target_name['first_name']} {target_name['last_name']}\n" \
f"Возраст, указанный в профиле - {target_age}.\n" \
f"Однако, мы полагаем, что настоящий возраст: {predicted} \n" \
f"Мода: {mode}"
if settings.log_needed:
logging.info(
f"analyze_response^Answered to {by}. Request: {message.chat.id}"
)
if not launched_by_tests:
bot.send_message(message.chat.id, response)
return 1
def test_get_friends_ages(target: str, wrong,
comment): # It shouldn't give wrong answer
"""
Test age_analyzer.get_friends_ages function
"""
ages = age_analyzer.get_friends_ages(target)
assert ages != wrong, comment
def analyze_and_insert(target, model, force_upgrade=False):
"""
Analyze and insert, update if needed
"""
connection = psycopg2.connect(database=settings.db_name,
user=settings.db_login,
password=settings.db_pass,
host=settings.db_ip,
port=settings.db_port)
target_id = age_analyzer.get_id_by_domain(target)
domain = age_analyzer.get_domain_by_id(target_id)
was_analyzed_recently = check_was_analyzed_recently(domain)
was_analyzed_ever = check_was_analyzed_ever(domain)
# print(was_analyzed_ever)
# print(was_analyzed_recently)
# print(f"target_domain: {domain}")
if was_analyzed_recently:
# print("1")
return get_age_from_database(domain)
elif was_analyzed_ever and force_upgrade:
# print("2")
return upgrade(domain, model)
else:
# print("3")
ages = age_analyzer.get_friends_ages(domain)
# print(domain)
if ages == "PC" or not ages:
return -1 # Profile closed
estimated_age = model._query(ages)
name = age_analyzer.get_name(domain)
mean = round(st.mean(ages), 2)
hmean = round(st.harmonic_mean(ages), 2)
mode = round(neuroanalyzer.find_average_mode(ages), 2)
median = round(st.median(ages), 2)
std = round(neuroanalyzer.find_std(ages), 2)
friends_cnt = len(ages)
today = datetime.datetime.now()
vk_age = age_analyzer.get_age(domain)
query = (
f"insert into analyzed("
f"id, domain, first_name, last_name, estimated_age, mean, mode, harmonic_mean,"
f"median, std, friends_cnt, verified, last_check, vk_age) values ("
f"%s, %s, %s, %s, %s, %s, %s, %s, %s,"
f"%s, %s, False, current_date, %s"
f")")
print(query)
cur = connection.cursor()
cur.execute(query, [
target_id, domain, name['first_name'], name['last_name'],
estimated_age, mean, mode, hmean, median, std, friends_cnt, vk_age
])
connection.commit()
connection.close()
# print(estimated_age)
return estimated_age
def estimate_age_recursive(target, model=neural_network):
target_friends = age_analyzer.get_friends(target)
target_id = age_analyzer.get_id_by_domain(target)
estimated_ages = []
if isinstance(target_friends, int):
return -1 # Profile closed
for now in target_friends:
now_ages = age_analyzer.get_friends_ages(now)
if isinstance(now_ages, list) and now != target_id:
if len(now_ages) != 0:
estimated_ages.append(model.query(now_ages, False, False))
answer = model.query(estimated_ages)
return answer
def analyze_recursive(message, launched_by_tests=False, model=neural_network):
"""
Answer to user on /recursive command
:param message: Message by decorator
:param launched_by_tests: Is it launched by tests
:param model: AgeRegressor model
:return: Code 1 if everything was ok, Code 2 if profile was closed/not exists, Code 3 - wrong format
"""
by = f"{message.chat.first_name} {message.chat.last_name} ({message.chat.id})"
try:
target = (message.text.split(' '))[1]
except IndexError:
if settings.log_needed:
logging.info(f"recursive_response^Wrong format. Requested by {by}")
logging.error(f"recursive_error")
if not launched_by_tests:
bot.send_message(message.chat.id, "Введите по формату\n"
"/analyze {id}")
return 3
if settings.log_needed:
logging.info(f"recursive_request^{by} wants to analyze {target}".encode("ascii", errors='xmlcharrefreplace'))
ages = age_analyzer.get_friends_ages(target)
if age_analyzer.is_profile_closed(target) or ages == "PC":
if settings.log_needed:
logging.info(f"recursive_response^{target} - no profile. Requested by {by}")
if not launched_by_tests:
bot.send_message(message.chat.id, "Страница закрыта или не существует. Попробуйте еще раз.")
return 2
else:
if not launched_by_tests:
bot.send_message(message.chat.id, f"Мы начали анализировать {target}")
if settings.log_needed:
logging.info(f"recursive_analyzing^Started analyze {target}. Requested by {by}.")
target_name = age_analyzer.get_name(target)
target_age = age_analyzer.get_age(target)
if target_age == -1:
target_age = "не указан"
predicted_recursive = recursive_estimate.estimate_age_recursive(target, model=model)
predicted = model._query(ages)
response = f"Мы проанализировали {target_name['first_name']} {target_name['last_name']}\n" \
f"Возраст, указанный в профиле - {target_age}.\n" \
f"Однако, оценив возраст рекурсивно, мы полагаем, что настоящий возраст: {predicted_recursive} \n" \
f"При стандартном способе оценки: {predicted}"
if settings.log_needed:
logging.info(f"analyze_response^Answered to {by}. Request: {message.chat.id}")
if not launched_by_tests:
bot.send_message(message.chat.id, response)
return 1
def fill_friends_age(data: pd.DataFrame):
"""Function fills friends ages' Mean, Median, etc.
:param data: pd.DataFrame, raw data
"""
for i in range(0, len(data)):
if analyzer.is_profile_closed(data['ID'][int(i)]):
print(f"{data['ID'][i]} is closed")
else:
ages = analyzer.get_friends_ages(data["ID"][i])
print(f"ID: {data['ID'][i]}. Ages: {ages} ")
data["Mean"][i] = round(st.mean(ages), 2)
data["Mode"][i] = round(find_average_mode(ages), 2)
data["Harmonic Mean"][i] = round(st.harmonic_mean(ages), 2)
data["Median"][i] = round(st.median(ages), 2)
ages_np = np.array(ages)
data["std"][i] = round(np.std(ages_np), 2)
df = pd.DataFrame(data)
return df
def upgrade(domain, model):
"""
Update person in the table
:return: person's age.
"""
target_id = age_analyzer.get_id_by_domain(domain)
domain = age_analyzer.get_domain_by_id(target_id)
connection = psycopg2.connect(database=settings.db_name,
user=settings.db_login,
password=settings.db_pass,
host=settings.db_ip,
port=settings.db_port)
cur = connection.cursor()
ages = age_analyzer.get_friends_ages(target_id)
estimated_age = model._query(ages)
name = age_analyzer.get_name(target_id)
mean = round(st.mean(ages), 2)
hmean = round(st.harmonic_mean(ages), 2)
mode = round(neuroanalyzer.find_average_mode(ages), 2)
median = round(st.median(ages), 2)
std = round(neuroanalyzer.find_std(ages), 2)
vk_age = age_analyzer.get_age(target_id)
friends_cnt = len(ages)
query = (f"UPDATE analyzed SET estimated_age = %s WHERE id=\'%s\';"
f"UPDATE analyzed SET mean = %s WHERE id=\'%s\';"
f"UPDATE analyzed SET harmonic_mean = %s WHERE id=\'%s\';"
f"UPDATE analyzed SET mode = %s WHERE id=\'%s\';"
f"UPDATE analyzed SET median = %s WHERE id=\'%s\';"
f"UPDATE analyzed SET std = %s WHERE id=\'%s\';"
f"UPDATE analyzed SET last_check = current_date WHERE id=\'%s\';"
f"UPDATE analyzed SET friends_cnt = %s WHERE id=\'%s\';"
f"UPDATE analyzed SET first_name = \'%s\' WHERE id=\'%s\';"
f"UPDATE analyzed SET last_name = \'%s\' WHERE id=\'%s\';"
f"UPDATE analyzed SET domain = \'%s\' WHERE id = \'%s\'")
cur.executescript(query, [
estimated_age, target_id, mean, target_id, hmean, target_id, mode,
target_id, median, target_id, std, target_id, target_id, friends_cnt,
target_id, name['first_name'], target_id, name['last_name'], target_id,
domain, target_id
])
connection.commit()
connection.close()
return estimated_age
df = csv_connect.fill_friends_age(df)
df = csv_connect.fill_vk_age(df)
df.fillna(-1.0, inplace=True)
df.to_csv('age_research1.csv', index=False)
if TRAIN_MODEL:
df = pd.read_csv('age_research1.csv')
model = neuroanalyzer.AgeRegressor()
model.train_with_raw_data(df)
model.save_model('neuronet.sav')
if ANALYZE:
print("Input target's ID:", end=" ")
target = input()
model = neuroanalyzer.AgeRegressor()
# neural_network.train_with_raw_data(df_with_data)
# File should be formatted like:
# ID,Real Age,VK Age,Mean,Harmonic Mean,Mode,Median,std'
# OR:
# Bot loads neural network from file
model.open_model(settings.neural_network_file)
ages = age_analyzer.get_friends_ages(target=target)
name = age_analyzer.get_name(target=target)
try:
predicted = model.query(ages)
answer = f"Neural network thinks that {name['first_name']} {name['last_name']}" \
f"({target}) age is {predicted}"
except TypeError:
answer = "Profile closed"
print(answer)
elif BOT:
telegram_bot.launch()