async def trad(msg: Message) -> None:
parola = utils.to_user_id(utils.remove_accents(msg.arg.lower()))
if parola == "":
await msg.reply("Cosa devo tradurre?")
return
results: List[Dict[str, str]] = []
for i in TRANSLATIONS:
for j in TRANSLATIONS[i]:
if utils.to_user_id(utils.remove_accents(j["en"].lower())) == parola:
results.append({"trad": j["it"], "cat": i})
elif utils.to_user_id(utils.remove_accents(j["it"].lower())) == parola:
results.append({"trad": j["en"], "cat": i})
if results:
if len(results) == 1:
await msg.reply(results[0]["trad"])
return
resultstext = ""
for k in results:
if resultstext != "":
resultstext += ", "
resultstext += "{trad} ({cat})".format(trad=k["trad"], cat=k["cat"])
await msg.reply(resultstext)
return
await msg.reply("Non trovato")
def printRequest(self, printRequest):
fileName = "{}-{}.ps".format(printRequest['author'],
datetime.now().isoformat())
command = 'a2ps --output="{}" --pretty-print="{}" --header="{}" \
--left-footer --footer="{}" --center-title="{}" \
--line-numbers=1'
title = get_key_or_substitute(printRequest, 'problem', "Source code")
institute = get_key_or_substitute(printRequest, 'institute', "")
team = get_key_or_substitute(printRequest, 'team', "")
room = "room " + get_key_or_substitute(printRequest, 'room', "?")
computer = "computer "\
+ get_key_or_substitute(printRequest, 'computer', "?")
path = os.path.join(settings.OUTPUT['OUTPUT_DIR'], fileName)
cmd = command.format(
path,
printRequest['language'].lower(),
remove_accents(printRequest['contest']),
printRequest['author'] + ' ' + institute + ' ' + team +
' ' + room + ' ' + computer,
title
)
proc = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE, shell=True)
source = remove_accents(printRequest['source']).encode('utf-8')
proc.communicate(input=source)
proc.wait()
def user(bot, event, *args):
"""find people by name"""
search = " ".join(args)
if not search:
raise Help(_("supply search term"))
search_lower = search.strip().lower()
search_upper = search.strip().upper()
segments = [
hangups.ChatMessageSegment(
_('results for user named "{}":').format(search), is_bold=True),
hangups.ChatMessageSegment(
'\n', hangups.hangouts_pb2.SEGMENT_TYPE_LINE_BREAK)
]
all_known_users = {}
for chat_id in bot.memory["user_data"]:
all_known_users[chat_id] = bot.get_hangups_user(chat_id)
for u in sorted(all_known_users.values(),
key=lambda x: x.full_name.split()[-1]):
fullname_lower = u.full_name.lower()
fullname_upper = u.full_name.upper()
unspaced_lower = re.sub(r'\s+', '', fullname_lower)
unspaced_upper = re.sub(r'\s+', '', u.full_name.upper())
if (search_lower in fullname_lower or search_lower in unspaced_lower
# XXX: turkish alphabet special case: converstion works better when uppercase
or search_upper in remove_accents(fullname_upper) or
search_upper in remove_accents(unspaced_upper)):
link = 'https://plus.google.com/u/0/{}/about'.format(u.id_.chat_id)
segments.append(
hangups.ChatMessageSegment(
u.full_name,
hangups.hangouts_pb2.SEGMENT_TYPE_LINK,
link_target=link))
if u.emails:
segments.append(hangups.ChatMessageSegment(' ('))
segments.append(
hangups.ChatMessageSegment(
u.emails[0],
hangups.hangouts_pb2.SEGMENT_TYPE_LINK,
link_target='mailto:{}'.format(u.emails[0])))
segments.append(hangups.ChatMessageSegment(')'))
segments.append(
hangups.ChatMessageSegment(' ... {}'.format(u.id_.chat_id)))
segments.append(
hangups.ChatMessageSegment(
'\n', hangups.hangouts_pb2.SEGMENT_TYPE_LINE_BREAK))
return segments
def user(bot, event, username, *args):
"""find people by name"""
username_lower = username.strip().lower()
username_upper = username.strip().upper()
segments = [hangups.ChatMessageSegment(_('results for user named "{}":').format(username),
is_bold=True),
hangups.ChatMessageSegment('\n', hangups.SegmentType.LINE_BREAK)]
all_known_users = {}
for chat_id in bot.memory["user_data"]:
all_known_users[chat_id] = bot.get_hangups_user(chat_id)
for u in sorted(all_known_users.values(), key=lambda x: x.full_name.split()[-1]):
if (not username_lower in u.full_name.lower() and
not username_upper in remove_accents(u.full_name.upper())):
continue
link = 'https://plus.google.com/u/0/{}/about'.format(u.id_.chat_id)
segments.append(hangups.ChatMessageSegment(u.full_name, hangups.SegmentType.LINK,
link_target=link))
if u.emails:
segments.append(hangups.ChatMessageSegment(' ('))
segments.append(hangups.ChatMessageSegment(u.emails[0], hangups.SegmentType.LINK,
link_target='mailto:{}'.format(u.emails[0])))
segments.append(hangups.ChatMessageSegment(')'))
segments.append(hangups.ChatMessageSegment(' ... {}'.format(u.id_.chat_id)))
segments.append(hangups.ChatMessageSegment('\n', hangups.SegmentType.LINE_BREAK))
bot.send_message_segments(event.conv, segments)
def phanbiet(dict_input: dict) -> int:
"""Phân biệt các post cho thuê phòng trọ hoặc phòng trong căn hộ cho thuê.
BĐS có mục đích cho thuê hoặc sang nhượng.
"""
realestate_type = dict_input['realestate_type']
content = dict_input['content']
if (realestate_type == 8):
new_content = remove_accents(content)
# Nếu chứa các từ "day nha/phong tro" hoặc "dang cho thue" thì trả về -1
regex_not_type8 = "day (nha |phong )?tro|dang cho thue"
not_type8= re.search(regex_not_type8, new_content)
# Keywords of type 8: phong tro, phong cho thue, phong trong can ho cho thue, cho thue phong
regex_type8 = "phong tro|phong (trong can ho )?(cho thue)|cho thue phong"
result_type8 = re.search(regex_type8, new_content)
if (not_type8 is not None):
realestate_type = -1
elif (result_type8 is not None):
realestate_type = 8
return realestate_type
def __accent_filter(self, text: str) -> (str, dict):
normalized = utils.remove_accents(text)
accent_locs = {}
for i, c in enumerate(text):
if c != normalized[i]:
accent_locs[i] = c
return normalized, accent_locs
def user(bot, event, username, *args):
"""find people by name"""
username_lower = username.strip().lower()
username_upper = username.strip().upper()
segments = [hangups.ChatMessageSegment(_('results for user named "{}":').format(username),
is_bold=True),
hangups.ChatMessageSegment('\n', hangups.SegmentType.LINE_BREAK)]
all_known_users = {}
for chat_id in bot.memory["user_data"]:
all_known_users[chat_id] = bot.get_hangups_user(chat_id)
for u in sorted(all_known_users.values(), key=lambda x: x.full_name.split()[-1]):
if (not username_lower in u.full_name.lower() and
not username_upper in remove_accents(u.full_name.upper())):
continue
link = 'https://plus.google.com/u/0/{}/about'.format(u.id_.chat_id)
segments.append(hangups.ChatMessageSegment(u.full_name, hangups.SegmentType.LINK,
link_target=link))
if u.emails:
segments.append(hangups.ChatMessageSegment(' ('))
segments.append(hangups.ChatMessageSegment(u.emails[0], hangups.SegmentType.LINK,
link_target='mailto:{}'.format(u.emails[0])))
segments.append(hangups.ChatMessageSegment(')'))
segments.append(hangups.ChatMessageSegment(' ... {}'.format(u.id_.chat_id)))
segments.append(hangups.ChatMessageSegment('\n', hangups.SegmentType.LINE_BREAK))
yield from bot.coro_send_message(event.conv, segments)
def extract_content(self):
classes = self.extract_classes()
for klass in classes[1:]: # Exclude ONLINE CLASS
folder_name = remove_accents(klass['class'])
create_folder(folder_name)
print('Extracting Class: {0}'.format(klass['class']))
self.browser.get('https://unipac-bomdespacho.blackboard.com{0}'.format(klass['href']))
self.browser.find_element_by_id('header::0-whatsNewView::CO').click() # Open content list
block_class_contents = self.browser.find_element_by_id('block::0-whatsNewView::CO')
class_contents = block_class_contents.find_elements_by_css_selector(
"a[onclick*='nautilus_utils.actionSelected']"
)
i_content = 0
for i_content in range(i_content, len(class_contents)):
try:
block_classes_contents = self.browser.find_element_by_id('block::0-whatsNewView::CO')
class_contents = block_classes_contents.find_elements_by_css_selector(
"a[onclick*='nautilus_utils.actionSelected']"
)
class_contents[i_content].click()
self.check_visibility(By.CLASS_NAME, "individualContent-link")
file_link = self.browser.find_element_by_class_name('individualContent-link').get_attribute('href')
cookies = self.browser.get_cookies()
download(cookies, file_link, folder_name)
self.browser.back()
self.check_visibility(By.ID, "block::0-whatsNewView::CO")
except TimeoutException:
print("Error in: {0} - {1}".format(klass['class'], klass['href']))
async def shitpost(msg: Message) -> None:
if msg.room is None:
return
phrase = utils.remove_accents(msg.arg.strip())
if len(phrase) > 50:
await msg.reply("Testo troppo lungo")
return
text0 = ""
text1 = ""
text2 = ""
if phrase == "":
if not msg.room.is_private:
return
phrase = "SHITPOST"
if not msg.room.is_private and ("x" in phrase or "X" in phrase):
phrase = "lolno"
for i in phrase:
if i in LETTERS:
if text0 != "":
text0 += " "
text1 += " "
text2 += " "
text0 += LETTERS[i][0]
text1 += LETTERS[i][1]
text2 += LETTERS[i][2]
html = '<pre style="margin: 0; overflow-x: auto">{}<br>{}<br>{}</pre>'
await msg.reply_htmlbox(html.format(text0, text1, text2))
def phanbiet(dict_input: dict) -> int:
"""Phân biệt đất nền (10).
Arguments:
inp_str {dict} -- dict chứa 3 thông tin sau:
id : id của bài đăng
content : content của bài đăng
realestate_type : loại bất động sản
Returns:
int - realestate_type mới của bài đăng
"""
realestate_type = dict_input['realestate_type']
# code goes here
if (realestate_type == 1):
dict_content = dict_input['content']
dict_content = remove_accents(dict_content)
pattern1 = "dat\s+nen\s|dat\s+nen\s+tho\s+cu|dat\s+du\s+an|dat\s+xay\s+dung|khu\s+du\s+an|nen\s+dat|phan\s+lo|dat_nen"
result1 = re.search(pattern1, dict_content)
"""Test xem có chứa từ khoá của đất thổ cư và nông nghiệp không
pattern2 = "len\s+tho\s+cu|len\s+100%\s+tho\s+cu|dat\s+tho\s+cu|dat\s+tc|dat\s+o\s+tai\s+(do\s+thi|nong\s+thon)|dat\s+trong\s+cay|dat\s+nong\s+nghiep|dat\s+lam\s+nghiep"
result2 = re.search(pattern2, dict_content)
if (result1 is not None):
print("{0} {1}".format(dict_input['id'], result2 is not None))
"""
if (result1 is not None):
realestate_type = 10
############################
# Return
############################
return realestate_type
def test_remove_accents(self):
string = "This is â wíred stríng"
expected = "This is a wired string"
result = remove_accents(string)
assert result == expected
def phanbiet(dict_input: dict) -> int:
"""Phân biệt căn hộ dịch vụ .
Arguments:
inp_str {dict} -- dict chứa 3 thông tin sau:
id : id của bài đăng
content : content của bài đăng
realestate_type : loại bất động sản
Returns:
int - realestate_type mới của bài đăng
"""
realestate_type = dict_input['realestate_type']
## code goes here
content = dict_input['content']
if (realestate_type == 3):
# remove Vietnameses accent from content using built-in function
new_content = remove_accents(content)
# find all the post that has such keyword by using regex rules
regex_rule = "can\s+ho\s+dich\s+vu|chdv"
result = re.search(regex_rule, new_content)
if (result is not None):
realestate_type = 2
else:
realestate_type = 3
############################
## Return
############################
return realestate_type
async def shitpost(conn: Connection, room: Optional[str], user: str, arg: str) -> None:
if room is None:
return
message = utils.remove_accents(arg.strip())
if len(message) > 50:
await conn.send_reply(room, user, "Testo troppo lungo")
return
text0 = ""
text1 = ""
text2 = ""
if message == "":
if not utils.is_private(conn, room):
return
message = "SHITPOST"
if not utils.is_private(conn, room) and ("x" in message or "X" in message):
message = "lolno"
for i in message:
if i in LETTERS:
if text0 != "":
text0 += " "
text1 += " "
text2 += " "
text0 += LETTERS[i][0]
text1 += LETTERS[i][1]
text2 += LETTERS[i][2]
html = '<pre style="margin: 0; overflow-x: auto">{}<br>{}<br>{}</pre>'
await conn.send_htmlbox(room, user, html.format(text0, text1, text2))
def printRequest(self, printRequest):
title = get_key_or_substitute(printRequest, 'problem', "Source code")
institute = get_key_or_substitute(printRequest, 'institute', "")
team = get_key_or_substitute(printRequest, 'team', "")
room = "room " + get_key_or_substitute(printRequest, 'room', "?")
computer = "computer "\
+ get_key_or_substitute(printRequest, 'computer', "?")
self.__print(
printer=settings.OUTPUT['PRINTER_NAME'],
language=printRequest['language'].lower(),
header=remove_accents(printRequest['contest']),
footer=printRequest['author'] + ' ' + institute + ' ' + team +
' ' + room + ' ' + computer,
title=title,
source=remove_accents(printRequest['source'])
)
def normalise_CSV_column_name(column_name):
# remove parentheses and their content
column_name = re.sub(r'\([^)]*\)', '', column_name)
# replace spaces with an underscore
column_name = re.sub(r' +', '_', column_name)
# remove accents and convert to lower case
column_name = remove_accents(column_name).lower()
return column_name
def _handle_keyword(bot, event, command, include_event_user=False):
"""handle keyword"""
if event.user.is_self:
return
_populate_keywords(bot, event)
users_in_chat = event.conv.users
"""check if synced room and syncing is enabled
if its a valid syncroom, get a list of all unique users across all rooms"""
if bot.get_config_option('syncing_enabled'):
syncouts = bot.get_config_option('sync_rooms') or []
for sync_room_list in syncouts:
if event.conv_id in sync_room_list:
for syncedroom in sync_room_list:
if event.conv_id not in syncedroom:
users_in_chat += bot.get_users_in_conversation(syncedroom)
users_in_chat = list(set(users_in_chat)) # make unique
event_text = re.sub(r"\s+", " ", event.text)
event_text_lower = event.text.lower()
for user in users_in_chat:
chat_id = user.id_.chat_id
try:
if _internal.keywords[chat_id] and ( not chat_id in event.user.id_.chat_id
or include_event_user ):
for phrase in _internal.keywords[chat_id]:
regexphrase = r"(^|\b| )" + re.escape(phrase) + r"($|\b)"
if re.search(regexphrase, event_text, re.IGNORECASE):
"""XXX: suppress alerts if it appears to be a valid mention to same user
logic condensed from the detection function in the mentions plugin, we may
miss some use-cases, but this should account for "most" of them"""
if 'plugins.mentions' in sys.modules:
_phrase_lower = phrase.lower()
_mention = "@" + _phrase_lower
if (_mention + " ") in event_text_lower or event_text_lower.endswith(_mention):
user = bot.get_hangups_user(chat_id)
_normalised_full_name_lower = remove_accents(user.full_name.lower())
if( _phrase_lower in _normalised_full_name_lower
or _phrase_lower in _normalised_full_name_lower.replace(" ", "")
or _phrase_lower in _normalised_full_name_lower.replace(" ", "_") ):
# part of name mention: skip
logger.debug("subscription matched full name fragment {}, skipping".format(user.full_name))
continue
if bot.memory.exists(['user_data', chat_id, "nickname"]):
_nickname = bot.memory.get_by_path(['user_data', chat_id, "nickname"])
if _phrase_lower == _nickname.lower():
# nickname mention: skip
logger.debug("subscription matched exact nickname {}, skipping".format(_nickname))
continue
yield from _send_notification(bot, event, phrase, user)
except KeyError:
# User probably hasn't subscribed to anything
continue
def user(bot, event, *args):
"""find people by name"""
search = " ".join(args)
if not search:
raise ValueError(_("supply search term"))
search_lower = search.strip().lower()
search_upper = search.strip().upper()
segments = [hangups.ChatMessageSegment(_('results for user named "{}":').format(search),
is_bold=True),
hangups.ChatMessageSegment('\n', hangups.SegmentType.LINE_BREAK)]
all_known_users = {}
for chat_id in bot.memory["user_data"]:
all_known_users[chat_id] = bot.get_hangups_user(chat_id)
for u in sorted(all_known_users.values(), key=lambda x: x.full_name.split()[-1]):
fullname_lower = u.full_name.lower()
fullname_upper = u.full_name.upper()
unspaced_lower = re.sub(r'\s+', '', fullname_lower)
unspaced_upper = re.sub(r'\s+', '', u.full_name.upper())
if( search_lower in fullname_lower
or search_lower in unspaced_lower
# XXX: turkish alphabet special case: converstion works better when uppercase
or search_upper in remove_accents(fullname_upper)
or search_upper in remove_accents(unspaced_upper) ):
link = 'https://plus.google.com/u/0/{}/about'.format(u.id_.chat_id)
segments.append(hangups.ChatMessageSegment(u.full_name, hangups.SegmentType.LINK,
link_target=link))
if u.emails:
segments.append(hangups.ChatMessageSegment(' ('))
segments.append(hangups.ChatMessageSegment(u.emails[0], hangups.SegmentType.LINK,
link_target='mailto:{}'.format(u.emails[0])))
segments.append(hangups.ChatMessageSegment(')'))
segments.append(hangups.ChatMessageSegment(' ... {}'.format(u.id_.chat_id)))
segments.append(hangups.ChatMessageSegment('\n', hangups.SegmentType.LINE_BREAK))
yield from bot.coro_send_message(event.conv, segments)
async def get_company_by_name(name: str):
# On recherche les noms commençant par 'name' et plus large si insuffisant
if name:
NB_RESULT_TO_RETURN = 5
# On remplace les accents par des lettres standards pour faciliter la recherche
name = remove_accents(name)
# On supprime les caractères spéciaux et on passe en minuscule
name = re.sub('[^A-Za-z0-9 ]+', '', name)
name = name.strip().lower()
df_name = df_companies_name.copy()
# On ne garde que les premières lettres
df_search = df_companies_name.str.slice(stop=len(name))
# On cherche les meilleures correspondances
df_search = df_search.apply(lambda x: SequenceMatcher(None, name, x).ratio())
# On trie les résultats pour avoir les meilleurs résultats en tête de liste
df_search.sort_values(inplace=True, ascending=False)
# On accèpte une correspondance à 90%
df_best_result = df_search[df_search >= 0.9]
nb_best_result = df_best_result.count() # Nombre de correspondance
if nb_best_result > NB_RESULT_TO_RETURN:
# On a assez de résultat à proposer
df_result = df_companies.iloc[df_best_result.index[:NB_RESULT_TO_RETURN]]
# print(df_result.to_json())
return JSONResponse(content=df_result['name'].to_json())
else:
# Il nous faut un peu plus de résultats, donc on cherche des correspondances plus larges
if nb_best_result > 0:
# Pour l'instant on prend au moins les résultats déjà trouvés
df_result = df_companies.iloc[df_best_result.index[:nb_best_result]]
# On supprime les noms déjà utilsés
df_name.drop(df_best_result.index[:nb_best_result], inplace=True)
# On cherche les meilleures correspondances
df_search = df_name.apply(lambda x: SequenceMatcher(None, name, x).ratio())
# On trie les résultats pour avoir les meilleurs résultats en tête de liste
df_search.sort_values(inplace=True, ascending=False)
# On ne garde que le nombre de résultats qui nous manquaient
df_search = df_companies.iloc[df_search.index[:NB_RESULT_TO_RETURN - nb_best_result]]
if nb_best_result > 0:
df_result = pd.concat([df_result, df_search])
else:
df_result = df_search
# print(df_result.to_json())
return JSONResponse(content=df_result['name'].to_json())
def extract_content(self):
doc = self.get_html_doc()
content_div = doc.xpath('//div[@class="content_nota"]')[0]
p_list = content_div.findall('p')
if not p_list:
p_list = content_div.findall('div')
ws_re = re.compile('\s\s+')
content_ = lambda p: p.text_content().strip()
p_text_list = [ws_re.sub(' ', content_(p)) for p in p_list]
content = unicode('\n\n'.join(p_text_list))
return utils.remove_accents(content.strip())
def clean(text): text = remove_accents(text) text = expand_contractions(text) text = handle_units(text) text = convert_word_to_number(text) text = remove_punctuation(text) doc = nlp(text) text = perform_spell_check(doc) doc = nlp(text) text = convert_plural_to_singular(doc) return text
def get_box_scores(date, team1, team2, period='GAME', stat_type='BASIC'):
date = pd.to_datetime(date)
suffix = get_game_suffix(date, team1, team2).replace('/', '%2F')
r1 = get(f'https://widgets.sports-reference.com/wg.fcgi?css=1&site=bbr&url={suffix}&div=div_box-{team1}-{period.lower()}-{stat_type.lower()}')
r2 = get(f'https://widgets.sports-reference.com/wg.fcgi?css=1&site=bbr&url={suffix}&div=div_box-{team2}-{period.lower()}-{stat_type.lower()}')
if r1.status_code==200 and r2.status_code==200:
soup = BeautifulSoup(r1.content, 'html.parser')
table = soup.find('table')
df1 = pd.read_html(str(table))[0]
df1.columns = list(map(lambda x: x[1], list(df1.columns)))
df1.rename(columns = {'Starters': 'PLAYER'}, inplace=True)
df1['PLAYER'] = df1['PLAYER'].apply(lambda name: remove_accents(name, team1, date.year))
reserve_index = df1[df1['PLAYER']=='Reserves'].index[0]
df1 = df1.drop(reserve_index).reset_index().drop('index', axis=1)
soup = BeautifulSoup(r2.content, 'html.parser')
table = soup.find('table')
df2 = pd.read_html(str(table))[0]
df2.columns = list(map(lambda x: x[1], list(df2.columns)))
df2.rename(columns = {'Starters': 'PLAYER'}, inplace=True)
df2['PLAYER'] = df2['PLAYER'].apply(lambda name: remove_accents(name, team2, date.year))
reserve_index = df2[df2['PLAYER']=='Reserves'].index[0]
df2 = df2.drop(reserve_index).reset_index().drop('index', axis=1)
return {team1: df1, team2: df2}
def get_box_scores(date, team1, team2, period='GAME', stat_type='BASIC'):
date = pd.to_datetime(date)
suffix = get_game_suffix(date, team1, team2).replace('/', '%2F')
r1 = get(
f'https://widgets.sports-reference.com/wg.fcgi?css=1&site=bbr&url={suffix}&div=div_box-{team1}-{period.lower()}-{stat_type.lower()}'
)
r2 = get(
f'https://widgets.sports-reference.com/wg.fcgi?css=1&site=bbr&url={suffix}&div=div_box-{team2}-{period.lower()}-{stat_type.lower()}'
)
dfs = []
if r1.status_code == 200 and r2.status_code == 200:
for rq in (r1, r2):
soup = BeautifulSoup(rq.content, 'html.parser')
table = soup.find('table')
raw_df = pd.read_html(str(table))[0]
df = _process_box(raw_df)
if rq == r1:
df['PLAYER'] = df['PLAYER'].apply(
lambda name: remove_accents(name, team1, date.year))
if rq == r2:
df['PLAYER'] = df['PLAYER'].apply(
lambda name: remove_accents(name, team2, date.year))
dfs.append(df)
return {team1: dfs[0], team2: dfs[1]}
def phanbiet(dict_input: dict) -> int:
"""Distinguish "dat tho cu" from other "dat" based on provided keywords
The keywords we are using are:
- dat tho cu
- dat tc
- dat o tai do thi
- dat o tai nong thon
- mat tien
- dat kiet
Arguments:
inp_str {dict} -- dict chứa 3 thông tin sau:
id : id của bài đăng
content : content của bài đăng
realestate_type : loại bất động sản
Returns:
int - realestate_type mới của bài đăng
"""
# Firstly, get needed information from input
realestate_type = dict_input['realestate_type']
content = dict_input['content']
if (realestate_type == 1):
# remove Vietnameses accent from content using built-in function
# `remove_accents` in utils.py
new_content = remove_accents(content)
# find all the post that has such keyword by using regex rules
regex_rule = "dat\s+tho\s+cu|dat\s+tc|dat\s+o\s+tai\s+(do\s+thi|nong\s+thon)|mat\s+tien|dat\s+kiet"
result = re.search(regex_rule, new_content)
if (result is not None):
realestate_type = 1
else:
# Not dat tho cu
realestate_type = -1
else:
# not the part we have to categorize
realestate_type = -1
############################
## Return
############################
return realestate_type
def get_roster(team, season_end_year):
r = get(
f'https://www.basketball-reference.com/teams/{team}/{season_end_year}.html'
)
df = None
if r.status_code == 200:
soup = BeautifulSoup(r.content, 'html.parser')
table = soup.find('table')
df = pd.read_html(str(table))[0]
df.columns = [
'NUMBER', 'PLAYER', 'POS', 'HEIGHT', 'WEIGHT', 'BIRTH_DATE',
'NATIONALITY', 'EXPERIENCE', 'COLLEGE'
]
df['PLAYER'] = df['PLAYER'].apply(
lambda name: remove_accents(name, team, season_end_year))
df['BIRTH_DATE'] = df['BIRTH_DATE'].apply(lambda x: pd.to_datetime(x))
df['NATIONALITY'] = df['NATIONALITY'].str.upper()
return df
async def guessthemon(msg: Message) -> None:
db = Database.open("veekun")
with db.get_session() as session:
# Retrieve a random pokemon
# Hangman words can only have ascii letters, so a few pokemon names wouldn't be
# parsed correctly. Punctuation signs are preserved.
# Some exceptions pass silently for simplicity, i.e. Nidoran♂ / Nidoran♀.
invalid_identifiers = ("porygon2", )
species = (session.query(v.PokemonSpecies).filter(
v.PokemonSpecies.identifier.notin_(invalid_identifiers)).order_by(
func.random()).first())
if not species:
raise SQLAlchemyError("Missing PokemonSpecies data")
# Get localized pokemon name
species_name = next(
(i.name for i in species.pokemon_species_name
if i.local_language_id == msg.language_id),
None,
)
if species_name is None:
raise SQLAlchemyError(
f"PokemonSpecies row {species.id}: no {msg.language} localization"
)
species_name = utils.remove_accents(species_name)
# Get pokedex flavor text
dex_entries = [
i.flavor_text for i in species.pokemon_species_flavor_text
if i.language_id == msg.language_id and len(i.flavor_text) <= 150
]
if not dex_entries: # This might fail but practically it never should
return
dex_entry = random.choice(dex_entries)
# Hide pokemon name from flavor text
regexp = re.compile(re.escape(species_name), re.IGNORECASE)
dex_entry = regexp.sub("???", dex_entry)
# Flavor text strings usually have unneeded newlines
dex_entry = dex_entry.replace("\n", " ")
await msg.reply(f"/hangman create {species_name}, {dex_entry}",
escape=False)
def get_roster_stats(team,
season_end_year,
data_format='PER_GAME',
playoffs=False):
if playoffs:
period = 'playoffs'
else:
period = 'leagues'
selector = data_format.lower()
r = get(
f'https://widgets.sports-reference.com/wg.fcgi?css=1&site=bbr&url=%2F{period}%2FNBA_{season_end_year}_{selector}.html&div=div_{selector}_stats'
)
df = None
possible_teams = [team]
for s in TEAM_SETS:
if team in s:
possible_teams = s
if r.status_code == 200:
soup = BeautifulSoup(r.content, 'html.parser')
table = soup.find('table')
df2 = pd.read_html(str(table))[0]
for index, row in df2.iterrows():
if row['Tm'] in possible_teams:
if df is None:
df = pd.DataFrame(columns=list(row.index) + ['SEASON'])
row['SEASON'] = f'{season_end_year-1}-{str(season_end_year)[2:]}'
df = df.append(row)
df.rename(columns={
'Player': 'PLAYER',
'Age': 'AGE',
'Tm': 'TEAM',
'Pos': 'POS'
},
inplace=True)
df['PLAYER'] = df['PLAYER'].apply(
lambda name: remove_accents(name, team, season_end_year))
df = df.reset_index().drop(['Rk', 'index'], axis=1)
return df
def get_roster(team, season_end_year):
r = get(
f'https://www.basketball-reference.com/teams/{team}/{season_end_year}.html'
)
df = None
if r.status_code == 200:
soup = BeautifulSoup(r.content, 'html.parser')
table = soup.find('table')
df = pd.read_html(str(table))[0]
df.columns = [
'NUMBER', 'PLAYER', 'POS', 'HEIGHT', 'WEIGHT', 'BIRTH_DATE',
'NATIONALITY', 'EXPERIENCE', 'COLLEGE'
]
# remove rows with no player name (this was the issue above)
df = df[df['PLAYER'].notna()]
df['PLAYER'] = df['PLAYER'].apply(
lambda name: remove_accents(name, team, season_end_year))
# handle rows with empty fields but with a player name.
df['BIRTH_DATE'] = df['BIRTH_DATE'].apply(lambda x: pd.to_datetime(x)
if pd.notna(x) else pd.NaT)
df['NATIONALITY'] = df['NATIONALITY'].apply(lambda x: x.upper()
if pd.notna(x) else '')
return df
def _handle_keyword(bot, event, command, include_event_user=False):
"""handle keyword"""
if event.user.is_self:
return
_populate_keywords(bot, event)
users_in_chat = event.conv.users
"""check if synced room and syncing is enabled
if its a valid syncroom, get a list of all unique users across all rooms"""
if bot.get_config_option('syncing_enabled'):
syncouts = bot.get_config_option('sync_rooms') or []
for sync_room_list in syncouts:
if event.conv_id in sync_room_list:
for syncedroom in sync_room_list:
if event.conv_id not in syncedroom:
users_in_chat += bot.get_users_in_conversation(
syncedroom)
users_in_chat = list(set(users_in_chat)) # make unique
event_text = re.sub(r"\s+", " ", event.text)
event_text_lower = event.text.lower()
for user in users_in_chat:
chat_id = user.id_.chat_id
try:
if _internal.keywords[chat_id] and (
not chat_id in event.user.id_.chat_id
or include_event_user):
for phrase in _internal.keywords[chat_id]:
regexphrase = r"(^|\b| )" + re.escape(phrase) + r"($|\b)"
if re.search(regexphrase, event_text, re.IGNORECASE):
"""XXX: suppress alerts if it appears to be a valid mention to same user
logic condensed from the detection function in the mentions plugin, we may
miss some use-cases, but this should account for "most" of them"""
if 'plugins.mentions' in sys.modules:
_phrase_lower = phrase.lower()
_mention = "@" + _phrase_lower
if (
_mention + " "
) in event_text_lower or event_text_lower.endswith(
_mention):
user = bot.get_hangups_user(chat_id)
_normalised_full_name_lower = remove_accents(
user.full_name.lower())
if (_phrase_lower
in _normalised_full_name_lower
or _phrase_lower
in _normalised_full_name_lower.replace(
" ", "") or _phrase_lower
in _normalised_full_name_lower.replace(
" ", "_")):
# part of name mention: skip
logger.debug(
"subscription matched full name fragment {}, skipping"
.format(user.full_name))
continue
if bot.memory.exists(
['user_data', chat_id, "nickname"]):
_nickname = bot.memory.get_by_path(
['user_data', chat_id, "nickname"])
if _phrase_lower == _nickname.lower():
# nickname mention: skip
logger.debug(
"subscription matched exact nickname {}, skipping"
.format(_nickname))
continue
yield from _send_notification(bot, event, phrase, user)
except KeyError:
# User probably hasn't subscribed to anything
continue
def mention(bot, event, *args):
"""alert a @mentioned user"""
"""allow mentions to be disabled via global or per-conversation config"""
config_mentions_enabled = False if bot.get_config_suboption(
event.conv.id_, 'mentions.enabled') is False else True
if not config_mentions_enabled:
logger.info("mentions explicitly disabled by config for {}".format(
event.conv_id))
return
"""minimum length check for @mention"""
minimum_length = bot.get_config_suboption(event.conv_id,
'mentionminlength')
if not minimum_length:
minimum_length = 2
username = args[0].strip()
if len(username) < minimum_length:
logger.debug("@mention from {} ({}) too short (== '{}')".format(
event.user.full_name, event.user.id_.chat_id, username))
return
users_in_chat = event.conv.users
mention_chat_ids = []
"""sync room support"""
if bot.get_config_option('syncing_enabled'):
sync_room_list = bot.get_config_option('sync_rooms')
if sync_room_list:
"""scan through each room group"""
for rooms_group in sync_room_list:
if event.conv_id in rooms_group:
"""current conversation is part of a syncroom group, add "external" users"""
for syncedroom in rooms_group:
if event.conv_id is not syncedroom:
users_in_chat += bot.get_users_in_conversation(
syncedroom)
users_in_chat = list(set(users_in_chat)) # make unique
logger.debug(
"@mention in a syncroom: {} user(s) present".format(
len(users_in_chat)))
break
"""
/bot mention <fragment> test
"""
noisy_mention_test = False
if len(args) == 2 and args[1] == "test":
noisy_mention_test = True
initiator_has_dnd = _user_has_dnd(bot, event.user.id_.chat_id)
"""
quidproquo: users can only @mention if they themselves are @mentionable (i.e. have a 1-on-1 with the bot)
"""
conv_1on1_initiator = yield from bot.get_1to1(
event.user.id_.chat_id, context={'initiator_convid': event.conv_id})
if bot.get_config_option("mentionquidproquo"):
if conv_1on1_initiator:
if initiator_has_dnd:
logger.info("quidproquo: user {} ({}) has DND active".format(
event.user.full_name, event.user.id_.chat_id))
if noisy_mention_test or bot.get_config_suboption(
event.conv_id, 'mentionerrors'):
yield from bot.coro_send_message(
event.conv,
_("<b>{}</b>, you cannot @mention anyone until your DND status is toggled off."
).format(event.user.full_name))
return
else:
logger.debug("quidproquo: user {} ({}) has 1-on-1".format(
event.user.full_name, event.user.id_.chat_id))
else:
logger.info("quidproquo: user {} ({}) has no 1-on-1".format(
event.user.full_name, event.user.id_.chat_id))
if noisy_mention_test or bot.get_config_suboption(
event.conv_id, 'mentionerrors'):
yield from bot.coro_send_message(
event.conv,
_("<b>{}</b> cannot @mention anyone until they say something to me first."
).format(event.user.full_name))
return
"""track mention statistics"""
user_tracking = {
"mentioned": [],
"ignored": [],
"failed": {
"pushbullet": [],
"one2one": [],
}
}
"""
begin mentioning users as long as they exist in the current conversation...
"""
conversation_name = bot.conversations.get_name(event.conv)
logger.info("@mention '{}' in '{}' ({})".format(username,
conversation_name,
event.conv.id_))
username_lower = username.lower()
username_upper = username.upper()
"""is @all available globally/per-conversation/initiator?"""
if username_lower == "all":
if not bot.get_config_suboption(event.conv.id_, 'mentionall'):
"""global toggle is off/not set, check admins"""
logger.debug(
"@all in {}: disabled/unset global/per-conversation".format(
event.conv.id_))
admins_list = bot.get_config_suboption(event.conv_id, 'admins')
if event.user_id.chat_id not in admins_list:
"""initiator is not an admin, check whitelist"""
logger.debug("@all in {}: user {} ({}) is not admin".format(
event.conv.id_, event.user.full_name,
event.user.id_.chat_id))
all_whitelist = bot.get_config_suboption(
event.conv_id, 'mentionallwhitelist')
if all_whitelist is None or event.user_id.chat_id not in all_whitelist:
logger.warning("@all in {}: user {} ({}) blocked".format(
event.conv.id_, event.user.full_name,
event.user.id_.chat_id))
if conv_1on1_initiator:
yield from bot.coro_send_message(
conv_1on1_initiator,
_("You are not allowed to mention all users in <b>{}</b>"
).format(conversation_name))
if noisy_mention_test or bot.get_config_suboption(
event.conv_id, 'mentionerrors'):
yield from bot.coro_send_message(
event.conv,
_("<b>{}</b> blocked from mentioning all users").
format(event.user.full_name))
return
else:
logger.info(
"@all in {}: allowed, {} ({}) is whitelisted".format(
event.conv.id_, event.user.full_name,
event.user.id_.chat_id))
else:
logger.info("@all in {}: allowed, {} ({}) is an admin".format(
event.conv.id_, event.user.full_name,
event.user.id_.chat_id))
else:
logger.info("@all in {}: enabled global/per-conversation".format(
event.conv.id_))
"""generate a list of users to be @mentioned"""
exact_nickname_matches = []
exact_fragment_matches = []
mention_list = []
for u in users_in_chat:
# mentions also checks nicknames if one is configured
# exact matches only! see following IF block
nickname = ""
nickname_lower = ""
if bot.memory.exists(['user_data', u.id_.chat_id, "nickname"]):
nickname = bot.memory.get_by_path(
['user_data', u.id_.chat_id, "nickname"])
nickname_lower = nickname.lower()
_normalised_full_name_upper = remove_accents(u.full_name.upper())
if (username_lower == "all"
or username_lower in u.full_name.replace(" ", "").lower()
or username_upper in _normalised_full_name_upper.replace(
" ", "")
or username_lower in u.full_name.replace(" ", "_").lower()
or username_upper in _normalised_full_name_upper.replace(
" ", "_") or username_lower == nickname_lower
or username in u.full_name.split(" ")):
logger.info("user {} ({}) is present".format(
u.full_name, u.id_.chat_id))
if u.is_self:
"""bot cannot be @mentioned"""
logger.debug("suppressing bot mention by {} ({})".format(
event.user.full_name, event.user.id_.chat_id))
continue
if u.id_.chat_id == event.user.id_.chat_id and username_lower == "all":
"""prevent initiating user from receiving duplicate @all"""
logger.debug("suppressing @all for {} ({})".format(
event.user.full_name, event.user.id_.chat_id))
continue
if u.id_.chat_id == event.user.id_.chat_id and not noisy_mention_test:
"""prevent initiating user from mentioning themselves"""
logger.debug("suppressing @self for {} ({})".format(
event.user.full_name, event.user.id_.chat_id))
continue
if u.id_.chat_id in mention_chat_ids:
"""prevent most duplicate mentions (in the case of syncouts)"""
logger.debug(
"suppressing duplicate mention for {} ({})".format(
event.user.full_name, event.user.id_.chat_id))
continue
if bot.memory.exists(["donotdisturb"]):
if _user_has_dnd(bot, u.id_.chat_id):
logger.info("suppressing @mention for {} ({})".format(
u.full_name, u.id_.chat_id))
user_tracking["ignored"].append(u.full_name)
continue
if username_lower == nickname_lower:
if u not in exact_nickname_matches:
exact_nickname_matches.append(u)
if (username in u.full_name.split(" ") or username_upper
in _normalised_full_name_upper.split(" ")):
if u not in exact_fragment_matches:
exact_fragment_matches.append(u)
if u not in mention_list:
mention_list.append(u)
if len(exact_nickname_matches) == 1:
"""prioritise exact nickname matches"""
logger.info("prioritising nickname match for {}".format(
exact_nickname_matches[0].full_name))
mention_list = exact_nickname_matches
elif len(exact_fragment_matches) == 1:
"""prioritise case-sensitive fragment matches"""
logger.info(
"prioritising single case-sensitive fragment match for {}".format(
exact_fragment_matches[0].full_name))
mention_list = exact_fragment_matches
elif len(exact_fragment_matches) > 1 and len(exact_fragment_matches) < len(
mention_list):
logger.info(
"prioritising multiple case-sensitive fragment match for {}".
format(exact_fragment_matches[0].full_name))
mention_list = exact_fragment_matches
if len(mention_list) > 1 and username_lower != "all":
send_multiple_user_message = True
if bot.memory.exists([
'user_data', event.user.id_.chat_id,
"mentionmultipleusermessage"
]):
send_multiple_user_message = bot.memory.get_by_path([
'user_data', event.user.id_.chat_id,
"mentionmultipleusermessage"
])
if send_multiple_user_message or noisy_mention_test:
if conv_1on1_initiator:
text_html = _(
'{} users would be mentioned with "@{}"! Be more specific. List of matching users:<br />'
).format(len(mention_list), username, conversation_name)
for u in mention_list:
text_html += u.full_name
if bot.memory.exists(
['user_data', u.id_.chat_id, "nickname"]):
text_html += ' (' + bot.memory.get_by_path(
['user_data', u.id_.chat_id, "nickname"]) + ')'
text_html += '<br />'
text_html += "<br /><em>To toggle this message on/off, use <b>/bot bemorespecific</b></em>"
yield from bot.coro_send_message(conv_1on1_initiator,
text_html)
logger.warning(
"@{} not sent due to multiple recipients".format(username_lower))
return #SHORT-CIRCUIT
"""support for reprocessor
override the source name by defining event._external_source"""
source_name = event.user.full_name
if hasattr(event, '_external_source'):
source_name = event._external_source
"""send @mention alerts"""
for u in mention_list:
alert_via_1on1 = True
"""pushbullet integration"""
if bot.memory.exists(['user_data', u.id_.chat_id, "pushbullet"]):
pushbullet_config = bot.memory.get_by_path(
['user_data', u.id_.chat_id, "pushbullet"])
if pushbullet_config is not None:
if pushbullet_config["api"] is not None:
success = False
try:
pb = PushBullet(pushbullet_config["api"])
push = pb.push_link(
title=_("{} mentioned you in {}").format(
source_name, conversation_name),
body=event.text,
url='https://hangouts.google.com/chat/{}'.format(
event.conv.id_))
if isinstance(push, tuple):
# backward-compatibility for pushbullet library < 0.8.0
success = push[0]
elif isinstance(push, dict):
success = True
else:
raise TypeError(
"unknown return from pushbullet library: {}".
format(push))
except Exception as e:
logger.exception("pushbullet error")
if success:
user_tracking["mentioned"].append(u.full_name)
logger.info("{} ({}) alerted via pushbullet".format(
u.full_name, u.id_.chat_id))
alert_via_1on1 = False # disable 1on1 alert
else:
user_tracking["failed"]["pushbullet"].append(
u.full_name)
logger.warning(
"pushbullet alert failed for {} ({})".format(
u.full_name, u.id_.chat_id))
if alert_via_1on1:
"""send alert with 1on1 conversation"""
conv_1on1 = yield from bot.get_1to1(
u.id_.chat_id, context={'initiator_convid': event.conv_id})
if username_lower == "all":
message_mentioned = _(
"<b>{}</b> @mentioned ALL in <i>{}</i>:<br />{}")
else:
message_mentioned = _(
"<b>{}</b> @mentioned you in <i>{}</i>:<br />{}")
if conv_1on1:
yield from bot.coro_send_message(
conv_1on1,
message_mentioned.format(source_name, conversation_name,
event.text)
) # prevent internal parser from removing <tags>
mention_chat_ids.append(u.id_.chat_id)
user_tracking["mentioned"].append(u.full_name)
logger.info("{} ({}) alerted via 1on1 ({})".format(
u.full_name, u.id_.chat_id, conv_1on1.id_))
else:
user_tracking["failed"]["one2one"].append(u.full_name)
if bot.get_config_suboption(event.conv_id, 'mentionerrors'):
yield from bot.coro_send_message(
event.conv,
_("@mention didn't work for <b>{}</b>. User must say something to me first."
).format(u.full_name))
logger.warning(
"user {} ({}) could not be alerted via 1on1".format(
u.full_name, u.id_.chat_id))
if noisy_mention_test:
text_html = _("<b>@mentions:</b><br />")
if len(user_tracking["failed"]["one2one"]) > 0:
text_html = text_html + _("1-to-1 fail: <i>{}</i><br />").format(
", ".join(user_tracking["failed"]["one2one"]))
if len(user_tracking["failed"]["pushbullet"]) > 0:
text_html = text_html + _(
"PushBullet fail: <i>{}</i><br />").format(", ".join(
user_tracking["failed"]["pushbullet"]))
if len(user_tracking["ignored"]) > 0:
text_html = text_html + _("Ignored (DND): <i>{}</i><br />").format(
", ".join(user_tracking["ignored"]))
if len(user_tracking["mentioned"]) > 0:
text_html = text_html + _("Alerted: <i>{}</i><br />").format(
", ".join(user_tracking["mentioned"]))
else:
text_html = text_html + _(
"Nobody was successfully @mentioned ;-(<br />")
if len(user_tracking["failed"]["one2one"]) > 0:
text_html = text_html + _(
"Users failing 1-to-1 need to say something to me privately first.<br />"
)
yield from bot.coro_send_message(event.conv, text_html)
def mention(bot, event, *args):
"""alert a @mentioned user"""
"""allow mentions to be disabled via global or per-conversation config"""
config_mentions_enabled = False if bot.get_config_suboption(event.conv.id_, 'mentions.enabled') is False else True
if not config_mentions_enabled:
logger.info("mentions explicitly disabled by config for {}".format(event.conv_id))
return
"""minimum length check for @mention"""
minimum_length = bot.get_config_suboption(event.conv_id, 'mentionminlength')
if not minimum_length:
minimum_length = 2
username = args[0].strip()
if len(username) < minimum_length:
logger.debug("@mention from {} ({}) too short (== '{}')".format(event.user.full_name, event.user.id_.chat_id, username))
return
users_in_chat = event.conv.users
mention_chat_ids = []
"""sync room support"""
if bot.get_config_option('syncing_enabled'):
sync_room_list = bot.get_config_option('sync_rooms')
if sync_room_list:
"""scan through each room group"""
for rooms_group in sync_room_list:
if event.conv_id in rooms_group:
"""current conversation is part of a syncroom group, add "external" users"""
for syncedroom in rooms_group:
if event.conv_id is not syncedroom:
users_in_chat += bot.get_users_in_conversation(syncedroom)
users_in_chat = list(set(users_in_chat)) # make unique
logger.debug("@mention in a syncroom: {} user(s) present".format(len(users_in_chat)))
break
"""
/bot mention <fragment> test
"""
noisy_mention_test = False
if len(args) == 2 and args[1] == "test":
noisy_mention_test = True
initiator_has_dnd = _user_has_dnd(bot, event.user.id_.chat_id)
"""
quidproquo: users can only @mention if they themselves are @mentionable (i.e. have a 1-on-1 with the bot)
"""
conv_1on1_initiator = yield from bot.get_1to1(event.user.id_.chat_id, context={ 'initiator_convid': event.conv_id })
if bot.get_config_option("mentionquidproquo"):
if conv_1on1_initiator:
if initiator_has_dnd:
logger.info("quidproquo: user {} ({}) has DND active".format(event.user.full_name, event.user.id_.chat_id))
if noisy_mention_test or bot.get_config_suboption(event.conv_id, 'mentionerrors'):
yield from bot.coro_send_message(
event.conv,
_("<b>{}</b>, you cannot @mention anyone until your DND status is toggled off.").format(
event.user.full_name))
return
else:
logger.debug("quidproquo: user {} ({}) has 1-on-1".format(event.user.full_name, event.user.id_.chat_id))
else:
logger.info("quidproquo: user {} ({}) has no 1-on-1".format(event.user.full_name, event.user.id_.chat_id))
if noisy_mention_test or bot.get_config_suboption(event.conv_id, 'mentionerrors'):
yield from bot.coro_send_message(
event.conv,
_("<b>{}</b> cannot @mention anyone until they say something to me first.").format(
event.user.full_name))
return
"""track mention statistics"""
user_tracking = {
"mentioned":[],
"ignored":[],
"failed": {
"pushbullet": [],
"one2one": [],
}
}
"""
begin mentioning users as long as they exist in the current conversation...
"""
conversation_name = bot.conversations.get_name(event.conv)
logger.info("@mention '{}' in '{}' ({})".format(username, conversation_name, event.conv.id_))
username_lower = username.lower()
username_upper = username.upper()
"""is @all available globally/per-conversation/initiator?"""
if username_lower == "all":
if not bot.get_config_suboption(event.conv.id_, 'mentionall'):
"""global toggle is off/not set, check admins"""
logger.debug("@all in {}: disabled/unset global/per-conversation".format(event.conv.id_))
admins_list = bot.get_config_suboption(event.conv_id, 'admins')
if event.user_id.chat_id not in admins_list:
"""initiator is not an admin, check whitelist"""
logger.debug("@all in {}: user {} ({}) is not admin".format(event.conv.id_, event.user.full_name, event.user.id_.chat_id))
all_whitelist = bot.get_config_suboption(event.conv_id, 'mentionallwhitelist')
if all_whitelist is None or event.user_id.chat_id not in all_whitelist:
logger.warning("@all in {}: user {} ({}) blocked".format(event.conv.id_, event.user.full_name, event.user.id_.chat_id))
if conv_1on1_initiator:
yield from bot.coro_send_message(
conv_1on1_initiator,
_("You are not allowed to mention all users in <b>{}</b>").format(
conversation_name))
if noisy_mention_test or bot.get_config_suboption(event.conv_id, 'mentionerrors'):
yield from bot.coro_send_message(
event.conv,
_("<b>{}</b> blocked from mentioning all users").format(
event.user.full_name))
return
else:
logger.info("@all in {}: allowed, {} ({}) is whitelisted".format(event.conv.id_, event.user.full_name, event.user.id_.chat_id))
else:
logger.info("@all in {}: allowed, {} ({}) is an admin".format(event.conv.id_, event.user.full_name, event.user.id_.chat_id))
else:
logger.info("@all in {}: enabled global/per-conversation".format(event.conv.id_))
"""generate a list of users to be @mentioned"""
exact_nickname_matches = []
exact_fragment_matches = []
mention_list = []
for u in users_in_chat:
# mentions also checks nicknames if one is configured
# exact matches only! see following IF block
nickname = ""
nickname_lower = ""
if bot.memory.exists(['user_data', u.id_.chat_id, "nickname"]):
nickname = bot.memory.get_by_path(['user_data', u.id_.chat_id, "nickname"])
nickname_lower = nickname.lower()
_normalised_full_name_upper = remove_accents(u.full_name.upper())
if (username_lower == "all" or
username_lower in u.full_name.replace(" ", "").lower() or
username_upper in _normalised_full_name_upper.replace(" ", "") or
username_lower in u.full_name.replace(" ", "_").lower() or
username_upper in _normalised_full_name_upper.replace(" ", "_") or
username_lower == nickname_lower or
username in u.full_name.split(" ")):
logger.info("user {} ({}) is present".format(u.full_name, u.id_.chat_id))
if u.is_self:
"""bot cannot be @mentioned"""
logger.debug("suppressing bot mention by {} ({})".format(event.user.full_name, event.user.id_.chat_id))
continue
if u.id_.chat_id == event.user.id_.chat_id and username_lower == "all":
"""prevent initiating user from receiving duplicate @all"""
logger.debug("suppressing @all for {} ({})".format(event.user.full_name, event.user.id_.chat_id))
continue
if u.id_.chat_id == event.user.id_.chat_id and not noisy_mention_test:
"""prevent initiating user from mentioning themselves"""
logger.debug("suppressing @self for {} ({})".format(event.user.full_name, event.user.id_.chat_id))
continue
if u.id_.chat_id in mention_chat_ids:
"""prevent most duplicate mentions (in the case of syncouts)"""
logger.debug("suppressing duplicate mention for {} ({})".format(event.user.full_name, event.user.id_.chat_id))
continue
if bot.memory.exists(["donotdisturb"]):
if _user_has_dnd(bot, u.id_.chat_id):
logger.info("suppressing @mention for {} ({})".format(u.full_name, u.id_.chat_id))
user_tracking["ignored"].append(u.full_name)
continue
if username_lower == nickname_lower:
if u not in exact_nickname_matches:
exact_nickname_matches.append(u)
if (username in u.full_name.split(" ") or
username_upper in _normalised_full_name_upper.split(" ")):
if u not in exact_fragment_matches:
exact_fragment_matches.append(u)
if u not in mention_list:
mention_list.append(u)
if len(exact_nickname_matches) == 1:
"""prioritise exact nickname matches"""
logger.info("prioritising nickname match for {}".format(exact_nickname_matches[0].full_name))
mention_list = exact_nickname_matches
elif len(exact_fragment_matches) == 1:
"""prioritise case-sensitive fragment matches"""
logger.info("prioritising single case-sensitive fragment match for {}".format(exact_fragment_matches[0].full_name))
mention_list = exact_fragment_matches
elif len(exact_fragment_matches) > 1 and len(exact_fragment_matches) < len(mention_list):
logger.info("prioritising multiple case-sensitive fragment match for {}".format(exact_fragment_matches[0].full_name))
mention_list = exact_fragment_matches
if len(mention_list) > 1 and username_lower != "all":
send_multiple_user_message = True
if bot.memory.exists(['user_data', event.user.id_.chat_id, "mentionmultipleusermessage"]):
send_multiple_user_message = bot.memory.get_by_path(['user_data', event.user.id_.chat_id, "mentionmultipleusermessage"])
if send_multiple_user_message or noisy_mention_test:
if conv_1on1_initiator:
text_html = _('{} users would be mentioned with "@{}"! Be more specific. List of matching users:<br />').format(
len(mention_list), username, conversation_name)
for u in mention_list:
text_html += u.full_name
if bot.memory.exists(['user_data', u.id_.chat_id, "nickname"]):
text_html += ' (' + bot.memory.get_by_path(['user_data', u.id_.chat_id, "nickname"]) + ')'
text_html += '<br />'
text_html += "<br /><em>To toggle this message on/off, use <b>/bot bemorespecific</b></em>"
yield from bot.coro_send_message(conv_1on1_initiator, text_html)
logger.warning("@{} not sent due to multiple recipients".format(username_lower))
return #SHORT-CIRCUIT
"""support for reprocessor
override the source name by defining event._external_source"""
source_name = event.user.full_name
if hasattr(event, '_external_source'):
source_name = event._external_source
"""send @mention alerts"""
for u in mention_list:
alert_via_1on1 = True
"""pushbullet integration"""
if bot.memory.exists(['user_data', u.id_.chat_id, "pushbullet"]):
pushbullet_config = bot.memory.get_by_path(['user_data', u.id_.chat_id, "pushbullet"])
if pushbullet_config is not None:
if pushbullet_config["api"] is not None:
success = False
try:
pb = PushBullet(pushbullet_config["api"])
push = pb.push_link(
title = _("{} mentioned you in {}").format(source_name, conversation_name),
body=event.text,
url='https://hangouts.google.com/chat/{}'.format(event.conv.id_) )
if isinstance(push, tuple):
# backward-compatibility for pushbullet library < 0.8.0
success = push[0]
elif isinstance(push, dict):
success = True
else:
raise TypeError("unknown return from pushbullet library: {}".format(push))
except Exception as e:
logger.exception("pushbullet error")
if success:
user_tracking["mentioned"].append(u.full_name)
logger.info("{} ({}) alerted via pushbullet".format(u.full_name, u.id_.chat_id))
alert_via_1on1 = False # disable 1on1 alert
else:
user_tracking["failed"]["pushbullet"].append(u.full_name)
logger.warning("pushbullet alert failed for {} ({})".format(u.full_name, u.id_.chat_id))
if alert_via_1on1:
"""send alert with 1on1 conversation"""
conv_1on1 = yield from bot.get_1to1(u.id_.chat_id, context={ 'initiator_convid': event.conv_id })
if username_lower == "all":
message_mentioned = _("<b>{}</b> @mentioned ALL in <i>{}</i>:<br />{}")
else:
message_mentioned = _("<b>{}</b> @mentioned you in <i>{}</i>:<br />{}")
if conv_1on1:
yield from bot.coro_send_message(
conv_1on1,
message_mentioned.format(
source_name,
conversation_name,
event.text)) # prevent internal parser from removing <tags>
mention_chat_ids.append(u.id_.chat_id)
user_tracking["mentioned"].append(u.full_name)
logger.info("{} ({}) alerted via 1on1 ({})".format(u.full_name, u.id_.chat_id, conv_1on1.id_))
else:
user_tracking["failed"]["one2one"].append(u.full_name)
if bot.get_config_suboption(event.conv_id, 'mentionerrors'):
yield from bot.coro_send_message(
event.conv,
_("@mention didn't work for <b>{}</b>. User must say something to me first.").format(
u.full_name))
logger.warning("user {} ({}) could not be alerted via 1on1".format(u.full_name, u.id_.chat_id))
if noisy_mention_test:
text_html = _("<b>@mentions:</b><br />")
if len(user_tracking["failed"]["one2one"]) > 0:
text_html = text_html + _("1-to-1 fail: <i>{}</i><br />").format(", ".join(user_tracking["failed"]["one2one"]))
if len(user_tracking["failed"]["pushbullet"]) > 0:
text_html = text_html + _("PushBullet fail: <i>{}</i><br />").format(", ".join(user_tracking["failed"]["pushbullet"]))
if len(user_tracking["ignored"]) > 0:
text_html = text_html + _("Ignored (DND): <i>{}</i><br />").format(", ".join(user_tracking["ignored"]))
if len(user_tracking["mentioned"]) > 0:
text_html = text_html + _("Alerted: <i>{}</i><br />").format(", ".join(user_tracking["mentioned"]))
else:
text_html = text_html + _("Nobody was successfully @mentioned ;-(<br />")
if len(user_tracking["failed"]["one2one"]) > 0:
text_html = text_html + _("Users failing 1-to-1 need to say something to me privately first.<br />")
yield from bot.coro_send_message(event.conv, text_html)
def xacdinh(dict_input: dict) -> float:
"""Phân biệt cái gì đó ahihi, mọi người sửa lại dòng này cho đúng nhé.
Arguments:
inp_str {dict} -- dict chứa 3 thông tin sau:
id : id của bài đăng
content : content của bài đăng
realestate_type : loại bất động sản
floor : số tầng
Returns:
float - gía trị của biến area_cal
Như trong doc thì có 3 loại diện tích:
"""
area_cal = 0
## code goes here
removed_accent_str = remove_accents(dict_input["content"])
copy_removed_accent_str = removed_accent_str
copy_removed_accent_str = re.sub(r"\d*\s?tang","",removed_accent_str)
removed_accent_str = re.sub("m2","",removed_accent_str)
removed_accent_str = re.sub(r"(\d*\s?tang)","",removed_accent_str)
# #print(removed_accent_str)
#Patterns
dtcn_patterns = re.compile(r'\b(dtcn|dien tich cong nhan|dt cong nhan|cong nhan|cn|dien tich so|so do|cong nhan)\:?', re.IGNORECASE)
dt_patterns = re.compile(r'\b(dien tich|dt|dien tich dat|dt dat)\s?\:?')
dt_patterns_2 = re.compile(r'\d+[.,]?\d*\s?(m2|m)?\s?[xX*]\s?\d+[.,]?\d*', re.IGNORECASE)
dtxd_patterns = re.compile(r'\b(dien tich xay dung|dien tich san|dt san|dt xay dung|dtxd|dtsd)\:?', re.IGNORECASE)
dtxd_patterns_2 = re.compile(r'\d+[.,]?\d*\s?(m2|m)?\s?[xX*]\s?\d+[.,]?\d*', re.IGNORECASE)
hectare_patterns = re.compile(r'ha', re.IGNORECASE)
dt_area_patterns = re.compile(r'\s?\d+[.,]?\d*\s?[^a-z0-9*.()\-,]?')
m2_patterns = re.compile(r'\d+[.,]?\d*\s?(m2|m)')
m2_patterns_2 = re.compile(r'\d+[.,]?\d*\s?m2')
m2_area_patterns = re.compile(r'\d+[.,]?\d*')
ngang_dai_patterns = re.compile(r'ngang\s?\d+[.,]?\d*[m]?\s?[*x,]?\s?dai\s?\d+[.,]?\d*')
dtxd_area_patterns = re.compile(r'\d+[.,]?\d*')
dtcn_area_patterns = re.compile(r'\d+[.,]?\d*')
# Check for dien tich cong nhan
if dtcn_patterns.search(removed_accent_str):
#print('dtcn!!!')
dtcn_index = dtcn_patterns.search(removed_accent_str).span()
# #print(str(removed_accent_str[dtcn_index[0]:]) + " ")
dtcn_string = removed_accent_str[dtcn_index[0] : dtcn_index[1] + 20]
# #print(dtcn_area_patterns.search(dtcn_string))
if (dtcn_area_patterns.search(dtcn_string)):
dtcn_area_index = dtcn_area_patterns.search(dtcn_string).span()
dtcn_area = dtcn_string[dtcn_area_index[0] : dtcn_area_index[1]]
dtcn_area = re.sub(',', '.', dtcn_area)
area_cal = float(dtcn_area)
if dt_patterns_2.search(dtcn_string):
dt_area_index = dt_patterns_2.search(dtcn_string).span()
dims = dt_area_patterns.findall(dtcn_string[dt_area_index[0]:dt_area_index[1]+3])
#print(dtcn_string[dt_area_index[0]:dt_area_index[1]+3])
# #print(dims)
area = 1
for dim in dims:
dim = re.sub(',', '.', dim)
#print(dim)
area *= float(dim)
area_cal = area
#print(area_cal)
# Check for dien tich/ dien tich dat/ dien tich xay dung
if area_cal == 0 and dt_patterns.search(removed_accent_str):
#print('dt/ dt dat!!!!\n')
# #print(dt_patterns.search(removed_accent_str))
dt_index = dt_patterns.search(removed_accent_str).span()
dt_string = removed_accent_str[dt_index[0]:]
# #print(dt_string)
#############OLD METHOD
# if dt_area_patterns.search(dt_string):
# dt_area_index = dt_area_patterns.search(dt_string).span()
# dt_area = dt_string[dt_area_index[0] : dt_area_index[1]]
# #print(dt_area)
# #print("*")
# dt_area = re.sub(',', '.', dt_area)
# # #print(dt_area)
# area_cal = float(dt_area)
# if hectare_patterns.search(dt_string[dt_area_index[0] : dt_area_index[1]+3]):
# area_cal *= 100
##############ARRAY METHOD
if m2_patterns_2.search(copy_removed_accent_str):
area = 1
#print(copy_removed_accent_str)
areas = m2_patterns_2.finditer(copy_removed_accent_str)
area_array = []
i = 0
for area in areas:
index = area.span()
area_array.append(copy_removed_accent_str[index[0]: index[1]])
area_array[i] = re.sub('m2','',area_array[i])
area_array[i] = re.sub('m','',area_array[i])
area_array[i] = re.sub(',','.',area_array[i])
area_array[i] = float(area_array[i])
#print(area_array[i])
i += 1
#print("******")
area_cal = min(area_array)
#print(area_cal)
#print('*******')
if dt_patterns_2.search(dt_string):
dt_area_index = dt_patterns_2.search(dt_string).span()
dims = dt_area_patterns.findall(dt_string[dt_area_index[0]:dt_area_index[1]+3])
#print(dt_string[dt_area_index[0]:dt_area_index[1]+3])
# #print(dims)
area = 1
for dim in dims:
dim = re.sub(',', '.', dim)
#print(dim)
area *= float(dim)
area_cal = area
#print(area_cal)
if area_cal == 0:
##Check for dien tich xay dung
if dtxd_patterns.search(removed_accent_str):
#print('dt xay dung!!!')
dtxd_index = dtxd_patterns.search(removed_accent_str).span()
dtxd_string = removed_accent_str[dtxd_index[0]:]
#print(dtxd_string)
dtxd_area_index = dtxd_patterns_2.search(dtxd_string).span()
dims = dtxd_area_patterns.findall(dtxd_string[dtxd_area_index[0]:dtxd_area_index[1]])
#print(dtxd_string[dtxd_area_index[0]:dtxd_area_index[1]])
#print(dims)
area = 1
for dim in dims:
dim = re.sub(',', '.', dim)
#print(dim)
area *= float(dim)
#print(area_cal)
##Last resort
if (area_cal == 0):
#print('Last resort!')
#print(copy_removed_accent_str)
if m2_patterns.search(copy_removed_accent_str):
m2_index = m2_patterns.search(copy_removed_accent_str).span()
m2_string = copy_removed_accent_str[m2_index[0] : m2_index[1]]
#print(m2_string)
#print('*')
# #print(copy_removed_accent_str)
m2_area_patterns_index = m2_area_patterns.search(m2_string).span()
m2_area = m2_string[m2_area_patterns_index[0]:m2_area_patterns_index[1]]
m2_area = re.sub(',', '.', m2_area)
area_cal = float(m2_area)
#print(area_cal)
if dt_patterns_2.search(removed_accent_str):
dt_area_index = dt_patterns_2.search(removed_accent_str).span()
dims = dt_area_patterns.findall(removed_accent_str[dt_area_index[0]:dt_area_index[1]+3])
#print(removed_accent_str[dt_area_index[0]:dt_area_index[1]+3])
# #print(dims)
area = 1
for dim in dims:
dim = re.sub(',', '.', dim)
#print(dim)
area *= float(dim)
area_cal = area
elif ngang_dai_patterns.search(removed_accent_str):
#print('Ngang dai')
dt_area_index = ngang_dai_patterns.search(removed_accent_str).span()
dims = dt_area_patterns.findall(removed_accent_str[dt_area_index[0]:dt_area_index[1]+3])
#print(removed_accent_str[dt_area_index[0]:dt_area_index[1]+3])
# #print(dims)
area = 1
for dim in dims:
dim = re.sub(',', '.', dim)
#print(dim)
area *= float(dim)
area_cal = area
#print(area_cal)
############################
## Return
############################
return area_cal
def _format_name(name):
return re.sub(r"[^0-9a-z]+", "", utils.remove_accents(name).lower())
def load_stop_words(self):
stop_words = [
utils.remove_accents(w.decode('utf-8'))
for w in nltk.corpus.stopwords.words(
'spanish')]
return stop_words
async def learnset(conn: Connection, room: Optional[str], user: str, arg: str) -> None:
args = arg.split(",")
if len(args) < 2:
return
pokemon = utils.to_user_id(utils.remove_accents(args[0].lower()))
version_group = utils.to_user_id(utils.remove_accents(args[1].lower()))
db = Database("veekun")
sql = "SELECT id FROM version_groups WHERE identifier = ?"
version_group_id = db.execute(sql, [version_group]).fetchone()
if version_group_id is None:
sql = "SELECT version_group_id FROM versions WHERE identifier = ?"
version_group_id = db.execute(sql, [version_group]).fetchone()
if version_group_id is None:
return
version_group_id = version_group_id[0]
sql = """SELECT pokemon_moves.version_group_id, pokemon_moves.pokemon_move_method_id,
(SELECT GROUP_CONCAT(IFNULL(version_names.name, ''), '/')
FROM versions
LEFT JOIN version_names ON version_names.version_id = versions.id AND version_names.local_language_id = 9
WHERE versions.version_group_id = pokemon_moves.version_group_id
ORDER BY versions.id) AS version_group,
IFNULL(move_names.name, '') AS move_name,
IFNULL(pokemon_move_method_prose.name, '') AS method_name,
IFNULL(pokemon_moves.level, 0) AS level, IFNULL(item_names.name, '') AS machine
FROM pokemon_species
LEFT JOIN pokemon ON pokemon.species_id = pokemon_species.id
LEFT JOIN pokemon_moves ON pokemon_moves.pokemon_id = pokemon.id
JOIN version_groups ON version_groups.id = pokemon_moves.version_group_id
JOIN moves ON moves.id = pokemon_moves.move_id
LEFT JOIN move_names ON move_names.move_id = moves.id AND move_names.local_language_id = 9
JOIN pokemon_move_methods ON pokemon_move_methods.id = pokemon_moves.pokemon_move_method_id
LEFT JOIN pokemon_move_method_prose ON pokemon_move_method_prose.pokemon_move_method_id = pokemon_move_methods.id AND pokemon_move_method_prose.local_language_id = 9
LEFT JOIN machines ON machines.move_id = moves.id AND pokemon_move_methods.id = 4 AND machines.version_group_id = version_groups.id
LEFT JOIN item_names ON item_names.item_id = machines.item_id AND item_names.local_language_id = 9
WHERE pokemon_species.identifier = ? AND version_groups.id = ?
ORDER BY pokemon_moves.pokemon_move_method_id, pokemon_moves.level, machines.machine_number, move_names.name"""
html = ""
current_move_method_id = 0
for row in db.execute(sql, [pokemon, version_group_id]):
if current_move_method_id != row["pokemon_move_method_id"]:
if current_move_method_id != 0:
html += "</tbody></table>"
html += "</details>"
html += (
"<details><summary><b><big>"
+ utils.html_escape(row["method_name"])
+ "</big></b></summary>"
)
html += '<table style="margin: 5px 0"><tbody>'
html += "<tr>"
html += " <th>Move</th>"
if row["pokemon_move_method_id"] == 1: # level-up
html += " <th>Level</th>"
elif row["pokemon_move_method_id"] == 2: # egg
pass
elif row["pokemon_move_method_id"] == 4: # machine
html += " <th>Machine</th>"
html += "</tr>"
current_move_method_id = row["pokemon_move_method_id"]
html += "<tr>"
html += " <td>" + utils.html_escape(row["move_name"]) + "</td>"
if current_move_method_id == 1: # level-up
html += (
' <td style="text-align: right">'
+ utils.html_escape(str(row["level"]))
+ "</td>"
)
elif current_move_method_id == 2: # egg
pass
elif current_move_method_id == 4: # machine
html += " <td>" + utils.html_escape(row["machine"]) + "</td>"
html += "</tr>"
if current_move_method_id != 0:
html += "</tbody></table>"
html += "</details>"
if not html:
await conn.send_reply(room, user, "Nessun dato")
return
await conn.send_htmlbox(room, user, '<div class="ladder">' + html + "</div>")
def _clean(s):
return ''.join(e for e in remove_accents(s).lower()
if e.isalnum() or e in ['-'])
async def learnset(msg: Message) -> None:
if len(msg.args) < 2:
return
pokemon_id = utils.to_user_id(utils.remove_accents(msg.args[0].lower()))
version = utils.to_user_id(utils.remove_accents(msg.args[1].lower()))
db = Database.open("veekun")
with db.get_session() as session:
version_group_id: Optional[int] = (
session.query(v.VersionGroups.id) # type: ignore # sqlalchemy
.filter_by(identifier=version)
.scalar()
)
if version_group_id is None:
version_group_id = (
session.query(v.Versions.version_group_id) # type: ignore # sqlalchemy
.filter_by(identifier=version)
.scalar()
)
if version_group_id is None:
return
class MovesDict(TypedDict):
name: str
level: Optional[int]
machine: Optional[str]
class ResultsDict(TypedDict):
name: str
moves: List[MovesDict]
results: Dict[int, ResultsDict] = {}
pokemon_species = (
session.query(v.PokemonSpecies) # type: ignore # sqlalchemy
.options(
joinedload(v.PokemonSpecies.pokemon)
.joinedload(v.Pokemon.pokemon_moves)
.joinedload(v.PokemonMoves.version_group)
.raiseload("*")
)
.filter_by(identifier=pokemon_id)
.first()
)
if pokemon_species:
for pokemon in pokemon_species.pokemon:
for pokemon_move in pokemon.pokemon_moves:
version_group = pokemon_move.version_group
if version_group.id != version_group_id:
continue
move = pokemon_move.move
move_name = next(
(i.name for i in move.move_names if i.local_language_id == 9),
"",
)
method = pokemon_move.pokemon_move_method
method_name = next(
(
i.name
for i in method.pokemon_move_method_prose
if i.local_language_id == 9
),
"",
)
data: MovesDict = {
"name": move_name,
"level": None,
"machine": None,
}
if method.id == 1: # level-up
data["level"] = pokemon_move.level
elif method.id == 4: # machine
machine = next(
(
i
for i in move.machines
if i.version_group_id == version_group.id
),
None,
)
if machine:
machine_name = next(
(
i.name
for i in machine.item.item_names
if i.local_language_id == 9
),
None,
)
data["machine"] = machine_name
if method.id not in results:
results[method.id] = {"name": method_name, "moves": []}
results[method.id]["moves"].append(data)
for method_id in sorted(results.keys()):
if method_id == 1: # level-up
results[method_id]["moves"].sort(key=lambda x: (x["level"], x["name"]))
elif method_id == 4: # machine
results[method_id]["moves"].sort(
key=lambda x: (x["machine"], x["name"])
)
else:
results[method_id]["moves"].sort(key=lambda x: x["name"])
html = utils.render_template(
"commands/learnsets.html", methods=sorted(results.keys()), results=results
)
if not html:
await msg.reply("Nessun dato")
return
await msg.reply_htmlbox('<div class="ladder">' + html + "</div>")
if __name__ == '__main__':
try:
target = sys.argv[1]
except IndexError:
print('You need to specify a target')
print('python3 compare_images.py <icon-name>')
sys.exit()
image_png = Image.open('./images/' + target + '.png')
matched_diff = sys.maxsize
match = ''
match_pos = None
for i, (captcha_name, captcha_image) in enumerate(
break_captcha(READ_CAPTCHA_PATH, target, image_png)):
for name, image in list_individual_images(PATH_EXISTENT):
if remove_accents(target) not in remove_accents(name):
continue
captcha_part = READ_CAPTCHA_PATH + captcha_name + '_' + str(
i) + '.png'
existent_image_path = PATH_EXISTENT + name
diff = compare_image(existent_image_path, captcha_part)
if diff < matched_diff:
matched_diff = diff
match = existent_image_path
match_pos = i
# print(matched_diff, match, match_pos)
print(match_pos)
