def dealWithTokensSelection(p, token_index=None):
# coming here form 30/31 (via add) or 40 (via /number)
sentence_word_tokens, sentence_tokens = sentAnno.getCurrentSentenceWordTokensSentenceTokens(
p)
#logging.debug('token_index: ' + str(token_index))
if token_index != None:
p.selectedTokens[token_index] = not p.selectedTokens[token_index]
text_msg = renderSentenceWithMweHighlights(p, sentence_word_tokens,
p.selectedTokens,
person.getMarkdownChar(p))
if sum(p.selectedTokens) < 2:
text_msg += "\n\n" + "Please select at least 2 tokens you want to include in the MWE."
kb = segmentTokenButtons(p, sentence_tokens, p.selectedTokens,
p.maxCharsPerLine)
logging.debug('in dealWithTokensSelection state: ' + str(p.state))
if sum(p.selectedTokens) > 1:
logging.debug('person.getSelectedTokensIndexTuple(p): ' +
str(person.getSelectedTokensIndexTuple(p)))
logging.debug('p.currentMweTmp: ' + str(p.currentMweTmp))
if p.currentMweTmp:
if person.getSelectedTokensIndexTuple(p) != tuple(p.currentMweTmp):
#via /number press
kb.append([CONFIRM_CHANNGES])
else:
# via + add
kb.append([CHOOSE_CAT_BUTTON])
kb.append([BACK_BUTTON])
tell(p.chat_id, text_msg, kb=kb, markdown=True)
person.setState(p, 30)
def changeConfidenceCurrentMWE(p, confidence):
currentSentenceIndex = person.getCurrentSentenceIndex(p)
indexTuple = person.getSelectedTokensIndexTuple(p)
sa = getOrInsertSentenceAnnotation(p.language, currentSentenceIndex,
p.chat_id)
sa.mwes[indexTuple]['CONFIDENCE'] = confidence
p.selectedMwes = sa.mwes # mirroring on person #
sa.put()
def askConfidenceLevel(p):
indexes = person.getSelectedTokensIndexTuple(p)
sentAnno.getCategoryAndConfidence(p, indexes)
mwe_flat = renderMwe(p, indexes, True, True)
text_msg = "You have selected the following MWE:\n" + mwe_flat + "\n\n"
text_msg += "Which NEW confidence level do you want to assign to it? Please enter a number between 1 and 10"
kb = util.makeArray2D(NUMBERS_0_10[1:], 5)
kb.append([BACK_BUTTON])
tell(p.chat_id, text_msg, kb, markdown=True)
person.setState(p, 70)
def dealWithChangeConfidenceLevel(p, newConf):
currentMweTuple = person.getSelectedTokensIndexTuple(p)
#currentMweTuple = tuple(p.currentMweTmp)
oldCat, oldConf = sentAnno.getCategoryAndConfidence(p, currentMweTuple)
sentAnno.changeConfidenceCurrentMWE(p, newConf) #mirroring on person
oldConfEmoji = NUMBERS_0_10[oldConf]
newConfEmoji = NUMBERS_0_10[newConf]
tell(p.chat_id,
CHECK + " Successfully changed MWE confidence from " + oldConfEmoji +
" to " + newConfEmoji,
markdown=True)
displayCurrentSentence(p)
def appendMwe(p, mweLabel, confidence=DEFAULT_CONFIDENCE):
currentSentenceIndex = person.getCurrentSentenceIndex(p)
sa = getOrInsertSentenceAnnotation(p.language, currentSentenceIndex,
p.chat_id)
if not sa.mwes:
sa.mwes = {}
indexTuple = person.getSelectedTokensIndexTuple(p)
if CONFIDENCE_ACTIVE:
sa.mwes[indexTuple] = {'CATEGORY': mweLabel, 'CONFIDENCE': confidence}
else:
sa.mwes[indexTuple] = {'CATEGORY': mweLabel}
p.selectedMwes = sa.mwes # mirroring on person #
sa.put()
def dealWithChangeMwe(p):
currentMweTmpTuple = tuple(p.currentMweTmp)
category, confidence = sentAnno.getCategoryAndConfidence(
p, currentMweTmpTuple)
mwe_flat_old = renderMwe(p, currentMweTmpTuple, True, False)
new_indexes = person.getSelectedTokensIndexTuple(p)
sentAnno.removeMwe(p, currentMweTmpTuple) #mirroring on person
#person.removeMwe(p, currentMweTmpTuple)
sentAnno.appendMwe(p, category) #mirroring on person
#person.appendMwe(p, category)
mwe_flat_new = renderMwe(p, new_indexes, True, False)
tell(p.chat_id,
CHECK + " Successfully changed MWE:\n" + OUT_BOX + ' ' +
mwe_flat_old + "\n" + IN_BOX + ' ' + mwe_flat_new + "\n",
markdown=True)
displayCurrentSentence(p)
def askAnnotationLabel(p):
indexes = person.getSelectedTokensIndexTuple(p)
logging.debug("in askAnnotationLabel. last state: " + str(p.last_state))
changeCategory = p.state == 40
mwe_flat = renderMwe(p, indexes, changeCategory, False)
text_msg = renderSentence(p) + "\n\n"
text_msg += "You have selected the following MWE:\n" + mwe_flat + "\n\n"
if changeCategory:
text_msg += "which NEW category do you want to assign to it?"
else:
text_msg += "which category do you want to assign to it?"
kb = util.segmentArrayOnMaxChars(categories.ANNOTATION_CATS,
p.maxCharsPerLine)
#kb = categories.ANNOTATION_CATS_SPLIT_ON_CHAR #sentences.ANNOTATION_CATS_3PL
kb.append([BACK_BUTTON])
tell(p.chat_id, text_msg, kb=kb, markdown=True)
person.setState(p, 31)
def post(self):
urlfetch.set_default_fetch_deadline(60)
body = json.loads(self.request.body)
logging.info('request body:')
logging.info(body)
self.response.write(json.dumps(body))
# update_id = body['update_id']
message = body['message']
#message_id = message.get('message_id')
# date = message.get('date')
if "chat" not in message:
return
# fr = message.get('from')
chat = message['chat']
chat_id = chat['id']
if "first_name" not in chat:
return
text = message.get('text').encode('utf-8') if "text" in message else ""
name = chat["first_name"].encode('utf-8')
last_name = chat["last_name"].encode(
'utf-8') if "last_name" in chat else "-"
username = chat["username"] if "username" in chat else "-"
#location = message["location"] if "location" in message else None
#logging.debug('location: ' + str(location))
def reply(msg=None, kb=None, hideKb=True, markdown=False):
tell(chat_id, msg, kb, hideKb, markdown)
p = ndb.Key(Person, str(chat_id)).get()
if p is None:
# new user
logging.info("Text: " + text)
if text == '/help':
reply(INSTRUCTIONS)
elif text.startswith("/start"):
tell_masters("New user: "******"Hi " + name + ", " +
"welcome to the Parseme Shared Task annotation!")
restart(p)
else:
reply("Something didn't work... please contact @kercos")
else:
# known user
person.updateUsername(p, username)
if text == '/state':
if p.state in STATES:
reply("You are in state " + str(p.state) + ": " +
STATES[p.state])
else:
reply("You are in state " + str(p.state))
elif text.startswith("/start"):
reply("Hi " + name + ", " +
"welcome back to the Parseme Shared Task annotation!")
person.reInitializePerson(p, chat_id, name, last_name,
username)
restart(p)
#-------------
# SETTINGS
#-------------
elif p.state == -1:
# settings
if text == BACK_BUTTON:
restart(p)
elif text == HIGHTLIGHTING_MODE_BUTTON:
goToSettingsHighlightingMode(p)
# state -11
elif text == BUTTONS_DENSITY_BUTTON:
goToSettingsButtonsDensity(p)
# state -12
else:
reply(FROWNING_FACE + " Sorry, I can't understand...")
elif p.state == -11:
# settings -> hightlighting mode
if text == BACK_BUTTON:
goToSetting(p)
elif text in person.MARKDOWN_OPTIONS.keys():
person.setHighlightingMode(p, text)
reply("Successfully set highlighting mode to " +
person.getHighlightedString(p, text),
markdown=True)
goToSetting(p)
#state -1
else:
reply(FROWNING_FACE + " Sorry, I can't understand...")
elif p.state == -12:
# settings -> buttons density
if text == BACK_BUTTON:
goToSetting(p)
elif util.RepresentsInt(text):
charPerLine = int(text)
tot_sentences = sentAnno.totalSentPersonCurrentLang(p)
if charPerLine >= MIN_CHARS_PER_LINE and charPerLine <= MAX_CHARS_PER_LINE:
person.setMaxCharsPerLine(p, charPerLine)
goToSettingsButtonsDensity(p)
# state -1
else:
reply(
"Sorry, not a valid index. Please enter an index between "
+ person.getHighlightedString(
p, str(MIN_CHARS_PER_LINE)) + " and " +
person.getHighlightedString(
p, str(MAX_CHARS_PER_LINE)) + ".")
else:
reply(FROWNING_FACE + " Sorry, I can't understand...")
# -------------
# INITIAL STATE
# -------------
elif p.state == 0:
# INITIAL STATE
if text in ['/help', HELP_BUTTON]:
reply(INSTRUCTIONS)
elif text == SETTINGS_BUTTON:
goToSetting(p)
#state -1
elif text == INFO_BUTTON:
textMsg = "Number of annotated sentences per language:\n"
for lang, flag in sentAnno.LANGUAGE_FLAGS.iteritems():
progress = str(person.getLanguageProgress(p, lang))
total = str(sentAnno.totalSentLang(lang))
textMsg += flag + ' ' + progress + '/' + total + '\n'
reply(textMsg)
elif text == ANNOTATE_BUTTON_EN:
person.setLanguage(p, 'ENGLISH')
person.importSelectedMwe(p)
displayCurrentSentence(p)
#state 1
elif text == ANNOTATE_BUTTON_IR:
person.setLanguage(p, 'FARSI')
person.importSelectedMwe(p)
displayCurrentSentence(p)
# state 1
elif chat_id in key.MASTER_CHAT_ID:
if text == '/test':
logging.debug('test')
#description = object.getObjectsById(4866725958909952)
#logging.debug('description: ' + description)
#logging.debug('type: ' + str(type(description)))
#reply(description)
#reply("Méthode de Français")
#deferred.defer(tell_fede, "Hello, world!")
elif text == '/testMarkdown':
reply(
"aaaa*xxxx*zzzzz\naaaa_xxxx_zzzz\naaaa`xxxx`zzzz",
markdown=True)
#reply("fdsadfa [dfsafd] dsfasdf a", markdown=True)
#reply("fdasfd *df\\*as* fa fsa", markdown=True)
#text = "*" + util.escapeMarkdown("`[8]") + "*"
#reply(text, markdown=True)
#logging.debug("reply text: " + text)
elif text == '/testBitMap':
#bitmaptest.testBitMap(100, 20)
bitmaptest.testBitMap(5000, 500)
reply('Successfully finished test (see log).')
elif text.startswith('/broadcast ') and len(text) > 11:
msg = text[11:] #.encode('utf-8')
deferred.defer(broadcast, msg)
elif text == '/restartAllUsers':
deferred.defer(
restartAllUsers,
"ParsemeBot v.2 is out! " + GRINNING_FACE)
elif text == '/resetAllUsers':
c = resetAllUsers()
reply("Resetted users: " + str(c))
else:
reply(FROWNING_FACE + " Sorry, I can't understand...")
#setLanguage(d.language)
else:
reply(FROWNING_FACE +
"Sorry I didn't understand you ...\n" +
"Press the HELP button to get more info.")
# --------------------------
# DISPLAY CURRENT SENTENCE
# --------------------------
elif p.state == 1:
# display current sentence
if text == BACK_BUTTON:
restart(p)
elif text == '/progress':
reply("Annotated sentence(s) for " + p.language + ': ' +
str(person.getLanguageProgress(p)) + '/' +
str(sentAnno.totalSentPersonCurrentLang(p)))
elif text == '/nextna':
index = person.getNextNonAnnSentIndex(p)
if index:
jumpToSentence(p, index + 1)
else:
reply(EXCLAMATION + " No next non-annotated sentence")
elif text == '/prevna':
index = person.getPrevNonAnnSentIndex(p)
if index:
jumpToSentence(p, index + 1)
else:
reply(EXCLAMATION +
" No previous non-annotated sentence")
elif text == ADD_MWE_BUTTON:
prepareToSelectTokens(p)
#state 30
elif text == NO_MWE_BUTTON and p.selectedMwes == None:
reply(
CHECK +
" The current sentence has been marked without MWEs.")
sentAnno.setEmptyMwes(p) #mirroring on person
#person.setEmptyMwes(p)
person.setCurrentSentenceAnnotated(p)
displayCurrentSentence(p)
# state 1
elif text == MOD_MWE_BUTTON and p.selectedMwes != None:
# reply(UNDER_CONSTRUCTION + " currently working on this.")
askWhichAnnotationToModify(p)
# state 50
elif text == REMOVE_MWE_BUTTON and p.selectedMwes != None:
askWhichAnnotationToRemove(p)
# state 60
elif text == NEXT_BUTTON:
if person.hasNextSentece(p):
person.goToNextSentence(p)
person.importSelectedMwe(p)
displayCurrentSentence(p)
#state 1
else:
displayCurrentSentence(p)
reply(EXCLAMATION + " You reached the last sentence")
# state 1
# state 1
elif text == PREV_BUTTON:
if person.hasPreviousSentece(p):
person.goToPrevSentence(p)
person.importSelectedMwe(p)
displayCurrentSentence(p)
#state 1
else:
displayCurrentSentence(p)
reply(EXCLAMATION + " You reached the first sentence")
# state 1
# state 1
elif text == GO_TO_BUTTON:
tot_sentences = sentAnno.totalSentPersonCurrentLang(p)
reply(
"Please insert the index of the sentence you want to jumpt to, between "
+ person.getHighlightedString(p, str(1)) + " and " +
person.getHighlightedString(p, str(tot_sentences)) +
" (you can also enter the number without pressing the "
+ JUMP_ARROW + " button).",
markdown=True)
elif text.startswith('/') and util.RepresentsInt(text[1:]):
mwe_number = int(text[1:])
jumpToModifySingleMwe(p, mwe_number)
#state 40
elif util.RepresentsInt(text):
index = int(text)
tot_sentences = sentAnno.totalSentPersonCurrentLang(p)
if index > 0 and index <= tot_sentences:
jumpToSentence(p, index)
# state 1
else:
reply(
EXCLAMATION +
"Not a valid index. If you want to jump to a specific sentence, "
"please enter an index between " +
person.getHighlightedString(p, str(1)) +
" and " + person.getHighlightedString(
p, str(tot_sentences)) + '.',
markdown=True)
else:
reply(FROWNING_FACE + " Sorry, I can't understand...")
elif p.state == 30:
# add MWE -> select tokens
if text == BACK_BUTTON:
displayCurrentSentence(p)
# state 1
elif text.endswith(']'):
token_number_str = text[text.rindex('[') + 1:-1]
#logging.debug('number_str: ' + token_number_str)
if util.RepresentsInt(token_number_str):
token_number = int(token_number_str)
if token_number > 0 and token_number <= len(
p.selectedTokens):
dealWithTokensSelection(p, token_number - 1)
#state 30
else:
reply("Not a valid token.")
else:
reply("Not a valid token.")
elif text == CHOOSE_CAT_BUTTON:
currentMwes = p.selectedMwes
if currentMwes and person.getSelectedTokensIndexTuple(
p) in currentMwes:
reply(EXCLAMATION + ' MWE has been already inserted!')
else:
askAnnotationLabel(p)
# state 31
elif text == CONFIRM_CHANNGES:
dealWithChangeMwe(p)
# state 1
else:
reply(FROWNING_FACE + " Sorry, I can't understand...")
elif p.state == 31:
# add MWE -> select category
if text == BACK_BUTTON:
if p.last_state == 40:
displayCurrentSentence(p)
# state 1
else:
dealWithTokensSelection(p)
# state 30
elif text in categories.ANNOTATION_CATS:
indexes = person.getSelectedTokensIndexTuple(p)
mwe_flat = renderMwe(p, indexes, False, False)
reply(CHECK + " The MWE " + mwe_flat +
" has been marked with category " +
person.getHighlightedString(p, text) + '\n',
markdown=True)
sentAnno.appendMwe(p, text) #mirroring on person
#person.appendMwe(p, text)
person.setCurrentSentenceAnnotated(p)
displayCurrentSentence(p)
# state 1
else:
reply(FROWNING_FACE + " Sorry, I can't understand...")
elif p.state == 40:
# user has selected a single MWE via the /number option
if text == BACK_BUTTON:
displayCurrentSentence(p)
# state 1
elif text == CHANGE_CAT_BUTTON:
askAnnotationLabel(p)
# state 31
elif text == REMOVE_BUTTON:
mwe_to_remove = person.getSelectedTokensIndexTuple(p)
mwe_flat = renderMwe(p, mwe_to_remove, True, False)
#logging.debug("after REMOVE_MWE_BUTTON: " + str(mwe_to_remove))
#logging.debug("mwe_to_remove: " + str(mwe_to_remove))
#logging.debug("current mwes: " + str(p.selectedMwes))
sentAnno.removeMwe(p, mwe_to_remove) #mirroring on person
#person.removeMwe(p, mwe_to_remove)
tell(p.chat_id,
CHECK + " The MWE " + mwe_flat + " has been removed!",
markdown=True)
displayCurrentSentence(p)
# state 1
elif text == CHANGE_MWE_ELEMENTS:
p.currentMweTmp = person.getSelectedTokensIndexTuple(p)
dealWithTokensSelection(p)
# state 30
elif text == CHANGE_MWE_CONFIDENCE:
askConfidenceLevel(p)
# state 70
else:
reply(FROWNING_FACE + " Sorry, I can't understand...")
elif p.state == 50:
# modify MWE
if text == BACK_BUTTON:
displayCurrentSentence(p)
# state 1
elif util.RepresentsInt(text):
mwe_number = int(text)
jumpToModifySingleMwe(p, mwe_number)
# state 40
else:
reply(FROWNING_FACE + " Sorry, I can't understand...")
elif p.state == 60:
# remove MWE
if text == BACK_BUTTON:
displayCurrentSentence(p)
# state 1
elif util.RepresentsInt(text):
mwe_number = int(text)
dealWithRemoveMwe(p, mwe_number)
# state 1
else:
reply(FROWNING_FACE + " Sorry, I can't understand...")
elif p.state == 70:
# change confidence level MWE
if text == BACK_BUTTON:
displayCurrentSentence(p)
# state 1
elif util.RepresentsInt(text):
newConf = int(text)
dealWithChangeConfidenceLevel(p, newConf)
# state 1
elif text in NUMBERS_0_10[1:]:
newConf = NUMBERS_0_10.index(text)
dealWithChangeConfidenceLevel(p, newConf)
# state 1
else:
reply(FROWNING_FACE + " Sorry, I can't understand...")
else:
reply("There is a problem (" + str(p.state).encode('utf-8') +
"). Report this to @kercos" + '\n')
restart(p)