def predict(self, text, original_text, pos):
tokens = tokenize_utterance(text)
entities = self.model.extract_entities(tokens)
def default_entity_mapping(entity):
range = entity[0]
ind = []
for i in range:
ind.append(i)
return {
"tag": entity[1],
"entity":
" ".join(original_text.split()[range[0]:range[-1] + 1]),
"start": ind[0],
"end": ind[-1] + 1,
"resolvedTo": {
'baseEntity':
" ".join(original_text.split()[range[0]:range[-1] + 1])
}
}
default_entities = compact(list(map(default_entity_mapping, entities)))
return default_entities, None
def front_back(original_nos):
original_nos = tokenize_utterance(original_nos)
front = original_nos[0]
length = len(original_nos)
o = ''
if (original_nos[len(original_nos) - 1] == ','):
original_nos.pop()
for e in original_nos:
o += e
return front, length, o
def ir_entity_tags(store, serviceid, model_type, engine,
syn_processed_truecased_text, entity_tags, datasources_map):
if entity_tags is not None:
text_tokens = syn_processed_truecased_text.split()
indexes_to_delete = []
text_tokens_final = []
for entity_each in entity_tags:
start = entity_each["start"]
end = entity_each["end"]
text_tokens[start] = entity_each["tag"]
while (end - start > 1):
start = start + 1
indexes_to_delete.append(start)
for index, token_each in enumerate(text_tokens):
if index not in indexes_to_delete:
text_tokens_final.append(token_each)
text = " ".join(text_tokens_final)
else:
text = syn_processed_truecased_text
tokens = tokenize_utterance(text)
distinct_token_list = get(datasources_map,
"distinct_token_list",
default=[])
if distinct_token_list is None:
distinct_token_list = []
tokens = list(set(tokens))
synonym_config = get(datasources_map, "synonymMappings", default=[])
for item in synonym_config:
distinct_token_list.append(item['word'])
distinct_token_list += [i for i in item['synonym']]
distinct_token_list = list(set(distinct_token_list))
distinct_token_list = [x.upper() for x in distinct_token_list]
tokens = [x.upper() for x in tokens]
flag = 0
for i in range(len(tokens)):
if tokens[i] in distinct_token_list:
flag = 1
break
if (len(distinct_token_list) > 0) and (flag == 0):
prediction = "No intent"
intent_list = get(datasources_map, "intents", default=[])
if len(intent_list) > 0:
probabilities = [get_proba(intent_list)]
else:
probabilities = [{"No intent": "100.0%"}]
else:
prediction, probabilities = store.get_intent(serviceid, model_type,
engine, text)
return prediction, probabilities
def change_case(self, utterance):
try:
tokensCorrect = tokenize_utterance(utterance)
tokens = [token.lower() for token in tokensCorrect]
wordCasingLookup = self.get_model("wordCasingLookup", "NGRAM",
None)
uniDist = self.get_model("uniDist", "NGRAM", None)
backwardBiDist = self.get_model("backwardBiDist", "NGRAM", None)
forwardBiDist = self.get_model("forwardBiDist", "NGRAM", None)
trigramDist = self.get_model("trigramDist", "NGRAM", None)
tokensTrueCase = convert_case(tokens, 'title', wordCasingLookup,
uniDist, backwardBiDist,
forwardBiDist, trigramDist)
return " ".join(tokensTrueCase), " ".join(tokensCorrect)
except:
logger.info(traceback.print_exc())
def stopword_remover(text, language):
# stop_words = set(stopwords.words(language))
stop_words = get_stopwords(language)
# if language == "english":
# stop_words = ["it", "it’s", "its", "this", "that", "that’ll", "these", "those", "am", "is", "are", "was",
# "were", "be", "been", "have", "has", "had", "having", "do", "does", "did", "doing", "a", "an",
# "the", "and", "but", "if", "or", "as", "of", "at", "by", "for", "to", "so", "than", "too",
# "should", "should’ve", "ain", "aren", "aren’t", "couldn", "couldn’t", "didn", "didn’t", "doesn",
# "doesn’t", "hadn", "hadn’t", "hasn", "hasn’t", "haven", "haven’t", "isn", "isn’t", "mightn",
# "mightn’t", "mustn", "mustn’t", "needn", "needn’t", "shall", "shouldn", "shouldn’t", "wasn",
# "wasn’t", "weren", "weren’t", "won’t", "wouldn", "wouldn’t"]
# else:
# stop_words = []
word_tokens = tokenize_utterance(text)
filtered_sentence = []
for w in word_tokens:
if w.lower() not in stop_words:
filtered_sentence.append(w)
return filtered_sentence
def parse_date(self, user_input):
""" handles user input, checks for main keywords and passes these keywords as
parameters to specific functions. """
tokens_correct = tokenize_utterance(user_input)
user_input = " ".join(tokens_correct)
actual_data = user_input
user_input = user_input.lower()
case_sensitive_input = actual_data.split(' ')
word_list = user_input.split(' ')
del_pos = []
data_content = []
supporting_data_content = []
result_data = []
# identify main keywords in user data and store it in 'data_content' array with it's index value.
# 'data_content' format - eg. ['day after tomorrow', [], main_data2, [start_index2, end_index2], ... ]
main_words = propertyFile.main_words
for i in range(len(word_list)):
words = word_list[i]
try:
# handle main keywords with multiple words (day after tomorrow, new year, etc)
if word_list[i] == 'day':
try:
check_data = False
if (word_list[i+1] == 'after' and word_list[i+2] == 'tomorrow') or (word_list[i+1] == 'before' and word_list[i+2] == 'yesterday'):
position = [i, i+3]
check_data = True
else:
main_words.index(word_list[i])
position = [i, i+1]
except:
main_words.index(word_list[i])
position = [i, i+1]
if check_data:
words = word_list[i] + ' ' + word_list[i+1] + ' ' + word_list[i+2]
del_pos.append(i+1)
del_pos.append(i+2)
elif word_list[i] == 'tomorrow' or word_list[i] == 'today' or word_list[i] == 'yesterday':
try:
check_data = False
if (word_list[i+1] == 'morning') or (word_list[i+1] == 'night') or (word_list[i+1] == 'afternoon') or (word_list[i+1] == 'evening'):
position = [i, i+2]
check_data = True
else:
main_words.index(word_list[i])
position = [i, i+1]
except:
main_words.index(word_list[i])
position = [i, i+1]
if check_data:
words = word_list[i] + ' ' + word_list[i+1]
del_pos.append(i+1)
elif word_list[i] == 'new':
try:
check_data = False
if word_list[i+1] == 'year':
position = [i, i+2]
check_data = True
else:
main_words.index(word_list[i])
position = [i, i+1]
except:
main_words.index(word_list[i])
position = [i, i+1]
if check_data:
words = word_list[i] + ' ' + word_list[i+1]
del_pos.append(i+1)
elif word_list[i] == 'now':
try:
if (word_list[i-1] == 'from'):
position = []
else:
words.index(word_list[i])
position = [i, i+1]
except:
main_words.index(word_list[i])
position = [i, i+1]
else:
main_words.index(word_list[i])
position = [i, i+1]
except:
position = []
if len(position) > 0 and position[0] not in del_pos:
data_content.append(words)
data_content.append(position)
# identify supporting keywords in user data and store it in 'supporting_data_content' array with it's index value.
# 'supporting_data_content' format - [supp_data1, [start_index1, end_index1], supp_data2, [start_index2, end_index2], ... ]
for i in range(len(word_list)):
word = word_list[i]
check_data = False
try:
# handle supporting keywords with multiple words (eg. from now)
if word_list[i] == 'from':
try:
if word_list[i+1] == 'now':
position = [i, i+1]
check_data = True
else:
position = []
except:
propertyFile.supporting_words.index(word_list[i])
position = [i, i+1]
if check_data:
word = word_list[i] + ' ' + word_list[i+1]
del_pos.append(i+1)
else:
propertyFile.supporting_words.index(word_list[i])
position = [i, i+1]
except:
position = []
if len(position) > 0 and position[0] not in del_pos:
supporting_data_content.append(word)
supporting_data_content.append(position)
# recognize numbers (even words eg. forty five) in user input and store in array 'days'.
try:
days = stringToNumHandlingFunc(user_input)
except:
days = []
# check if user input is in past tense
check_past_tense = self.check_past_tense_in_input(user_input)
# handle all the identified main keywords
for i in range(0, len(data_content), 2):
if data_content[i] in propertyFile.common_words:
timestamp_data_common_words = self.handle_common_words(data_content[i])
timestamp_data_common_words[0]['position'] = data_content[i + 1]
timestamp_data_common_words[0]['words'] = self.get_words(case_sensitive_input, data_content[i + 1][0],
data_content[i + 1][-1])
result_data.append(timestamp_data_common_words[0])
elif data_content[i] in propertyFile.festival_names:
least_diff = []
check_diff = False
check_diff_sec = True
if len(supporting_data_content) > 0:
for j in range(0, len(supporting_data_content), 2):
pos = data_content[i + 1][0]
if ((supporting_data_content[j] in propertyFile.selected_supporting_words) and (
supporting_data_content[j + 1][0] < pos)):
diff = data_content[i + 1][0] - supporting_data_content[j + 1][0]
if diff < 5:
if diff == 1:
check_diff = False
else:
check_diff = True
if len(least_diff) == 0:
least_diff.append(supporting_data_content[j])
least_diff.append(supporting_data_content[j + 1])
else:
if (least_diff[1][0] < supporting_data_content[j + 1][0]):
least_diff = []
least_diff.append(supporting_data_content[j])
least_diff.append(supporting_data_content[j + 1])
check_diff_sec = False
if len(least_diff) > 0:
timestamp_data_fun = self.handle_festival_names(least_diff[0] + ' ' + data_content[i], check_past_tense)
if check_diff and check_diff_sec:
timestamp_data_fun[0]['position'] = data_content[i + 1]
timestamp_data_fun[0]['words'] = self.convert_list_tostring(
case_sensitive_input[data_content[i + 1][0]:data_content[i + 1][-1]])
else:
timestamp_data_fun[0]['position'] = [least_diff[1][0], data_content[i + 1][1]]
timestamp_data_fun[0]['words'] = self.convert_list_tostring(
case_sensitive_input[least_diff[1][0]:least_diff[1][1]])
timestamp_data_fun[0]['words'] += ' ' + self.convert_list_tostring(
case_sensitive_input[data_content[i + 1][0]:data_content[i + 1][-1]])
else:
timestamp_data_fun = self.handle_festival_names(data_content[i], check_past_tense)
timestamp_data_fun[0]['position'] = data_content[i + 1]
timestamp_data_fun[0]['words'] = self.get_words(case_sensitive_input, data_content[i + 1][0],
data_content[i + 1][1])
else:
timestamp_data_fun = self.handle_festival_names(data_content[i], check_past_tense)
timestamp_data_fun[0]['position'] = data_content[i + 1]
timestamp_data_fun[0]['words'] = self.get_words(case_sensitive_input, data_content[i + 1][0],
data_content[i + 1][1])
result_data.append(timestamp_data_fun[0])
elif data_content[i] in propertyFile.week_day_words:
least_diff = []
check_diff = False
check_diff_sec = True
if len(supporting_data_content) > 0:
for j in range(0, len(supporting_data_content), 2):
pos = data_content[i + 1][0]
if ((supporting_data_content[j] in propertyFile.selected_supporting_words) and (
supporting_data_content[j + 1][0] < pos)):
diff = data_content[i + 1][0] - supporting_data_content[j + 1][0]
if diff < 5:
if diff == 1:
check_diff = False
else:
check_diff = True
if len(least_diff) == 0:
least_diff.append(supporting_data_content[j])
least_diff.append(supporting_data_content[j + 1])
else:
if (least_diff[1][0] < supporting_data_content[j + 1][0]):
least_diff = []
least_diff.append(supporting_data_content[j])
least_diff.append(supporting_data_content[j + 1])
check_diff_sec = False
if len(least_diff) > 0:
timestamp_data_week = self.handle_weekdays(least_diff[0] + ' ' + data_content[i], check_past_tense)
if check_diff and check_diff_sec:
timestamp_data_week[0]['position'] = data_content[i + 1]
timestamp_data_week[0]['words'] = self.convert_list_tostring(
case_sensitive_input[data_content[i + 1][0]:data_content[i + 1][1]])
else:
timestamp_data_week[0]['position'] = [least_diff[1][0], data_content[i + 1][1]]
timestamp_data_week[0]['words'] = self.convert_list_tostring(
case_sensitive_input[least_diff[1][0]:least_diff[1][1]])
timestamp_data_week[0]['words'] += ' ' + self.convert_list_tostring(
case_sensitive_input[data_content[i + 1][0]:data_content[i + 1][1]])
else:
timestamp_data_week = self.handle_weekdays(data_content[i], check_past_tense)
timestamp_data_week[0]['position'] = data_content[i + 1]
timestamp_data_week[0]['words'] = self.get_words(case_sensitive_input, data_content[i + 1][0],
data_content[i + 1][1])
else:
timestamp_data_week = self.handle_weekdays(data_content[i], check_past_tense)
timestamp_data_week[0]['position'] = data_content[i + 1]
timestamp_data_week[0]['words'] = self.get_words(case_sensitive_input, data_content[i + 1][0],
data_content[i + 1][1])
result_data.append(timestamp_data_week[0])
elif data_content[i] in propertyFile.time_period_words:
least_diff = []
valid_num = []
check_diff = False
check_diff_sec = True
if len(days) > 0:
no_days_arr = []
for j in range(len(days)):
arr = list(days[j].values())
no_days_arr.extend(arr)
for k in range(0, len(no_days_arr), 4):
if no_days_arr[k + 3] + 1 == data_content[i + 1][0]:
no_day = no_days_arr[k + 1]
valid_num.append(no_days_arr[k])
valid_num.append(no_days_arr[k + 1])
valid_num.append(no_days_arr[k + 2])
valid_num.append(no_days_arr[k + 3])
if len(valid_num) == 0:
no_day = 1
else:
no_day = 1
if len(supporting_data_content) > 0:
for j in range(0, len(supporting_data_content), 2):
pos = data_content[i + 1][0]
pos_supp = supporting_data_content[j + 1][0]
# checking for first set of supporting data (eg. next, previous, last).
# these supporting data occur before main data. eg. next year
if ((supporting_data_content[j] in propertyFile.first_part_supporting_words) and (pos_supp < pos)):
diff = pos - pos_supp
if diff <= 5:
if (diff > 1 and len(valid_num) == 0) or (diff > 2 and len(valid_num) != 0):
check_diff = True
if len(least_diff) == 0:
least_diff.append(supporting_data_content[j])
least_diff.append(supporting_data_content[j + 1])
else:
if (least_diff[1][0] < pos_supp):
least_diff = []
least_diff.append(supporting_data_content[j])
least_diff.append(supporting_data_content[j + 1])
check_diff_sec = False
# checking for second set of supporting data (eg. ago, from now ) and set check_second_supp as True
# these supporting data occur after main data. eg. year ago
elif ((supporting_data_content[j] in propertyFile.second_part_supporting_words) and (pos_supp > pos)):
diff = pos_supp - pos
if diff <= 5:
if (diff > 1 and len(valid_num) == 0) or (diff > 2 and len(valid_num) != 0):
check_diff = True
if len(least_diff) == 0:
least_diff.append(supporting_data_content[j])
least_diff.append(supporting_data_content[j + 1])
else:
if (least_diff[1][0] < pos_supp):
least_diff = []
least_diff.append(supporting_data_content[j])
least_diff.append(supporting_data_content[j + 1])
if len(least_diff) > 0:
least_pos_supp = least_diff[1][0]
if (least_diff[0] in propertyFile.first_part_supporting_words):
if check_diff and check_diff_sec:
if len(valid_num) > 0:
timestamp_data_week = self.handle_time_period(least_diff[0] + ' ' + data_content[i],
check_past_tense, no_day)
if valid_num[2] == valid_num[3]:
timestamp_data_week[0]['words'] = self.convert_list_tostring(
case_sensitive_input[valid_num[2]])
else:
timestamp_data_week[0]['words'] = self.convert_list_tostring(
case_sensitive_input[valid_num[2]:valid_num[3] + 1])
timestamp_data_week[0]['words'] += ' ' + self.convert_list_tostring(
case_sensitive_input[data_content[i + 1][0]:data_content[i + 1][1]])
timestamp_data_week[0]['position'] = [valid_num[2], data_content[i + 1][1]]
else:
timestamp_data_week = self.handle_time_period(least_diff[0] + ' ' + data_content[i],
check_past_tense, no_day)
timestamp_data_week[0]['words'] = self.convert_list_tostring(
case_sensitive_input[data_content[i + 1][0]:data_content[i + 1][1]])
timestamp_data_week[0]['position'] = data_content[i + 1]
else:
if len(valid_num) > 0:
timestamp_data_week = self.handle_time_period(least_diff[0] + ' ' + data_content[i],
check_past_tense, no_day)
timestamp_data_week[0]['words'] = self.convert_list_tostring(
case_sensitive_input[least_pos_supp])
if valid_num[2] == valid_num[3]:
timestamp_data_week[0]['words'] += ' ' + self.convert_list_tostring(
case_sensitive_input[valid_num[2]])
else:
timestamp_data_week[0]['words'] += ' ' + self.convert_list_tostring(
case_sensitive_input[valid_num[2]:valid_num[3] + 1])
timestamp_data_week[0]['words'] += ' ' + self.convert_list_tostring(
case_sensitive_input[data_content[i + 1][0]:data_content[i + 1][1]])
timestamp_data_week[0]['position'] = [least_pos_supp, data_content[i + 1][1]]
else:
timestamp_data_week = self.handle_time_period(least_diff[0] + ' ' + data_content[i],
check_past_tense, no_day)
timestamp_data_week[0]['words'] = self.convert_list_tostring(
case_sensitive_input[least_pos_supp])
timestamp_data_week[0]['words'] += ' ' + self.convert_list_tostring(
case_sensitive_input[data_content[i + 1][0]:data_content[i + 1][1]])
timestamp_data_week[0]['position'] = [least_pos_supp, data_content[i + 1][1]]
else:
if check_diff and check_diff_sec:
if len(valid_num) > 0:
timestamp_data_week = self.handle_time_period(least_diff[0] + ' ' + data_content[i],
check_past_tense, no_day)
timestamp_data_week[0]['position'] = [valid_num[2], data_content[i + 1][1]]
if valid_num[2] == valid_num[3]:
timestamp_data_week[0]['words'] = self.convert_list_tostring(
case_sensitive_input[valid_num[2]])
else:
timestamp_data_week[0]['words'] = self.convert_list_tostring(
case_sensitive_input[valid_num[2]:valid_num[3] + 1])
timestamp_data_week[0]['words'] += ' ' + self.convert_list_tostring(
case_sensitive_input[data_content[i + 1][0]:data_content[i + 1][1]])
else:
timestamp_data_week = self.handle_time_period(least_diff[0] + ' ' + data_content[i],
check_past_tense, no_day)
timestamp_data_week[0]['position'] = data_content[i + 1]
timestamp_data_week[0]['words'] = self.convert_list_tostring(
case_sensitive_input[data_content[i + 1][0]:data_content[i + 1][1]])
else:
if len(valid_num) > 0:
timestamp_data_week = self.handle_time_period(least_diff[0] + ' ' + data_content[i],
check_past_tense, no_day)
if len(least_diff[0].split(' ')) > 1:
timestamp_data_week[0]['position'] = [valid_num[2], least_pos_supp + 2]
if valid_num[2] == valid_num[3]:
timestamp_data_week[0]['words'] = self.convert_list_tostring(
case_sensitive_input[valid_num[2]])
else:
timestamp_data_week[0]['words'] = self.convert_list_tostring(
case_sensitive_input[valid_num[2]:valid_num[3] + 1])
timestamp_data_week[0]['words'] += ' ' + self.convert_list_tostring(
case_sensitive_input[data_content[i + 1][0]:data_content[i + 1][1]])
timestamp_data_week[0]['words'] += ' ' + self.convert_list_tostring(
case_sensitive_input[least_diff[1][0]:least_diff[1][1] + 1])
else:
timestamp_data_week[0]['position'] = [valid_num[2], least_pos_supp + 1]
if valid_num[2] == valid_num[3]:
timestamp_data_week[0]['words'] = self.convert_list_tostring(
case_sensitive_input[valid_num[2]])
else:
timestamp_data_week[0]['words'] = self.convert_list_tostring(
case_sensitive_input[valid_num[2]:valid_num[3] + 1])
timestamp_data_week[0]['words'] += ' ' + self.convert_list_tostring(
case_sensitive_input[data_content[i + 1][0]:data_content[i + 1][1]])
timestamp_data_week[0]['words'] += ' ' + self.convert_list_tostring(
case_sensitive_input[least_pos_supp])
else:
timestamp_data_week = self.handle_time_period(least_diff[0] + ' ' + data_content[i],
check_past_tense, no_day)
if len(least_diff[0].split(' ')) > 1:
timestamp_data_week[0]['position'] = [data_content[i + 1][0], least_pos_supp + 2]
timestamp_data_week[0]['words'] = self.convert_list_tostring(
case_sensitive_input[data_content[i + 1][0]:data_content[i + 1][1]])
timestamp_data_week[0]['words'] += ' ' + self.convert_list_tostring(
case_sensitive_input[least_diff[1][0]:least_diff[1][1]])
else:
timestamp_data_week[0]['position'] = [data_content[i + 1][0], least_pos_supp + 1]
timestamp_data_week[0]['words'] = self.convert_list_tostring(
case_sensitive_input[data_content[i + 1][0]:data_content[i + 1][1]])
timestamp_data_week[0]['words'] += ' ' + self.convert_list_tostring(
case_sensitive_input[least_diff[1][0]:least_diff[1][1]])
else:
if len(valid_num) > 0:
timestamp_data_week = self.handle_time_period(data_content[i], check_past_tense, no_day)
timestamp_data_week[0]['position'] = [valid_num[2], data_content[i + 1][1]]
timestamp_data_week[0]['words'] = self.get_words(case_sensitive_input, valid_num[2],
data_content[i + 1][1])
else:
timestamp_data_week = self.handle_time_period(data_content[i], check_past_tense, no_day)
timestamp_data_week[0]['position'] = data_content[i + 1]
timestamp_data_week[0]['words'] = self.get_words(case_sensitive_input, data_content[i + 1][0],
data_content[i + 1][1])
timestamp_data_week[0]['timestamp'] = ''
result_data.append(timestamp_data_week[0])
else:
if len(valid_num) > 0:
timestamp_data_week = self.handle_time_period(data_content[i], check_past_tense, no_day)
timestamp_data_week[0]['words'] = self.get_words(case_sensitive_input, valid_num[2],
data_content[i + 1][1])
timestamp_data_week[0]['position'] = [valid_num[2], data_content[i + 1][1]]
else:
timestamp_data_week = self.handle_time_period(data_content[i], check_past_tense, no_day)
timestamp_data_week[0]['words'] = self.get_words(case_sensitive_input, data_content[i + 1][0],
data_content[i + 1][1])
timestamp_data_week[0]['position'] = data_content[i + 1]
timestamp_data_week[0]['timestamp'] = ''
result_data.append(timestamp_data_week[0])
# python package 'datefinder' in function 'handle_dates' checks for date and returns timestamp
timestamp_data_date = self.handle_dates(actual_data)
if len(timestamp_data_date) > 0:
for i in range(len(timestamp_data_date)):
result_data.append(timestamp_data_date[i])
return result_data
def stringToNumHandlingFunc(s):
def text2int(textnum, numwords=None):
if numwords is None:
numwords = OrderedDict()
if not numwords:
units = [
"zero", "one", "two", "three", "four", "five", "six", "seven",
"eight", "nine", "ten", "eleven", "twelve", "thirteen",
"fourteen", "fifteen", "sixteen", "seventeen", "eighteen",
"nineteen"
]
tens = [
"", "", "twenty", "thirty", "forty", "fifty", "sixty", "seventy",
"eighty", "ninety"
]
scales = ["hundred", "thousand", "million", "billion", "trillion"]
numwords["and"] = (1, 0)
for idx, word in enumerate(units):
numwords[word] = (1, idx)
for idx, word in enumerate(tens):
numwords[word] = (1, idx * 10)
for idx, word in enumerate(scales):
numwords[word] = (10**(idx * 3 or 2), 0)
current = result = 0
for word in textnum.split():
if word not in numwords:
raise Exception("Illegal word: " + word)
scale, increment = numwords[word]
current = current * scale + increment
if scale > 100:
result += current
current = 0
number = result + current
return number
original = s
if (len(s) > 0):
if (s[len(s) - 1] != '.'):
s = s + '.'
ot = tokenize_utterance(s)
otl = s.lower()
otl = tokenize_utterance(otl)
comma_number = ''
numbers = [
'zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight',
'nine', 'ten', 'eleven', 'twelve', 'thirteen', 'fourteen', 'fifteen',
'sixteen', 'seventeen', 'eighteen', 'nineteen', 'twenty', 'thirty',
'forty', 'fifty', 'sixty', 'seventy', 'eighty', 'ninety', 'hundred',
'thousand', 'million', 'billion', 'point', 'and'
]
units_and_tens = [
'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine',
'ten', 'eleven', 'twelve', 'thirteen', 'fourteen', 'fifteen',
'sixteen', 'seventeen', 'eighteen', 'nineteen', 'twenty', 'thirty',
'forty', 'fifty', 'sixty', 'seventy', 'eighty', 'ninety'
]
multiples = ['billion', 'hundred', 'thousand', 'million']
list2 = []
word = ''
z = 1
list1 = otl
data = OrderedDict()
not_decimal = 0
comma_numbers = []
s_index = OrderedDict()
s_index_copy = OrderedDict()
s_index_count = 0
for i in range(0, len(otl)):
s_index[i] = otl[i]
s_index_copy[i] = ot[i]
s_index_count += 1
end_comma_numbers = 0
i = 0
output_list = []
_and_ = 0
count = 0
def front_back(original_nos):
original_nos = tokenize_utterance(original_nos)
front = original_nos[0]
length = len(original_nos)
o = ''
if (original_nos[len(original_nos) - 1] == ','):
original_nos.pop()
for e in original_nos:
o += e
return front, length, o
def dict_to_list(k, start, length):
s_index2 = OrderedDict()
for c, v in list(s_index.items()):
s_index2[c] = v
start_index = -1
end_index = -1
first = 0
for key, value in list(s_index.items()):
if (start == value and first == 0):
start_index = key
first = 1
elif (first == 0):
del s_index2[key]
end_index = start_index + (length - 1)
if (start_index == -1 or end_index == -1):
return -1, -1
else:
for j in range(start_index, end_index + 1):
del s_index2[j]
s_index.clear()
for c, v in list(s_index2.items()):
s_index[c] = v
return start_index, end_index
ot2 = []
while (i < len(list1) and count < s_index_count):
I = 0
digit = 0
flag_continue = 0
flag_number = 0
if (re.match("^[0-9]+\.[0-9]+$", list1[i]) is not None
or re.match("^[0-9]+$", list1[i]) is not None
or re.match("\.[0-9]+$", list1[i]) is not None):
if (list1[i + 1] in multiples):
multiplier = float(w2n.word_to_num(list1[i + 1]))
list2.append(list1[i] + " " + list1[i + 1])
list2.append(multiplier * float(list1[i]))
ot2.append(ot[i] + " " + ot[i + 1])
flag_number = 3
elif (list1[i + 1] in numbers):
list2.append(list1[i])
flag_number = 2
I = 1
if (list1[i + 1] == ','):
comma_numbers = []
k = i
while (re.match("^[0-9]+\.[0-9]+$", list1[k]) is not None
or re.match("^[0-9]+$", list1[k]) is not None
or list1[k] == ','
or re.match("\.[0-9]+$", list1[k]) is not None):
comma_numbers.append(list1[k])
k += 1
k = 0
compare_temp = ""
full_string = ""
stop = 0
while (k <= len(comma_numbers) - 1):
compare_temp += comma_numbers[k]
if (compare_temp in original):
full_string = compare_temp
k += 1
if (compare_temp not in original):
compare2 = compare_temp[0:len(compare_temp) -
len(comma_numbers[k])]
if (len(compare2) != 0):
compare_temp = compare2 + ' '
stop += 1
else:
compare_temp = ''
if (stop > 50):
k += 1
nos = ''
original_nos = ''
flag = 0
length = 0
for z in full_string:
if (z == ' ' or length == len(full_string) - 1):
if (re.match("^[0-9]+$", z) is not None):
nos = nos + z
original_nos += z
if ('.' in nos and len(nos) != 0):
front, length2, original_nos = front_back(
original_nos)
data['text'] = original_nos
data['digit'] = float(nos)
start, end = dict_to_list(count, front, length2)
data['start_index'] = start
data['end_index'] = end
output_list.append(data)
count += 1
elif (len(nos) != 0):
front, length2, original_nos = front_back(
original_nos)
data['text'] = original_nos
data['digit'] = int(nos)
start, end = dict_to_list(count, front, length2)
data['start_index'] = start
data['end_index'] = end
output_list.append(data)
count += 1
file_name = 'resolving_numbers.json'
nos = ''
comma_number = full_string
original_nos = ''
data = OrderedDict()
elif (z != ',' or z == '.'):
original_nos += z
nos = nos + z
elif (z == ','):
original_nos += z
length += 1
full_string = ""
flag_number = 5
elif (re.match("^[0-9]+\.[0-9]+$", list1[i]) is not None
or re.match("\.[0-9]+$", list1[i]) is not None):
if (list1[i + 1] not in numbers):
list2.append(list1[i])
flag_number = 2
if (re.match("^[0-9]+$", list1[i]) is not None
and list1[i + 1] not in numbers and flag_number != 5):
list2.append(list1[i])
flag_number = 2
I = 1
if (list1[i] in numbers and flag_number != 5):
if (list1[i + 1] == 'and' or list1[i + 1] in 'And'):
if (list1[i] in numbers and list1[i + 2] in numbers
and _and_ == 0 and list1[i] not in units_and_tens):
list2.append(list1[i])
ot2.append(ot[i])
flag_number = 1
_and_ += 1
else:
list2.append(list1[i])
ot2.append(ot[i])
flag_number = 0
elif (list1[i] == 'and' or list1[i] == 'And'):
if (_and_ == 1):
flag_number = 1
list2.append(list1[i])
ot2.append(ot[i])
else:
flag_number = 0
elif (list1[i + 1] not in numbers):
list2.append(list1[i])
ot2.append(ot[i])
if (len(list2) == 1):
flag_number = 4
else:
flag_number = 0
else:
list2.append(list1[i])
ot2.append(ot[i])
flag_number = 1
if (flag_number == 0):
if (len(list2) != 0):
s = " ".join(list2)
ot2 = " ".join(ot2)
data['text'] = ot2
data['digit'] = text2int(s)
start, end = dict_to_list(count, list2[0], len(list2))
data['start_index'] = start
data['end_index'] = end
_and_ = 0
output_list.append(data)
if (start != end):
count += end - start + 1
else:
count += 1
data = OrderedDict()
list2 = []
ot2 = []
elif (re.match(";|,| |/|\?|\"|\]|\[|\(|\)|\{|\}|\t|\n|\v|\f|\r",
list1[i]) is not None or list1[i] == '.'
or list1[i] == "'"):
nothing = 0
else:
start, end = dict_to_list(count, list1[i], 1)
if (start == -1 and end == -1):
do_nothing = 1
else:
count = start + 1
elif (flag_number == 2):
if (len(list2) != 0):
if (I == 1):
data['text'] = list2[0]
data['digit'] = int(list2[0])
start, end = dict_to_list(count, list2[0], 1)
data['start_index'] = start
data['end_index'] = end
count += 1
else:
data['text'] = list2[0]
data['digit'] = float(list2[0])
start, end = dict_to_list(count, list2[0], 1)
data['start_index'] = start
data['end_index'] = end
count += 1
output_list.append(data)
data = OrderedDict()
list2 = []
elif (flag_number == 3):
if (len(list2) != 0):
list3 = list2[0].split(' ')
data['text'] = ot2[0]
data['digit'] = list2[1]
start, end = dict_to_list(count, list3[0], len(list3))
count += 2
data['start_index'] = start
data['end_index'] = end
output_list.append(data)
data = OrderedDict()
list2 = []
ot2 = []
i += 1
elif (flag_number == 4):
if (len(list2) != 0):
data['text'] = ot2[0]
data['digit'] = w2n.word_to_num(list2[0])
start, end = dict_to_list(count, list2[0], 1)
data['start_index'] = start
data['end_index'] = end
output_list.append(data)
count += 1
data = OrderedDict()
list2 = []
ot2 = []
for comma in comma_number:
if (comma == ','):
i += 2
comma_number = ''
i += 1
return output_list