def predict(self, text, original_text, pos): tokens = tokenize_utterance(text) entities = self.model.extract_entities(tokens) def default_entity_mapping(entity): range = entity[0] ind = [] for i in range: ind.append(i) return { "tag": entity[1], "entity": " ".join(original_text.split()[range[0]:range[-1] + 1]), "start": ind[0], "end": ind[-1] + 1, "resolvedTo": { 'baseEntity': " ".join(original_text.split()[range[0]:range[-1] + 1]) } } default_entities = compact(list(map(default_entity_mapping, entities))) return default_entities, None
def front_back(original_nos): original_nos = tokenize_utterance(original_nos) front = original_nos[0] length = len(original_nos) o = '' if (original_nos[len(original_nos) - 1] == ','): original_nos.pop() for e in original_nos: o += e return front, length, o
def ir_entity_tags(store, serviceid, model_type, engine, syn_processed_truecased_text, entity_tags, datasources_map): if entity_tags is not None: text_tokens = syn_processed_truecased_text.split() indexes_to_delete = [] text_tokens_final = [] for entity_each in entity_tags: start = entity_each["start"] end = entity_each["end"] text_tokens[start] = entity_each["tag"] while (end - start > 1): start = start + 1 indexes_to_delete.append(start) for index, token_each in enumerate(text_tokens): if index not in indexes_to_delete: text_tokens_final.append(token_each) text = " ".join(text_tokens_final) else: text = syn_processed_truecased_text tokens = tokenize_utterance(text) distinct_token_list = get(datasources_map, "distinct_token_list", default=[]) if distinct_token_list is None: distinct_token_list = [] tokens = list(set(tokens)) synonym_config = get(datasources_map, "synonymMappings", default=[]) for item in synonym_config: distinct_token_list.append(item['word']) distinct_token_list += [i for i in item['synonym']] distinct_token_list = list(set(distinct_token_list)) distinct_token_list = [x.upper() for x in distinct_token_list] tokens = [x.upper() for x in tokens] flag = 0 for i in range(len(tokens)): if tokens[i] in distinct_token_list: flag = 1 break if (len(distinct_token_list) > 0) and (flag == 0): prediction = "No intent" intent_list = get(datasources_map, "intents", default=[]) if len(intent_list) > 0: probabilities = [get_proba(intent_list)] else: probabilities = [{"No intent": "100.0%"}] else: prediction, probabilities = store.get_intent(serviceid, model_type, engine, text) return prediction, probabilities
def change_case(self, utterance): try: tokensCorrect = tokenize_utterance(utterance) tokens = [token.lower() for token in tokensCorrect] wordCasingLookup = self.get_model("wordCasingLookup", "NGRAM", None) uniDist = self.get_model("uniDist", "NGRAM", None) backwardBiDist = self.get_model("backwardBiDist", "NGRAM", None) forwardBiDist = self.get_model("forwardBiDist", "NGRAM", None) trigramDist = self.get_model("trigramDist", "NGRAM", None) tokensTrueCase = convert_case(tokens, 'title', wordCasingLookup, uniDist, backwardBiDist, forwardBiDist, trigramDist) return " ".join(tokensTrueCase), " ".join(tokensCorrect) except: logger.info(traceback.print_exc())
def stopword_remover(text, language): # stop_words = set(stopwords.words(language)) stop_words = get_stopwords(language) # if language == "english": # stop_words = ["it", "it’s", "its", "this", "that", "that’ll", "these", "those", "am", "is", "are", "was", # "were", "be", "been", "have", "has", "had", "having", "do", "does", "did", "doing", "a", "an", # "the", "and", "but", "if", "or", "as", "of", "at", "by", "for", "to", "so", "than", "too", # "should", "should’ve", "ain", "aren", "aren’t", "couldn", "couldn’t", "didn", "didn’t", "doesn", # "doesn’t", "hadn", "hadn’t", "hasn", "hasn’t", "haven", "haven’t", "isn", "isn’t", "mightn", # "mightn’t", "mustn", "mustn’t", "needn", "needn’t", "shall", "shouldn", "shouldn’t", "wasn", # "wasn’t", "weren", "weren’t", "won’t", "wouldn", "wouldn’t"] # else: # stop_words = [] word_tokens = tokenize_utterance(text) filtered_sentence = [] for w in word_tokens: if w.lower() not in stop_words: filtered_sentence.append(w) return filtered_sentence
def parse_date(self, user_input): """ handles user input, checks for main keywords and passes these keywords as parameters to specific functions. """ tokens_correct = tokenize_utterance(user_input) user_input = " ".join(tokens_correct) actual_data = user_input user_input = user_input.lower() case_sensitive_input = actual_data.split(' ') word_list = user_input.split(' ') del_pos = [] data_content = [] supporting_data_content = [] result_data = [] # identify main keywords in user data and store it in 'data_content' array with it's index value. # 'data_content' format - eg. ['day after tomorrow', [], main_data2, [start_index2, end_index2], ... ] main_words = propertyFile.main_words for i in range(len(word_list)): words = word_list[i] try: # handle main keywords with multiple words (day after tomorrow, new year, etc) if word_list[i] == 'day': try: check_data = False if (word_list[i+1] == 'after' and word_list[i+2] == 'tomorrow') or (word_list[i+1] == 'before' and word_list[i+2] == 'yesterday'): position = [i, i+3] check_data = True else: main_words.index(word_list[i]) position = [i, i+1] except: main_words.index(word_list[i]) position = [i, i+1] if check_data: words = word_list[i] + ' ' + word_list[i+1] + ' ' + word_list[i+2] del_pos.append(i+1) del_pos.append(i+2) elif word_list[i] == 'tomorrow' or word_list[i] == 'today' or word_list[i] == 'yesterday': try: check_data = False if (word_list[i+1] == 'morning') or (word_list[i+1] == 'night') or (word_list[i+1] == 'afternoon') or (word_list[i+1] == 'evening'): position = [i, i+2] check_data = True else: main_words.index(word_list[i]) position = [i, i+1] except: main_words.index(word_list[i]) position = [i, i+1] if check_data: words = word_list[i] + ' ' + word_list[i+1] del_pos.append(i+1) elif word_list[i] == 'new': try: check_data = False if word_list[i+1] == 'year': position = [i, i+2] check_data = True else: main_words.index(word_list[i]) position = [i, i+1] except: main_words.index(word_list[i]) position = [i, i+1] if check_data: words = word_list[i] + ' ' + word_list[i+1] del_pos.append(i+1) elif word_list[i] == 'now': try: if (word_list[i-1] == 'from'): position = [] else: words.index(word_list[i]) position = [i, i+1] except: main_words.index(word_list[i]) position = [i, i+1] else: main_words.index(word_list[i]) position = [i, i+1] except: position = [] if len(position) > 0 and position[0] not in del_pos: data_content.append(words) data_content.append(position) # identify supporting keywords in user data and store it in 'supporting_data_content' array with it's index value. # 'supporting_data_content' format - [supp_data1, [start_index1, end_index1], supp_data2, [start_index2, end_index2], ... ] for i in range(len(word_list)): word = word_list[i] check_data = False try: # handle supporting keywords with multiple words (eg. from now) if word_list[i] == 'from': try: if word_list[i+1] == 'now': position = [i, i+1] check_data = True else: position = [] except: propertyFile.supporting_words.index(word_list[i]) position = [i, i+1] if check_data: word = word_list[i] + ' ' + word_list[i+1] del_pos.append(i+1) else: propertyFile.supporting_words.index(word_list[i]) position = [i, i+1] except: position = [] if len(position) > 0 and position[0] not in del_pos: supporting_data_content.append(word) supporting_data_content.append(position) # recognize numbers (even words eg. forty five) in user input and store in array 'days'. try: days = stringToNumHandlingFunc(user_input) except: days = [] # check if user input is in past tense check_past_tense = self.check_past_tense_in_input(user_input) # handle all the identified main keywords for i in range(0, len(data_content), 2): if data_content[i] in propertyFile.common_words: timestamp_data_common_words = self.handle_common_words(data_content[i]) timestamp_data_common_words[0]['position'] = data_content[i + 1] timestamp_data_common_words[0]['words'] = self.get_words(case_sensitive_input, data_content[i + 1][0], data_content[i + 1][-1]) result_data.append(timestamp_data_common_words[0]) elif data_content[i] in propertyFile.festival_names: least_diff = [] check_diff = False check_diff_sec = True if len(supporting_data_content) > 0: for j in range(0, len(supporting_data_content), 2): pos = data_content[i + 1][0] if ((supporting_data_content[j] in propertyFile.selected_supporting_words) and ( supporting_data_content[j + 1][0] < pos)): diff = data_content[i + 1][0] - supporting_data_content[j + 1][0] if diff < 5: if diff == 1: check_diff = False else: check_diff = True if len(least_diff) == 0: least_diff.append(supporting_data_content[j]) least_diff.append(supporting_data_content[j + 1]) else: if (least_diff[1][0] < supporting_data_content[j + 1][0]): least_diff = [] least_diff.append(supporting_data_content[j]) least_diff.append(supporting_data_content[j + 1]) check_diff_sec = False if len(least_diff) > 0: timestamp_data_fun = self.handle_festival_names(least_diff[0] + ' ' + data_content[i], check_past_tense) if check_diff and check_diff_sec: timestamp_data_fun[0]['position'] = data_content[i + 1] timestamp_data_fun[0]['words'] = self.convert_list_tostring( case_sensitive_input[data_content[i + 1][0]:data_content[i + 1][-1]]) else: timestamp_data_fun[0]['position'] = [least_diff[1][0], data_content[i + 1][1]] timestamp_data_fun[0]['words'] = self.convert_list_tostring( case_sensitive_input[least_diff[1][0]:least_diff[1][1]]) timestamp_data_fun[0]['words'] += ' ' + self.convert_list_tostring( case_sensitive_input[data_content[i + 1][0]:data_content[i + 1][-1]]) else: timestamp_data_fun = self.handle_festival_names(data_content[i], check_past_tense) timestamp_data_fun[0]['position'] = data_content[i + 1] timestamp_data_fun[0]['words'] = self.get_words(case_sensitive_input, data_content[i + 1][0], data_content[i + 1][1]) else: timestamp_data_fun = self.handle_festival_names(data_content[i], check_past_tense) timestamp_data_fun[0]['position'] = data_content[i + 1] timestamp_data_fun[0]['words'] = self.get_words(case_sensitive_input, data_content[i + 1][0], data_content[i + 1][1]) result_data.append(timestamp_data_fun[0]) elif data_content[i] in propertyFile.week_day_words: least_diff = [] check_diff = False check_diff_sec = True if len(supporting_data_content) > 0: for j in range(0, len(supporting_data_content), 2): pos = data_content[i + 1][0] if ((supporting_data_content[j] in propertyFile.selected_supporting_words) and ( supporting_data_content[j + 1][0] < pos)): diff = data_content[i + 1][0] - supporting_data_content[j + 1][0] if diff < 5: if diff == 1: check_diff = False else: check_diff = True if len(least_diff) == 0: least_diff.append(supporting_data_content[j]) least_diff.append(supporting_data_content[j + 1]) else: if (least_diff[1][0] < supporting_data_content[j + 1][0]): least_diff = [] least_diff.append(supporting_data_content[j]) least_diff.append(supporting_data_content[j + 1]) check_diff_sec = False if len(least_diff) > 0: timestamp_data_week = self.handle_weekdays(least_diff[0] + ' ' + data_content[i], check_past_tense) if check_diff and check_diff_sec: timestamp_data_week[0]['position'] = data_content[i + 1] timestamp_data_week[0]['words'] = self.convert_list_tostring( case_sensitive_input[data_content[i + 1][0]:data_content[i + 1][1]]) else: timestamp_data_week[0]['position'] = [least_diff[1][0], data_content[i + 1][1]] timestamp_data_week[0]['words'] = self.convert_list_tostring( case_sensitive_input[least_diff[1][0]:least_diff[1][1]]) timestamp_data_week[0]['words'] += ' ' + self.convert_list_tostring( case_sensitive_input[data_content[i + 1][0]:data_content[i + 1][1]]) else: timestamp_data_week = self.handle_weekdays(data_content[i], check_past_tense) timestamp_data_week[0]['position'] = data_content[i + 1] timestamp_data_week[0]['words'] = self.get_words(case_sensitive_input, data_content[i + 1][0], data_content[i + 1][1]) else: timestamp_data_week = self.handle_weekdays(data_content[i], check_past_tense) timestamp_data_week[0]['position'] = data_content[i + 1] timestamp_data_week[0]['words'] = self.get_words(case_sensitive_input, data_content[i + 1][0], data_content[i + 1][1]) result_data.append(timestamp_data_week[0]) elif data_content[i] in propertyFile.time_period_words: least_diff = [] valid_num = [] check_diff = False check_diff_sec = True if len(days) > 0: no_days_arr = [] for j in range(len(days)): arr = list(days[j].values()) no_days_arr.extend(arr) for k in range(0, len(no_days_arr), 4): if no_days_arr[k + 3] + 1 == data_content[i + 1][0]: no_day = no_days_arr[k + 1] valid_num.append(no_days_arr[k]) valid_num.append(no_days_arr[k + 1]) valid_num.append(no_days_arr[k + 2]) valid_num.append(no_days_arr[k + 3]) if len(valid_num) == 0: no_day = 1 else: no_day = 1 if len(supporting_data_content) > 0: for j in range(0, len(supporting_data_content), 2): pos = data_content[i + 1][0] pos_supp = supporting_data_content[j + 1][0] # checking for first set of supporting data (eg. next, previous, last). # these supporting data occur before main data. eg. next year if ((supporting_data_content[j] in propertyFile.first_part_supporting_words) and (pos_supp < pos)): diff = pos - pos_supp if diff <= 5: if (diff > 1 and len(valid_num) == 0) or (diff > 2 and len(valid_num) != 0): check_diff = True if len(least_diff) == 0: least_diff.append(supporting_data_content[j]) least_diff.append(supporting_data_content[j + 1]) else: if (least_diff[1][0] < pos_supp): least_diff = [] least_diff.append(supporting_data_content[j]) least_diff.append(supporting_data_content[j + 1]) check_diff_sec = False # checking for second set of supporting data (eg. ago, from now ) and set check_second_supp as True # these supporting data occur after main data. eg. year ago elif ((supporting_data_content[j] in propertyFile.second_part_supporting_words) and (pos_supp > pos)): diff = pos_supp - pos if diff <= 5: if (diff > 1 and len(valid_num) == 0) or (diff > 2 and len(valid_num) != 0): check_diff = True if len(least_diff) == 0: least_diff.append(supporting_data_content[j]) least_diff.append(supporting_data_content[j + 1]) else: if (least_diff[1][0] < pos_supp): least_diff = [] least_diff.append(supporting_data_content[j]) least_diff.append(supporting_data_content[j + 1]) if len(least_diff) > 0: least_pos_supp = least_diff[1][0] if (least_diff[0] in propertyFile.first_part_supporting_words): if check_diff and check_diff_sec: if len(valid_num) > 0: timestamp_data_week = self.handle_time_period(least_diff[0] + ' ' + data_content[i], check_past_tense, no_day) if valid_num[2] == valid_num[3]: timestamp_data_week[0]['words'] = self.convert_list_tostring( case_sensitive_input[valid_num[2]]) else: timestamp_data_week[0]['words'] = self.convert_list_tostring( case_sensitive_input[valid_num[2]:valid_num[3] + 1]) timestamp_data_week[0]['words'] += ' ' + self.convert_list_tostring( case_sensitive_input[data_content[i + 1][0]:data_content[i + 1][1]]) timestamp_data_week[0]['position'] = [valid_num[2], data_content[i + 1][1]] else: timestamp_data_week = self.handle_time_period(least_diff[0] + ' ' + data_content[i], check_past_tense, no_day) timestamp_data_week[0]['words'] = self.convert_list_tostring( case_sensitive_input[data_content[i + 1][0]:data_content[i + 1][1]]) timestamp_data_week[0]['position'] = data_content[i + 1] else: if len(valid_num) > 0: timestamp_data_week = self.handle_time_period(least_diff[0] + ' ' + data_content[i], check_past_tense, no_day) timestamp_data_week[0]['words'] = self.convert_list_tostring( case_sensitive_input[least_pos_supp]) if valid_num[2] == valid_num[3]: timestamp_data_week[0]['words'] += ' ' + self.convert_list_tostring( case_sensitive_input[valid_num[2]]) else: timestamp_data_week[0]['words'] += ' ' + self.convert_list_tostring( case_sensitive_input[valid_num[2]:valid_num[3] + 1]) timestamp_data_week[0]['words'] += ' ' + self.convert_list_tostring( case_sensitive_input[data_content[i + 1][0]:data_content[i + 1][1]]) timestamp_data_week[0]['position'] = [least_pos_supp, data_content[i + 1][1]] else: timestamp_data_week = self.handle_time_period(least_diff[0] + ' ' + data_content[i], check_past_tense, no_day) timestamp_data_week[0]['words'] = self.convert_list_tostring( case_sensitive_input[least_pos_supp]) timestamp_data_week[0]['words'] += ' ' + self.convert_list_tostring( case_sensitive_input[data_content[i + 1][0]:data_content[i + 1][1]]) timestamp_data_week[0]['position'] = [least_pos_supp, data_content[i + 1][1]] else: if check_diff and check_diff_sec: if len(valid_num) > 0: timestamp_data_week = self.handle_time_period(least_diff[0] + ' ' + data_content[i], check_past_tense, no_day) timestamp_data_week[0]['position'] = [valid_num[2], data_content[i + 1][1]] if valid_num[2] == valid_num[3]: timestamp_data_week[0]['words'] = self.convert_list_tostring( case_sensitive_input[valid_num[2]]) else: timestamp_data_week[0]['words'] = self.convert_list_tostring( case_sensitive_input[valid_num[2]:valid_num[3] + 1]) timestamp_data_week[0]['words'] += ' ' + self.convert_list_tostring( case_sensitive_input[data_content[i + 1][0]:data_content[i + 1][1]]) else: timestamp_data_week = self.handle_time_period(least_diff[0] + ' ' + data_content[i], check_past_tense, no_day) timestamp_data_week[0]['position'] = data_content[i + 1] timestamp_data_week[0]['words'] = self.convert_list_tostring( case_sensitive_input[data_content[i + 1][0]:data_content[i + 1][1]]) else: if len(valid_num) > 0: timestamp_data_week = self.handle_time_period(least_diff[0] + ' ' + data_content[i], check_past_tense, no_day) if len(least_diff[0].split(' ')) > 1: timestamp_data_week[0]['position'] = [valid_num[2], least_pos_supp + 2] if valid_num[2] == valid_num[3]: timestamp_data_week[0]['words'] = self.convert_list_tostring( case_sensitive_input[valid_num[2]]) else: timestamp_data_week[0]['words'] = self.convert_list_tostring( case_sensitive_input[valid_num[2]:valid_num[3] + 1]) timestamp_data_week[0]['words'] += ' ' + self.convert_list_tostring( case_sensitive_input[data_content[i + 1][0]:data_content[i + 1][1]]) timestamp_data_week[0]['words'] += ' ' + self.convert_list_tostring( case_sensitive_input[least_diff[1][0]:least_diff[1][1] + 1]) else: timestamp_data_week[0]['position'] = [valid_num[2], least_pos_supp + 1] if valid_num[2] == valid_num[3]: timestamp_data_week[0]['words'] = self.convert_list_tostring( case_sensitive_input[valid_num[2]]) else: timestamp_data_week[0]['words'] = self.convert_list_tostring( case_sensitive_input[valid_num[2]:valid_num[3] + 1]) timestamp_data_week[0]['words'] += ' ' + self.convert_list_tostring( case_sensitive_input[data_content[i + 1][0]:data_content[i + 1][1]]) timestamp_data_week[0]['words'] += ' ' + self.convert_list_tostring( case_sensitive_input[least_pos_supp]) else: timestamp_data_week = self.handle_time_period(least_diff[0] + ' ' + data_content[i], check_past_tense, no_day) if len(least_diff[0].split(' ')) > 1: timestamp_data_week[0]['position'] = [data_content[i + 1][0], least_pos_supp + 2] timestamp_data_week[0]['words'] = self.convert_list_tostring( case_sensitive_input[data_content[i + 1][0]:data_content[i + 1][1]]) timestamp_data_week[0]['words'] += ' ' + self.convert_list_tostring( case_sensitive_input[least_diff[1][0]:least_diff[1][1]]) else: timestamp_data_week[0]['position'] = [data_content[i + 1][0], least_pos_supp + 1] timestamp_data_week[0]['words'] = self.convert_list_tostring( case_sensitive_input[data_content[i + 1][0]:data_content[i + 1][1]]) timestamp_data_week[0]['words'] += ' ' + self.convert_list_tostring( case_sensitive_input[least_diff[1][0]:least_diff[1][1]]) else: if len(valid_num) > 0: timestamp_data_week = self.handle_time_period(data_content[i], check_past_tense, no_day) timestamp_data_week[0]['position'] = [valid_num[2], data_content[i + 1][1]] timestamp_data_week[0]['words'] = self.get_words(case_sensitive_input, valid_num[2], data_content[i + 1][1]) else: timestamp_data_week = self.handle_time_period(data_content[i], check_past_tense, no_day) timestamp_data_week[0]['position'] = data_content[i + 1] timestamp_data_week[0]['words'] = self.get_words(case_sensitive_input, data_content[i + 1][0], data_content[i + 1][1]) timestamp_data_week[0]['timestamp'] = '' result_data.append(timestamp_data_week[0]) else: if len(valid_num) > 0: timestamp_data_week = self.handle_time_period(data_content[i], check_past_tense, no_day) timestamp_data_week[0]['words'] = self.get_words(case_sensitive_input, valid_num[2], data_content[i + 1][1]) timestamp_data_week[0]['position'] = [valid_num[2], data_content[i + 1][1]] else: timestamp_data_week = self.handle_time_period(data_content[i], check_past_tense, no_day) timestamp_data_week[0]['words'] = self.get_words(case_sensitive_input, data_content[i + 1][0], data_content[i + 1][1]) timestamp_data_week[0]['position'] = data_content[i + 1] timestamp_data_week[0]['timestamp'] = '' result_data.append(timestamp_data_week[0]) # python package 'datefinder' in function 'handle_dates' checks for date and returns timestamp timestamp_data_date = self.handle_dates(actual_data) if len(timestamp_data_date) > 0: for i in range(len(timestamp_data_date)): result_data.append(timestamp_data_date[i]) return result_data
def stringToNumHandlingFunc(s): def text2int(textnum, numwords=None): if numwords is None: numwords = OrderedDict() if not numwords: units = [ "zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen", "sixteen", "seventeen", "eighteen", "nineteen" ] tens = [ "", "", "twenty", "thirty", "forty", "fifty", "sixty", "seventy", "eighty", "ninety" ] scales = ["hundred", "thousand", "million", "billion", "trillion"] numwords["and"] = (1, 0) for idx, word in enumerate(units): numwords[word] = (1, idx) for idx, word in enumerate(tens): numwords[word] = (1, idx * 10) for idx, word in enumerate(scales): numwords[word] = (10**(idx * 3 or 2), 0) current = result = 0 for word in textnum.split(): if word not in numwords: raise Exception("Illegal word: " + word) scale, increment = numwords[word] current = current * scale + increment if scale > 100: result += current current = 0 number = result + current return number original = s if (len(s) > 0): if (s[len(s) - 1] != '.'): s = s + '.' ot = tokenize_utterance(s) otl = s.lower() otl = tokenize_utterance(otl) comma_number = '' numbers = [ 'zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten', 'eleven', 'twelve', 'thirteen', 'fourteen', 'fifteen', 'sixteen', 'seventeen', 'eighteen', 'nineteen', 'twenty', 'thirty', 'forty', 'fifty', 'sixty', 'seventy', 'eighty', 'ninety', 'hundred', 'thousand', 'million', 'billion', 'point', 'and' ] units_and_tens = [ 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten', 'eleven', 'twelve', 'thirteen', 'fourteen', 'fifteen', 'sixteen', 'seventeen', 'eighteen', 'nineteen', 'twenty', 'thirty', 'forty', 'fifty', 'sixty', 'seventy', 'eighty', 'ninety' ] multiples = ['billion', 'hundred', 'thousand', 'million'] list2 = [] word = '' z = 1 list1 = otl data = OrderedDict() not_decimal = 0 comma_numbers = [] s_index = OrderedDict() s_index_copy = OrderedDict() s_index_count = 0 for i in range(0, len(otl)): s_index[i] = otl[i] s_index_copy[i] = ot[i] s_index_count += 1 end_comma_numbers = 0 i = 0 output_list = [] _and_ = 0 count = 0 def front_back(original_nos): original_nos = tokenize_utterance(original_nos) front = original_nos[0] length = len(original_nos) o = '' if (original_nos[len(original_nos) - 1] == ','): original_nos.pop() for e in original_nos: o += e return front, length, o def dict_to_list(k, start, length): s_index2 = OrderedDict() for c, v in list(s_index.items()): s_index2[c] = v start_index = -1 end_index = -1 first = 0 for key, value in list(s_index.items()): if (start == value and first == 0): start_index = key first = 1 elif (first == 0): del s_index2[key] end_index = start_index + (length - 1) if (start_index == -1 or end_index == -1): return -1, -1 else: for j in range(start_index, end_index + 1): del s_index2[j] s_index.clear() for c, v in list(s_index2.items()): s_index[c] = v return start_index, end_index ot2 = [] while (i < len(list1) and count < s_index_count): I = 0 digit = 0 flag_continue = 0 flag_number = 0 if (re.match("^[0-9]+\.[0-9]+$", list1[i]) is not None or re.match("^[0-9]+$", list1[i]) is not None or re.match("\.[0-9]+$", list1[i]) is not None): if (list1[i + 1] in multiples): multiplier = float(w2n.word_to_num(list1[i + 1])) list2.append(list1[i] + " " + list1[i + 1]) list2.append(multiplier * float(list1[i])) ot2.append(ot[i] + " " + ot[i + 1]) flag_number = 3 elif (list1[i + 1] in numbers): list2.append(list1[i]) flag_number = 2 I = 1 if (list1[i + 1] == ','): comma_numbers = [] k = i while (re.match("^[0-9]+\.[0-9]+$", list1[k]) is not None or re.match("^[0-9]+$", list1[k]) is not None or list1[k] == ',' or re.match("\.[0-9]+$", list1[k]) is not None): comma_numbers.append(list1[k]) k += 1 k = 0 compare_temp = "" full_string = "" stop = 0 while (k <= len(comma_numbers) - 1): compare_temp += comma_numbers[k] if (compare_temp in original): full_string = compare_temp k += 1 if (compare_temp not in original): compare2 = compare_temp[0:len(compare_temp) - len(comma_numbers[k])] if (len(compare2) != 0): compare_temp = compare2 + ' ' stop += 1 else: compare_temp = '' if (stop > 50): k += 1 nos = '' original_nos = '' flag = 0 length = 0 for z in full_string: if (z == ' ' or length == len(full_string) - 1): if (re.match("^[0-9]+$", z) is not None): nos = nos + z original_nos += z if ('.' in nos and len(nos) != 0): front, length2, original_nos = front_back( original_nos) data['text'] = original_nos data['digit'] = float(nos) start, end = dict_to_list(count, front, length2) data['start_index'] = start data['end_index'] = end output_list.append(data) count += 1 elif (len(nos) != 0): front, length2, original_nos = front_back( original_nos) data['text'] = original_nos data['digit'] = int(nos) start, end = dict_to_list(count, front, length2) data['start_index'] = start data['end_index'] = end output_list.append(data) count += 1 file_name = 'resolving_numbers.json' nos = '' comma_number = full_string original_nos = '' data = OrderedDict() elif (z != ',' or z == '.'): original_nos += z nos = nos + z elif (z == ','): original_nos += z length += 1 full_string = "" flag_number = 5 elif (re.match("^[0-9]+\.[0-9]+$", list1[i]) is not None or re.match("\.[0-9]+$", list1[i]) is not None): if (list1[i + 1] not in numbers): list2.append(list1[i]) flag_number = 2 if (re.match("^[0-9]+$", list1[i]) is not None and list1[i + 1] not in numbers and flag_number != 5): list2.append(list1[i]) flag_number = 2 I = 1 if (list1[i] in numbers and flag_number != 5): if (list1[i + 1] == 'and' or list1[i + 1] in 'And'): if (list1[i] in numbers and list1[i + 2] in numbers and _and_ == 0 and list1[i] not in units_and_tens): list2.append(list1[i]) ot2.append(ot[i]) flag_number = 1 _and_ += 1 else: list2.append(list1[i]) ot2.append(ot[i]) flag_number = 0 elif (list1[i] == 'and' or list1[i] == 'And'): if (_and_ == 1): flag_number = 1 list2.append(list1[i]) ot2.append(ot[i]) else: flag_number = 0 elif (list1[i + 1] not in numbers): list2.append(list1[i]) ot2.append(ot[i]) if (len(list2) == 1): flag_number = 4 else: flag_number = 0 else: list2.append(list1[i]) ot2.append(ot[i]) flag_number = 1 if (flag_number == 0): if (len(list2) != 0): s = " ".join(list2) ot2 = " ".join(ot2) data['text'] = ot2 data['digit'] = text2int(s) start, end = dict_to_list(count, list2[0], len(list2)) data['start_index'] = start data['end_index'] = end _and_ = 0 output_list.append(data) if (start != end): count += end - start + 1 else: count += 1 data = OrderedDict() list2 = [] ot2 = [] elif (re.match(";|,| |/|\?|\"|\]|\[|\(|\)|\{|\}|\t|\n|\v|\f|\r", list1[i]) is not None or list1[i] == '.' or list1[i] == "'"): nothing = 0 else: start, end = dict_to_list(count, list1[i], 1) if (start == -1 and end == -1): do_nothing = 1 else: count = start + 1 elif (flag_number == 2): if (len(list2) != 0): if (I == 1): data['text'] = list2[0] data['digit'] = int(list2[0]) start, end = dict_to_list(count, list2[0], 1) data['start_index'] = start data['end_index'] = end count += 1 else: data['text'] = list2[0] data['digit'] = float(list2[0]) start, end = dict_to_list(count, list2[0], 1) data['start_index'] = start data['end_index'] = end count += 1 output_list.append(data) data = OrderedDict() list2 = [] elif (flag_number == 3): if (len(list2) != 0): list3 = list2[0].split(' ') data['text'] = ot2[0] data['digit'] = list2[1] start, end = dict_to_list(count, list3[0], len(list3)) count += 2 data['start_index'] = start data['end_index'] = end output_list.append(data) data = OrderedDict() list2 = [] ot2 = [] i += 1 elif (flag_number == 4): if (len(list2) != 0): data['text'] = ot2[0] data['digit'] = w2n.word_to_num(list2[0]) start, end = dict_to_list(count, list2[0], 1) data['start_index'] = start data['end_index'] = end output_list.append(data) count += 1 data = OrderedDict() list2 = [] ot2 = [] for comma in comma_number: if (comma == ','): i += 2 comma_number = '' i += 1 return output_list