def run_test(self):
message = testcase["message"]
unit_type = testcase.get("unit_type", None)
number_detector_object = NumberDetector(entity_name="number",
language=language,
unit_type=unit_type)
number_detector_object.set_min_max_digits(
min_digit=testcase.get('min_digit',
number_detector_object.min_digit),
max_digit=testcase.get('max_digit',
number_detector_object.max_digit))
number_dicts, spans = number_detector_object.detect_entity(message)
expected_number_dicts, expected_spans = parse_expected_outputs(
testcase["outputs"])
expected_outputs = list(
six.moves.zip(expected_number_dicts, expected_spans))
prefix = failure_string_prefix.format(message=message,
language=language)
self.assertEqual(
len(number_dicts), len(spans), prefix +
u"Returned numbers and original_texts have different lengths")
self.assertEqual(
len(spans), len(expected_outputs), prefix +
u"Returned numbers and expected_outputs have different lengths"
)
for output in six.moves.zip(number_dicts, spans):
self.assertIn(
output, expected_outputs,
prefix + u"{got} not in {expected_outputs}".format(
got=output, expected_outputs=expected_outputs))
def test_en_number_detection_for_integer_number(self):
"""
Number detection for english language for integer number like '100', '2'
"""
message = u'100 got selected for interview'
number_detector_object = NumberDetector(entity_name=self.entity_name,
language='en')
number_dicts, original_texts = number_detector_object.detect_entity(
message)
zipped = zip(number_dicts, original_texts)
self.assertEqual(len(zipped), 1)
self.assertIn(({'value': '100', 'unit': None}, u'100'), zipped)
def test_en_number_detection_for_integer_number_with_unit(self):
"""
Number detection for english language for integer number with units like 'Rs100', '2Rs'
"""
message = u'rs.100 is the application charger'
number_detector_object = NumberDetector(entity_name=self.entity_name,
language='en')
number_dicts, original_texts = number_detector_object.detect_entity(
message)
zipped = zip(number_dicts, original_texts)
self.assertEqual(len(zipped), 1)
self.assertIn(({'value': '100', 'unit': 'rupees'}, 'rs.100'), zipped)
def test_en_number_detection_for_decimal_number_with_scale_and_unit(self):
"""
Number detection for english language for decimal number with scale like '1.2 thousand', '2.2k' excluding unit
"""
message = 'I bought a car toy for 2.3k rupees'
number_detector_object = NumberDetector(entity_name=self.entity_name,
language='en')
number_dicts, original_texts = number_detector_object.detect_entity(
message)
zipped = list(zip(number_dicts, original_texts))
self.assertEqual(len(zipped), 1)
self.assertIn(({'value': '2300', 'unit': None}, u'2.3k'), zipped)
def test_en_number_detection_for_decimal_number_with_scale(self):
"""
Number detection for english language for decimal number with scale like '1.2 thousand', '2.2k', '1.4m'
"""
message = 'my monthly salary is 2.2k'
number_detector_object = NumberDetector(entity_name=self.entity_name,
language='en')
number_dicts, original_texts = number_detector_object.detect_entity(
message)
zipped = zip(number_dicts, original_texts)
self.assertEqual(len(zipped), 1)
self.assertIn(({'value': '2200', 'unit': None}, u'2.2k'), zipped)
def test_en_number_detection_for_integer_number_with_scale(self):
"""
Number detection for english language for integer number with scale like '1 thousand', '1k', '1m'
"""
message = '1 thousand men were killed in war'
number_detector_object = NumberDetector(entity_name=self.entity_name,
language='en')
number_dicts, original_texts = number_detector_object.detect_entity(
message)
zipped = zip(number_dicts, original_texts)
self.assertEqual(len(zipped), 1)
self.assertIn(({'value': '1000', 'unit': None}, u'1 thousand'), zipped)
def test_en_number_detection_for_decimal_number(self):
"""
Number detection for english language for decimal number like '100.2'
"""
message = u'Todays temperature is 11.2 degree celsius'
number_detector_object = NumberDetector(entity_name=self.entity_name,
language='en')
number_dicts, original_texts = number_detector_object.detect_entity(
message)
zipped = zip(number_dicts, original_texts)
self.assertEqual(len(zipped), 1)
self.assertIn(({'value': '11.2', 'unit': None}, u'11.2'), zipped)
def resolve_numerals(text, language) -> str:
"""
Uses NumberDetector to resolve numeric occurrences in text for both English and Hindi.
Args:
text (str): processed string with numerals and character constants fixed
language (str): Language for NumberDetector
Returns:
processed_text (str): modified text
"""
processed_text = text
number_detector = NumberDetector('asr_dummy', language=language)
# FIXME: Detection fails if text starts with '0' since number detector discards it
detected_numerals, original_texts = number_detector.detect_entity(
text=text)
detected_numerals_hi, original_texts_hi = number_detector.detect_entity(
text=text, language='hi')
detected_numerals.extend(detected_numerals_hi)
original_texts.extend(original_texts_hi)
for number, original_text in zip(detected_numerals, original_texts):
substitution_reg = re.compile(re.escape(original_text), re.IGNORECASE)
processed_text = substitution_reg.sub(
number[NUMBER_DETECTION_RETURN_DICT_VALUE], processed_text)
return processed_text
def test_en_number_detection_for_decimal_number_with_scale_and_unit_and_different_unit_type_given(
self):
"""
Number detection for english language for decimal number with scale like '1.2 thousand rupees', 'Rupees 2.2k'
"""
message = 'I buys 2.3k kg mango'
number_detector_object = NumberDetector(entity_name=self.entity_name,
language='en',
unit_type='currency')
number_dicts, original_texts = number_detector_object.detect_entity(
message)
zipped = list(zip(number_dicts, original_texts))
self.assertEqual(len(zipped), 0)
def test_en_number_detection_for_integer_number_with_scale_and_unit(self):
"""
Number detection for english language for integer number with scale and unit like 'Rs 1 thousand', '1k Rs'
"""
message = 'i need 1 thousand rupees'
number_detector_object = NumberDetector(entity_name=self.entity_name,
language='en')
number_dicts, original_texts = number_detector_object.detect_entity(
message)
zipped = zip(number_dicts, original_texts)
self.assertEqual(len(zipped), 1)
self.assertIn(({
'value': '1000',
'unit': 'rupees'
}, u'1 thousand rupees'), zipped)
def test_en_number_detection_for_decimal_number_with_unit(self):
"""
Number detection for english language for decimal number with unit like '10.2k rupees'
"""
message = u'my monthly salary is 10.12k rupees'
number_detector_object = NumberDetector(entity_name=self.entity_name,
language='en')
number_dicts, original_texts = number_detector_object.detect_entity(
message)
zipped = zip(number_dicts, original_texts)
self.assertEqual(len(zipped), 1)
self.assertIn(({
'value': '10120',
'unit': 'rupees'
}, u'10.12k rupees'), zipped)
def test_en_number_detection_for_decimal_number_with_scale_and_unit_and_unit_type_given(
self):
"""
Number detection for english language for decimal number with scale like '1.2 thousand rupees', 'Rupees 2.2k'
"""
message = 'I bought a car toy for 2.3k rupees'
number_detector_object = NumberDetector(entity_name=self.entity_name,
language='en',
unit_type='currency')
number_dicts, original_texts = number_detector_object.detect_entity(
message)
zipped = zip(number_dicts, original_texts)
self.assertEqual(len(zipped), 1)
self.assertIn(({
'value': '2300',
'unit': 'rupees'
}, u'2.3k rupees'), zipped)
class BaseNumberRangeDetector(object):
def __init__(self,
entity_name,
language,
data_directory_path,
unit_type=None):
"""
Standard Number detection class, read data from language data path and help to detect number ranges like min
and max value from given number range text for given languages.
Args:
entity_name (str): entity_name: string by which the detected number would be replaced
language (str): language code of text
data_directory_path (str): path of data folder for given language
unit_type (str, optional): number unit types like weight, currency, temperature, used to detect number with
specific unit type only. If None, it will detect all number ranges irrespective
of units. You can see all unit types supported inside number detection
language data with filename unit.csv.
"""
self.text = ''
self.tagged_text = ''
self.processed_text = ''
self.entity_name = entity_name
self.tag = '__' + entity_name + '__'
self.range_variants_map = {}
self.unit_type = unit_type
self.min_range_prefix_variants = None
self.min_range_suffix_variants = None
self.max_range_prefix_variants = None
self.max_range_suffix_variants = None
self.min_max_range_variants = None
self.number_detected_map = {}
self.number_detector = NumberDetector(entity_name=entity_name,
language=language)
self.number_detector.set_min_max_digits(1, 100)
# Method to initialise regex params
self._init_regex_for_range(data_directory_path)
# Variable to define default order in which detector will work
self.detector_preferences = [
self._detect_min_max_num_range,
self._detect_min_num_range_with_prefix_variants,
self._detect_min_num_range_with_suffix_variants,
self._detect_max_num_range_with_prefix_variants,
self._detect_max_num_range_with_suffix_variants
]
def _init_regex_for_range(self, data_directory_path):
"""
Initialise params which hold variants of keywords defining whether a given number range in text contains
min value, max value or both.
Params:
min_range_start_variants (list): List of keywords which occur before min value in text
min_range_end_variants (list): List of keywords which occur after min value in text
max_range_start_variants (list): List of keywords which occur before max value in text
max_range_end_variants (list): List of keywords which occur after max value in text
min_max_range_variants (list): List of keywords which occur in between min and max value in text
Args:
data_directory_path (str): Data directory path
Returns:
None
"""
number_range_df = pd.read_csv(os.path.join(
data_directory_path,
numeral_constant.NUMBER_RANGE_KEYWORD_FILE_NAME),
encoding='utf-8')
for index, row in number_range_df.iterrows():
range_variants = get_list_from_pipe_sep_string(
row[numeral_constant.COLUMN_NUMBER_RANGE_VARIANTS])
for variant in range_variants:
self.range_variants_map[variant] = \
NumberRangeVariant(position=row[numeral_constant.COLUMN_NUMBER_RANGE_POSITION],
range_type=row[numeral_constant.COLUMN_NUMBER_RANGE_RANGE_TYPE])
self.min_range_prefix_variants = [
re.escape(variant)
for variant, value in self.range_variants_map.items()
if (value.position == -1
and value.range_type == numeral_constant.NUMBER_RANGE_MIN_TYPE)
]
self.min_range_suffix_variants = [
re.escape(variant)
for variant, value in self.range_variants_map.items()
if (value.position == 1
and value.range_type == numeral_constant.NUMBER_RANGE_MIN_TYPE)
]
self.max_range_prefix_variants = [
re.escape(variant)
for variant, value in self.range_variants_map.items()
if (value.position == -1
and value.range_type == numeral_constant.NUMBER_RANGE_MAX_TYPE)
]
self.max_range_suffix_variants = [
re.escape(variant)
for variant, value in self.range_variants_map.items()
if (value.position == 1
and value.range_type == numeral_constant.NUMBER_RANGE_MAX_TYPE)
]
self.min_max_range_variants = [
re.escape(variant)
for variant, value in self.range_variants_map.items()
if (value.position == 0 and value.range_type ==
numeral_constant.NUMBER_RANGE_MIN_MAX_TYPE)
]
def _tag_number_in_text(self, processed_text):
"""
replace number in text with number tag from number_detected_map
Args:
processed_text (str): processed text
Returns:
(str): text with number replaced with tag
Examples:
>>> text = 'i want to buy 3 apples and more than two bananas'
>>> number_detected_map = {'__number__0': ({'value': '2', 'unit': None}, 'two'),
'__number__1': ({'value': '3', 'unit': None}, '3')}
>>> self._tag_number_in_text(text)
i want to buy __number__1 apples and more than __number__0 bananas
"""
tagged_number_text = processed_text
sorted_number_detected_map = sorted(
self.number_detected_map.items(),
key=lambda kv: len(kv[1].original_text),
reverse=True)
for number_tag in sorted_number_detected_map:
tagged_number_text = tagged_number_text.replace(
number_tag[1].original_text, number_tag[0], 1)
return tagged_number_text
def _get_number_tag_dict(self):
"""
Method to create number tag dict. Its run number detection on text and create a dict having number tag as key
and value as tuple of entity value and original text.
Returns:
(dict): dict containing number tag and their corresponding value and original text
Examples:
>>> text = 'I want 12 dozen banana'
>>> self._get_number_tag_dict()
{'__number_1': ({'value': 12, 'unit': None}, '12')}
"""
detected_number_dict = {}
entity_value_list, original_text_list = self.number_detector.detect_entity(
self.processed_text)
for index, (entity_value, original_text) in enumerate(
zip(entity_value_list, original_text_list)):
detected_number_dict[numeral_constant.NUMBER_REPLACE_TEXT +
str(index)] = ValueTextPair(
entity_value=entity_value,
original_text=original_text)
return detected_number_dict
def _get_original_text_from_tagged_text(self, number_tag_text):
"""
Return original text value of number tag from number detected map
Args:
number_tag_text (str): tagged number
Returns:
(str or None): Original value of tagged number if found else None
"""
original = number_tag_text
for number_tag in self.number_detected_map:
original = original.replace(
number_tag, self.number_detected_map[number_tag].original_text)
if original == number_tag_text:
return None
return original
def detect_number_range(self, text):
"""
Detect number-range from number range text. Run through list of detectors defined in detector_preferences in
the preferences.
Args:
text(str): text string
Returns:
(tuple): a tuple containing
(list): list containing detected numeric text
(list): list containing original numeral text
"""
self.text = text
self.tagged_text = text
self.processed_text = text
self.number_detected_map = self._get_number_tag_dict()
self.processed_text = self._tag_number_in_text(text)
number_list, original_list = None, None
for detector in self.detector_preferences:
number_list, original_list = detector(number_list, original_list)
self._update_tagged_text(original_list)
return number_list, original_list
def _get_number_range(self, min_part_match, max_part_match, full_match):
"""
Update number_range_list and original_list by finding entity value of number tag and original text from
number_detected_map
Args:
min_part_match (str or None): tagged min number
max_part_match (str or None): tagged max number
full_match (str): text matching regex
Returns:
(tuple): a tuple containing
(list): list containing detected numeric text
(list): list containing original numeral text
"""
number_range = None
original_text = None
if full_match not in self.processed_text:
return number_range, original_text
entity_value_min, entity_value_max, entity_unit = None, None, None
if min_part_match and min_part_match in self.number_detected_map:
entity_dict = self.number_detected_map[min_part_match].entity_value
entity_value_min = entity_dict[
numeral_constant.NUMBER_DETECTION_RETURN_DICT_VALUE]
entity_unit = entity_dict[
numeral_constant.NUMBER_DETECTION_RETURN_DICT_UNIT]
if max_part_match and max_part_match in self.number_detected_map:
entity_dict = self.number_detected_map[max_part_match].entity_value
entity_value_max = entity_dict[
numeral_constant.NUMBER_DETECTION_RETURN_DICT_VALUE]
entity_unit = entity_dict[
numeral_constant.NUMBER_DETECTION_RETURN_DICT_UNIT]
if self.unit_type and (entity_unit is None or
self.number_detector.get_unit_type(entity_unit)
!= self.unit_type):
return number_range, original_text
original_text = self._get_original_text_from_tagged_text(full_match)
if (entity_value_min or entity_value_max) and original_text:
self.processed_text = self.processed_text.replace(
full_match.strip(), '', 1)
original_text = original_text.strip()
number_range = {
numeral_constant.NUMBER_RANGE_MIN_VALUE: entity_value_min,
numeral_constant.NUMBER_RANGE_MAX_VALUE: entity_value_max,
numeral_constant.NUMBER_RANGE_VALUE_UNIT: entity_unit
}
return number_range, original_text
def _detect_min_num_range_with_prefix_variants(self,
number_range_list=None,
original_list=None):
"""
Method to detect number range containing only min value and keywords which identify value as min present
before them. Example - More than 2 {'more than' => keyword, '2' => min value},
At least seven hundred rupees {'At least' => keyword, 'seven hundred rupees'=>min value}
Args:
number_range_list (list):
original_list (list):
Returns:
(tuple): a tuple containing
(list): list containing detected numeric text
(list): list containing original numeral text
"""
number_range_list = number_range_list or []
original_list = original_list or []
if self.min_range_prefix_variants:
min_prefix_choices = '|'.join(self.min_range_prefix_variants)
min_range_start_pattern = re.compile(
ur'((?:{min_prefix_choices})\s+({number}\d+))'.format(
number=numeral_constant.NUMBER_REPLACE_TEXT,
min_prefix_choices=min_prefix_choices), re.UNICODE)
number_range_matches = min_range_start_pattern.findall(
self.processed_text)
for match in number_range_matches:
number_range, original_text = self._get_number_range(
min_part_match=match[1],
max_part_match=None,
full_match=match[0])
if number_range and original_text:
number_range_list.append(number_range)
original_list.append(original_text)
return number_range_list, original_list
def _detect_min_num_range_with_suffix_variants(self,
number_range_list=None,
original_list=None):
"""
Method to detect number range containing only min value and keywords which identify value as min present
after them.
Args:
number_range_list (list):
original_list (list):
Returns:
(tuple): a tuple containing
(list): list containing detected numeric text
(list): list containing original numeral text
"""
number_range_list = number_range_list or []
original_list = original_list or []
if self.min_range_suffix_variants:
min_suffix_choices = '|'.join(self.min_range_suffix_variants)
min_range_end_pattern = re.compile(
ur'(({number}\d+)\s+(?:{min_suffix_choices}))'.format(
number=numeral_constant.NUMBER_REPLACE_TEXT,
min_suffix_choices=min_suffix_choices), re.UNICODE)
number_range_matches = min_range_end_pattern.findall(
self.processed_text)
for match in number_range_matches:
number_range, original_text = self._get_number_range(
min_part_match=match[1],
max_part_match=None,
full_match=match[0])
if number_range and original_text:
number_range_list.append(number_range)
original_list.append(original_text)
return number_range_list, original_list
def _detect_max_num_range_with_prefix_variants(self,
number_range_list=None,
original_list=None):
"""
Method to detect number range containing only max value and keywords which identify value as min present
before them. Example - less than 2 {'less than' => keyword, '2' => max value},
At most seven hundred rupees {'At most' => keyword, 'seven hundred rupees'=>min value}
Args:
number_range_list (list):
original_list (list):
Returns:
(tuple): a tuple containing
(list): list containing detected numeric text
(list): list containing original numeral text
"""
number_range_list = number_range_list or []
original_list = original_list or []
if self.max_range_prefix_variants:
max_prefix_choices = '|'.join(self.max_range_prefix_variants)
max_range_start_pattern = re.compile(
ur'((?:{max_prefix_choices})\s+({number}\d+))'.format(
number=numeral_constant.NUMBER_REPLACE_TEXT,
max_prefix_choices=max_prefix_choices), re.UNICODE)
number_range_matches = max_range_start_pattern.findall(
self.processed_text)
for match in number_range_matches:
number_range, original_text = self._get_number_range(
min_part_match=None,
max_part_match=match[1],
full_match=match[0])
if number_range and original_text:
number_range_list.append(number_range)
original_list.append(original_text)
return number_range_list, original_list
def _detect_max_num_range_with_suffix_variants(self,
number_range_list=None,
original_list=None):
"""
Method to detect number range containing only max value and keywords which identify value as min present
after them.
Args:
number_range_list (list):
original_list (list):
Returns:
(tuple): a tuple containing
(list): list containing detected numeric text
(list): list containing original numeral text
"""
number_range_list = number_range_list or []
original_list = original_list or []
if self.max_range_suffix_variants:
max_suffix_choices = '|'.join(self.max_range_suffix_variants)
max_range_end_pattern = re.compile(
ur'(({number}\d+)\s+(?:{max_suffix_choices}))'.format(
number=numeral_constant.NUMBER_REPLACE_TEXT,
max_suffix_choices=max_suffix_choices), re.UNICODE)
number_range_matches = max_range_end_pattern.findall(
self.processed_text)
for match in number_range_matches:
number_range, original_text = self._get_number_range(
min_part_match=None,
max_part_match=match[1],
full_match=match[0])
if number_range and original_text:
number_range_list.append(number_range)
original_list.append(original_text)
return number_range_list, original_list
def _detect_min_max_num_range(self,
number_range_list=None,
original_list=None):
"""
Method to detect number range containing both min and max value and keywords them present in between
Example - 2000 to 30000 {'to' => keyword, '2000' => min value, '30000' => ,max_value},
2k-3k hundred rupees {'-' => keyword, '2k' => min value, '3k' => ,max_value}
Args:
number_range_list (list):
original_list (list):
Returns:
(tuple): a tuple containing
(list): list containing detected numeric text
(list): list containing original numeral text
"""
number_range_list = number_range_list or []
original_list = original_list or []
if self.min_max_range_variants:
min_max_choices = '|'.join(self.min_max_range_variants)
min_max_range_pattern = re.compile(
ur'(({number}\d+)\s*(?:{min_max_choices})\s*'
ur'({number}\d+))'.format(
number=numeral_constant.NUMBER_REPLACE_TEXT,
min_max_choices=min_max_choices), re.UNICODE)
number_range_matches = min_max_range_pattern.findall(
self.processed_text)
for match in number_range_matches:
number_range, original_text = self._get_number_range(
min_part_match=match[1],
max_part_match=match[2],
full_match=match[0])
if number_range and original_text:
number_range_list.append(number_range)
original_list.append(original_text)
return number_range_list, original_list
def _update_tagged_text(self, original_number_list):
"""
Replaces detected date with tag generated from entity_name used to initialize the object with
A final string with all dates replaced will be stored in object's tagged_text attribute
A string with all dates removed will be stored in object's processed_text attribute
Args:
original_number_list (list): list of substrings of original text to be replaced with tag
created from entity_name
"""
for detected_text in original_number_list:
self.tagged_text = self.tagged_text.replace(
detected_text, self.tag)
def get_currency(text,detected_lang):
from ner_v2.detectors.numeral.number.number_detection import NumberDetector
detector = NumberDetector(entity_name='number', language=detected_lang,unit_type='currency')
number = detector.detect_entity(text)
return number