def run_test(self): message = testcase["message"] unit_type = testcase.get("unit_type", None) number_detector_object = NumberDetector(entity_name="number", language=language, unit_type=unit_type) number_detector_object.set_min_max_digits( min_digit=testcase.get('min_digit', number_detector_object.min_digit), max_digit=testcase.get('max_digit', number_detector_object.max_digit)) number_dicts, spans = number_detector_object.detect_entity(message) expected_number_dicts, expected_spans = parse_expected_outputs( testcase["outputs"]) expected_outputs = list( six.moves.zip(expected_number_dicts, expected_spans)) prefix = failure_string_prefix.format(message=message, language=language) self.assertEqual( len(number_dicts), len(spans), prefix + u"Returned numbers and original_texts have different lengths") self.assertEqual( len(spans), len(expected_outputs), prefix + u"Returned numbers and expected_outputs have different lengths" ) for output in six.moves.zip(number_dicts, spans): self.assertIn( output, expected_outputs, prefix + u"{got} not in {expected_outputs}".format( got=output, expected_outputs=expected_outputs))
def test_en_number_detection_for_integer_number(self): """ Number detection for english language for integer number like '100', '2' """ message = u'100 got selected for interview' number_detector_object = NumberDetector(entity_name=self.entity_name, language='en') number_dicts, original_texts = number_detector_object.detect_entity( message) zipped = zip(number_dicts, original_texts) self.assertEqual(len(zipped), 1) self.assertIn(({'value': '100', 'unit': None}, u'100'), zipped)
def test_en_number_detection_for_integer_number_with_unit(self): """ Number detection for english language for integer number with units like 'Rs100', '2Rs' """ message = u'rs.100 is the application charger' number_detector_object = NumberDetector(entity_name=self.entity_name, language='en') number_dicts, original_texts = number_detector_object.detect_entity( message) zipped = zip(number_dicts, original_texts) self.assertEqual(len(zipped), 1) self.assertIn(({'value': '100', 'unit': 'rupees'}, 'rs.100'), zipped)
def test_en_number_detection_for_decimal_number_with_scale_and_unit(self): """ Number detection for english language for decimal number with scale like '1.2 thousand', '2.2k' excluding unit """ message = 'I bought a car toy for 2.3k rupees' number_detector_object = NumberDetector(entity_name=self.entity_name, language='en') number_dicts, original_texts = number_detector_object.detect_entity( message) zipped = list(zip(number_dicts, original_texts)) self.assertEqual(len(zipped), 1) self.assertIn(({'value': '2300', 'unit': None}, u'2.3k'), zipped)
def test_en_number_detection_for_decimal_number_with_scale(self): """ Number detection for english language for decimal number with scale like '1.2 thousand', '2.2k', '1.4m' """ message = 'my monthly salary is 2.2k' number_detector_object = NumberDetector(entity_name=self.entity_name, language='en') number_dicts, original_texts = number_detector_object.detect_entity( message) zipped = zip(number_dicts, original_texts) self.assertEqual(len(zipped), 1) self.assertIn(({'value': '2200', 'unit': None}, u'2.2k'), zipped)
def test_en_number_detection_for_integer_number_with_scale(self): """ Number detection for english language for integer number with scale like '1 thousand', '1k', '1m' """ message = '1 thousand men were killed in war' number_detector_object = NumberDetector(entity_name=self.entity_name, language='en') number_dicts, original_texts = number_detector_object.detect_entity( message) zipped = zip(number_dicts, original_texts) self.assertEqual(len(zipped), 1) self.assertIn(({'value': '1000', 'unit': None}, u'1 thousand'), zipped)
def test_en_number_detection_for_decimal_number(self): """ Number detection for english language for decimal number like '100.2' """ message = u'Todays temperature is 11.2 degree celsius' number_detector_object = NumberDetector(entity_name=self.entity_name, language='en') number_dicts, original_texts = number_detector_object.detect_entity( message) zipped = zip(number_dicts, original_texts) self.assertEqual(len(zipped), 1) self.assertIn(({'value': '11.2', 'unit': None}, u'11.2'), zipped)
def __init__(self, entity_name, language, data_directory_path, unit_type=None): """ Standard Number detection class, read data from language data path and help to detect number ranges like min and max value from given number range text for given languages. Args: entity_name (str): entity_name: string by which the detected number would be replaced language (str): language code of text data_directory_path (str): path of data folder for given language unit_type (str, optional): number unit types like weight, currency, temperature, used to detect number with specific unit type only. If None, it will detect all number ranges irrespective of units. You can see all unit types supported inside number detection language data with filename unit.csv. """ self.text = '' self.tagged_text = '' self.processed_text = '' self.entity_name = entity_name self.tag = '__' + entity_name + '__' self.range_variants_map = {} self.unit_type = unit_type self.language = language self.min_range_prefix_variants = None self.min_range_suffix_variants = None self.max_range_prefix_variants = None self.max_range_suffix_variants = None self.min_max_range_variants = None self.number_detected_map = {} self.number_detector = NumberDetector(entity_name=entity_name, language=language, unit_type=unit_type, detect_without_unit=True) self.number_detector.set_min_max_digits(1, 100) # Method to initialise regex params self._init_regex_for_range(data_directory_path) # Variable to define default order in which detector will work self.detector_preferences = [ self._detect_min_max_num_range, self._detect_min_num_range_with_prefix_variants, self._detect_min_num_range_with_suffix_variants, self._detect_max_num_range_with_prefix_variants, self._detect_max_num_range_with_suffix_variants, self._detect_absolute_number ]
def test_en_number_detection_for_decimal_number_with_scale_and_unit_and_different_unit_type_given( self): """ Number detection for english language for decimal number with scale like '1.2 thousand rupees', 'Rupees 2.2k' """ message = 'I buys 2.3k kg mango' number_detector_object = NumberDetector(entity_name=self.entity_name, language='en', unit_type='currency') number_dicts, original_texts = number_detector_object.detect_entity( message) zipped = list(zip(number_dicts, original_texts)) self.assertEqual(len(zipped), 0)
def test_en_number_detection_for_integer_number_with_scale_and_unit(self): """ Number detection for english language for integer number with scale and unit like 'Rs 1 thousand', '1k Rs' """ message = 'i need 1 thousand rupees' number_detector_object = NumberDetector(entity_name=self.entity_name, language='en') number_dicts, original_texts = number_detector_object.detect_entity( message) zipped = zip(number_dicts, original_texts) self.assertEqual(len(zipped), 1) self.assertIn(({ 'value': '1000', 'unit': 'rupees' }, u'1 thousand rupees'), zipped)
def test_en_number_detection_for_decimal_number_with_unit(self): """ Number detection for english language for decimal number with unit like '10.2k rupees' """ message = u'my monthly salary is 10.12k rupees' number_detector_object = NumberDetector(entity_name=self.entity_name, language='en') number_dicts, original_texts = number_detector_object.detect_entity( message) zipped = zip(number_dicts, original_texts) self.assertEqual(len(zipped), 1) self.assertIn(({ 'value': '10120', 'unit': 'rupees' }, u'10.12k rupees'), zipped)
def test_en_number_detection_for_decimal_number_with_scale_and_unit_and_unit_type_given( self): """ Number detection for english language for decimal number with scale like '1.2 thousand rupees', 'Rupees 2.2k' """ message = 'I bought a car toy for 2.3k rupees' number_detector_object = NumberDetector(entity_name=self.entity_name, language='en', unit_type='currency') number_dicts, original_texts = number_detector_object.detect_entity( message) zipped = zip(number_dicts, original_texts) self.assertEqual(len(zipped), 1) self.assertIn(({ 'value': '2300', 'unit': 'rupees' }, u'2.3k rupees'), zipped)
def resolve_numerals(text, language) -> str: """ Uses NumberDetector to resolve numeric occurrences in text for both English and Hindi. Args: text (str): processed string with numerals and character constants fixed language (str): Language for NumberDetector Returns: processed_text (str): modified text """ processed_text = text number_detector = NumberDetector('asr_dummy', language=language) # FIXME: Detection fails if text starts with '0' since number detector discards it detected_numerals, original_texts = number_detector.detect_entity( text=text) detected_numerals_hi, original_texts_hi = number_detector.detect_entity( text=text, language='hi') detected_numerals.extend(detected_numerals_hi) original_texts.extend(original_texts_hi) for number, original_text in zip(detected_numerals, original_texts): substitution_reg = re.compile(re.escape(original_text), re.IGNORECASE) processed_text = substitution_reg.sub( number[NUMBER_DETECTION_RETURN_DICT_VALUE], processed_text) return processed_text
def number(request): """Use NumberDetector to detect numerals Attributes: request: url parameters: request params: message (str): natural text on which detection logic is to be run. Note if structured value is detection is run on structured value instead of message entity_name (str): name of the entity. Also acts as elastic-search dictionary name if entity uses elastic-search lookup structured_value (str): Value obtained from any structured elements. Note if structured value is detection is run on structured value instead of message (For example, UI elements like form, payload, etc) fallback_value (str): If the detection logic fails to detect any value either from structured_value or message then we return a fallback_value as an output. bot_message (str): previous message from a bot/agent. unit_type(str): restrict number range to detect for some unit types like 'currency', 'temperature' min_digit (str): min digit max_digit (str): max digit Returns: dict or None: dictionary containing entity_value, original_text and detection; entity_value is in itself a dict with its keys varying from entity to entity Example: message = "I want to purchase 30 units of mobile and 40 units of Television" entity_name = 'number_of_unit' structured_value = None fallback_value = None bot_message = None unit_type = None output = get_number(message=message, entity_name=entity_name, structured_value=structured_value, fallback_value=fallback_value, bot_message=bot_message, min_digit=1, max_digit=2) print output >> [{'detection': 'message', 'original_text': '30', 'entity_value': {'value': '30', 'unit': None}}, {'detection': 'message', 'original_text': '40', 'entity_value': {'value': '40', 'unit': None}}] message = "I want to reserve a table for 3 people" entity_name = 'number_of_people' structured_value = None fallback_value = None bot_message = None unit_type = None min_digit=1 max_digit=6 output = number(request) print output >> [{'detection': 'message', 'original_text': 'for 3 people', 'entity_value': {'value': '3', 'unit': 'people'}}] """ try: parameters_dict = get_parameters_dictionary(request) ner_logger.debug('Start: %s ' % parameters_dict[PARAMETER_ENTITY_NAME]) number_detection = NumberDetector(entity_name=parameters_dict[PARAMETER_ENTITY_NAME], language=parameters_dict[PARAMETER_SOURCE_LANGUAGE], unit_type=parameters_dict[PARAMETER_NUMBER_UNIT_TYPE]) if parameters_dict[PARAMETER_MIN_DIGITS] and parameters_dict[PARAMETER_MAX_DIGITS]: min_digit = int(parameters_dict[PARAMETER_MIN_DIGITS]) max_digit = int(parameters_dict[PARAMETER_MAX_DIGITS]) number_detection.set_min_max_digits(min_digit=min_digit, max_digit=max_digit) entity_output = number_detection.detect(message=parameters_dict[PARAMETER_MESSAGE], structured_value=parameters_dict[PARAMETER_STRUCTURED_VALUE], fallback_value=parameters_dict[PARAMETER_FALLBACK_VALUE], bot_message=parameters_dict[PARAMETER_BOT_MESSAGE]) ner_logger.debug('Finished %s : %s ' % (parameters_dict[PARAMETER_ENTITY_NAME], entity_output)) except TypeError as e: ner_logger.exception('Exception for numeric: %s ' % e) return HttpResponse(status=500) return HttpResponse(json.dumps({'data': entity_output}), content_type='application/json')
def get_currency(text,detected_lang): from ner_v2.detectors.numeral.number.number_detection import NumberDetector detector = NumberDetector(entity_name='number', language=detected_lang,unit_type='currency') number = detector.detect_entity(text) return number