def test_validate_extraTokens_2(self): entity_definition = { 'patterns': [[['defined_wrong']]], 'extraTokens': ['defined_wrong'] } errors = find_errors_in_entity_definition(entity_definition) assert errors == ['The property "extraTokens" must be a tuple!']
def test_valid_definition(self): def dumb_function(arg): return arg def dumb_validator(arg): return True entity_definition = { 'patterns': [[['DEFINED_CORRECT']]], 'extraTokens': ({ 'label': 'DEFINED_CORRECT', 'values': ('defined') }, ), 'extraCleaning': { 'DEFINED_CORRECT': dumb_function }, 'spacing': { 'NUM': '', 'NUM_ORD': '', 'default': ' ', }, 'ontological_tags': False, 'entityCleaning': dumb_function, 'entityValidation': dumb_validator, 'collapsiblePatterns': (), } errors = find_errors_in_entity_definition(entity_definition) assert errors == []
def test_validate_many_errors(self): entity_definition = { 'patterns': [[['defined_wrong']]], 'extraTokens': ({ 'label': 'correctly', 'values': ('defined') }, ), 'extraCleaning': { 'NUM': '' }, 'spacing': '', 'entityValidation': '', 'collapsiblePatterns': '', 'entityCleaning': '', 'ontological_tags': '' } errors = find_errors_in_entity_definition(entity_definition) assert errors == [ 'The token "correctly" is defined in "extraTokens" but is not used in any "patterns"', 'The token "defined_wrong" is used in "patterns" but is not a built in token, ' 'please define "defined_wrong" in "extraTokens"', 'Each property in "extraCleaning" must be a function!', 'The property "spacing" must be a dict!', 'The property "ontological_tags" must be a bool!', 'The property "entityCleaning" must be a function!', 'The property "entityValidation" must be a function!', 'The property "collapsiblePatterns" must be a tuple!' ]
def test_validate_extra_tokens_and_pattern_tokens_tuple_of_strings(self): entity_definition = { 'patterns': [[['defined_wrong']]], 'extraTokens': ('defined_wrong', ) } errors = find_errors_in_entity_definition(entity_definition) assert errors == [ 'Your "extraTokens" must be a tuple of dictionaries!' ]
def test_validate_extra_tokens_and_pattern_tokens_valid(self): # this function should finally pass entity_definition = { 'patterns': [[['DEFINED_CORRECT']]], 'extraTokens': ({ 'label': 'DEFINED_CORRECT', 'values': ('defined') }, ) } assert [] == find_errors_in_entity_definition(entity_definition)
def test_do_symbol_type_checks_entityCleaning(self): entity_definition = { 'patterns': [[['DEFINED_CORRECT']]], 'extraTokens': ({ 'label': 'DEFINED_CORRECT', 'values': ('defined') }, ), 'entityCleaning': '' } errors = find_errors_in_entity_definition(entity_definition) assert errors == ['The property "entityCleaning" must be a function!']
def test_do_symbol_type_checks_ontological_tags(self): entity_definition = { 'patterns': [[['DEFINED_CORRECT']]], 'extraTokens': ({ 'label': 'DEFINED_CORRECT', 'values': ('defined') }, ), 'ontological_tags': '' } errors = find_errors_in_entity_definition(entity_definition) assert errors == ['The property "ontological_tags" must be a bool!']
def test_validate_extraCleaning_is_dict(self): entity_definition = { 'patterns': [[['DEFINED_CORRECT']]], 'extraTokens': ({ 'label': 'DEFINED_CORRECT', 'values': ('defined') }, ), 'extraCleaning': '' } assert ['The property "extraCleaning" must be a dict!' ] == find_errors_in_entity_definition(entity_definition)
def test_validate_extra_tokens_and_pattern_tokens_missing_label_prop(self): entity_definition = { 'patterns': [[['defined_wrong']]], 'extraTokens': ({ 'values': ('defined_wrong') }, ) } errors = find_errors_in_entity_definition(entity_definition) assert errors == [ 'Your "extraTokens" dictionaries must contain a "label" property!' ]
def test_validate_collapsiblePatterns(self): entity_definition = { 'patterns': [[['DEFINED_CORRECT']]], 'extraTokens': ({ 'label': 'DEFINED_CORRECT', 'values': ('defined') }, ), 'collapsiblePatterns': '' } errors = find_errors_in_entity_definition(entity_definition) assert errors == [ 'The property "collapsiblePatterns" must be a tuple!' ]
def _validate_individual_entity(entity): entity_name = os.path.split(entity)[-1].replace(".py", "") entity_module = import_module(entity_name) definition_errors = [] if 'ENTITY_DEFINITION' in dir(entity_module): print('Checking entity definition {0:.<35}'.format(entity_name), end='') errors = find_errors_in_entity_definition( entity_module.ENTITY_DEFINITION) _log_entity_definition_error_results(errors) definition_errors.extend(errors) if definition_errors: raise Exception( 'Please fix all entity definition errors before running discovery!' )
def test_validate_extraCleaning_is_functions(self): entity_definition = { 'patterns': [[['DEFINED_CORRECT']]], 'extraTokens': ({ 'label': 'DEFINED_CORRECT', 'values': ('defined') }, ), 'extraCleaning': { 'NUM': '' } } errors = find_errors_in_entity_definition(entity_definition) assert errors == [ 'Each property in "extraCleaning" must be a function!' ]
def test_validate_extra_tokens_and_pattern_tokens_tokens_not_defined(self): entity_definition = { 'patterns': [[['defined_wrong']]], 'extraTokens': ({ 'label': 'correctly', 'values': ('defined') }, ) } errors = find_errors_in_entity_definition(entity_definition) expected_errors = [ 'The token "correctly" is defined in "extraTokens" but is not used in any "patterns"', 'The token "defined_wrong" is used in "patterns" but is not a built in token, ' 'please define "defined_wrong" in "extraTokens"' ] assert errors == expected_errors
def test_importing_address_number(self): import sys import os from importlib import import_module directions_entity_folder_path = os.path.abspath( os.path.join('..', 'examples', 'directions', 'custom', 'entities')) sys.path.append(directions_entity_folder_path) entity = 'address_number.py' entity_name = entity.split("/")[-1].replace(".py", "") entity_module = import_module(entity_name) if 'ENTITY_DEFINITION' in dir(entity_module): errors = find_errors_in_entity_definition( entity_module.ENTITY_DEFINITION) assert errors == []
def test_entity_definition_is_dictionary(self): entity_definition = 'this is a string' errors = find_errors_in_entity_definition(entity_definition) assert errors == ['Your ENTITY_DEFINITION must be a dictionary!']
def test_patterns_property_exists(self): entity_definition = {'missing': 'keys'} errors = find_errors_in_entity_definition(entity_definition) assert errors == [ 'Your ENTITY_DEFINITION must contain a "patterns" property!' ]
def test_validate_extra_tokens_and_pattern_tokens(self): entity_definition = {'patterns': [[['what_am_i']]]} errors = find_errors_in_entity_definition(entity_definition) assert errors == [ 'The token "what_am_i" is used in "patterns" but is not a built in token, please define "what_am_i" in "extraTokens"' ]
def test_patterns_property_has_correct_value_valid(self): # this finally has a correct patterns value entity_definition = {'patterns': [[['NUM']]]} errors = find_errors_in_entity_definition(entity_definition) assert errors == []
def test_patterns_property_has_correct_value_nested_list(self): entity_definition = {'patterns': [['still wrong stuff']]} errors = find_errors_in_entity_definition(entity_definition) assert errors == [ 'The "patterns" property of ENTITY_DEFINITION must be a list of lists of lists of strings. e.g. [[["NUM"], ["LETTER"]]]' ]