def test_replace_keywords(self): """For each of the test case initialize a new KeywordProcessor. Add the keywords the test case to KeywordProcessor. Replace keywords and check if they match the expected result for the test case. """ for test_id, test_case in enumerate(self.test_cases): keyword_replacer = KeywordProcessor() keyword_replacer.add_keywords_from_dict(test_case['keyword_dict']) new_sentence = keyword_replacer.replace_keywords( test_case['sentence']) replaced_sentence = test_case['sentence'] keyword_mapping = {} for val in test_case['keyword_dict']: for value in test_case['keyword_dict'][val]: keyword_mapping[value] = val for key in sorted(keyword_mapping, key=len, reverse=True): lowercase = re.compile(r'(?<!\w){}(?!\w)'.format( re.escape(key))) replaced_sentence = lowercase.sub(keyword_mapping[key], replaced_sentence) self.assertEqual( new_sentence, replaced_sentence, "new_sentence don't match the expected results for test case: {}" .format(test_id))
def test_dictionary_loading(self): keyword_processor = KeywordProcessor() keyword_dict = { "java": ["java_2e", "java programing"], "product management": ["product management techniques", "product management"] } keyword_processor.add_keywords_from_dict(keyword_dict) sentence = 'I know java_2e and product management techniques' keywords_extracted = keyword_processor.extract_keywords(sentence) self.assertEqual(keywords_extracted, ['java', 'product management'], "Failed file format one test") sentence_new = keyword_processor.replace_keywords(sentence) self.assertEqual(sentence_new, "I know java and product management", "Failed file format one test")
def test_extract_keywords(self): """For each of the test case initialize a new KeywordProcessor. Add the keywords the test case to KeywordProcessor. Extract keywords and check if they match the expected result for the test case. """ for test_id, test_case in enumerate(self.test_cases): keyword_processor = KeywordProcessor() keyword_processor.add_keywords_from_dict(test_case['keyword_dict']) keywords_extracted = keyword_processor.extract_keywords( test_case['sentence']) self.assertEqual( keywords_extracted, test_case['keywords'], "keywords_extracted don't match the expected results for test case: {}" .format(test_id))
from flashtext.keyword import KeywordProcessor keyword_processor = KeywordProcessor() # keyword_processor.add_keyword('SQL injection') keyword_processor.add_keyword('SQL injection', ('vulnerability type', 'SQL injection')) keyword_processor.add_keyword('cross-site scripting', ('vulnerability type', 'cross-site scripting')) keyword_processor.add_keyword('cross-site scripting', 'XSS') # keyword_processor.add_keyword('parameter') # keyword_processor.add_keyword('function') # keyword_processor.add_keyword('variable') keyword_dict = {"cross-site scripting": ["XSS"], "parametert": ["variabler"]} # {'clean_name': ['list of unclean names']} keyword_processor.add_keywords_from_dict(keyword_dict) # Or add keywords from a list: keyword_processor.add_keywords_from_list(["parameter", "function", "variable"]) ####keyword replacement # keyword_processor.add_keyword('cross-site scripting', 'XSS') # keyword_processor.replace_keywords('vulnerability is cross-site scripting') keyword_processor.extract_keywords( 'SQL injection vulnerability in the update_zone function in catalog/admin/geo_zones.php in osCommerce Online Merchant 2.3.3.4 and earlier allows remote administrators to execute arbitrary SQL commands via the zID parameter in a list action. ' )