def test_replace_keywords(self):
"""For each of the test case initialize a new KeywordProcessor.
Add the keywords the test case to KeywordProcessor.
Replace keywords and check if they match the expected result for the test case.
"""
for test_id, test_case in enumerate(self.test_cases):
keyword_replacer = KeywordProcessor()
keyword_replacer.add_keywords_from_dict(test_case['keyword_dict'])
new_sentence = keyword_replacer.replace_keywords(
test_case['sentence'])
replaced_sentence = test_case['sentence']
keyword_mapping = {}
for val in test_case['keyword_dict']:
for value in test_case['keyword_dict'][val]:
keyword_mapping[value] = val
for key in sorted(keyword_mapping, key=len, reverse=True):
lowercase = re.compile(r'(?<!\w){}(?!\w)'.format(
re.escape(key)))
replaced_sentence = lowercase.sub(keyword_mapping[key],
replaced_sentence)
self.assertEqual(
new_sentence, replaced_sentence,
"new_sentence don't match the expected results for test case: {}"
.format(test_id))
def test_dictionary_loading(self):
keyword_processor = KeywordProcessor()
keyword_dict = {
"java": ["java_2e", "java programing"],
"product management": ["product management techniques", "product management"]
}
keyword_processor.add_keywords_from_dict(keyword_dict)
sentence = 'I know java_2e and product management techniques'
keywords_extracted = keyword_processor.extract_keywords(sentence)
self.assertEqual(keywords_extracted, ['java', 'product management'],
"Failed file format one test")
sentence_new = keyword_processor.replace_keywords(sentence)
self.assertEqual(sentence_new, "I know java and product management",
"Failed file format one test")
def test_extract_keywords(self):
"""For each of the test case initialize a new KeywordProcessor.
Add the keywords the test case to KeywordProcessor.
Extract keywords and check if they match the expected result for the test case.
"""
for test_id, test_case in enumerate(self.test_cases):
keyword_processor = KeywordProcessor()
keyword_processor.add_keywords_from_dict(test_case['keyword_dict'])
keywords_extracted = keyword_processor.extract_keywords(
test_case['sentence'])
self.assertEqual(
keywords_extracted, test_case['keywords'],
"keywords_extracted don't match the expected results for test case: {}"
.format(test_id))
from flashtext.keyword import KeywordProcessor
keyword_processor = KeywordProcessor()
# keyword_processor.add_keyword('SQL injection')
keyword_processor.add_keyword('SQL injection',
('vulnerability type', 'SQL injection'))
keyword_processor.add_keyword('cross-site scripting',
('vulnerability type', 'cross-site scripting'))
keyword_processor.add_keyword('cross-site scripting', 'XSS')
# keyword_processor.add_keyword('parameter')
# keyword_processor.add_keyword('function')
# keyword_processor.add_keyword('variable')
keyword_dict = {"cross-site scripting": ["XSS"], "parametert": ["variabler"]}
# {'clean_name': ['list of unclean names']}
keyword_processor.add_keywords_from_dict(keyword_dict)
# Or add keywords from a list:
keyword_processor.add_keywords_from_list(["parameter", "function", "variable"])
####keyword replacement
# keyword_processor.add_keyword('cross-site scripting', 'XSS')
# keyword_processor.replace_keywords('vulnerability is cross-site scripting')
keyword_processor.extract_keywords(
'SQL injection vulnerability in the update_zone function in catalog/admin/geo_zones.php in osCommerce Online Merchant 2.3.3.4 and earlier allows remote administrators to execute arbitrary SQL commands via the zID parameter in a list action. '
)