class SoSKeywordParser(SoSCleanerParser):
"""Handles parsing for user provided keywords
"""
name = 'Keyword Parser'
map_file_key = 'keyword_map'
def __init__(self, config, keywords=None, keyword_file=None):
self.mapping = SoSKeywordMap()
self.user_keywords = []
super(SoSKeywordParser, self).__init__(config)
for _keyword in self.mapping.dataset.keys():
self.user_keywords.append(_keyword)
if keywords:
for keyword in keywords:
if keyword not in self.user_keywords:
# pre-generate an obfuscation mapping for each keyword
# this is necessary for cases where filenames are being
# obfuscated before or instead of file content
self.mapping.get(keyword.lower())
self.user_keywords.append(keyword)
if keyword_file and os.path.exists(keyword_file):
with open(keyword_file, 'r') as kwf:
self.user_keywords.extend(kwf.read().splitlines())
def _parse_line(self, line):
return line, 0
def __init__(self, conf_file=None, keywords=None):
self.mapping = SoSKeywordMap()
self.user_keywords = []
super(SoSKeywordParser, self).__init__(conf_file)
for _keyword in self.mapping.dataset.keys():
self.user_keywords.append(_keyword)
if keywords:
self.user_keywords.extend(keywords)
def __init__(self, conf_file=None, keywords=None, keyword_file=None):
self.mapping = SoSKeywordMap()
self.user_keywords = []
super(SoSKeywordParser, self).__init__(conf_file)
for _keyword in self.mapping.dataset.keys():
self.user_keywords.append(_keyword)
if keywords:
self.user_keywords.extend(keywords)
if keyword_file and os.path.exists(keyword_file):
with open(keyword_file, 'r') as kwf:
self.user_keywords.extend(kwf.read().splitlines())
class SoSKeywordParser(SoSCleanerParser):
"""Handles parsing for user provided keywords
"""
name = 'Keyword Parser'
map_file_key = 'keyword_map'
prep_map_file = ''
def __init__(self, conf_file=None, keywords=None, keyword_file=None):
self.mapping = SoSKeywordMap()
self.user_keywords = []
super(SoSKeywordParser, self).__init__(conf_file)
for _keyword in self.mapping.dataset.keys():
self.user_keywords.append(_keyword)
if keywords:
self.user_keywords.extend(keywords)
if keyword_file and os.path.exists(keyword_file):
with open(keyword_file, 'r') as kwf:
self.user_keywords.extend(kwf.read().splitlines())
def parse_line(self, line):
count = 0
for keyword in self.user_keywords:
if keyword in line:
line = line.replace(keyword, self.mapping.get(keyword))
count += 1
return line, count
def __init__(self, config, keywords=None, keyword_file=None):
self.mapping = SoSKeywordMap()
self.user_keywords = []
super(SoSKeywordParser, self).__init__(config)
for _keyword in self.mapping.dataset.keys():
self.user_keywords.append(_keyword)
if keywords:
for keyword in keywords:
if keyword not in self.user_keywords:
# pre-generate an obfuscation mapping for each keyword
# this is necessary for cases where filenames are being
# obfuscated before or instead of file content
self.mapping.get(keyword)
self.user_keywords.append(keyword)
if keyword_file and os.path.exists(keyword_file):
with open(keyword_file, 'r') as kwf:
self.user_keywords.extend(kwf.read().splitlines())
def setUp(self):
self.mac_map = SoSMacMap()
self.ip_map = SoSIPMap()
self.host_map = SoSHostnameMap()
self.host_map.load_domains_from_options(['redhat.com'])
self.kw_map = SoSKeywordMap()
class CleanerMapTests(unittest.TestCase):
def setUp(self):
self.mac_map = SoSMacMap()
self.ip_map = SoSIPMap()
self.host_map = SoSHostnameMap()
self.host_map.load_domains_from_options(['redhat.com'])
self.kw_map = SoSKeywordMap()
def test_mac_map_obfuscate_valid_v4(self):
_test = self.mac_map.get('12:34:56:78:90:ab')
self.assertNotEqual(_test, '12:34:56:78:90:ab')
def test_mac_map_obfuscate_valid_v6(self):
_test = self.mac_map.get('12:34:56:ff:fe:78:90:ab')
self.assertNotEqual(_test, '12:34:56:ff:fe:78:90:ab')
def test_mac_map_obfuscate_valid_v6_quad(self):
_test = self.mac_map.get('1234:56ff:fe78:90ab')
self.assertNotEqual(_test, '1234:56ff:fe78:90ab')
def test_mac_map_skip_ignores(self):
_test = self.mac_map.get('ff:ff:ff:ff:ff:ff')
self.assertEquals(_test, 'ff:ff:ff:ff:ff:ff')
def test_mac_map_avoid_duplicate_obfuscation(self):
_test = self.mac_map.get('ab:cd:ef:fe:dc:ba')
_dup = self.mac_map.get(_test)
self.assertEquals(_test, _dup)
def test_ip_map_obfuscate_v4_with_cidr(self):
_test = self.ip_map.get('192.168.1.0/24')
self.assertNotEqual(_test, '192.168.1.0/24')
def test_ip_map_obfuscate_no_cidr(self):
_test = self.ip_map.get('192.168.2.2')
self.assertNotEqual(_test, '192.168.2.2')
def test_ip_map_obfuscate_same_subnet(self):
_net = ip_interface(self.ip_map.get('192.168.3.0/24'))
_test = ip_interface(self.ip_map.get('192.168.3.1'))
self.assertTrue(_test.ip in _net.network)
def test_ip_map_get_same_with_or_without_cidr(self):
_hostwsub = self.ip_map.get('192.168.4.1/24')
_hostnosub = self.ip_map.get('192.168.4.1')
self.assertEqual(_hostwsub.split('/')[0], _hostnosub)
def test_ip_skip_ignores(self):
_test = self.ip_map.get('127.0.0.1')
self.assertEquals(_test, '127.0.0.1')
def test_hostname_obfuscate_domain_options(self):
_test = self.host_map.get('www.redhat.com')
self.assertNotEqual(_test, 'www.redhat.com')
def test_hostname_obfuscate_same_item(self):
_test1 = self.host_map.get('example.redhat.com')
_test2 = self.host_map.get('example.redhat.com')
self.assertEqual(_test1, _test2)
def test_hostname_obfuscate_just_domain(self):
_test = self.host_map.get('redhat.com')
self.assertEqual(_test, 'obfuscateddomain0.com')
def test_hostname_no_obfuscate_non_loaded_domain(self):
_test = self.host_map.get('foobar.com')
self.assertEqual(_test, 'foobar.com')
def test_hostname_no_obfuscate_non_loaded_fqdn(self):
_test = self.host_map.get('example.foobar.com')
self.assertEqual(_test, 'example.foobar.com')
def test_keyword_single(self):
_test = self.kw_map.get('foobar')
self.assertEqual(_test, 'obfuscatedword0')
def setUp(self):
self.mac_map = SoSMacMap()
self.ip_map = SoSIPMap()
self.host_map = SoSHostnameMap(['redhat.com'])
self.kw_map = SoSKeywordMap()