def test_multithreading(self):
path = get_path('numbers.json')
policy = parse_redaction_policy_from_file(path)
assert_equal("asdf####fdas### H#ll# w#rld", policy.redact("asdf1234fdas666 H3ll0 w0rld"))
errors = []
lock = threading.Lock()
regex = re.compile(r"[0-9]")
class TestThread(threading.Thread):
def run(self):
for i in range(500):
message = u''.join(random_utf8_char() for _ in range(128))
redacted_message = policy.redact(message)
if regex.search(redacted_message):
with lock:
errors.append((message, redacted_message))
break
threads = []
for i in range(10):
threads.append(TestThread())
for thread in threads:
thread.start()
for thread in threads:
thread.join()
assert_equal(errors, [])
def test_parse_redaction_policy_from_file(self):
with tempfile.NamedTemporaryFile() as f:
json.dump({
'version': 1,
'rules': [
{
'description': 'redact passwords',
'trigger': 'password='******'search': 'password="******"',
'replace': 'password="******"',
},
{
'description': 'redact social security numbers',
'search': '\d{3}-\d{2}-\d{4}',
'replace': 'XXX-XX-XXXX',
},
]
}, f)
f.flush()
policy = parse_redaction_policy_from_file(f.name)
assert_equal(policy.rules, [
RedactionRule(u'password='******'password="******"', u'password="******"'),
RedactionRule(None, u'\d{3}-\d{2}-\d{4}', u'XXX-XX-XXXX'),
])
def test_multithreading(self):
path = get_path('numbers.json')
policy = parse_redaction_policy_from_file(path)
assert_equal("asdf####fdas### H#ll# w#rld", policy.redact("asdf1234fdas666 H3ll0 w0rld"))
errors = []
lock = threading.Lock()
regex = re.compile(r"[0-9]")
class TestThread(threading.Thread):
def run(self):
for i in xrange(500):
message = u''.join(random_utf8_char() for _ in xrange(128))
redacted_message = policy.redact(message)
if regex.search(redacted_message):
with lock:
errors.append((message, redacted_message))
break
threads = []
for i in xrange(10):
threads.append(TestThread())
for thread in threads:
thread.start()
for thread in threads:
thread.join()
assert_equal(errors, [])
def test_real_rules(self):
path = get_path('real-1.json')
policy = parse_redaction_policy_from_file(path)
messages = [
("Hello, world", "Hello, world"),
("CC 1234-2345-3456-4576", "CC XXXX-XXXX-XXXX-XXXX"),
("CC 1234234534654576", "CC XXXXXXXXXXXXXXXX"),
("CC 1234,2345,3456,4576", "CC XXXX-XXXX-XXXX-XXXX"),
("SSN 123-45-6789", "SSN XXX-XX-XXXX"),
("SSN 123456789", "SSN XXXXXXXXX"),
("My password=Hello123", "My password=xxxxx"),
("Host www.cloudera.com", "Host HOSTNAME.REDACTED"),
("www.c1-foo.org rules!", "HOSTNAME.REDACTED rules!"),
("IP1 8.8.8.8", "IP1 0.0.0.0"),
("IP2 192.168.0.1", "IP2 0.0.0.0"),
("My email is [email protected]",
"My email is [email protected]"),
("[email protected] is interesting",
"[email protected] is interesting"),
("Multi 1234-2345-3456-4567\nLine 123-45-6789",
"Multi XXXX-XXXX-XXXX-XXXX\nLine XXX-XX-XXXX"),
]
for message, redacted_message in messages:
assert_equal(redacted_message, policy.redact(message))
def test_parse_redaction_policy_from_file(self):
with tempfile.NamedTemporaryFile() as f:
json.dump({
'version': 1,
'rules': [
{
'description': 'redact passwords',
'trigger': 'password='******'search': 'password="******"',
'replace': 'password="******"',
},
{
'description': 'redact social security numbers',
'search': '\d{3}-\d{2}-\d{4}',
'replace': 'XXX-XX-XXXX',
},
]
}, f)
f.flush()
policy = parse_redaction_policy_from_file(f.name)
assert_equal(policy.rules, [
RedactionRule(u'password='******'password="******"', u'password="******"'),
RedactionRule(None, u'\d{3}-\d{2}-\d{4}', u'XXX-XX-XXXX'),
])
def test_ordering(self):
path = get_path('ordering-1.json')
policy = parse_redaction_policy_from_file(path)
messages = [
("Hello, world", "Hello, world"),
("one", "four"),
("This one is a nice one", "This four is a nice four"),
("Please help me: ten", "Please help me: thirteen"),
("HappY abc", "HappY stu"),
]
for message, redacted_message in messages:
assert_equal(redacted_message, policy.redact(message))
def test_ordering(self):
path = get_path('ordering-1.json')
policy = parse_redaction_policy_from_file(path)
messages = [
("Hello, world", "Hello, world"),
("one", "four"),
("This one is a nice one", "This four is a nice four"),
("Please help me: ten", "Please help me: thirteen"),
("HappY abc", "HappY stu"),
]
for message, redacted_message in messages:
assert_equal(redacted_message, policy.redact(message))
def test_unicode_strings(self):
path = get_path('real-1.json')
policy = parse_redaction_policy_from_file(path)
messages = [
("äöüß 123-45-6789", "äöüß XXX-XX-XXXX"),
("你好阿基尔 1234234534654576", "你好阿基尔 XXXXXXXXXXXXXXXX"),
("ã 你好 1234,2345,3456,4576", "ã 你好 XXXX-XXXX-XXXX-XXXX"),
]
for message, redacted_message in messages:
message_to_redact = smart_str(message)
self.logger.debug("Message to redact : %s " % message_to_redact)
self.logger.debug("Message after redact : %s " %
policy.redact(message_to_redact))
assert_equal(redacted_message, policy.redact(message_to_redact))
def test_case_sensitivity(self):
path = get_path('case-1.json')
policy = parse_redaction_policy_from_file(path)
messages = [
("Hello, world", "Hello, world"),
("Say aAa! aaa! AAAAAA!", "Say bbb! bbb! bbbbbb!"),
("I like dddogs. dDd", "I like dddogs. dDd"),
("Cccats. Dddogs", "Cccats. eEeogs"),
("Trigger fff gGg", "Trigger fff gGg"),
("Trigger fFf Ggg", "Trigger fFf Ggg"),
("Trigger fFf gGg", "Trigger fFf hHh"),
]
for message, redacted_message in messages:
assert_equal(redacted_message, policy.redact(message))
def test_back_refs(self):
path = get_path('replace-1.json')
policy = parse_redaction_policy_from_file(path)
messages = [
("Hello, world", "Hello, world"),
("1234-2345-3456-4576", "XXXX-XXXX-XXXX-4576"),
("Words www.gmail.com is cool", "Words HOSTNAME.REDACTED.com is cool"),
("short.org", "HOSTNAME.REDACTED.org"),
("long.n4me.h-1.co.fr", "HOSTNAME.REDACTED.fr"),
("Ping 192.168.0.1", "Ping 0.192.1.168"),
("Magic word", "word: Magic word, word"),
]
for message, redacted_message in messages:
assert_equal(redacted_message, policy.redact(message))
def test_back_refs(self):
path = get_path('replace-1.json')
policy = parse_redaction_policy_from_file(path)
messages = [
("Hello, world", "Hello, world"),
("1234-2345-3456-4576", "XXXX-XXXX-XXXX-4576"),
("Words www.gmail.com is cool", "Words HOSTNAME.REDACTED.com is cool"),
("short.org", "HOSTNAME.REDACTED.org"),
("long.n4me.h-1.co.fr", "HOSTNAME.REDACTED.fr"),
("Ping 192.168.0.1", "Ping 0.192.1.168"),
("Magic word", "word: Magic word, word"),
]
for message, redacted_message in messages:
assert_equal(redacted_message, policy.redact(message))
def test_case_sensitivity(self):
path = get_path('case-1.json')
policy = parse_redaction_policy_from_file(path)
messages = [
("Hello, world", "Hello, world"),
("Say aAa! aaa! AAAAAA!", "Say bbb! bbb! bbbbbb!"),
("I like dddogs. dDd", "I like dddogs. dDd"),
("Cccats. Dddogs", "Cccats. eEeogs"),
("Trigger fff gGg", "Trigger fff gGg"),
("Trigger fFf Ggg", "Trigger fFf Ggg"),
("Trigger fFf gGg", "Trigger fFf hHh"),
]
for message, redacted_message in messages:
assert_equal(redacted_message, policy.redact(message))
def test_is_dir(self): path = '/tmp' parse_redaction_policy_from_file(path)
def test_is_not_json(self):
path = get_path('not-json.json')
parse_redaction_policy_from_file(path)
def test_no_version(self):
path = get_path('no-version.json')
parse_redaction_policy_from_file(path)
def test_no_brace(self):
path = get_path('no-brace.json')
parse_redaction_policy_from_file(path)
def test_huge_rules(self):
path = get_path('huge-1.json')
policy = parse_redaction_policy_from_file(path)
assert_equal("This string is not redadted", policy.redact(MESSAGE))
def test_alpha_version(self):
path = get_path('alpha-version.json')
parse_redaction_policy_from_file(path)
def test_int_version(self):
path = get_path('verint.json')
policy = parse_redaction_policy_from_file(path)
assert_equal("Hxllx, wxrld", policy.redact("Hello, world"))
def test_empty_rules(self):
path = get_path('empty-rules.json')
policy = parse_redaction_policy_from_file(path)
assert_equal(MESSAGE, policy.redact(MESSAGE))
def test_extra_attr(self):
path = get_path('extra-attr.json')
parse_redaction_policy_from_file(path)
def test_bad_regex(self):
path = get_path('bad-regex.json')
parse_redaction_policy_from_file(path)
def test_does_not_exist(self):
path = get_path('thisfiledoesnotexist.json')
parse_redaction_policy_from_file(path)
def test_unknown_version(self):
path = get_path('unknown-version.json')
parse_redaction_policy_from_file(path)
def test_basic_good1(self):
path = get_path('good-1.json')
policy = parse_redaction_policy_from_file(path)
assert_equal("Hxllx, wxrld", policy.redact("Hello, world"))
def test_no_search(self):
path = get_path('no-search.json')
parse_redaction_policy_from_file(path)
