def register_log_filtering(rules): """ `add_redaction_filter` injects the redaction filter into all of the `logger` handlers. This must be called after all of the handlers have been added to `logger`, otherwise those handlers may expose unredacted strings. """ global_redaction_engine.add_rules(rules) logfilter.add_log_redaction_filter_to_logger(global_redaction_engine, logging.root)
def register_log_filtering(policy):
"""
`add_redaction_filter` injects the redaction filter into all of the `logger`
handlers. This must be called after all of the handlers have been added to
`logger`, otherwise those handlers may expose unredacted strings.
"""
if policy:
global_redaction_engine.add_policy(policy)
logfilter.add_log_redaction_filter_to_logger(global_redaction_engine, logging.root)
def setUpClass(cls):
cls.logger = logging.getLogger(cls.__name__)
cls.handler = MockLoggingHandler()
cls.logger.addHandler(cls.handler)
engine = RedactionEngine([
RedactionRule('password='******'password="******"', 'password="******"'),
RedactionRule('ssn=', 'ssn=\d{3}-\d{2}-\d{4}', 'ssn=XXX-XX-XXXX'),
])
add_log_redaction_filter_to_logger(engine, cls.logger)
def setUpClass(cls):
cls.logger = logging.getLogger(cls.__name__)
cls.handler = MockLoggingHandler()
cls.logger.addHandler(cls.handler)
policy = RedactionPolicy([
RedactionRule('password='******'password="******"', 'password="******"'),
RedactionRule('ssn=', 'ssn=\d{3}-\d{2}-\d{4}', 'ssn=XXX-XX-XXXX'),
])
engine = RedactionEngine([policy])
add_log_redaction_filter_to_logger(engine, cls.logger)
def test_redact_statements(self):
old_policies = redaction.global_redaction_engine.policies
redaction.global_redaction_engine.policies = [
RedactionPolicy([
RedactionRule('', 'ssn=\d{3}-\d{2}-\d{4}', 'ssn=XXX-XX-XXXX'),
])
]
logfilter.add_log_redaction_filter_to_logger(redaction.global_redaction_engine, logging.root)
sensitive_query = 'SELECT "ssn=123-45-6789"'
redacted_query = 'SELECT "ssn=XXX-XX-XXXX"'
nonsensitive_query = 'SELECT "hello"'
snippets = [
{
'status': 'ready',
'viewSettings': {
'sqlDialect': True,
'snippetImage': '/static/beeswax/art/icon_beeswax_48.png',
'placeHolder': 'Example: SELECT * FROM tablename, or press CTRL + space',
'aceMode': 'ace/mode/hive'
},
'id': '10a29cda-063f-1439-4836-d0c460154075',
'statement_raw': sensitive_query,
'statement': sensitive_query,
'type': 'hive'
},
{
'status': 'ready',
'viewSettings': {
'sqlDialect': True,
'snippetImage': '/static/impala/art/icon_impala_48.png',
'placeHolder': 'Example: SELECT * FROM tablename, or press CTRL + space',
'aceMode': 'ace/mode/impala'
},
'id': 'e17d195a-beb5-76bf-7489-a9896eeda67a',
'statement_raw': sensitive_query,
'statement': sensitive_query,
'type': 'impala'
},
{
'status': 'ready',
'viewSettings': {
'sqlDialect': True,
'snippetImage': '/static/beeswax/art/icon_beeswax_48.png',
'placeHolder': 'Example: SELECT * FROM tablename, or press CTRL + space',
'aceMode': 'ace/mode/hive'
},
'id': '10a29cda-063f-1439-4836-d0c460154075',
'statement_raw': nonsensitive_query,
'statement': nonsensitive_query,
'type': 'hive'
},
]
try:
self.document2.type = 'notebook'
self.document2.update_data({'snippets': snippets})
self.document2.search = sensitive_query
self.document2.save()
saved_snippets = self.document2.data_dict['snippets']
# Make sure redacted queries are redacted.
assert_equal(redacted_query, saved_snippets[0]['statement'])
assert_equal(redacted_query, saved_snippets[0]['statement_raw'])
assert_equal(True, saved_snippets[0]['is_redacted'])
assert_equal(redacted_query, saved_snippets[1]['statement'])
assert_equal(redacted_query, saved_snippets[1]['statement_raw'])
assert_equal(True, saved_snippets[1]['is_redacted'])
document = Document2.objects.get(pk=self.document2.pk)
assert_equal(redacted_query, document.search)
# Make sure unredacted queries are not redacted.
assert_equal(nonsensitive_query, saved_snippets[2]['statement'])
assert_equal(nonsensitive_query, saved_snippets[2]['statement_raw'])
assert_false('is_redacted' in saved_snippets[2])
finally:
redaction.global_redaction_engine.policies = old_policies
def test_redact_statements(self):
old_policies = redaction.global_redaction_engine.policies
redaction.global_redaction_engine.policies = [
RedactionPolicy([
RedactionRule('', 'ssn=\d{3}-\d{2}-\d{4}', 'ssn=XXX-XX-XXXX'),
])
]
logfilter.add_log_redaction_filter_to_logger(redaction.global_redaction_engine, logging.root)
sensitive_query = 'SELECT "ssn=123-45-6789"'
redacted_query = 'SELECT "ssn=XXX-XX-XXXX"'
nonsensitive_query = 'SELECT "hello"'
snippets = [
{
'status': 'ready',
'viewSettings': {
'sqlDialect': True,
'snippetImage': '/static/beeswax/art/icon_beeswax_48.png',
'placeHolder': 'Example: SELECT * FROM tablename, or press CTRL + space',
'aceMode': 'ace/mode/hive'
},
'id': '10a29cda-063f-1439-4836-d0c460154075',
'statement_raw': sensitive_query,
'statement': sensitive_query,
'type': 'hive'
},
{
'status': 'ready',
'viewSettings': {
'sqlDialect': True,
'snippetImage': '/static/impala/art/icon_impala_48.png',
'placeHolder': 'Example: SELECT * FROM tablename, or press CTRL + space',
'aceMode': 'ace/mode/impala'
},
'id': 'e17d195a-beb5-76bf-7489-a9896eeda67a',
'statement_raw': sensitive_query,
'statement': sensitive_query,
'type': 'impala'
},
{
'status': 'ready',
'viewSettings': {
'sqlDialect': True,
'snippetImage': '/static/beeswax/art/icon_beeswax_48.png',
'placeHolder': 'Example: SELECT * FROM tablename, or press CTRL + space',
'aceMode': 'ace/mode/hive'
},
'id': '10a29cda-063f-1439-4836-d0c460154075',
'statement_raw': nonsensitive_query,
'statement': nonsensitive_query,
'type': 'hive'
},
]
try:
self.document2.type = 'notebook'
self.document2.update_data({'snippets': snippets})
self.document2.save()
saved_snippets = self.document2.data_dict['snippets']
# Make sure redacted queries are redacted.
assert_equal(redacted_query, saved_snippets[0]['statement'])
assert_equal(redacted_query, saved_snippets[0]['statement_raw'])
assert_equal(True, saved_snippets[0]['is_redacted'])
assert_equal(redacted_query, saved_snippets[1]['statement'])
assert_equal(redacted_query, saved_snippets[1]['statement_raw'])
assert_equal(True, saved_snippets[1]['is_redacted'])
# Make sure unredacted queries are not redacted.
assert_equal(nonsensitive_query, saved_snippets[2]['statement'])
assert_equal(nonsensitive_query, saved_snippets[2]['statement_raw'])
assert_false('is_redacted' in saved_snippets[2])
finally:
redaction.global_redaction_engine.policies = old_policies
def test_redact_statements(self):
old_policies = redaction.global_redaction_engine.policies
redaction.global_redaction_engine.policies = [
RedactionPolicy([RedactionRule("", "ssn=\d{3}-\d{2}-\d{4}", "ssn=XXX-XX-XXXX")])
]
logfilter.add_log_redaction_filter_to_logger(redaction.global_redaction_engine, logging.root)
sensitive_query = 'SELECT "ssn=123-45-6789"'
redacted_query = 'SELECT "ssn=XXX-XX-XXXX"'
nonsensitive_query = 'SELECT "hello"'
snippets = [
{
"status": "ready",
"viewSettings": {
"sqlDialect": True,
"snippetImage": "/static/beeswax/art/icon_beeswax_48.png",
"placeHolder": "Example: SELECT * FROM tablename, or press CTRL + space",
"aceMode": "ace/mode/hive",
},
"id": "10a29cda-063f-1439-4836-d0c460154075",
"statement_raw": sensitive_query,
"statement": sensitive_query,
"type": "hive",
},
{
"status": "ready",
"viewSettings": {
"sqlDialect": True,
"snippetImage": "/static/impala/art/icon_impala_48.png",
"placeHolder": "Example: SELECT * FROM tablename, or press CTRL + space",
"aceMode": "ace/mode/impala",
},
"id": "e17d195a-beb5-76bf-7489-a9896eeda67a",
"statement_raw": sensitive_query,
"statement": sensitive_query,
"type": "impala",
},
{
"status": "ready",
"viewSettings": {
"sqlDialect": True,
"snippetImage": "/static/beeswax/art/icon_beeswax_48.png",
"placeHolder": "Example: SELECT * FROM tablename, or press CTRL + space",
"aceMode": "ace/mode/hive",
},
"id": "10a29cda-063f-1439-4836-d0c460154075",
"statement_raw": nonsensitive_query,
"statement": nonsensitive_query,
"type": "hive",
},
]
try:
self.document2.type = "notebook"
self.document2.update_data({"snippets": snippets})
self.document2.save()
saved_snippets = self.document2.data_dict["snippets"]
# Make sure redacted queries are redacted.
assert_equal(redacted_query, saved_snippets[0]["statement"])
assert_equal(redacted_query, saved_snippets[0]["statement_raw"])
assert_equal(True, saved_snippets[0]["is_redacted"])
assert_equal(redacted_query, saved_snippets[1]["statement"])
assert_equal(redacted_query, saved_snippets[1]["statement_raw"])
assert_equal(True, saved_snippets[1]["is_redacted"])
# Make sure unredacted queries are not redacted.
assert_equal(nonsensitive_query, saved_snippets[2]["statement"])
assert_equal(nonsensitive_query, saved_snippets[2]["statement_raw"])
assert_false("is_redacted" in saved_snippets[2])
finally:
redaction.global_redaction_engine.policies = old_policies
