def perform_drift_detection(session, expect_folder, update):
"""
Perform Drift Detection
:type session: neo4j_session
:param session: session with infrastructure information
:type expect_folder: string
:param expect_folder: detector directory
:type update: Decides whether or not to update the graph
:param update: boolean
:return: list of tuples of drift dictionaries and detectors
"""
drift_info_detector_pairs = []
for root, _, filenames in os.walk(expect_folder):
for filename in filenames:
file_path = os.path.join(root, filename)
try:
detector = load_detector_from_json_file(file_path)
for drift_info in detector.run(session, update):
drift_info_detector_pairs.append((drift_info, detector))
if update:
try:
write_detector_to_json_file(detector, file_path)
except ValidationError as err:
msg = "Unable to save DriftDetector from file {0}.\n{1}".format(
file_path, err.messages)
logger.exception(msg)
except ValidationError as err:
msg = "Unable to create DriftDetector from file {0}.\n{1}".format(
file_path, err.messages)
logger.exception(msg)
return drift_info_detector_pairs
def test_drift_from_multiple_properties():
"""
Tests fields with multiple properties handles correctly.
:return:
"""
mock_session = MagicMock()
mock_boltstatementresult = MagicMock()
key_1 = "key_1"
key_2 = "key_2"
key_3 = "key_3"
results = [
{key_1: "1", key_2: "8", key_3: ["15", "22", "29"]},
{key_1: "2", key_2: "9", key_3: ["16", "23", "30"]},
{key_1: "3", key_2: "10", key_3: ["17", "24", "31"]},
{key_1: "4", key_2: "11", key_3: ["18", "25", "32"]},
{key_1: "5", key_2: "12", key_3: ["19", "26", "33"]},
{key_1: "6", key_2: "13", key_3: ["20", "27", "34"]},
{key_1: "7", key_2: "14", key_3: ["21", "28", "35"]}
]
mock_boltstatementresult.__getitem__.side_effect = results.__getitem__
mock_boltstatementresult.__iter__.side_effect = results.__iter__
mock_session.run.return_value = mock_boltstatementresult
detector = load_detector_from_json_file("tests/data/detectors/test_multiple_properties.json")
drifts = []
for it in detector.run(mock_session, False):
drifts.append(it)
mock_session.run.assert_called_with(detector.validation_query)
print(drifts)
assert {key_1: "7", key_2: "14", key_3: ["21", "28", "35"]} in drifts
assert {key_1: "3", key_2: "10", key_3: ["17", "24", "31"]} not in drifts
def test_detector_no_drift():
"""
Test that a detector that detects no drift returns none.
:return:
"""
mock_session = MagicMock()
mock_boltstatementresult = MagicMock()
key = "key"
results = [
{key: "1"},
{key: "2"},
{key: "3"},
{key: "4"},
{key: "5"},
{key: "6"},
]
mock_boltstatementresult.__getitem__.side_effect = results.__getitem__
mock_boltstatementresult.__iter__.side_effect = results.__iter__
mock_session.run.return_value = mock_boltstatementresult
detector = load_detector_from_json_file("tests/data/detectors/test_expectations.json")
drifts = []
for it in detector.run(mock_session, False):
drifts.append(it)
mock_session.run.assert_called_with(detector.validation_query)
assert not drifts
def test_detector_multiple_expectations():
"""
Test that multiple fields runs properly.
:return:
"""
key_1 = "baseline_tag"
key_2 = "other_tag"
mock_session = MagicMock()
mock_boltstatementresult = MagicMock()
results = [
{key_1: "1", key_2: "8"},
{key_1: "2", key_2: "9"},
{key_1: "3", key_2: "10"},
{key_1: "4", key_2: "11"},
{key_1: "5", key_2: "12"},
{key_1: "6", key_2: "13"},
{key_1: "7", key_2: "14"}
]
mock_boltstatementresult.__getitem__.side_effect = results.__getitem__
mock_boltstatementresult.__iter__.side_effect = results.__iter__
mock_session.run.return_value = mock_boltstatementresult
detector = load_detector_from_json_file("tests/data/detectors/test_multiple_expectations.json")
drifts = []
for it in detector.run(mock_session, False):
drifts.append(it)
mock_session.run.assert_called_with(detector.validation_query)
assert {key_1: "7", key_2: "14"} in drifts
def test_json_loader():
"""
Tests loading schema passes
:return:
"""
filepath = "tests/data/detectors/test_expectations.json"
detector = load_detector_from_json_file(filepath)
assert detector.name == "Test-Expectations"
assert detector.validation_query == "MATCH (d) RETURN d.test"
assert str(detector.detector_type) == "DriftDetectorType.EXPOSURE"
assert detector.expectations == [['1'], ['2'], ['3'], ['4'], ['5'], ['6']]