def test_basic_drift_detection():
"""
Tests that drift detection works.
"""
data = FileSystem.load("tests/data/test_cli_detectors/detector/1.json")
start_state = StateSchema().load(data)
data = FileSystem.load("tests/data/test_cli_detectors/detector/2.json")
end_state = StateSchema().load(data)
new_results, missing_results = perform_drift_detection(
start_state, end_state)
new_results = [
drift_info_detector_pair[0] for drift_info_detector_pair in new_results
]
missing_results = [
drift_info_detector_pair[0]
for drift_info_detector_pair in missing_results
]
assert {
'd.test': '36',
'd.test2': '37',
'd.test3': ['38', '39', '40']
} in new_results
assert {
'd.test': '7',
'd.test2': '14',
'd.test3': ['21', '28', '35']
} in missing_results
def test_get_state_detectors(neo4j_session):
data = [
["1", "8", ["15", "22", "29"]],
["2", "9", ["16", "23", "30"]],
["3", "10", ["17", "24", "31"]],
["4", "11", ["18", "25", "32"]],
["5", "12", ["19", "26", "33"]],
["6", "13", ["20", "27", "34"]],
["7", "14", ["21", "28", "35"]],
["36", "37", ["38", "39", "40"]],
]
ingest_nodes = """
MERGE (person:Person{test: {test}})
SET person.test2 = {test2},
person.test3 = {test3}
"""
for node in data:
test = node[0]
test2 = node[1]
test3 = node[2]
neo4j_session.run(
ingest_nodes,
test=test,
test2=test2,
test3=test3,
)
query_directory = "tests/data/test_update_detectors/test_detector"
state_serializer = StateSchema()
shortcut_serializer = ShortcutSchema()
storage = FileSystem
file_1 = str(datetime.datetime(2019, 1, 1, 0, 0, 2)) + ".json"
file_2 = str(datetime.datetime(2019, 1, 1, 0, 0, 1)) + ".json"
get_query_state(neo4j_session, query_directory, state_serializer, storage,
file_1)
add_shortcut(FileSystem(), ShortcutSchema(), query_directory,
"most-recent", file_1)
detector_1_data = FileSystem.load(os.path.join(query_directory, file_1))
detector_1 = state_serializer.load(detector_1_data)
detector_2_data = FileSystem.load(os.path.join(query_directory, file_2))
detector_2 = state_serializer.load(detector_2_data)
assert detector_1.name == detector_2.name
assert detector_1.validation_query == detector_2.validation_query
assert detector_1.properties == detector_2.properties
assert detector_1.results == detector_2.results
shortcut_data = FileSystem.load(
os.path.join(query_directory, "shortcut.json"))
shortcut = shortcut_serializer.load(shortcut_data)
assert shortcut.shortcuts['most-recent'] == file_1
shortcut_data = shortcut_serializer.dump(shortcut)
FileSystem.write(shortcut_data,
os.path.join(query_directory, "shortcut.json"))
def test_basic_drift_detection():
"""
Tests that drift detection works.
"""
data = FileSystem.load("tests/data/test_cli_detectors/detector/1.json")
start_state = StateSchema().load(data)
data = FileSystem.load("tests/data/test_cli_detectors/detector/2.json")
end_state = StateSchema().load(data)
new_results, missing_results = perform_drift_detection(start_state, end_state)
assert ['36', '37', ['38', '39', '40']] in new_results
assert ['7', '14', ['21', '28', '35']] in missing_results
def run_drift_detection(config):
try:
if not valid_directory(config.query_directory):
logger.error("Invalid Drift Detection Directory")
return
state_serializer = StateSchema()
shortcut_serializer = ShortcutSchema()
shortcut_data = FileSystem.load(os.path.join(config.query_directory, "shortcut.json"))
shortcut = shortcut_serializer.load(shortcut_data)
start_state_data = FileSystem.load(
os.path.join(
config.query_directory, shortcut.shortcuts.get(
config.start_state,
config.start_state,
),
),
)
start_state = state_serializer.load(start_state_data)
end_state_data = FileSystem.load(
os.path.join(
config.query_directory, shortcut.shortcuts.get(
config.end_state,
config.end_state,
),
),
)
end_state = state_serializer.load(end_state_data)
new_results, missing_results = perform_drift_detection(start_state, end_state)
report_drift_new(new_results)
report_drift_missing(missing_results)
except ValidationError as err:
msg = "Unable to create DriftStates from files {},{} for \n{} in directory {}.".format(
config.start_state,
config.end_state,
err.messages,
config.query_directory,
)
logger.exception(msg)
except ValueError as err:
msg = "Unable to create DriftStates from files {},{} for \n{} in directory {}.".format(
config.start_state,
config.end_state,
err,
config.query_directory,
)
logger.exception(msg)
def test_drift_from_multiple_properties():
"""
Test fields with multiple properties handles correctly.
:return:
"""
mock_session = MagicMock()
mock_boltstatementresult = MagicMock()
key_1 = "d.test"
key_2 = "d.test2"
key_3 = "d.test3"
results = [{
key_1: "1",
key_2: "8",
key_3: ["15", "22", "29"]
}, {
key_1: "2",
key_2: "9",
key_3: ["16", "23", "30"]
}, {
key_1: "3",
key_2: "10",
key_3: ["17", "24", "31"]
}, {
key_1: "4",
key_2: "11",
key_3: ["18", "25", "32"]
}, {
key_1: "5",
key_2: "12",
key_3: ["19", "26", "33"]
}, {
key_1: "6",
key_2: "13",
key_3: ["20", "27", "34"]
}, {
key_1: "7",
key_2: "14",
key_3: ["21", "28", "35"]
}]
mock_boltstatementresult.__getitem__.side_effect = results.__getitem__
mock_boltstatementresult.__iter__.side_effect = results.__iter__
mock_session.run.return_value = mock_boltstatementresult
data = FileSystem.load(
"tests/data/detectors/test_multiple_properties.json")
state_old = StateSchema().load(data)
state_new = State(state_old.name, state_old.validation_query,
state_old.properties, [])
get_state(mock_session, state_new)
state_new.properties = state_old.properties
drifts = compare_states(state_old, state_new)
mock_session.run.assert_called_with(state_new.validation_query)
print(drifts)
assert {key_1: "7", key_2: "14", key_3: ["21", "28", "35"]} in drifts[0]
assert {
key_1: "3",
key_2: "10",
key_3: ["17", "24", "31"]
} not in drifts[0]
def test_drift_detection_errors():
data = FileSystem.load("tests/data/test_cli_detectors/detector/1.json")
start_state = StateSchema().load(data)
data = FileSystem.load("tests/data/test_cli_detectors/detector/2.json")
end_state = StateSchema().load(data)
start_state.name = "Wrong Name"
with pytest.raises(ValueError):
perform_drift_detection(start_state, end_state)
start_state = StateSchema().load(data)
start_state.properties = ["Incorrect", "Properties"]
with pytest.raises(ValueError):
perform_drift_detection(start_state, end_state)
start_state = StateSchema().load(data)
start_state.validation_query = "Invalid Validation Query"
with pytest.raises(ValueError):
perform_drift_detection(start_state, end_state)
def test_state_multiple_expectations():
"""
Test that multiple fields runs properly.
:return:
"""
key_1 = "d.test"
key_2 = "d.test2"
mock_session = MagicMock()
mock_boltstatementresult = MagicMock()
results = [
{
key_1: "1",
key_2: "8"
},
{
key_1: "2",
key_2: "9"
},
{
key_1: "3",
key_2: "10"
},
{
key_1: "4",
key_2: "11"
},
{
key_1: "5",
key_2: "12"
},
{
key_1: "6",
key_2: "13"
},
{
key_1: "7",
key_2: "14"
},
]
mock_boltstatementresult.__getitem__.side_effect = results.__getitem__
mock_boltstatementresult.__iter__.side_effect = results.__iter__
mock_session.run.return_value = mock_boltstatementresult
data = FileSystem.load(
"tests/data/detectors/test_multiple_expectations.json")
state_old = StateSchema().load(data)
state_new = State(state_old.name, state_old.validation_query,
state_old.properties, [])
get_state(mock_session, state_new)
state_new.properties = state_old.properties
drifts = compare_states(state_old, state_new)
mock_session.run.assert_called_with(state_new.validation_query)
assert {key_1: "7", key_2: "14"} in drifts[0]
def test_json_loader():
"""
Test loading schema passes
:return:
"""
filepath = "tests/data/detectors/test_expectations.json"
data = FileSystem.load(filepath)
state = StateSchema().load(data)
assert state.name == "Test-Expectations"
assert state.validation_query == "MATCH (d) RETURN d.test"
assert state.results == [['1'], ['2'], ['3'], ['4'], ['5'], ['6']]
def test_state_differences():
"""
Test that state_differences picks up new drift
:return:
"""
filepath = "tests/data/detectors/test_expectations.json"
data = FileSystem.load(filepath)
state_1 = StateSchema().load(data)
state_2 = StateSchema().load(data)
state_2.results.append(["7"])
drift_info_state_pairs = compare_states(state_1, state_2)
assert ({'d.test': "7"}, state_2) in drift_info_state_pairs
def test_nonexistent_shortcuts():
cli = CLI(prog="cartography-detectdrift")
directory = "tests/data/test_cli_detectors/detector"
alias = "test_shortcut"
file = "3.json"
shortcut_path = os.path.join(directory, "shortcut.json")
cli.main([
"add-shortcut", "--query-directory", directory, "--shortcut", alias,
"--file", file
])
shortcut_data = FileSystem.load(shortcut_path)
shortcut = ShortcutSchema().load(shortcut_data)
with pytest.raises(KeyError):
shortcut.shortcuts[alias]
def test_compare_states():
"""
Test that differences between two states is recorded
:return:
"""
filepath = "tests/data/detectors/test_expectations.json"
data = FileSystem.load(filepath)
state_1 = StateSchema().load(data)
state_2 = StateSchema().load(data)
state_1.results.append(["7"])
state_2.results.append(["8"])
new, missing = perform_drift_detection(state_1, state_2)
assert ({'d.test': "7"}, state_1) in missing
assert ({'d.test': "8"}, state_2) in new
def test_state_picks_up_drift():
"""
Test that a state that detects drift.
:return:
"""
key = "d.test"
mock_session = MagicMock()
mock_boltstatementresult = MagicMock()
results = [
{
key: "1"
},
{
key: "2"
},
{
key: "3"
},
{
key: "4"
},
{
key: "5"
},
{
key: "6"
},
{
key: "7"
},
]
mock_boltstatementresult.__getitem__.side_effect = results.__getitem__
mock_boltstatementresult.__iter__.side_effect = results.__iter__
mock_session.run.return_value = mock_boltstatementresult
data = FileSystem.load("tests/data/detectors/test_expectations.json")
state_old = StateSchema().load(data)
state_new = State(state_old.name, state_old.validation_query,
state_old.properties, [])
get_state(mock_session, state_new)
state_new.properties = state_old.properties
drifts = compare_states(state_old, state_new)
mock_session.run.assert_called_with(state_new.validation_query)
assert drifts
assert drifts[0][0] == {key: "7"}
def test_basic_add_shortcuts():
"""
Tests that the CLI can add shortcuts.
"""
cli = CLI(prog="cartography-detectdrift")
directory = "tests/data/test_cli_detectors/detector"
alias = "test_shortcut"
file = "1.json"
shortcut_path = directory + '/shortcut.json'
cli.main([
"add-shortcut", "--query-directory", directory, "--shortcut", alias,
"--file", file
])
shortcut_data = FileSystem.load(shortcut_path)
shortcut = ShortcutSchema().load(shortcut_data)
assert shortcut.shortcuts[alias] == file
shortcut.shortcuts.pop(alias)
shortcut_data = ShortcutSchema().dump(shortcut)
FileSystem.write(shortcut_data, shortcut_path)
def test_shortcut_fails_when_shortcut_exists():
"""
Tests add_shortcut fails when shortcuts exist.
"""
cli = CLI(prog="cartography-detectdrift")
directory = "tests/data/test_cli_detectors/detector"
alias = "2.json"
filename = "1.json"
cli.main([
"add-shortcut",
"--query-directory",
directory,
"--shortcut",
alias,
"--file",
filename,
])
shortcut_path = directory + '/shortcut.json'
shortcut_data = FileSystem.load(shortcut_path)
shortcut = ShortcutSchema().load(shortcut_data)
with pytest.raises(KeyError):
shortcut.shortcuts[alias]
def test_use_shortcuts_for_shortcuts():
"""
Tests add_shortcut can parse shortcuts.
"""
cli = CLI(prog="cartography-detectdrift")
directory = "tests/data/test_cli_detectors/detector"
alias = "test_shortcut"
alias_2 = "test_shortcut_2"
filename = "1.json"
shortcut_path = directory + '/shortcut.json'
cli.main([
"add-shortcut",
"--query-directory",
directory,
"--shortcut",
alias,
"--file",
filename,
])
cli.main([
"add-shortcut",
"--query-directory",
directory,
"--shortcut",
alias_2,
"--file",
alias,
])
shortcut_data = FileSystem.load(shortcut_path)
shortcut = ShortcutSchema().load(shortcut_data)
assert shortcut.shortcuts[alias] == filename
assert shortcut.shortcuts[alias_2] == filename
# Return shortcut back to its original state.
shortcut.shortcuts.pop(alias)
shortcut.shortcuts.pop(alias_2)
shortcut_data = ShortcutSchema().dump(shortcut)
FileSystem.write(shortcut_data, shortcut_path)