def test_get_state_detectors(neo4j_session):
data = [
["1", "8", ["15", "22", "29"]],
["2", "9", ["16", "23", "30"]],
["3", "10", ["17", "24", "31"]],
["4", "11", ["18", "25", "32"]],
["5", "12", ["19", "26", "33"]],
["6", "13", ["20", "27", "34"]],
["7", "14", ["21", "28", "35"]],
["36", "37", ["38", "39", "40"]],
]
ingest_nodes = """
MERGE (person:Person{test: {test}})
SET person.test2 = {test2},
person.test3 = {test3}
"""
for node in data:
test = node[0]
test2 = node[1]
test3 = node[2]
neo4j_session.run(
ingest_nodes,
test=test,
test2=test2,
test3=test3,
)
query_directory = "tests/data/test_update_detectors/test_detector"
state_serializer = StateSchema()
shortcut_serializer = ShortcutSchema()
storage = FileSystem
file_1 = str(datetime.datetime(2019, 1, 1, 0, 0, 2)) + ".json"
file_2 = str(datetime.datetime(2019, 1, 1, 0, 0, 1)) + ".json"
get_query_state(neo4j_session, query_directory, state_serializer, storage,
file_1)
add_shortcut(FileSystem(), ShortcutSchema(), query_directory,
"most-recent", file_1)
detector_1_data = FileSystem.load(os.path.join(query_directory, file_1))
detector_1 = state_serializer.load(detector_1_data)
detector_2_data = FileSystem.load(os.path.join(query_directory, file_2))
detector_2 = state_serializer.load(detector_2_data)
assert detector_1.name == detector_2.name
assert detector_1.validation_query == detector_2.validation_query
assert detector_1.properties == detector_2.properties
assert detector_1.results == detector_2.results
shortcut_data = FileSystem.load(
os.path.join(query_directory, "shortcut.json"))
shortcut = shortcut_serializer.load(shortcut_data)
assert shortcut.shortcuts['most-recent'] == file_1
shortcut_data = shortcut_serializer.dump(shortcut)
FileSystem.write(shortcut_data,
os.path.join(query_directory, "shortcut.json"))
def run_drift_detection(config):
try:
if not valid_directory(config.query_directory):
logger.error("Invalid Drift Detection Directory")
return
state_serializer = StateSchema()
shortcut_serializer = ShortcutSchema()
shortcut_data = FileSystem.load(os.path.join(config.query_directory, "shortcut.json"))
shortcut = shortcut_serializer.load(shortcut_data)
start_state_data = FileSystem.load(
os.path.join(
config.query_directory, shortcut.shortcuts.get(
config.start_state,
config.start_state,
),
),
)
start_state = state_serializer.load(start_state_data)
end_state_data = FileSystem.load(
os.path.join(
config.query_directory, shortcut.shortcuts.get(
config.end_state,
config.end_state,
),
),
)
end_state = state_serializer.load(end_state_data)
new_results, missing_results = perform_drift_detection(start_state, end_state)
report_drift_new(new_results)
report_drift_missing(missing_results)
except ValidationError as err:
msg = "Unable to create DriftStates from files {},{} for \n{} in directory {}.".format(
config.start_state,
config.end_state,
err.messages,
config.query_directory,
)
logger.exception(msg)
except ValueError as err:
msg = "Unable to create DriftStates from files {},{} for \n{} in directory {}.".format(
config.start_state,
config.end_state,
err,
config.query_directory,
)
logger.exception(msg)
def test_basic_add_shortcuts():
"""
Tests that the CLI can add shortcuts.
"""
cli = CLI(prog="cartography-detectdrift")
directory = "tests/data/test_cli_detectors/detector"
alias = "test_shortcut"
file = "1.json"
shortcut_path = directory + '/shortcut.json'
cli.main([
"add-shortcut", "--query-directory", directory, "--shortcut", alias,
"--file", file
])
shortcut_data = FileSystem.load(shortcut_path)
shortcut = ShortcutSchema().load(shortcut_data)
assert shortcut.shortcuts[alias] == file
shortcut.shortcuts.pop(alias)
shortcut_data = ShortcutSchema().dump(shortcut)
FileSystem.write(shortcut_data, shortcut_path)
def test_use_shortcuts_for_shortcuts():
"""
Tests add_shortcut can parse shortcuts.
"""
cli = CLI(prog="cartography-detectdrift")
directory = "tests/data/test_cli_detectors/detector"
alias = "test_shortcut"
alias_2 = "test_shortcut_2"
filename = "1.json"
shortcut_path = directory + '/shortcut.json'
cli.main([
"add-shortcut",
"--query-directory",
directory,
"--shortcut",
alias,
"--file",
filename,
])
cli.main([
"add-shortcut",
"--query-directory",
directory,
"--shortcut",
alias_2,
"--file",
alias,
])
shortcut_data = FileSystem.load(shortcut_path)
shortcut = ShortcutSchema().load(shortcut_data)
assert shortcut.shortcuts[alias] == filename
assert shortcut.shortcuts[alias_2] == filename
# Return shortcut back to its original state.
shortcut.shortcuts.pop(alias)
shortcut.shortcuts.pop(alias_2)
shortcut_data = ShortcutSchema().dump(shortcut)
FileSystem.write(shortcut_data, shortcut_path)
def test_nonexistent_shortcuts():
cli = CLI(prog="cartography-detectdrift")
directory = "tests/data/test_cli_detectors/detector"
alias = "test_shortcut"
file = "3.json"
shortcut_path = os.path.join(directory, "shortcut.json")
cli.main([
"add-shortcut", "--query-directory", directory, "--shortcut", alias,
"--file", file
])
shortcut_data = FileSystem.load(shortcut_path)
shortcut = ShortcutSchema().load(shortcut_data)
with pytest.raises(KeyError):
shortcut.shortcuts[alias]
def run_add_shortcut(config):
"""
Runs add_shortcut from the command line. Does error handling.
:type config: Config Object
:param config: Config of adding shortcut
:return:
"""
if not valid_directory(config.query_directory):
logger.error("Invalid Drift Detection Directory")
return
try:
add_shortcut(FileSystem, ShortcutSchema(), config.query_directory,
config.shortcut, config.filename)
except ValidationError as err:
msg = "Could not load shortcut file from json file {} in query directory {}.".format(
err.messages,
config.query_directory,
)
logger.exception(msg)
def run_add_shortcut(config):
"""
Runs add_shortcut from the command line. Does error handling.
:type config: Config Object
:param config: Config of adding shortcut
:return:
"""
if not valid_directory(config.query_directory):
logger.error("Invalid Drift Detection Directory")
return
if not os.path.isfile(os.path.join(config.query_directory,
config.filename)):
msg = "File does not exist."
logger.error(msg)
return
try:
add_shortcut(FileSystem, ShortcutSchema(), config.query_directory,
config.shortcut, config.filename)
except ValidationError as err:
msg = "Could not load report_info file from {0}.".format(err.messages)
logger.exception(msg)
def test_shortcut_fails_when_shortcut_exists():
"""
Tests add_shortcut fails when shortcuts exist.
"""
cli = CLI(prog="cartography-detectdrift")
directory = "tests/data/test_cli_detectors/detector"
alias = "2.json"
filename = "1.json"
cli.main([
"add-shortcut",
"--query-directory",
directory,
"--shortcut",
alias,
"--file",
filename,
])
shortcut_path = directory + '/shortcut.json'
shortcut_data = FileSystem.load(shortcut_path)
shortcut = ShortcutSchema().load(shortcut_data)
with pytest.raises(KeyError):
shortcut.shortcuts[alias]
def run_get_states(config):
"""
Handles neo4j errors and then updates detectors.
:type config: Config Object
:param config: Config Object from CLI
:return:
"""
if not valid_directory(config.drift_detection_directory):
logger.error("Invalid Drift Detection Directory")
return
neo4j_auth = None
if config.neo4j_user or config.neo4j_password:
neo4j_auth = (config.neo4j_user, config.neo4j_password)
try:
neo4j_driver = GraphDatabase.driver(
config.neo4j_uri,
auth=neo4j_auth,
)
except neobolt.exceptions.ServiceUnavailable as e:
logger.debug("Error occurred during Neo4j connect.", exc_info=True)
logger.error(
("Unable to connect to Neo4j using the provided URI '%s', an error occurred: '%s'. Make sure the "
"Neo4j server is running and accessible from your network."),
config.neo4j_uri,
e,
)
return
except neobolt.exceptions.AuthError as e:
logger.debug("Error occurred during Neo4j auth.", exc_info=True)
if not neo4j_auth:
logger.error(
("Unable to auth to Neo4j, an error occurred: '%s'. driftdetect attempted to connect to Neo4j "
"without any auth. Check your Neo4j server settings to see if auth is required and, if it is, "
"provide driftdetect with a valid username and password."),
e,
)
else:
logger.error(
("Unable to auth to Neo4j, an error occurred: '%s'. driftdetect attempted to connect to Neo4j "
"with a username and password. Check your Neo4j server settings to see if the username and "
"password provided to driftdetect are valid credentials."),
e,
)
return
with neo4j_driver.session() as session:
filename = '.'.join([str(i) for i in time.gmtime()] + ["json"])
state_serializer = StateSchema()
shortcut_serializer = ShortcutSchema()
for query_directory in FileSystem.walk(
config.drift_detection_directory):
try:
get_query_state(session, query_directory, state_serializer,
FileSystem, filename)
add_shortcut(FileSystem, shortcut_serializer, query_directory,
'most-recent', filename)
except ValidationError as err:
msg = "Unable to create State for directory {}, with data \n{}".format(
query_directory,
err.messages,
)
logger.exception(msg)
except KeyError as err:
msg = f"Could not find {err} field in state template for directory {query_directory}."
logger.exception(msg)
except FileNotFoundError as err:
logger.exception(err)
except neobolt.exceptions.CypherSyntaxError as err:
logger.exception(err)
