def test_pure_memory_put_retrieve_events(self):
triples_data_set = [
[("CIM_Process", {
"Handle": 123
}), "priority", 255],
[
("CIM_Process", {
"Handle": 123
}),
"user",
("Win32_UserAccount", {
"Name": "my_user",
"Domain": "my_domain"
}),
],
]
files_updates_total_number = lib_event.store_events_as_json_triples_list(
triples_data_set)
new_graph = rdflib.Graph()
lib_kbase.retrieve_all_events_to_graph_then_clear(new_graph)
print("files_updates_total_number=", files_updates_total_number)
print("len(triples_list)=", len(new_graph))
#for one_triple in new_graph:
# print(" one_triple=", one_triple)
self.assertEqual(files_updates_total_number, 2)
# When the subject and object of a triple are urls, this triple is stored twice.
self.assertEqual(len(new_graph), 2)
def test_sqlalchemy_sqlite_write_two_urls(self):
graph_cleanup = rdflib.Graph()
lib_kbase.retrieve_all_events_to_graph_then_clear(graph_cleanup)
triples_count_a = 1000
test_graph_input_a = _create_dummy_graph(triples_count_a)
test_url_a = "http://dummy.xyz/url_a"
count_events_url_a = lib_kbase.write_graph_to_events(
test_url_a, test_graph_input_a)
actual_events_count_1 = lib_kbase.events_count()
self.assertEqual(actual_events_count_1, triples_count_a)
triples_count_b = 2000
test_graph_input_b = _create_dummy_graph(triples_count_b)
test_url_b = "http://dummy.xyz/url_b"
count_events_url_b = lib_kbase.write_graph_to_events(
test_url_b, test_graph_input_b)
actual_events_count_2 = lib_kbase.events_count()
self.assertEqual(actual_events_count_2,
triples_count_a + triples_count_b)
test_graph_output_a = rdflib.Graph()
count_events_output_a = lib_kbase.read_events_to_graph(
test_url_a, test_graph_output_a)
self.assertEqual(count_events_url_a, count_events_output_a)
actual_events_count_3 = lib_kbase.events_count()
self.assertEqual(actual_events_count_3, triples_count_b)
test_graph_output_b = rdflib.Graph()
count_events_output_b = lib_kbase.read_events_to_graph(
test_url_b, test_graph_output_b)
self.assertEqual(count_events_url_b, count_events_output_b)
actual_events_count_4 = lib_kbase.events_count()
self.assertEqual(actual_events_count_4, 0)
def test_sqlalchemy_write_twice_nourl_events_count(self):
"""Writes with no url then reads all."""
test_graph_input = rdflib.Graph().parse(
"tests/input_test_data/test_events_tcp_sockets.xml")
count_events = lib_kbase.write_graph_to_events(None, test_graph_input)
self.assertEqual(count_events, 43)
actual_events_count = lib_kbase.events_count()
self.assertEqual(count_events, actual_events_count)
count_events = lib_kbase.write_graph_to_events(None, test_graph_input)
self.assertEqual(count_events, 43)
actual_events_count = lib_kbase.events_count()
# TODO: BEWARE, WHY 63 WITH IN-MEMORY SQLITE ??
self.assertEqual(63, actual_events_count)
graph_from_sql_alchemy_again = rdflib.Graph()
lib_kbase.retrieve_all_events_to_graph_then_clear(
graph_from_sql_alchemy_again)
read_events_count = len(graph_from_sql_alchemy_again)
print("read_events_count=", read_events_count)
self.assertEqual(read_events_count, 43)
actual_events_count_again = lib_kbase.events_count()
self.assertEqual(0, actual_events_count_again)
def test_pure_memory_retrieve_all_events_empty(self):
"""This reads all events twice, and the second time it should return nothing. """
# To start with, this cleans all events.
new_graph = rdflib.Graph()
lib_kbase.retrieve_all_events_to_graph_then_clear(new_graph)
new_graph = rdflib.Graph()
lib_kbase.retrieve_all_events_to_graph_then_clear(new_graph)
self.assertEqual(len(new_graph), 0)
def test_pure_memory_write_two_urls_plus_none(self):
graph_cleanup = rdflib.Graph()
lib_kbase.retrieve_all_events_to_graph_then_clear(graph_cleanup)
triples_count_a = 10
test_graph_input_a = _create_dummy_graph(triples_count_a)
test_url_a = "http://dummy.xyz/url_a"
count_events_url_a = lib_kbase.write_graph_to_events(
test_url_a, test_graph_input_a)
self.assertEqual(count_events_url_a, triples_count_a)
actual_events_count_1 = lib_kbase.events_count()
self.assertEqual(actual_events_count_1, triples_count_a)
triples_count_b = 20
test_graph_input_b = _create_dummy_graph(triples_count_b)
test_url_b = "http://dummy.xyz/url_b"
count_events_url_b = lib_kbase.write_graph_to_events(
test_url_b, test_graph_input_b)
self.assertEqual(count_events_url_b,
max(triples_count_a, triples_count_b))
actual_events_count_2 = lib_kbase.events_count()
self.assertEqual(actual_events_count_2, count_events_url_b)
triples_count_z = 100
test_graph_input_z = _create_dummy_graph(triples_count_z)
count_events_url_z = lib_kbase.write_graph_to_events(
None, test_graph_input_z)
self.assertEqual(
count_events_url_z,
max(triples_count_a, triples_count_b, triples_count_z))
actual_events_count_3 = lib_kbase.events_count()
self.assertEqual(count_events_url_z, count_events_url_z)
test_graph_output_a = rdflib.Graph()
count_events_output_a = lib_kbase.read_events_to_graph(
test_url_a, test_graph_output_a)
self.assertEqual(count_events_url_a, count_events_output_a)
actual_events_count_3 = lib_kbase.events_count()
self.assertEqual(actual_events_count_3,
max(triples_count_b, triples_count_z))
test_graph_output_b = rdflib.Graph()
count_events_output_b = lib_kbase.read_events_to_graph(
test_url_b, test_graph_output_b)
self.assertEqual(count_events_url_b, count_events_output_b)
actual_events_count_4 = lib_kbase.events_count()
self.assertEqual(actual_events_count_4, triples_count_z)
test_graph_output_z = rdflib.Graph()
count_events_output_z = lib_kbase.retrieve_all_events_to_graph_then_clear(
test_graph_output_z)
self.assertEqual(count_events_url_z, count_events_output_z)
actual_events_count_5 = lib_kbase.events_count()
self.assertEqual(actual_events_count_5, 0)
def test_pure_memory_insert_write_nourl_read_all_subprocess(self):
triples_count = 100
test_graph_input = _create_dummy_graph(triples_count)
count_events = lib_kbase.write_graph_to_events(None, test_graph_input)
self.assertEqual(count_events, triples_count)
graph_whole_content = rdflib.Graph()
lib_kbase.retrieve_all_events_to_graph_then_clear(graph_whole_content)
input_triples = _graph_to_triples_set(test_graph_input)
output_triples = _graph_to_triples_set(graph_whole_content)
# The two lists of triples must be identical: Comparison of the string representations.
self.assertEqual(input_triples, output_triples)
def test_sqlalchemy_write_url_read_all_only(self):
"""Writes for an URL then reads all."""
test_graph_input = rdflib.Graph().parse(
"tests/input_test_data/test_events_tcp_sockets.xml")
# The URL is not important because it is not accessed.
# However, it must be correctly handled by rdflib when it creates a UriRef
test_url = "http://vps516494.ovh.net/Survol/survol/sources_types/Linux/tcp_sockets.py?xid=."
count_events = lib_kbase.write_graph_to_events(test_url,
test_graph_input)
self.assertEqual(count_events, 43)
graph_from_sql_alchemy = rdflib.Graph()
lib_kbase.retrieve_all_events_to_graph_then_clear(
graph_from_sql_alchemy)
self.assertEqual(count_events, len(graph_from_sql_alchemy))
def test_sqlalchemy_sqlite_write_mixed_read_all_subprocess(self):
"""Writes a RDF graph, then reads from another process."""
graph_cleanup = rdflib.Graph()
lib_kbase.retrieve_all_events_to_graph_then_clear(graph_cleanup)
triples_count = 1000
test_graph_input_nourl = _create_dummy_graph(triples_count)
count_events_nourl = lib_kbase.write_graph_to_events(
None, test_graph_input_nourl)
test_graph_input_url = rdflib.Graph().parse(
"tests/input_test_data/test_events_tcp_sockets.xml")
# The URL is not important because it is not accessed.
# However, it must be correctly handled by rdflib when it creates a UriRef
test_url = "http://vps516494.ovh.net/Survol/survol/sources_types/Linux/tcp_sockets.py?xid=."
count_events_url = lib_kbase.write_graph_to_events(
test_url, test_graph_input_url)
count_events = count_events_nourl + count_events_url
# This shared queue is used by a subprocess reading the events, to send them back to this process.
shared_queue = multiprocessing.Queue()
created_process = _create_subprocess(count_events, shared_queue,
self.sqlite_path)
# Reads the triples sent by the subprocess.
output_list = _read_triples_from_queue(count_events, shared_queue)
# The subprocess is not needed any longer.
created_process.terminate()
created_process.join()
print("Killing pid=", created_process.pid)
input_triples_nourl = _graph_to_triples_set(test_graph_input_nourl)
input_triples_url = _graph_to_triples_set(test_graph_input_url)
input_triples = input_triples_nourl | input_triples_url
output_triples = _graph_to_triples_set(output_list)
# The two lists of triples must be identical: Comparison of the string representations.
self.assertEqual(input_triples, output_triples)
def test_sqlalchemy_nourl_events_count(self):
"""Just writes."""
graph_cleanup = rdflib.Graph()
lib_kbase.retrieve_all_events_to_graph_then_clear(graph_cleanup)
test_graph_input = rdflib.Graph().parse(
"tests/input_test_data/test_events_tcp_sockets.xml")
count_events = lib_kbase.write_graph_to_events(None, test_graph_input)
self.assertEqual(count_events, 43)
actual_events_count = lib_kbase.events_count()
self.assertEqual(count_events, actual_events_count)
graph_from_sql_alchemy = rdflib.Graph()
lib_kbase.retrieve_all_events_to_graph_then_clear(
graph_from_sql_alchemy)
read_events_count = len(graph_from_sql_alchemy)
self.assertEqual(read_events_count, actual_events_count)
def test_sqlalchemy_read_all_twice(self):
"""Just writes."""
actual_events_count = lib_kbase.events_count()
graph_from_sql_alchemy = rdflib.Graph()
lib_kbase.retrieve_all_events_to_graph_then_clear(
graph_from_sql_alchemy)
read_events_count = len(graph_from_sql_alchemy)
self.assertEqual(read_events_count, actual_events_count)
events_count_after_read = lib_kbase.events_count()
self.assertEqual(events_count_after_read, 0)
# Second read empties all.
graph_from_sql_alchemy = rdflib.Graph()
lib_kbase.retrieve_all_events_to_graph_then_clear(
graph_from_sql_alchemy)
self.assertEqual(len(graph_from_sql_alchemy), 0)
def test_write_read_graph_to_events_with_other_nodes(self):
triples_data_set = [
[("CIM_Process", {
"Handle": 123
}), "ParentProcessId", 1],
[
("CIM_Directory", {
"Name": "/tmp"
}),
# CIM_DirectoryContainsFile.GroupComponent or CIM_DirectoryContainsFile.PartComponent
"CIM_DirectoryContainsFile",
("CIM_DataFile", {
"Name": "/tmp/anyfile.tmp"
})
]
]
updates_total_number = lib_event.store_events_as_json_triples_list(
triples_data_set)
test_graph_input = rdflib.Graph().parse(
"tests/input_test_data/test_events_enumerate_CIM_Process.xml")
test_url = "http://vps516494.ovh.net/Survol/survol/sources_types/enumerate_CIM_Process.py?xid=."
count_events = lib_kbase.write_graph_to_events(test_url,
test_graph_input)
self.assertEqual(count_events, 682)
test_graph_output = rdflib.Graph()
count_events_output = lib_kbase.read_events_to_graph(
test_url, test_graph_output)
self.assertEqual(count_events, count_events_output)
new_graph = rdflib.Graph()
lib_kbase.retrieve_all_events_to_graph_then_clear(new_graph)
print("files_updates_total_number=", updates_total_number)
print("len(triples_list)=", len(new_graph))
self.assertEqual(updates_total_number, 2)
self.assertEqual(len(new_graph), 2)
def test_sqlalchemy_url_events_count(self):
graph_cleanup = rdflib.Graph()
lib_kbase.retrieve_all_events_to_graph_then_clear(graph_cleanup)
test_graph_input = rdflib.Graph().parse(
"tests/input_test_data/test_events_tcp_sockets.xml")
test_url = "http://vps516494.ovh.net/Survol/survol/sources_types/Linux/tcp_sockets.py?xid=."
count_events = lib_kbase.write_graph_to_events(test_url,
test_graph_input)
self.assertEqual(count_events, 43)
actual_events_count = lib_kbase.events_count()
self.assertEqual(count_events, actual_events_count)
graph_from_sql_alchemy = rdflib.Graph()
lib_kbase.retrieve_all_events_to_graph_then_clear(
graph_from_sql_alchemy)
read_events_count = len(graph_from_sql_alchemy)
self.assertEqual(read_events_count, actual_events_count)
def Main():
lib_common.set_events_credentials()
# This can process remote hosts because it does not call any script, just shows them.
cgiEnv = lib_common.ScriptEnvironment()
logging.debug("Starting.")
grph = cgiEnv.GetGraph()
logging.debug("About to get events")
num_triples = lib_kbase.retrieve_all_events_to_graph_then_clear(grph)
logging.debug("num_triples=%d" % num_triples)
cgiEnv.OutCgiRdf()
def _read_all_events_from_graph(expected_triple_numbers, pqueue):
"""This function is used by EventsMultiprocessGraphTest.test_write_here_read_there to run
a subprocess which reads events (triples) from the shared graph which contains all events.
These triples are then returned with a shared queue, and the main process compares with the initial content."""
grph = rdflib.Graph()
num_triples = lib_kbase.retrieve_all_events_to_graph_then_clear(grph)
if num_triples != expected_triple_numbers:
# This is just an information, because a complete comparison is done later.
sys.stderr.write(
"_read_all_events_from_graph Wrong number of events:%d %d\n" %
(num_triples, expected_triple_numbers))
for one_triple in grph:
pqueue.put(one_triple)
