def test_pure_memory_put_retrieve_events(self): triples_data_set = [ [("CIM_Process", { "Handle": 123 }), "priority", 255], [ ("CIM_Process", { "Handle": 123 }), "user", ("Win32_UserAccount", { "Name": "my_user", "Domain": "my_domain" }), ], ] files_updates_total_number = lib_event.store_events_as_json_triples_list( triples_data_set) new_graph = rdflib.Graph() lib_kbase.retrieve_all_events_to_graph_then_clear(new_graph) print("files_updates_total_number=", files_updates_total_number) print("len(triples_list)=", len(new_graph)) #for one_triple in new_graph: # print(" one_triple=", one_triple) self.assertEqual(files_updates_total_number, 2) # When the subject and object of a triple are urls, this triple is stored twice. self.assertEqual(len(new_graph), 2)
def test_sqlalchemy_sqlite_write_two_urls(self): graph_cleanup = rdflib.Graph() lib_kbase.retrieve_all_events_to_graph_then_clear(graph_cleanup) triples_count_a = 1000 test_graph_input_a = _create_dummy_graph(triples_count_a) test_url_a = "http://dummy.xyz/url_a" count_events_url_a = lib_kbase.write_graph_to_events( test_url_a, test_graph_input_a) actual_events_count_1 = lib_kbase.events_count() self.assertEqual(actual_events_count_1, triples_count_a) triples_count_b = 2000 test_graph_input_b = _create_dummy_graph(triples_count_b) test_url_b = "http://dummy.xyz/url_b" count_events_url_b = lib_kbase.write_graph_to_events( test_url_b, test_graph_input_b) actual_events_count_2 = lib_kbase.events_count() self.assertEqual(actual_events_count_2, triples_count_a + triples_count_b) test_graph_output_a = rdflib.Graph() count_events_output_a = lib_kbase.read_events_to_graph( test_url_a, test_graph_output_a) self.assertEqual(count_events_url_a, count_events_output_a) actual_events_count_3 = lib_kbase.events_count() self.assertEqual(actual_events_count_3, triples_count_b) test_graph_output_b = rdflib.Graph() count_events_output_b = lib_kbase.read_events_to_graph( test_url_b, test_graph_output_b) self.assertEqual(count_events_url_b, count_events_output_b) actual_events_count_4 = lib_kbase.events_count() self.assertEqual(actual_events_count_4, 0)
def test_sqlalchemy_write_twice_nourl_events_count(self): """Writes with no url then reads all.""" test_graph_input = rdflib.Graph().parse( "tests/input_test_data/test_events_tcp_sockets.xml") count_events = lib_kbase.write_graph_to_events(None, test_graph_input) self.assertEqual(count_events, 43) actual_events_count = lib_kbase.events_count() self.assertEqual(count_events, actual_events_count) count_events = lib_kbase.write_graph_to_events(None, test_graph_input) self.assertEqual(count_events, 43) actual_events_count = lib_kbase.events_count() # TODO: BEWARE, WHY 63 WITH IN-MEMORY SQLITE ?? self.assertEqual(63, actual_events_count) graph_from_sql_alchemy_again = rdflib.Graph() lib_kbase.retrieve_all_events_to_graph_then_clear( graph_from_sql_alchemy_again) read_events_count = len(graph_from_sql_alchemy_again) print("read_events_count=", read_events_count) self.assertEqual(read_events_count, 43) actual_events_count_again = lib_kbase.events_count() self.assertEqual(0, actual_events_count_again)
def test_pure_memory_retrieve_all_events_empty(self): """This reads all events twice, and the second time it should return nothing. """ # To start with, this cleans all events. new_graph = rdflib.Graph() lib_kbase.retrieve_all_events_to_graph_then_clear(new_graph) new_graph = rdflib.Graph() lib_kbase.retrieve_all_events_to_graph_then_clear(new_graph) self.assertEqual(len(new_graph), 0)
def test_pure_memory_write_two_urls_plus_none(self): graph_cleanup = rdflib.Graph() lib_kbase.retrieve_all_events_to_graph_then_clear(graph_cleanup) triples_count_a = 10 test_graph_input_a = _create_dummy_graph(triples_count_a) test_url_a = "http://dummy.xyz/url_a" count_events_url_a = lib_kbase.write_graph_to_events( test_url_a, test_graph_input_a) self.assertEqual(count_events_url_a, triples_count_a) actual_events_count_1 = lib_kbase.events_count() self.assertEqual(actual_events_count_1, triples_count_a) triples_count_b = 20 test_graph_input_b = _create_dummy_graph(triples_count_b) test_url_b = "http://dummy.xyz/url_b" count_events_url_b = lib_kbase.write_graph_to_events( test_url_b, test_graph_input_b) self.assertEqual(count_events_url_b, max(triples_count_a, triples_count_b)) actual_events_count_2 = lib_kbase.events_count() self.assertEqual(actual_events_count_2, count_events_url_b) triples_count_z = 100 test_graph_input_z = _create_dummy_graph(triples_count_z) count_events_url_z = lib_kbase.write_graph_to_events( None, test_graph_input_z) self.assertEqual( count_events_url_z, max(triples_count_a, triples_count_b, triples_count_z)) actual_events_count_3 = lib_kbase.events_count() self.assertEqual(count_events_url_z, count_events_url_z) test_graph_output_a = rdflib.Graph() count_events_output_a = lib_kbase.read_events_to_graph( test_url_a, test_graph_output_a) self.assertEqual(count_events_url_a, count_events_output_a) actual_events_count_3 = lib_kbase.events_count() self.assertEqual(actual_events_count_3, max(triples_count_b, triples_count_z)) test_graph_output_b = rdflib.Graph() count_events_output_b = lib_kbase.read_events_to_graph( test_url_b, test_graph_output_b) self.assertEqual(count_events_url_b, count_events_output_b) actual_events_count_4 = lib_kbase.events_count() self.assertEqual(actual_events_count_4, triples_count_z) test_graph_output_z = rdflib.Graph() count_events_output_z = lib_kbase.retrieve_all_events_to_graph_then_clear( test_graph_output_z) self.assertEqual(count_events_url_z, count_events_output_z) actual_events_count_5 = lib_kbase.events_count() self.assertEqual(actual_events_count_5, 0)
def test_pure_memory_insert_write_nourl_read_all_subprocess(self): triples_count = 100 test_graph_input = _create_dummy_graph(triples_count) count_events = lib_kbase.write_graph_to_events(None, test_graph_input) self.assertEqual(count_events, triples_count) graph_whole_content = rdflib.Graph() lib_kbase.retrieve_all_events_to_graph_then_clear(graph_whole_content) input_triples = _graph_to_triples_set(test_graph_input) output_triples = _graph_to_triples_set(graph_whole_content) # The two lists of triples must be identical: Comparison of the string representations. self.assertEqual(input_triples, output_triples)
def test_sqlalchemy_write_url_read_all_only(self): """Writes for an URL then reads all.""" test_graph_input = rdflib.Graph().parse( "tests/input_test_data/test_events_tcp_sockets.xml") # The URL is not important because it is not accessed. # However, it must be correctly handled by rdflib when it creates a UriRef test_url = "http://vps516494.ovh.net/Survol/survol/sources_types/Linux/tcp_sockets.py?xid=." count_events = lib_kbase.write_graph_to_events(test_url, test_graph_input) self.assertEqual(count_events, 43) graph_from_sql_alchemy = rdflib.Graph() lib_kbase.retrieve_all_events_to_graph_then_clear( graph_from_sql_alchemy) self.assertEqual(count_events, len(graph_from_sql_alchemy))
def test_sqlalchemy_sqlite_write_mixed_read_all_subprocess(self): """Writes a RDF graph, then reads from another process.""" graph_cleanup = rdflib.Graph() lib_kbase.retrieve_all_events_to_graph_then_clear(graph_cleanup) triples_count = 1000 test_graph_input_nourl = _create_dummy_graph(triples_count) count_events_nourl = lib_kbase.write_graph_to_events( None, test_graph_input_nourl) test_graph_input_url = rdflib.Graph().parse( "tests/input_test_data/test_events_tcp_sockets.xml") # The URL is not important because it is not accessed. # However, it must be correctly handled by rdflib when it creates a UriRef test_url = "http://vps516494.ovh.net/Survol/survol/sources_types/Linux/tcp_sockets.py?xid=." count_events_url = lib_kbase.write_graph_to_events( test_url, test_graph_input_url) count_events = count_events_nourl + count_events_url # This shared queue is used by a subprocess reading the events, to send them back to this process. shared_queue = multiprocessing.Queue() created_process = _create_subprocess(count_events, shared_queue, self.sqlite_path) # Reads the triples sent by the subprocess. output_list = _read_triples_from_queue(count_events, shared_queue) # The subprocess is not needed any longer. created_process.terminate() created_process.join() print("Killing pid=", created_process.pid) input_triples_nourl = _graph_to_triples_set(test_graph_input_nourl) input_triples_url = _graph_to_triples_set(test_graph_input_url) input_triples = input_triples_nourl | input_triples_url output_triples = _graph_to_triples_set(output_list) # The two lists of triples must be identical: Comparison of the string representations. self.assertEqual(input_triples, output_triples)
def test_sqlalchemy_nourl_events_count(self): """Just writes.""" graph_cleanup = rdflib.Graph() lib_kbase.retrieve_all_events_to_graph_then_clear(graph_cleanup) test_graph_input = rdflib.Graph().parse( "tests/input_test_data/test_events_tcp_sockets.xml") count_events = lib_kbase.write_graph_to_events(None, test_graph_input) self.assertEqual(count_events, 43) actual_events_count = lib_kbase.events_count() self.assertEqual(count_events, actual_events_count) graph_from_sql_alchemy = rdflib.Graph() lib_kbase.retrieve_all_events_to_graph_then_clear( graph_from_sql_alchemy) read_events_count = len(graph_from_sql_alchemy) self.assertEqual(read_events_count, actual_events_count)
def test_sqlalchemy_read_all_twice(self): """Just writes.""" actual_events_count = lib_kbase.events_count() graph_from_sql_alchemy = rdflib.Graph() lib_kbase.retrieve_all_events_to_graph_then_clear( graph_from_sql_alchemy) read_events_count = len(graph_from_sql_alchemy) self.assertEqual(read_events_count, actual_events_count) events_count_after_read = lib_kbase.events_count() self.assertEqual(events_count_after_read, 0) # Second read empties all. graph_from_sql_alchemy = rdflib.Graph() lib_kbase.retrieve_all_events_to_graph_then_clear( graph_from_sql_alchemy) self.assertEqual(len(graph_from_sql_alchemy), 0)
def test_write_read_graph_to_events_with_other_nodes(self): triples_data_set = [ [("CIM_Process", { "Handle": 123 }), "ParentProcessId", 1], [ ("CIM_Directory", { "Name": "/tmp" }), # CIM_DirectoryContainsFile.GroupComponent or CIM_DirectoryContainsFile.PartComponent "CIM_DirectoryContainsFile", ("CIM_DataFile", { "Name": "/tmp/anyfile.tmp" }) ] ] updates_total_number = lib_event.store_events_as_json_triples_list( triples_data_set) test_graph_input = rdflib.Graph().parse( "tests/input_test_data/test_events_enumerate_CIM_Process.xml") test_url = "http://vps516494.ovh.net/Survol/survol/sources_types/enumerate_CIM_Process.py?xid=." count_events = lib_kbase.write_graph_to_events(test_url, test_graph_input) self.assertEqual(count_events, 682) test_graph_output = rdflib.Graph() count_events_output = lib_kbase.read_events_to_graph( test_url, test_graph_output) self.assertEqual(count_events, count_events_output) new_graph = rdflib.Graph() lib_kbase.retrieve_all_events_to_graph_then_clear(new_graph) print("files_updates_total_number=", updates_total_number) print("len(triples_list)=", len(new_graph)) self.assertEqual(updates_total_number, 2) self.assertEqual(len(new_graph), 2)
def test_sqlalchemy_url_events_count(self): graph_cleanup = rdflib.Graph() lib_kbase.retrieve_all_events_to_graph_then_clear(graph_cleanup) test_graph_input = rdflib.Graph().parse( "tests/input_test_data/test_events_tcp_sockets.xml") test_url = "http://vps516494.ovh.net/Survol/survol/sources_types/Linux/tcp_sockets.py?xid=." count_events = lib_kbase.write_graph_to_events(test_url, test_graph_input) self.assertEqual(count_events, 43) actual_events_count = lib_kbase.events_count() self.assertEqual(count_events, actual_events_count) graph_from_sql_alchemy = rdflib.Graph() lib_kbase.retrieve_all_events_to_graph_then_clear( graph_from_sql_alchemy) read_events_count = len(graph_from_sql_alchemy) self.assertEqual(read_events_count, actual_events_count)
def Main(): lib_common.set_events_credentials() # This can process remote hosts because it does not call any script, just shows them. cgiEnv = lib_common.ScriptEnvironment() logging.debug("Starting.") grph = cgiEnv.GetGraph() logging.debug("About to get events") num_triples = lib_kbase.retrieve_all_events_to_graph_then_clear(grph) logging.debug("num_triples=%d" % num_triples) cgiEnv.OutCgiRdf()
def _read_all_events_from_graph(expected_triple_numbers, pqueue): """This function is used by EventsMultiprocessGraphTest.test_write_here_read_there to run a subprocess which reads events (triples) from the shared graph which contains all events. These triples are then returned with a shared queue, and the main process compares with the initial content.""" grph = rdflib.Graph() num_triples = lib_kbase.retrieve_all_events_to_graph_then_clear(grph) if num_triples != expected_triple_numbers: # This is just an information, because a complete comparison is done later. sys.stderr.write( "_read_all_events_from_graph Wrong number of events:%d %d\n" % (num_triples, expected_triple_numbers)) for one_triple in grph: pqueue.put(one_triple)