def test_from_batch(self): """This searches the content of a process memory which contains a SQL memory.""" proc_open = _start_subprocess(sample_batch_script) child_stdin, child_stdout_and_stderr = (proc_open.stdin, proc_open.stdout) my_source_sql_queries = lib_client.SourceLocal( "sources_types/CIM_Process/memory_regex_search/scan_sql_queries.py", "CIM_Process", Handle=proc_open.pid) triple_sql_queries = my_source_sql_queries.get_triplestore() print(len(triple_sql_queries)) #self.assertEqual(len(triple_sql_queries.m_triplestore), 190) lst_matches = list(triple_sql_queries.get_instances()) print("Matches:", lst_matches) #self.assertEqual(len(lst_matches), 5) # Any string will do. child_stdin.write("Stop") print(lst_matches) proc_open.communicate()
def test_from_perl(self): """This searches the content of a process memory which contains a SQL memory.""" proc_open = _start_subprocess(_perl_path, sample_perl_script) my_source_filenames = lib_client.SourceLocal( "sources_types/CIM_Process/memory_regex_search/search_filenames.py", "CIM_Process", Handle=proc_open.pid) sys.stdout.flush() triple_filenames = my_source_filenames.get_triplestore() print("len(triple_filenames)=", len(triple_filenames)) filenames_set = set() for one_instance in triple_filenames.get_instances(): if type(one_instance).__name__ == 'CIM_DataFile': filenames_set.add(one_instance.Name) for one_filename in sorted(filenames_set): print(" ", one_filename) self.assertTrue(_perl_path in filenames_set) self.assertTrue( lib_util.standardized_file_path(windows_system32_cmd_exe) in filenames_set) proc_open.communicate()
def test_dirmenu_cim_process(self): my_source = lib_client.SourceLocal("entity_dirmenu_only.py", "CIM_Process", Handle=CurrentPid) triple_sql_queries = my_source.get_triplestore() print(len(triple_sql_queries))
def test_from_python(self): # Runs this process: It allocates a variable containing a SQL query, then it waits. proc_open = _start_subprocess(sys.executable, sample_python_script) my_source_sql_queries = lib_client.SourceLocal( "sources_types/CIM_Process/memory_regex_search/scan_sql_queries.py", "CIM_Process", Handle=proc_open.pid) triple_sql_queries = my_source_sql_queries.get_triplestore() print("len(triple_sql_queries)=", len(triple_sql_queries)) # This creates objects like: # "CIM_Process/embedded_sql_query.Query=SW5zZXJ0IGFuIGVudHJ5IGludG8gdGhlIGxpc3Qgb2Ygd2FybmluZ3MgZmlsdGVycyAoYXQgdGhlIGZyb250KS4=,Handle=15564" # Maybe, this is not the best representation. Possibly have a generic query ? Some inheritance ? # The specific detail about this query, is that it depends on the process. # This test focus on the parsing of memory, not on the queries representation. queries_set = set() for one_instance in triple_sql_queries.get_instances(): if type(one_instance).__name__ == 'CIM_Process/embedded_sql_query': self.assertEqual(one_instance.Handle, str(proc_open.pid)) decoded_query = lib_util.Base64Decode(one_instance.Query) queries_set.add(decoded_query) print("queries_set=", queries_set) self.assertTrue("select something from somewhere" in queries_set) self.assertTrue("select * from 'AnyTable'" in queries_set) self.assertTrue("select a,b,c from 'AnyTable'" in queries_set) self.assertTrue("select A.x,B.y from AnyTable A, OtherTable B" in queries_set) proc_open.communicate()
def test_dirmenu_cim_directory(self): my_source = lib_client.SourceLocal("entity_dirmenu_only.py", "CIM_Directory", Name=always_present_dir) triple_sql_queries = my_source.get_triplestore() print(len(triple_sql_queries))
def test_from_python2(self): """This searches the content of a process memory which contains a SQL memory.""" proc_open = _start_subprocess(sys.executable, sample_python_script) my_source_filenames = lib_client.SourceLocal( "sources_types/CIM_Process/memory_regex_search/search_filenames.py", "CIM_Process", Handle=proc_open.pid) triple_filenames = my_source_filenames.get_triplestore() filenames_set = set() for one_instance in triple_filenames.get_instances(): if type(one_instance).__name__ == 'CIM_DataFile': filenames_set.add(one_instance.Name) for one_filename in sorted(filenames_set): print(" ", one_filename) self.assertTrue( lib_util.standardized_file_path(windows_system32_cmd_exe) in filenames_set) self.assertTrue( lib_util.standardized_file_path(sys.executable) in filenames_set) # This filepath is calculated in the Python script file_name_with_slashes = os.path.join( os.path.dirname(sys.executable), "this_is_a_file_name_with_slashes.cpp").replace("\\", "/") self.assertTrue(file_name_with_slashes in filenames_set) tst_stdout, tst_stderr = proc_open.communicate() self.assertEqual(tst_stderr, None)
def test_associations_cim_process_executable_cim_process(self): my_source = lib_client.SourceLocal( "entity.py", "CIM_Process", Handle=CurrentPid, __associator_attribute__="CIM_ProcessExecutable.Antecedent", ) # It must return sys.executable triple_sql_queries = my_source.get_triplestore()
def test_associations_cim_directory_contains_file_part_component_dir(self): my_source = lib_client.SourceLocal( "entity.py", "CIM_Directory", Name=always_present_sub_dir, __associator_attribute__="CIM_DirectoryContainsFile.PartComponent", ) # It must return at least the current pid. triple_sql_queries = my_source.get_triplestore()
def test_associations_cim_process_executable_cim_datafile(self): my_source = lib_client.SourceLocal( "entity.py", "CIM_DataFile", Name=sys.executable, __associator_attribute__="CIM_ProcessExecutable.Dependent", ) # It must return at least the current pid. triple_sql_queries = my_source.get_triplestore()
def test_conformance_enumerate_Win32_UserAccount(self): """ Test of enumerate_Win32_UserAccount.py and corrected of attached ontology """ my_source_local = lib_client.SourceLocal( "sources_types/win32/enumerate_Win32_UserAccount.py") print("test_conformance_enumerate_Win32_UserAccount: query=%s" % my_source_local.create_url_query()) the_content_rdf = my_source_local.content_rdf() errors_list = self._check_rdf_url_ontology(the_content_rdf) self.assertEqual(errors_list, [])
def test_conformance_enumerate_CIM_LogicalDisk(self): """ Test of enumerate_CIM_LogicalDisk.py """ my_source_local = lib_client.SourceLocal( "sources_types/enumerate_CIM_LogicalDisk.py") print("test_conformance_enumerate_CIM_LogicalDisk: query=%s" % my_source_local.create_url_query()) the_content_rdf = my_source_local.content_rdf() errors_list = self._check_rdf_url_ontology(the_content_rdf) self.assertEqual(errors_list, [])
def test_from_perl(self): proc_open = _start_subprocess(_perl_path, sample_perl_script) my_source_urls = lib_client.SourceLocal( "sources_types/CIM_Process/memory_regex_search/search_urls.py", "CIM_Process", Handle=proc_open.pid) triple_urls = my_source_urls.get_triplestore() print("len(triple_urls)=", len(triple_urls)) proc_open.communicate()
def test_from_python(self): proc_open = _start_subprocess(sys.executable, sample_python_script) my_source_connection_strings = lib_client.SourceLocal( "sources_types/CIM_Process/memory_regex_search/search_connection_strings.py", "CIM_Process", Handle=proc_open.pid) triple_connection_strings = my_source_connection_strings.get_triplestore() print("len(triple_connection_strings_queries)=", len(triple_connection_strings)) proc_open.communicate()
def test_from_perl(self): """This searches COM classes ids in a process memory.""" proc_open = _start_subprocess(_perl_path, sample_perl_script) my_source_com_classes = lib_client.SourceLocal( "sources_types/CIM_Process/memory_regex_search/search_com_classes.py", "CIM_Process", Handle=proc_open.pid) triple_com_classes = my_source_com_classes.get_triplestore() print("len(triple_sql_queries)=", len(triple_com_classes)) proc_open.communicate()
def test_conformance_file_stat(self): """ This runs a filestat command and checks that the ontology included in the graph is correct. """ my_source_local = lib_client.SourceLocal( "sources_types/CIM_DataFile/file_stat.py", "CIM_DataFile", Name=always_present_file) print("test_conformance_file_stat: query=%s" % my_source_local.create_url_query()) the_content_rdf = my_source_local.content_rdf() errors_list = self._check_rdf_url_ontology(the_content_rdf) self.assertEqual(errors_list, [])
def test_from_perl(self): """This searches the content of a process memory which contains ODBC connection strings.""" proc_open = _start_subprocess(_perl_path, sample_perl_script) my_source_connection_strings = lib_client.SourceLocal( "sources_types/CIM_Process/memory_regex_search/search_connection_strings.py", "CIM_Process", Handle=proc_open.pid) triple_connection_strings = my_source_connection_strings.get_triplestore( ) print("len(triple_connection_strings_queries)=", len(triple_connection_strings)) proc_open.communicate()
def test_conformance_win32_NetLocalGroupGetMembers(self): """ Test of win32_NetLocalGroupGetMembers.py and the correctness of the ontology of classes and attributes. """ # The group "Users" is always here. my_source_local = lib_client.SourceLocal( "sources_types/Win32_Group/win32_NetLocalGroupGetMembers.py", "Win32_Group", Name="Users", Domain=CurrentDomainWin32) print("test_conformance_win32_NetLocalGroupGetMembers: query=%s" % my_source_local.create_url_query()) the_content_rdf = my_source_local.content_rdf() errors_list = self._check_rdf_url_ontology(the_content_rdf) self.assertEqual(errors_list, [])
def test_sqlite_query_dependencies_simple_select(self): """ This extracts the dependencies of a SQL query on a sqlite database. It also tests the correct encoding of a SQL query in a URL. """ query_clear = "select * from invoices" my_source = lib_client.SourceLocal( "sources_types/sqlite/query/sqlite_query_dependencies.py", "sqlite/query", File=self._chinook_db, Query=query_clear) the_graph = my_source.get_graph() for one_triple in the_graph.triples((None, None, None)): print(" ", one_triple) table_names = self._get_column_names_from_graph(the_graph) print("table_names=", table_names) self.assertEqual(table_names, {'*****@*****.**'})
def test_sqlite_query_dependencies_select_case_insensitive(self): """ This extracts the dependencies of a SQL query on a sqlite database. The query must be case-insensitive. It also tests the correct encoding of a SQL query in a URL. """ query_clear = "select * from Invoices, INVOICE_ITEMS where INVOICES.InvoiceId = Invoice_Items.InvoiceId" my_source = lib_client.SourceLocal( "sources_types/sqlite/query/sqlite_query_dependencies.py", "sqlite/query", File=self._chinook_db, Query=query_clear) the_graph = my_source.get_graph() table_names = self._get_column_names_from_graph(the_graph) print("table_names=", table_names) self.assertEqual(table_names, {'*****@*****.**', '*****@*****.**'})
def _get_urls_from_python_process(self): """This searches the content of a process memory which contains a SQL memory.""" proc_open = _start_subprocess(sys.executable, sample_python_script) # Short delay so the process has time enough to start and declare fill its memory space. time.sleep(0.5) my_source_urls = lib_client.SourceLocal( "sources_types/CIM_Process/memory_regex_search/search_urls.py", "CIM_Process", Handle=proc_open.pid) triple_urls = my_source_urls.get_triplestore() print("len(triple_urls)=", len(triple_urls)) urls_set = None for one_instance in triple_urls.get_instances(): if type(one_instance).__name__ == 'CIM_Process': print("one_instance.Handle=", one_instance.Handle) print("attrs=", one_instance.graph_attributes) # This flag because the URLs are added with the predicate pc.property_rdf_data_nolist1 # This is some sort of hard-code but it does not matter yet in a test program. urls_list = one_instance.graph_attributes['ns1:Data1'] print("urls_list=", urls_list) # There should be one process only. self.assertTrue(urls_set is None) urls_set = set(urls_list) # url_http_str = u"http://www.gnu.org/gnu/gnu.html" # url_http_bytes = b"https://pypi.org/help/" # url_https_bytes = b"https://www.python.org/about/" # url_https_str = u"https://www.perl.org/about.html" self.assertTrue("https://pypi.org" in urls_set) self.assertTrue("https://www.python.org" in urls_set) if sys.version_info >= (3, ): self.assertTrue("https://www.perl.org" in urls_set) self.assertTrue("http://www.gnu.org" in urls_set) break tst_stdout, tst_stderr = proc_open.communicate() self.assertEqual(tst_stderr, None) print("tst_stdout=", tst_stdout) return urls_set
def test_sqlite_tables_and_views(self): """ This extracts the tables and views of a sqllite database. """ my_source = lib_client.SourceLocal( "sources_types/sqlite/file/sqlite_tables_and_views.py", "sqlite/file", File=self._chinook_db) the_graph = my_source.get_graph() table_names = self._get_column_names_from_graph(the_graph) print("table_names=", table_names) self.assertEqual( table_names, { '*****@*****.**', '*****@*****.**', '*****@*****.**', '*****@*****.**', '*****@*****.**', '*****@*****.**', '*****@*****.**', '*****@*****.**', '*****@*****.**', '*****@*****.**', '*****@*****.**', '*****@*****.**', '*****@*****.**' })
def test_sqlite_table_fields(self): """ This extracts the columns of a table in a sqlite database. """ my_source = lib_client.SourceLocal( "sources_types/sqlite/table/sqlite_table_fields.py", "sqlite/table", File=self._chinook_db, Table="invoices") print("query=%s" % my_source.create_url_query()) the_graph = my_source.get_graph() column_class_node = lib_kbase.class_node_uriref("sqlite/column") column_nodes = [ column_node for column_node, _, _ in the_graph.triples(( None, rdflib.namespace.RDF.type, column_class_node)) ] column_names = set( str(column_name) for column_name in [ list( the_graph.triples((column_node, rdflib.namespace.RDFS.label, None)))[0][2] for column_node in column_nodes ]) print("column_names=", column_names) self.assertEqual( column_names, { '*****@*****.**', '*****@*****.**', '*****@*****.**', '*****@*****.**', '*****@*****.**', '*****@*****.**', '*****@*****.**', '*****@*****.**', '*****@*****.**' })
def test_from_perl(self): proc_open = _start_subprocess(_perl_path, sample_perl_script) my_source_sql_queries = lib_client.SourceLocal( "sources_types/CIM_Process/memory_regex_search/scan_sql_queries.py", "CIM_Process", Handle=proc_open.pid) triple_sql_queries = my_source_sql_queries.get_triplestore() queries_set = set() for one_instance in triple_sql_queries.get_instances(): if type(one_instance).__name__ == 'CIM_Process/embedded_sql_query': self.assertEqual(one_instance.Handle, str(proc_open.pid)) decoded_query = lib_util.Base64Decode(one_instance.Query) queries_set.add(decoded_query) print("queries_set=", queries_set) self.assertTrue("select column_a from table_a" in queries_set) self.assertTrue("select column_b from table_b" in queries_set) proc_open.communicate()
def CallbackSelect(self, grph, class_name, predicate_prefix, filtered_where_key_values): DEBUG( "SurvolCallbackSelect class_name=%s predicate_prefix=%s where_key_values=%s", class_name, predicate_prefix, str(filtered_where_key_values)) # Maybe there is a script: predicate_prefix="survol:CIM_DataFile/mapping_processes" prefix, colon, script_nickname = predicate_prefix.partition(":") DEBUG("SurvolCallbackSelect script_nickname=%s", script_nickname) if script_nickname: # For example: script_nickname="CIM_DataFile/mapping_processes" # Wildcards or directories are not accepted yet. script_name = "sources_types/" + script_nickname + ".py" DEBUG( "SurvolCallbackSelect script_name=%s filtered_where_key_values=%s", script_name, str(filtered_where_key_values)) # TODO: Check that there are enough parameters for this script ? my_source = lib_client.SourceLocal(script_name, class_name, **filtered_where_key_values) DEBUG("SurvolCallbackSelect my_source=%s", my_source) my_triplestore = my_source.get_triplestore() # This is returned anyway, as a triplestore that rdflib Sparql can work on. my_triplestore.copy_to_graph(grph) list_instances = my_triplestore.get_instances() # TODO: We filter only the objects of the right type, # TODO: ... but we lose all the other objects which could be stored in the output triplestore !!... DEBUG("SurvolCallbackSelect tp=%s class_name=%s", type(list_instances), class_name) DEBUG("SurvolCallbackSelect list_instances=%s", str(list_instances)) for one_instance in list_instances: WARNING( "SurvolCallbackSelect one_instance.__class__.__name__=%s", one_instance.__class__.__name__) if one_instance.__class__.__name__ == class_name: # 'CIM_DataFile.Name=/usr/lib/systemd/systemd-journald' instance_url = one_instance.__class__.__name__ + "." + one_instance.m_entity_id one_instance.m_key_value_pairs[ lib_kbase. PredicateIsDefinedBy] = lib_common.NodeLiteral( predicate_prefix) # Add it again, so the original Sparql query will work. one_instance.m_key_value_pairs[ lib_kbase.PredicateSeeAlso] = lib_common.NodeLiteral( predicate_prefix) DEBUG("SurvolCallbackSelect instance_url=%s", instance_url) yield (instance_url, one_instance.m_key_value_pairs) else: entity_module = lib_util.GetEntityModule(class_name) if not entity_module: raise Exception( "SurvolCallbackSelect: No module for class:%s" % class_name) try: enumerate_function = entity_module.SelectFromWhere except AttributeError: exc = sys.exc_info()[1] WARNING("No Enumerate for %s:%s", class_name, str(exc)) return iter_enumeration = enumerate_function(filtered_where_key_values) # for one_key_value_dict in iter_enumeration: for one_key_value_dict_nodes in iter_enumeration: class_ontology = lib_util.OntologyClassKeys(class_name) ontology_key_values = {} for key_node, value_node in one_key_value_dict_nodes.items(): key_str = lib_properties.PropToQName(key_node) if key_str in class_ontology: ontology_key_values[key_str] = str(value_node) # This reorders the attributes if needed. key_value_path = lib_util.KWArgsToEntityId( class_name, **ontology_key_values) # key_value_path = ".".join( '%s="%s"' % ( lib_properties.PropToQName(key), str(value) ) for key, value in one_key_value_dict_nodes.items() ) object_path = "SurvolLocalHost:" + class_name + "." + key_value_path one_key_value_dict_nodes[ lib_kbase.PredicateIsDefinedBy] = lib_common.NodeLiteral( predicate_prefix) # Add it again, so the original Sparql query will work. one_key_value_dict_nodes[ lib_kbase.PredicateSeeAlso] = lib_common.NodeLiteral( predicate_prefix) yield (object_path, one_key_value_dict_nodes)