def test_from_batch(self):
"""This searches the content of a process memory which contains a SQL memory."""
proc_open = _start_subprocess(sample_batch_script)
child_stdin, child_stdout_and_stderr = (proc_open.stdin,
proc_open.stdout)
my_source_sql_queries = lib_client.SourceLocal(
"sources_types/CIM_Process/memory_regex_search/scan_sql_queries.py",
"CIM_Process",
Handle=proc_open.pid)
triple_sql_queries = my_source_sql_queries.get_triplestore()
print(len(triple_sql_queries))
#self.assertEqual(len(triple_sql_queries.m_triplestore), 190)
lst_matches = list(triple_sql_queries.get_instances())
print("Matches:", lst_matches)
#self.assertEqual(len(lst_matches), 5)
# Any string will do.
child_stdin.write("Stop")
print(lst_matches)
proc_open.communicate()
def test_from_perl(self):
"""This searches the content of a process memory which contains a SQL memory."""
proc_open = _start_subprocess(_perl_path, sample_perl_script)
my_source_filenames = lib_client.SourceLocal(
"sources_types/CIM_Process/memory_regex_search/search_filenames.py",
"CIM_Process",
Handle=proc_open.pid)
sys.stdout.flush()
triple_filenames = my_source_filenames.get_triplestore()
print("len(triple_filenames)=", len(triple_filenames))
filenames_set = set()
for one_instance in triple_filenames.get_instances():
if type(one_instance).__name__ == 'CIM_DataFile':
filenames_set.add(one_instance.Name)
for one_filename in sorted(filenames_set):
print(" ", one_filename)
self.assertTrue(_perl_path in filenames_set)
self.assertTrue(
lib_util.standardized_file_path(windows_system32_cmd_exe) in
filenames_set)
proc_open.communicate()
def test_dirmenu_cim_process(self):
my_source = lib_client.SourceLocal("entity_dirmenu_only.py",
"CIM_Process",
Handle=CurrentPid)
triple_sql_queries = my_source.get_triplestore()
print(len(triple_sql_queries))
def test_from_python(self):
# Runs this process: It allocates a variable containing a SQL query, then it waits.
proc_open = _start_subprocess(sys.executable, sample_python_script)
my_source_sql_queries = lib_client.SourceLocal(
"sources_types/CIM_Process/memory_regex_search/scan_sql_queries.py",
"CIM_Process",
Handle=proc_open.pid)
triple_sql_queries = my_source_sql_queries.get_triplestore()
print("len(triple_sql_queries)=", len(triple_sql_queries))
# This creates objects like:
# "CIM_Process/embedded_sql_query.Query=SW5zZXJ0IGFuIGVudHJ5IGludG8gdGhlIGxpc3Qgb2Ygd2FybmluZ3MgZmlsdGVycyAoYXQgdGhlIGZyb250KS4=,Handle=15564"
# Maybe, this is not the best representation. Possibly have a generic query ? Some inheritance ?
# The specific detail about this query, is that it depends on the process.
# This test focus on the parsing of memory, not on the queries representation.
queries_set = set()
for one_instance in triple_sql_queries.get_instances():
if type(one_instance).__name__ == 'CIM_Process/embedded_sql_query':
self.assertEqual(one_instance.Handle, str(proc_open.pid))
decoded_query = lib_util.Base64Decode(one_instance.Query)
queries_set.add(decoded_query)
print("queries_set=", queries_set)
self.assertTrue("select something from somewhere" in queries_set)
self.assertTrue("select * from 'AnyTable'" in queries_set)
self.assertTrue("select a,b,c from 'AnyTable'" in queries_set)
self.assertTrue("select A.x,B.y from AnyTable A, OtherTable B" in queries_set)
proc_open.communicate()
def test_dirmenu_cim_directory(self):
my_source = lib_client.SourceLocal("entity_dirmenu_only.py",
"CIM_Directory",
Name=always_present_dir)
triple_sql_queries = my_source.get_triplestore()
print(len(triple_sql_queries))
def test_from_python2(self):
"""This searches the content of a process memory which contains a SQL memory."""
proc_open = _start_subprocess(sys.executable, sample_python_script)
my_source_filenames = lib_client.SourceLocal(
"sources_types/CIM_Process/memory_regex_search/search_filenames.py",
"CIM_Process",
Handle=proc_open.pid)
triple_filenames = my_source_filenames.get_triplestore()
filenames_set = set()
for one_instance in triple_filenames.get_instances():
if type(one_instance).__name__ == 'CIM_DataFile':
filenames_set.add(one_instance.Name)
for one_filename in sorted(filenames_set):
print(" ", one_filename)
self.assertTrue(
lib_util.standardized_file_path(windows_system32_cmd_exe) in
filenames_set)
self.assertTrue(
lib_util.standardized_file_path(sys.executable) in filenames_set)
# This filepath is calculated in the Python script
file_name_with_slashes = os.path.join(
os.path.dirname(sys.executable),
"this_is_a_file_name_with_slashes.cpp").replace("\\", "/")
self.assertTrue(file_name_with_slashes in filenames_set)
tst_stdout, tst_stderr = proc_open.communicate()
self.assertEqual(tst_stderr, None)
def test_associations_cim_process_executable_cim_process(self):
my_source = lib_client.SourceLocal(
"entity.py",
"CIM_Process",
Handle=CurrentPid,
__associator_attribute__="CIM_ProcessExecutable.Antecedent",
)
# It must return sys.executable
triple_sql_queries = my_source.get_triplestore()
def test_associations_cim_directory_contains_file_part_component_dir(self):
my_source = lib_client.SourceLocal(
"entity.py",
"CIM_Directory",
Name=always_present_sub_dir,
__associator_attribute__="CIM_DirectoryContainsFile.PartComponent",
)
# It must return at least the current pid.
triple_sql_queries = my_source.get_triplestore()
def test_associations_cim_process_executable_cim_datafile(self):
my_source = lib_client.SourceLocal(
"entity.py",
"CIM_DataFile",
Name=sys.executable,
__associator_attribute__="CIM_ProcessExecutable.Dependent",
)
# It must return at least the current pid.
triple_sql_queries = my_source.get_triplestore()
def test_conformance_enumerate_Win32_UserAccount(self):
"""
Test of enumerate_Win32_UserAccount.py and corrected of attached ontology
"""
my_source_local = lib_client.SourceLocal(
"sources_types/win32/enumerate_Win32_UserAccount.py")
print("test_conformance_enumerate_Win32_UserAccount: query=%s" %
my_source_local.create_url_query())
the_content_rdf = my_source_local.content_rdf()
errors_list = self._check_rdf_url_ontology(the_content_rdf)
self.assertEqual(errors_list, [])
def test_conformance_enumerate_CIM_LogicalDisk(self):
"""
Test of enumerate_CIM_LogicalDisk.py
"""
my_source_local = lib_client.SourceLocal(
"sources_types/enumerate_CIM_LogicalDisk.py")
print("test_conformance_enumerate_CIM_LogicalDisk: query=%s" %
my_source_local.create_url_query())
the_content_rdf = my_source_local.content_rdf()
errors_list = self._check_rdf_url_ontology(the_content_rdf)
self.assertEqual(errors_list, [])
def test_from_perl(self):
proc_open = _start_subprocess(_perl_path, sample_perl_script)
my_source_urls = lib_client.SourceLocal(
"sources_types/CIM_Process/memory_regex_search/search_urls.py",
"CIM_Process",
Handle=proc_open.pid)
triple_urls = my_source_urls.get_triplestore()
print("len(triple_urls)=", len(triple_urls))
proc_open.communicate()
def test_from_python(self):
proc_open = _start_subprocess(sys.executable, sample_python_script)
my_source_connection_strings = lib_client.SourceLocal(
"sources_types/CIM_Process/memory_regex_search/search_connection_strings.py",
"CIM_Process",
Handle=proc_open.pid)
triple_connection_strings = my_source_connection_strings.get_triplestore()
print("len(triple_connection_strings_queries)=", len(triple_connection_strings))
proc_open.communicate()
def test_from_perl(self):
"""This searches COM classes ids in a process memory."""
proc_open = _start_subprocess(_perl_path, sample_perl_script)
my_source_com_classes = lib_client.SourceLocal(
"sources_types/CIM_Process/memory_regex_search/search_com_classes.py",
"CIM_Process",
Handle=proc_open.pid)
triple_com_classes = my_source_com_classes.get_triplestore()
print("len(triple_sql_queries)=", len(triple_com_classes))
proc_open.communicate()
def test_conformance_file_stat(self):
"""
This runs a filestat command and checks that the ontology included in the graph is correct.
"""
my_source_local = lib_client.SourceLocal(
"sources_types/CIM_DataFile/file_stat.py",
"CIM_DataFile",
Name=always_present_file)
print("test_conformance_file_stat: query=%s" %
my_source_local.create_url_query())
the_content_rdf = my_source_local.content_rdf()
errors_list = self._check_rdf_url_ontology(the_content_rdf)
self.assertEqual(errors_list, [])
def test_from_perl(self):
"""This searches the content of a process memory which contains ODBC connection strings."""
proc_open = _start_subprocess(_perl_path, sample_perl_script)
my_source_connection_strings = lib_client.SourceLocal(
"sources_types/CIM_Process/memory_regex_search/search_connection_strings.py",
"CIM_Process",
Handle=proc_open.pid)
triple_connection_strings = my_source_connection_strings.get_triplestore(
)
print("len(triple_connection_strings_queries)=",
len(triple_connection_strings))
proc_open.communicate()
def test_conformance_win32_NetLocalGroupGetMembers(self):
"""
Test of win32_NetLocalGroupGetMembers.py and the correctness of the ontology of classes and attributes.
"""
# The group "Users" is always here.
my_source_local = lib_client.SourceLocal(
"sources_types/Win32_Group/win32_NetLocalGroupGetMembers.py",
"Win32_Group",
Name="Users",
Domain=CurrentDomainWin32)
print("test_conformance_win32_NetLocalGroupGetMembers: query=%s" %
my_source_local.create_url_query())
the_content_rdf = my_source_local.content_rdf()
errors_list = self._check_rdf_url_ontology(the_content_rdf)
self.assertEqual(errors_list, [])
def test_sqlite_query_dependencies_simple_select(self):
"""
This extracts the dependencies of a SQL query on a sqlite database.
It also tests the correct encoding of a SQL query in a URL.
"""
query_clear = "select * from invoices"
my_source = lib_client.SourceLocal(
"sources_types/sqlite/query/sqlite_query_dependencies.py",
"sqlite/query",
File=self._chinook_db,
Query=query_clear)
the_graph = my_source.get_graph()
for one_triple in the_graph.triples((None, None, None)):
print(" ", one_triple)
table_names = self._get_column_names_from_graph(the_graph)
print("table_names=", table_names)
self.assertEqual(table_names, {'*****@*****.**'})
def test_sqlite_query_dependencies_select_case_insensitive(self):
"""
This extracts the dependencies of a SQL query on a sqlite database.
The query must be case-insensitive.
It also tests the correct encoding of a SQL query in a URL.
"""
query_clear = "select * from Invoices, INVOICE_ITEMS where INVOICES.InvoiceId = Invoice_Items.InvoiceId"
my_source = lib_client.SourceLocal(
"sources_types/sqlite/query/sqlite_query_dependencies.py",
"sqlite/query",
File=self._chinook_db,
Query=query_clear)
the_graph = my_source.get_graph()
table_names = self._get_column_names_from_graph(the_graph)
print("table_names=", table_names)
self.assertEqual(table_names,
{'*****@*****.**', '*****@*****.**'})
def _get_urls_from_python_process(self):
"""This searches the content of a process memory which contains a SQL memory."""
proc_open = _start_subprocess(sys.executable, sample_python_script)
# Short delay so the process has time enough to start and declare fill its memory space.
time.sleep(0.5)
my_source_urls = lib_client.SourceLocal(
"sources_types/CIM_Process/memory_regex_search/search_urls.py",
"CIM_Process",
Handle=proc_open.pid)
triple_urls = my_source_urls.get_triplestore()
print("len(triple_urls)=", len(triple_urls))
urls_set = None
for one_instance in triple_urls.get_instances():
if type(one_instance).__name__ == 'CIM_Process':
print("one_instance.Handle=", one_instance.Handle)
print("attrs=", one_instance.graph_attributes)
# This flag because the URLs are added with the predicate pc.property_rdf_data_nolist1
# This is some sort of hard-code but it does not matter yet in a test program.
urls_list = one_instance.graph_attributes['ns1:Data1']
print("urls_list=", urls_list)
# There should be one process only.
self.assertTrue(urls_set is None)
urls_set = set(urls_list)
# url_http_str = u"http://www.gnu.org/gnu/gnu.html"
# url_http_bytes = b"https://pypi.org/help/"
# url_https_bytes = b"https://www.python.org/about/"
# url_https_str = u"https://www.perl.org/about.html"
self.assertTrue("https://pypi.org" in urls_set)
self.assertTrue("https://www.python.org" in urls_set)
if sys.version_info >= (3, ):
self.assertTrue("https://www.perl.org" in urls_set)
self.assertTrue("http://www.gnu.org" in urls_set)
break
tst_stdout, tst_stderr = proc_open.communicate()
self.assertEqual(tst_stderr, None)
print("tst_stdout=", tst_stdout)
return urls_set
def test_sqlite_tables_and_views(self):
"""
This extracts the tables and views of a sqllite database.
"""
my_source = lib_client.SourceLocal(
"sources_types/sqlite/file/sqlite_tables_and_views.py",
"sqlite/file",
File=self._chinook_db)
the_graph = my_source.get_graph()
table_names = self._get_column_names_from_graph(the_graph)
print("table_names=", table_names)
self.assertEqual(
table_names, {
'*****@*****.**', '*****@*****.**',
'*****@*****.**', '*****@*****.**',
'*****@*****.**', '*****@*****.**',
'*****@*****.**', '*****@*****.**',
'*****@*****.**', '*****@*****.**',
'*****@*****.**', '*****@*****.**',
'*****@*****.**'
})
def test_sqlite_table_fields(self):
"""
This extracts the columns of a table in a sqlite database.
"""
my_source = lib_client.SourceLocal(
"sources_types/sqlite/table/sqlite_table_fields.py",
"sqlite/table",
File=self._chinook_db,
Table="invoices")
print("query=%s" % my_source.create_url_query())
the_graph = my_source.get_graph()
column_class_node = lib_kbase.class_node_uriref("sqlite/column")
column_nodes = [
column_node for column_node, _, _ in the_graph.triples((
None, rdflib.namespace.RDF.type, column_class_node))
]
column_names = set(
str(column_name) for column_name in [
list(
the_graph.triples((column_node,
rdflib.namespace.RDFS.label,
None)))[0][2]
for column_node in column_nodes
])
print("column_names=", column_names)
self.assertEqual(
column_names, {
'*****@*****.**',
'*****@*****.**',
'*****@*****.**',
'*****@*****.**', '*****@*****.**',
'*****@*****.**',
'*****@*****.**',
'*****@*****.**',
'*****@*****.**'
})
def test_from_perl(self):
proc_open = _start_subprocess(_perl_path, sample_perl_script)
my_source_sql_queries = lib_client.SourceLocal(
"sources_types/CIM_Process/memory_regex_search/scan_sql_queries.py",
"CIM_Process",
Handle=proc_open.pid)
triple_sql_queries = my_source_sql_queries.get_triplestore()
queries_set = set()
for one_instance in triple_sql_queries.get_instances():
if type(one_instance).__name__ == 'CIM_Process/embedded_sql_query':
self.assertEqual(one_instance.Handle, str(proc_open.pid))
decoded_query = lib_util.Base64Decode(one_instance.Query)
queries_set.add(decoded_query)
print("queries_set=", queries_set)
self.assertTrue("select column_a from table_a" in queries_set)
self.assertTrue("select column_b from table_b" in queries_set)
proc_open.communicate()
def CallbackSelect(self, grph, class_name, predicate_prefix,
filtered_where_key_values):
DEBUG(
"SurvolCallbackSelect class_name=%s predicate_prefix=%s where_key_values=%s",
class_name, predicate_prefix, str(filtered_where_key_values))
# Maybe there is a script: predicate_prefix="survol:CIM_DataFile/mapping_processes"
prefix, colon, script_nickname = predicate_prefix.partition(":")
DEBUG("SurvolCallbackSelect script_nickname=%s", script_nickname)
if script_nickname:
# For example: script_nickname="CIM_DataFile/mapping_processes"
# Wildcards or directories are not accepted yet.
script_name = "sources_types/" + script_nickname + ".py"
DEBUG(
"SurvolCallbackSelect script_name=%s filtered_where_key_values=%s",
script_name, str(filtered_where_key_values))
# TODO: Check that there are enough parameters for this script ?
my_source = lib_client.SourceLocal(script_name, class_name,
**filtered_where_key_values)
DEBUG("SurvolCallbackSelect my_source=%s", my_source)
my_triplestore = my_source.get_triplestore()
# This is returned anyway, as a triplestore that rdflib Sparql can work on.
my_triplestore.copy_to_graph(grph)
list_instances = my_triplestore.get_instances()
# TODO: We filter only the objects of the right type,
# TODO: ... but we lose all the other objects which could be stored in the output triplestore !!...
DEBUG("SurvolCallbackSelect tp=%s class_name=%s",
type(list_instances), class_name)
DEBUG("SurvolCallbackSelect list_instances=%s",
str(list_instances))
for one_instance in list_instances:
WARNING(
"SurvolCallbackSelect one_instance.__class__.__name__=%s",
one_instance.__class__.__name__)
if one_instance.__class__.__name__ == class_name:
# 'CIM_DataFile.Name=/usr/lib/systemd/systemd-journald'
instance_url = one_instance.__class__.__name__ + "." + one_instance.m_entity_id
one_instance.m_key_value_pairs[
lib_kbase.
PredicateIsDefinedBy] = lib_common.NodeLiteral(
predicate_prefix)
# Add it again, so the original Sparql query will work.
one_instance.m_key_value_pairs[
lib_kbase.PredicateSeeAlso] = lib_common.NodeLiteral(
predicate_prefix)
DEBUG("SurvolCallbackSelect instance_url=%s", instance_url)
yield (instance_url, one_instance.m_key_value_pairs)
else:
entity_module = lib_util.GetEntityModule(class_name)
if not entity_module:
raise Exception(
"SurvolCallbackSelect: No module for class:%s" %
class_name)
try:
enumerate_function = entity_module.SelectFromWhere
except AttributeError:
exc = sys.exc_info()[1]
WARNING("No Enumerate for %s:%s", class_name, str(exc))
return
iter_enumeration = enumerate_function(filtered_where_key_values)
# for one_key_value_dict in iter_enumeration:
for one_key_value_dict_nodes in iter_enumeration:
class_ontology = lib_util.OntologyClassKeys(class_name)
ontology_key_values = {}
for key_node, value_node in one_key_value_dict_nodes.items():
key_str = lib_properties.PropToQName(key_node)
if key_str in class_ontology:
ontology_key_values[key_str] = str(value_node)
# This reorders the attributes if needed.
key_value_path = lib_util.KWArgsToEntityId(
class_name, **ontology_key_values)
# key_value_path = ".".join( '%s="%s"' % ( lib_properties.PropToQName(key), str(value) ) for key, value in one_key_value_dict_nodes.items() )
object_path = "SurvolLocalHost:" + class_name + "." + key_value_path
one_key_value_dict_nodes[
lib_kbase.PredicateIsDefinedBy] = lib_common.NodeLiteral(
predicate_prefix)
# Add it again, so the original Sparql query will work.
one_key_value_dict_nodes[
lib_kbase.PredicateSeeAlso] = lib_common.NodeLiteral(
predicate_prefix)
yield (object_path, one_key_value_dict_nodes)
