def test_from_python(self):
# Runs this process: It allocates a variable containing a SQL query, then it waits.
proc_open = _start_subprocess(sys.executable, sample_python_script)
my_source_sql_queries = lib_client.SourceLocal(
"sources_types/CIM_Process/memory_regex_search/scan_sql_queries.py",
"CIM_Process",
Handle=proc_open.pid)
triple_sql_queries = my_source_sql_queries.get_triplestore()
print("len(triple_sql_queries)=", len(triple_sql_queries))
# This creates objects like:
# "CIM_Process/embedded_sql_query.Query=SW5zZXJ0IGFuIGVudHJ5IGludG8gdGhlIGxpc3Qgb2Ygd2FybmluZ3MgZmlsdGVycyAoYXQgdGhlIGZyb250KS4=,Handle=15564"
# Maybe, this is not the best representation. Possibly have a generic query ? Some inheritance ?
# The specific detail about this query, is that it depends on the process.
# This test focus on the parsing of memory, not on the queries representation.
queries_set = set()
for one_instance in triple_sql_queries.get_instances():
if type(one_instance).__name__ == 'CIM_Process/embedded_sql_query':
self.assertEqual(one_instance.Handle, str(proc_open.pid))
decoded_query = lib_util.Base64Decode(one_instance.Query)
queries_set.add(decoded_query)
print("queries_set=", queries_set)
self.assertTrue("select something from somewhere" in queries_set)
self.assertTrue("select * from 'AnyTable'" in queries_set)
self.assertTrue("select a,b,c from 'AnyTable'" in queries_set)
self.assertTrue("select A.x,B.y from AnyTable A, OtherTable B" in queries_set)
proc_open.communicate()
def Main():
cgiEnv = lib_common.CgiEnv()
# "NtOpenObjectAuditAlarm%40C%3A\windows\system32\ntdll.dll"
# Filename is optional.
symbolFull = cgiEnv.GetId()
# The symbol is already demangled.
symbol_encode = cgiEnv.m_entity_id_dict["Name"]
# TODO: This should be packaged in lib_symbol.
symbolNam = lib_util.Base64Decode(symbol_encode)
filNam = cgiEnv.m_entity_id_dict["File"]
DEBUG("symbol=%s filNam=%s", symbolNam, filNam)
grph = cgiEnv.GetGraph()
symNode = lib_uris.gUriGen.SymbolUri(symbolNam, filNam)
if filNam:
filNode = lib_common.gUriGen.FileUri(filNam)
grph.add((filNode, pc.property_symbol_defined, symNode))
(fulNam, lstArgs) = lib_symbol.SymToArgs(symbolNam)
if lstArgs:
for arg in lstArgs:
# TODO: Order of arguments must not be changed.
argNode = lib_uris.gUriGen.ClassUri(arg, filNam)
grph.add((symNode, pc.property_argument, argNode))
cgiEnv.OutCgiRdf("LAYOUT_RECT", [pc.property_argument])
def EntityName(entity_ids_arr):
thePid = entity_ids_arr[1]
# sys.stderr.write("thePid=%s\n"%thePid)
sqlQuery = entity_ids_arr[0]
resu = lib_util.Base64Decode(sqlQuery)
# If the query contains double-quotes, it crashes Graphviz
resu = resu.replace('"',"'")
# resu = resu.replace('"','\\"')
return "Pid " + str(thePid) + ":" + resu
def EntityName(entity_ids_arr):
entity_id = entity_ids_arr[0]
try:
# Trailing padding.
resu = lib_util.Base64Decode(entity_id)
# TODO: Should have a more generic solution: i.e. always b64 encode CGI-incompatible strings.
# See lib_uris.SymbolUri which does the encoding.
resu = lib_util.html_escape(resu)
return resu
except TypeError:
exc = sys.exc_info()[1]
ERROR("CANNOT DECODE: symbol=(%s):%s", entity_id, str(exc))
return entity_id
def EntityName(entity_ids_arr):
entity_id = entity_ids_arr[0]
try:
# Trailing padding.
resu = lib_util.Base64Decode(entity_id)
# TODO: LE FAIRE AUSSI POUR LES AUTRES SYMBOLES.
resu = cgi.escape(resu)
return resu
except TypeError:
exc = sys.exc_info()[1]
sys.stderr.write("CANNOT DECODE: symbol=(%s):%s\n" %
(entity_id, str(exc)))
return entity_id
def EntityName(entity_ids_arr):
entity_id = entity_ids_arr[0]
try:
# Trailing padding.
# TODO: Encoding is done in lib_uris.ClassUri : The encoding should be more generic.
# TODO: ... and done only when the content is CGI-incompatible.
# TODO: Or do just like sources_types.sql.query.MakeUri
resu = lib_util.Base64Decode(entity_id)
resu = lib_util.html_escape(resu)
return resu
except TypeError as exc:
ERROR("CANNOT DECODE: class=(%s):%s", entity_id, str(exc))
return entity_id
def EntityName(entity_ids_arr):
entity_id = entity_ids_arr[0]
# PROBLEME: Double &kt;< !!!
# return entity_id
try:
# Trailing padding.
resu = lib_util.Base64Decode(entity_id)
resu = cgi.escape(resu)
return resu
except TypeError:
exc = sys.exc_info()[1]
sys.stderr.write("CANNOT DECODE: class=(%s):%s\n" %
(entity_id, str(exc)))
return entity_id
def EntityNameUtil(textPrefix, sqlQuery):
resu = lib_util.Base64Decode(sqlQuery)
resu = cgi.escape(resu)
resu = stripblanks(resu)
lenFilNam = len(textPrefix)
lenResu = len(resu)
lenTot = lenFilNam + lenResu
lenMaxi = 50
lenDiff = lenTot - lenMaxi
if lenDiff > 0:
lenResu -= lenDiff
if lenResu < 30:
lenResu = 30
return textPrefix + ":" + resu[:lenResu] + "..."
else:
return textPrefix + ":" + resu
def test_from_perl(self):
proc_open = _start_subprocess(_perl_path, sample_perl_script)
my_source_sql_queries = lib_client.SourceLocal(
"sources_types/CIM_Process/memory_regex_search/scan_sql_queries.py",
"CIM_Process",
Handle=proc_open.pid)
triple_sql_queries = my_source_sql_queries.get_triplestore()
queries_set = set()
for one_instance in triple_sql_queries.get_instances():
if type(one_instance).__name__ == 'CIM_Process/embedded_sql_query':
self.assertEqual(one_instance.Handle, str(proc_open.pid))
decoded_query = lib_util.Base64Decode(one_instance.Query)
queries_set.add(decoded_query)
print("queries_set=", queries_set)
self.assertTrue("select column_a from table_a" in queries_set)
self.assertTrue("select column_b from table_b" in queries_set)
proc_open.communicate()
def Main():
cgiEnv = lib_common.CgiEnv()
# "NtOpenObjectAuditAlarm%40C%3A\windows\system32\ntdll.dll"
# Filename is optional.
# The symbol is already demangled.
symbol_encode = cgiEnv.m_entity_id_dict["Name"]
# TODO: This should be packaged in lib_symbol.
symbolNam = lib_util.Base64Decode(symbol_encode)
filNam = cgiEnv.m_entity_id_dict["File"]
sys.stderr.write("symbol=%s filNam=%s\n" % (symbolNam, filNam))
grph = cgiEnv.GetGraph()
symNode = lib_uris.gUriGen.SymbolUri(symbolNam, filNam)
if filNam:
filNode = lib_common.gUriGen.FileUri(filNam)
grph.add((filNode, pc.property_symbol_defined, symNode))
versStr = VersionString(filNam)
grph.add((filNode, pc.property_information,
lib_common.NodeLiteral(versStr)))
sym = FindPESymbol(filNam, symbolNam)
if sym is not None:
# docTxt = getattr(sym,"__doc__").replace(r" ","")
# Non-breaking space: A0 10100000  
# docTxt = getattr(sym,"__doc__").replace(chr(160),"")
# Ca ne marche pas ...
docTxt = getattr(sym, "__doc__")
# This string is filled with spaces and CR which are translated into " ".
docTxt = re.sub('\s+', ' ', docTxt).strip()
grph.add((symNode, pc.property_information,
lib_common.NodeLiteral(docTxt)))
# Possible values are "name","offset","ordinal","forwarder"
try:
fwrd = getattr(sym, "forwarder")
grph.add((symNode, lib_common.MakeProp("Forwarder"),
lib_common.NodeLiteral(fwrd)))
except:
pass
try:
fwrd = getattr(sym, "ordinal")
grph.add((symNode, lib_common.MakeProp("Ordinal"),
lib_common.NodeLiteral(fwrd)))
except:
pass
(fulNam, lstArgs) = lib_symbol.SymToArgs(symbolNam)
if lstArgs:
for arg in lstArgs:
# TODO: Order of arguments must not be changed.
argNode = lib_uris.gUriGen.ClassUri(arg, filNam)
grph.add((symNode, pc.property_argument, argNode))
cgiEnv.OutCgiRdf("LAYOUT_RECT", [pc.property_argument])
def GetEnvArgs(cgiEnv):
sqlQuery_encode = cgiEnv.m_entity_id_dict["Query"]
sqlQuery = lib_util.Base64Decode(sqlQuery_encode)
return (sqlQuery)
def EntityName(entity_ids_arr):
resu = lib_util.Base64Decode(entity_ids_arr[0])
resu = cgi.escape(resu)
resu = stripblanks(resu)
return resu
def ValueDecode(self, valueCoded):
return lib_util.Base64Decode(valueCoded)
def Main():
orig_req_uri = lib_util.RequestUri()
# It initialises an implicit global object similar.
# When in the mode of global merging, the method "cgiEnv.OutCgiRdf()" does not generate anything,
# but simply stores the new cgiEnv in a global list..
# The script loops on the URLs passed as CGI parameters.
# The URLs are loaded and their content merged into the container lib_common.globalGraph
lib_common.CgiEnvMergeMode()
arguments = cgi.FieldStorage()
# The display mode is read now, otherwise the CGI arguments are later destroyed, in this script.
the_mode = lib_util.GuessDisplayMode()
logging.debug("the_mode=%s", the_mode)
# Concatenation of error messages of each script.
cumulated_error = ""
# This logic might be needed in lib_client.py
for urlfil in arguments.getlist("url"):
# The parameters are coded in base64, although we leave the possibility not to encode them,
# for compatibility with test scripts.
complete_url = lib_util.Base64Decode(urlfil)
logging.debug("complete_url=%s", complete_url)
# Only the URL without the arguments.
url_split = complete_url.split("?")
url_no_args = url_split[0]
if len(url_split) > 1:
cgi_query_string = url_split[1]
else:
cgi_query_string = ""
# The URL might be absolute or relative. Example:
# "survol/sources_types/enumerate_CIM_Process.py?xid=."
idx_htbin = url_no_args.find("sources_types/")
if idx_htbin == -1:
# This may be the main presentation page of a Survol, WMI or WBEM object. Example:
# "http://127.0.0.1:80/Survol/survol/entity.py?xid=CIM_Process.Handle=640"
survol_prefix = "survol/"
idx_survol = url_no_args.find(survol_prefix)
if idx_survol == -1:
# TODO: This happens if the URL is a main presentation page of an object,
# instead of a script: Something like "survol/entity.py/entity.py?xid=..."
# This should be fixed but is not an issue.
logging.warning("merge: SHOULD NOT HAPPEN url=%s",
complete_url)
url_path_short = "INVALID_MERGED_URL"
else:
# Just starts at the beginning of the script name: "entity.py", "entity_wmi.py", "entity_wbem.py".
url_path_short = url_no_args[idx_survol + len(survol_prefix):]
else:
url_path_short = url_no_args[idx_htbin:]
# url_path_short is the actual script to load.
url_dir_nam = os.path.dirname(url_path_short)
# The directory of the script is used to build a Python module name.
modu_nam = url_dir_nam.replace("/", ".")
url_fil_nam = os.path.basename(url_path_short)
logging.debug(
"url_path_short=%s url_dir_nam=%s modu_nam=%s url_fil_nam=%s",
url_path_short, url_dir_nam, modu_nam, url_fil_nam)
try:
# argDir="sources_types.win32" urlFileNam="enumerate_top_level_windows.py"
imported_mod = lib_util.GetScriptModule(modu_nam, url_fil_nam)
except Exception as exc:
logging.warning(
"Caught %s when loading modu_nam=%s url_fil_nam=%s", exc,
modu_nam, url_fil_nam)
continue
if not imported_mod:
cumulated_error = "merge_scripts.py Cannot import complete_url=%s" % complete_url
continue
try:
# The entire URL must be "injected" so the parameters will be properly parsed,
# when Main() call lib_util.RequestUri().
# The script passed as CGI parameter, believes that it is loaded as a plain URL.
url_unquote = lib_util.urllib_unquote(complete_url)
os.environ["REQUEST_URI"] = url_unquote
os.environ['SCRIPT_NAME'] = url_fil_nam
# "xid=EURO%5CLONL00111310@process:16580"
os.environ['QUERY_STRING'] = cgi_query_string
lib_common.enable_error_message(False)
# This executes the script: The new nodes and links are merged in a global RDF container.
imported_mod.Main()
except Exception as exc:
logging.warning(
"Caught %s when executing Main in modu_nam=%s url_fil_nam=%s",
exc, modu_nam, url_fil_nam)
if cumulated_error != "":
cumulated_error += " ; "
cumulated_error += " url=" + url_no_args + " / " + url_fil_nam + ":" + str(
exc)
continue
lib_common.enable_error_message(True)
os.environ["REQUEST_URI"] = orig_req_uri
# OutCgiRdf has been called by each script without writing anything,
# but the specific parameters per script are stored inside.
# Here, all the RDF nodes and links, loaded from each URL, and then merged in lib_common.globalGraph,
# are then transformed into the chosen output format.
lib_common.MergeOutCgiRdf(the_mode, cumulated_error)
def Main():
origReqUri = lib_util.RequestUri()
# There is only one cgiEnv and "cgiEnv.OutCgiRdf()" does not generate anything.
lib_common.CgiEnvMergeMode()
arguments = cgi.FieldStorage()
# The display mode is read now, otherwise the CGI arguments are later destroyed, in this script.
theMode = lib_util.GuessDisplayMode()
sys.stderr.write("merge_scripts.py theMode=%s\n" % (theMode))
# Concatenation of error messages of each script.
cumulatedError = ""
for urlfil in arguments.getlist("url"):
# The parameters are coded in base64, although we leave the possibility not to encode them,
# for compatibility with test scripts.
complete_url = lib_util.Base64Decode(urlfil)
sys.stderr.write("complete_url=%s\n" % complete_url)
# Only the URL without the arguments.
urlSplit = complete_url.split("?")
urlNoArgs = urlSplit[0]
if len(urlSplit) > 1:
cgiQueryString = urlSplit[1]
else:
cgiQueryString = ""
# The URL might be absolute or relative. Example:
# "survol/sources_types/enumerate_CIM_Process.py?xid=."
idxHtbin = urlNoArgs.find("sources_types/")
if idxHtbin == -1:
# This may be the main presentation page of a Survol, WMI or WBEM object. Example:
# "http://127.0.0.1:80/Survol/survol/entity.py?xid=CIM_Process.Handle=640"
survolPrefix = "survol/"
idxSurvol = urlNoArgs.find(survolPrefix)
if idxSurvol == -1:
# TODO: This happens if the URL is a main presentation page of an object,
# instead of a script: Something like "survol/entity.py/entity.py?xid=..."
# This should be fixed but is not an issue.
sys.stderr.write("merge: SHOULD NOT HAPPEN url=%s\n" %
complete_url)
urlPathShort = "INVALID_MERGED_URL"
else:
# Just starts at the beginning of the script name: "entity.py", "entity_wmi.py", "entity_wbem.py".
urlPathShort = urlNoArgs[idxSurvol + len(survolPrefix):]
else:
urlPathShort = urlNoArgs[idxHtbin:]
# urlPathShort is the actual script to load.
urlDirNam = os.path.dirname(urlPathShort)
moduNam = urlDirNam.replace("/", ".")
urlFilNam = os.path.basename(urlPathShort)
sys.stderr.write(
"urlPathShort=%s urlDirNam=%s moduNam=%s urlFilNam=%s\n" %
(urlPathShort, urlDirNam, moduNam, urlFilNam))
try:
# argDir="sources_types.win32" urlFileNam="enumerate_top_level_windows.py"
importedMod = lib_util.GetScriptModule(moduNam, urlFilNam)
except Exception:
errorMsg = sys.exc_info()[1]
sys.stderr.write(
"Caught %s when loading moduNam=%s urlFilNam=%s\n" %
(errorMsg, moduNam, urlFilNam))
continue
if not importedMod:
cumulatedError = "merge_scripts.py Cannot import complete_url=%s" % (
complete_url)
continue
try:
# The entire URL must be "injected" so the parameters will be properly parsed,
# when Main() call lib_util.RequestUri().
urlUnquote = lib_util.urllib_unquote(complete_url)
os.environ["REQUEST_URI"] = urlUnquote
os.environ['SCRIPT_NAME'] = urlFilNam
# "xid=EURO%5CLONL00111310@process:16580"
os.environ['QUERY_STRING'] = cgiQueryString
lib_common.ErrorMessageEnable(False)
importedMod.Main()
except Exception:
errorMsg = sys.exc_info()[1]
sys.stderr.write(
"Caught %s when executing Main in moduNam=%s urlFilNam=%s\n" %
(errorMsg, moduNam, urlFilNam))
if cumulatedError != "":
cumulatedError += " ; "
cumulatedError += " url=" + urlNoArgs + " / " + urlFilNam + ":" + str(
errorMsg)
continue
lib_common.ErrorMessageEnable(True)
os.environ["REQUEST_URI"] = origReqUri
# OutCgiRdf has been called by each script without writing anything,
# but the specific parameters per script are stored inside.
# TESTER AVEC CA:
# http://127.0.0.1:8000/survol/merge_scripts.py?url=aHRiaW4vc291cmNlc190eXBlcy9hZGRyL3NvY2tldF9ob3N0LnB5P3hpZD1hZGRyLklkJTNEMTkyLjE2OC4xLjg4JTNBc3No&url=aHRiaW4vc291cmNlc190eXBlcy9DSU1fQ29tcHV0ZXJTeXN0ZW0vaG9zdG5hbWVfbm1hcC5weT94aWQ9Q0lNX0NvbXB1dGVyU3lzdGVtLk5hbWUlM0RVbmtub3duLTMwLWI1LWMyLTAyLTBjLWI1LTI&url=aHRiaW4vc291cmNlc190eXBlcy9hZGRyL3NvY2tldF9ob3N0LnB5P3hpZD1hZGRyLklkJTNEMTkyLjE2OC4xLjg4JTNBc3ZybG9j&url=aHRiaW4vZW50aXR5LnB5P3hpZD1zbWJzaHIuSWQ9Ly8vL1dETXlDbG91ZE1pcnJvci9yY2hhdGVhdQ&url=aHRiaW4vc291cmNlc190eXBlcy9DSU1fQ29tcHV0ZXJTeXN0ZW0vY29ubmVjdGVkX3NvY2tldHMucHk_eGlkPUNJTV9Db21wdXRlclN5c3RlbS5OYW1lJTNEVW5rbm93bi0zMC1iNS1jMi0wMi0wYy1iNS0y&url=aHRiaW4vc291cmNlc190eXBlcy9DSU1fQ29tcHV0ZXJTeXN0ZW0vaG9zdG5hbWVfbm1hcC5weT94aWQ9Q0lNX0NvbXB1dGVyU3lzdGVtLk5hbWUlM0RVbmtub3duLTMwLWI1LWMyLTAyLTBjLWI1LTI&url=aHRiaW4vc291cmNlc190eXBlcy9ncm91cC9saW51eF91c2VyX2dyb3VwLnB5P3hpZD1ncm91cC5OYW1lJTNEYXBhY2hl&url=aHRiaW4vc291cmNlc190eXBlcy91c2VyL3VzZXJfbGludXhfaWQucHk_eGlkPXVzZXIuRG9tYWluJTNETG9jYWxIb3N0JTJDTmFtZSUzRGFwYWNoZQ&url=aHRiaW4vc291cmNlc190eXBlcy9hZGRyL3NvY2tldF9jb25uZWN0ZWRfcHJvY2Vzc2VzLnB5P3hpZD1hZGRyLklkJTNEMTkyLjE2OC4xLjg4JTNBdGVsbmV0&url=aHRiaW4vc291cmNlc190eXBlcy91c2VyL3VzZXJfcHJvY2Vzc2VzLnB5P3hpZD11c2VyLkRvbWFpbiUzRExvY2FsSG9zdCUyQ05hbWUlM0RhcGFjaGU&url=aHRiaW4vc291cmNlc190eXBlcy9DSU1fUHJvY2Vzcy9wcm9jZXNzX2N3ZC5weT94aWQ9Q0lNX1Byb2Nlc3MuSGFuZGxlJTNEMTQ3MDU&url=aHRiaW4vc291cmNlc190eXBlcy9hZGRyL3NvY2tldF9jb25uZWN0ZWRfcHJvY2Vzc2VzLnB5P3hpZD1hZGRyLklkJTNEMTkyLjE2OC4xLjg4JTNBc3No&url=aHRiaW4vc291cmNlc190eXBlcy9DSU1fUHJvY2Vzcy9wcm9jZXNzX2N3ZC5weT94aWQ9Q0lNX1Byb2Nlc3MuSGFuZGxlJTNEMTQ3MDU&url=aHRiaW4vc291cmNlc190eXBlcy9hZGRyL3NvY2tldF9jb25uZWN0ZWRfcHJvY2Vzc2VzLnB5P3hpZD1hZGRyLklkJTNEMTkyLjE2OC4xLjg4JTNBdGVsbmV0&url=aHRiaW4vc291cmNlc190eXBlcy9hZGRyL3NvY2tldF9jb25uZWN0ZWRfcHJvY2Vzc2VzLnB5P3hpZD1hZGRyLklkJTNEMTkyLjE2OC4xLjg4JTNBdGVsbmV0&url=aHRiaW4vc291cmNlc190eXBlcy9hZGRyL3NvY2tldF9jb25uZWN0ZWRfcHJvY2Vzc2VzLnB5P3hpZD1hZGRyLklkJTNEMTkyLjE2OC4xLjg4JTNBc3No&url=aHR0cDovLzE5Mi4xNjguMS44ODo4MC9TdXJ2b2wvaHRiaW4vZW50aXR5LnB5P3hpZD1DSU1fUHJvY2Vzcy5IYW5kbGU9MjA1MTI
lib_common.MergeOutCgiRdf(theMode, cumulatedError)