def test_from_python(self): # Runs this process: It allocates a variable containing a SQL query, then it waits. proc_open = _start_subprocess(sys.executable, sample_python_script) my_source_sql_queries = lib_client.SourceLocal( "sources_types/CIM_Process/memory_regex_search/scan_sql_queries.py", "CIM_Process", Handle=proc_open.pid) triple_sql_queries = my_source_sql_queries.get_triplestore() print("len(triple_sql_queries)=", len(triple_sql_queries)) # This creates objects like: # "CIM_Process/embedded_sql_query.Query=SW5zZXJ0IGFuIGVudHJ5IGludG8gdGhlIGxpc3Qgb2Ygd2FybmluZ3MgZmlsdGVycyAoYXQgdGhlIGZyb250KS4=,Handle=15564" # Maybe, this is not the best representation. Possibly have a generic query ? Some inheritance ? # The specific detail about this query, is that it depends on the process. # This test focus on the parsing of memory, not on the queries representation. queries_set = set() for one_instance in triple_sql_queries.get_instances(): if type(one_instance).__name__ == 'CIM_Process/embedded_sql_query': self.assertEqual(one_instance.Handle, str(proc_open.pid)) decoded_query = lib_util.Base64Decode(one_instance.Query) queries_set.add(decoded_query) print("queries_set=", queries_set) self.assertTrue("select something from somewhere" in queries_set) self.assertTrue("select * from 'AnyTable'" in queries_set) self.assertTrue("select a,b,c from 'AnyTable'" in queries_set) self.assertTrue("select A.x,B.y from AnyTable A, OtherTable B" in queries_set) proc_open.communicate()
def Main(): cgiEnv = lib_common.CgiEnv() # "NtOpenObjectAuditAlarm%40C%3A\windows\system32\ntdll.dll" # Filename is optional. symbolFull = cgiEnv.GetId() # The symbol is already demangled. symbol_encode = cgiEnv.m_entity_id_dict["Name"] # TODO: This should be packaged in lib_symbol. symbolNam = lib_util.Base64Decode(symbol_encode) filNam = cgiEnv.m_entity_id_dict["File"] DEBUG("symbol=%s filNam=%s", symbolNam, filNam) grph = cgiEnv.GetGraph() symNode = lib_uris.gUriGen.SymbolUri(symbolNam, filNam) if filNam: filNode = lib_common.gUriGen.FileUri(filNam) grph.add((filNode, pc.property_symbol_defined, symNode)) (fulNam, lstArgs) = lib_symbol.SymToArgs(symbolNam) if lstArgs: for arg in lstArgs: # TODO: Order of arguments must not be changed. argNode = lib_uris.gUriGen.ClassUri(arg, filNam) grph.add((symNode, pc.property_argument, argNode)) cgiEnv.OutCgiRdf("LAYOUT_RECT", [pc.property_argument])
def EntityName(entity_ids_arr): thePid = entity_ids_arr[1] # sys.stderr.write("thePid=%s\n"%thePid) sqlQuery = entity_ids_arr[0] resu = lib_util.Base64Decode(sqlQuery) # If the query contains double-quotes, it crashes Graphviz resu = resu.replace('"',"'") # resu = resu.replace('"','\\"') return "Pid " + str(thePid) + ":" + resu
def EntityName(entity_ids_arr): entity_id = entity_ids_arr[0] try: # Trailing padding. resu = lib_util.Base64Decode(entity_id) # TODO: Should have a more generic solution: i.e. always b64 encode CGI-incompatible strings. # See lib_uris.SymbolUri which does the encoding. resu = lib_util.html_escape(resu) return resu except TypeError: exc = sys.exc_info()[1] ERROR("CANNOT DECODE: symbol=(%s):%s", entity_id, str(exc)) return entity_id
def EntityName(entity_ids_arr): entity_id = entity_ids_arr[0] try: # Trailing padding. resu = lib_util.Base64Decode(entity_id) # TODO: LE FAIRE AUSSI POUR LES AUTRES SYMBOLES. resu = cgi.escape(resu) return resu except TypeError: exc = sys.exc_info()[1] sys.stderr.write("CANNOT DECODE: symbol=(%s):%s\n" % (entity_id, str(exc))) return entity_id
def EntityName(entity_ids_arr): entity_id = entity_ids_arr[0] try: # Trailing padding. # TODO: Encoding is done in lib_uris.ClassUri : The encoding should be more generic. # TODO: ... and done only when the content is CGI-incompatible. # TODO: Or do just like sources_types.sql.query.MakeUri resu = lib_util.Base64Decode(entity_id) resu = lib_util.html_escape(resu) return resu except TypeError as exc: ERROR("CANNOT DECODE: class=(%s):%s", entity_id, str(exc)) return entity_id
def EntityName(entity_ids_arr): entity_id = entity_ids_arr[0] # PROBLEME: Double &kt;< !!! # return entity_id try: # Trailing padding. resu = lib_util.Base64Decode(entity_id) resu = cgi.escape(resu) return resu except TypeError: exc = sys.exc_info()[1] sys.stderr.write("CANNOT DECODE: class=(%s):%s\n" % (entity_id, str(exc))) return entity_id
def EntityNameUtil(textPrefix, sqlQuery): resu = lib_util.Base64Decode(sqlQuery) resu = cgi.escape(resu) resu = stripblanks(resu) lenFilNam = len(textPrefix) lenResu = len(resu) lenTot = lenFilNam + lenResu lenMaxi = 50 lenDiff = lenTot - lenMaxi if lenDiff > 0: lenResu -= lenDiff if lenResu < 30: lenResu = 30 return textPrefix + ":" + resu[:lenResu] + "..." else: return textPrefix + ":" + resu
def test_from_perl(self): proc_open = _start_subprocess(_perl_path, sample_perl_script) my_source_sql_queries = lib_client.SourceLocal( "sources_types/CIM_Process/memory_regex_search/scan_sql_queries.py", "CIM_Process", Handle=proc_open.pid) triple_sql_queries = my_source_sql_queries.get_triplestore() queries_set = set() for one_instance in triple_sql_queries.get_instances(): if type(one_instance).__name__ == 'CIM_Process/embedded_sql_query': self.assertEqual(one_instance.Handle, str(proc_open.pid)) decoded_query = lib_util.Base64Decode(one_instance.Query) queries_set.add(decoded_query) print("queries_set=", queries_set) self.assertTrue("select column_a from table_a" in queries_set) self.assertTrue("select column_b from table_b" in queries_set) proc_open.communicate()
def Main(): cgiEnv = lib_common.CgiEnv() # "NtOpenObjectAuditAlarm%40C%3A\windows\system32\ntdll.dll" # Filename is optional. # The symbol is already demangled. symbol_encode = cgiEnv.m_entity_id_dict["Name"] # TODO: This should be packaged in lib_symbol. symbolNam = lib_util.Base64Decode(symbol_encode) filNam = cgiEnv.m_entity_id_dict["File"] sys.stderr.write("symbol=%s filNam=%s\n" % (symbolNam, filNam)) grph = cgiEnv.GetGraph() symNode = lib_uris.gUriGen.SymbolUri(symbolNam, filNam) if filNam: filNode = lib_common.gUriGen.FileUri(filNam) grph.add((filNode, pc.property_symbol_defined, symNode)) versStr = VersionString(filNam) grph.add((filNode, pc.property_information, lib_common.NodeLiteral(versStr))) sym = FindPESymbol(filNam, symbolNam) if sym is not None: # docTxt = getattr(sym,"__doc__").replace(r" ","") # Non-breaking space: A0 10100000   # docTxt = getattr(sym,"__doc__").replace(chr(160),"") # Ca ne marche pas ... docTxt = getattr(sym, "__doc__") # This string is filled with spaces and CR which are translated into " ". docTxt = re.sub('\s+', ' ', docTxt).strip() grph.add((symNode, pc.property_information, lib_common.NodeLiteral(docTxt))) # Possible values are "name","offset","ordinal","forwarder" try: fwrd = getattr(sym, "forwarder") grph.add((symNode, lib_common.MakeProp("Forwarder"), lib_common.NodeLiteral(fwrd))) except: pass try: fwrd = getattr(sym, "ordinal") grph.add((symNode, lib_common.MakeProp("Ordinal"), lib_common.NodeLiteral(fwrd))) except: pass (fulNam, lstArgs) = lib_symbol.SymToArgs(symbolNam) if lstArgs: for arg in lstArgs: # TODO: Order of arguments must not be changed. argNode = lib_uris.gUriGen.ClassUri(arg, filNam) grph.add((symNode, pc.property_argument, argNode)) cgiEnv.OutCgiRdf("LAYOUT_RECT", [pc.property_argument])
def GetEnvArgs(cgiEnv): sqlQuery_encode = cgiEnv.m_entity_id_dict["Query"] sqlQuery = lib_util.Base64Decode(sqlQuery_encode) return (sqlQuery)
def EntityName(entity_ids_arr): resu = lib_util.Base64Decode(entity_ids_arr[0]) resu = cgi.escape(resu) resu = stripblanks(resu) return resu
def ValueDecode(self, valueCoded): return lib_util.Base64Decode(valueCoded)
def Main(): orig_req_uri = lib_util.RequestUri() # It initialises an implicit global object similar. # When in the mode of global merging, the method "cgiEnv.OutCgiRdf()" does not generate anything, # but simply stores the new cgiEnv in a global list.. # The script loops on the URLs passed as CGI parameters. # The URLs are loaded and their content merged into the container lib_common.globalGraph lib_common.CgiEnvMergeMode() arguments = cgi.FieldStorage() # The display mode is read now, otherwise the CGI arguments are later destroyed, in this script. the_mode = lib_util.GuessDisplayMode() logging.debug("the_mode=%s", the_mode) # Concatenation of error messages of each script. cumulated_error = "" # This logic might be needed in lib_client.py for urlfil in arguments.getlist("url"): # The parameters are coded in base64, although we leave the possibility not to encode them, # for compatibility with test scripts. complete_url = lib_util.Base64Decode(urlfil) logging.debug("complete_url=%s", complete_url) # Only the URL without the arguments. url_split = complete_url.split("?") url_no_args = url_split[0] if len(url_split) > 1: cgi_query_string = url_split[1] else: cgi_query_string = "" # The URL might be absolute or relative. Example: # "survol/sources_types/enumerate_CIM_Process.py?xid=." idx_htbin = url_no_args.find("sources_types/") if idx_htbin == -1: # This may be the main presentation page of a Survol, WMI or WBEM object. Example: # "http://127.0.0.1:80/Survol/survol/entity.py?xid=CIM_Process.Handle=640" survol_prefix = "survol/" idx_survol = url_no_args.find(survol_prefix) if idx_survol == -1: # TODO: This happens if the URL is a main presentation page of an object, # instead of a script: Something like "survol/entity.py/entity.py?xid=..." # This should be fixed but is not an issue. logging.warning("merge: SHOULD NOT HAPPEN url=%s", complete_url) url_path_short = "INVALID_MERGED_URL" else: # Just starts at the beginning of the script name: "entity.py", "entity_wmi.py", "entity_wbem.py". url_path_short = url_no_args[idx_survol + len(survol_prefix):] else: url_path_short = url_no_args[idx_htbin:] # url_path_short is the actual script to load. url_dir_nam = os.path.dirname(url_path_short) # The directory of the script is used to build a Python module name. modu_nam = url_dir_nam.replace("/", ".") url_fil_nam = os.path.basename(url_path_short) logging.debug( "url_path_short=%s url_dir_nam=%s modu_nam=%s url_fil_nam=%s", url_path_short, url_dir_nam, modu_nam, url_fil_nam) try: # argDir="sources_types.win32" urlFileNam="enumerate_top_level_windows.py" imported_mod = lib_util.GetScriptModule(modu_nam, url_fil_nam) except Exception as exc: logging.warning( "Caught %s when loading modu_nam=%s url_fil_nam=%s", exc, modu_nam, url_fil_nam) continue if not imported_mod: cumulated_error = "merge_scripts.py Cannot import complete_url=%s" % complete_url continue try: # The entire URL must be "injected" so the parameters will be properly parsed, # when Main() call lib_util.RequestUri(). # The script passed as CGI parameter, believes that it is loaded as a plain URL. url_unquote = lib_util.urllib_unquote(complete_url) os.environ["REQUEST_URI"] = url_unquote os.environ['SCRIPT_NAME'] = url_fil_nam # "xid=EURO%5CLONL00111310@process:16580" os.environ['QUERY_STRING'] = cgi_query_string lib_common.enable_error_message(False) # This executes the script: The new nodes and links are merged in a global RDF container. imported_mod.Main() except Exception as exc: logging.warning( "Caught %s when executing Main in modu_nam=%s url_fil_nam=%s", exc, modu_nam, url_fil_nam) if cumulated_error != "": cumulated_error += " ; " cumulated_error += " url=" + url_no_args + " / " + url_fil_nam + ":" + str( exc) continue lib_common.enable_error_message(True) os.environ["REQUEST_URI"] = orig_req_uri # OutCgiRdf has been called by each script without writing anything, # but the specific parameters per script are stored inside. # Here, all the RDF nodes and links, loaded from each URL, and then merged in lib_common.globalGraph, # are then transformed into the chosen output format. lib_common.MergeOutCgiRdf(the_mode, cumulated_error)
def Main(): origReqUri = lib_util.RequestUri() # There is only one cgiEnv and "cgiEnv.OutCgiRdf()" does not generate anything. lib_common.CgiEnvMergeMode() arguments = cgi.FieldStorage() # The display mode is read now, otherwise the CGI arguments are later destroyed, in this script. theMode = lib_util.GuessDisplayMode() sys.stderr.write("merge_scripts.py theMode=%s\n" % (theMode)) # Concatenation of error messages of each script. cumulatedError = "" for urlfil in arguments.getlist("url"): # The parameters are coded in base64, although we leave the possibility not to encode them, # for compatibility with test scripts. complete_url = lib_util.Base64Decode(urlfil) sys.stderr.write("complete_url=%s\n" % complete_url) # Only the URL without the arguments. urlSplit = complete_url.split("?") urlNoArgs = urlSplit[0] if len(urlSplit) > 1: cgiQueryString = urlSplit[1] else: cgiQueryString = "" # The URL might be absolute or relative. Example: # "survol/sources_types/enumerate_CIM_Process.py?xid=." idxHtbin = urlNoArgs.find("sources_types/") if idxHtbin == -1: # This may be the main presentation page of a Survol, WMI or WBEM object. Example: # "http://127.0.0.1:80/Survol/survol/entity.py?xid=CIM_Process.Handle=640" survolPrefix = "survol/" idxSurvol = urlNoArgs.find(survolPrefix) if idxSurvol == -1: # TODO: This happens if the URL is a main presentation page of an object, # instead of a script: Something like "survol/entity.py/entity.py?xid=..." # This should be fixed but is not an issue. sys.stderr.write("merge: SHOULD NOT HAPPEN url=%s\n" % complete_url) urlPathShort = "INVALID_MERGED_URL" else: # Just starts at the beginning of the script name: "entity.py", "entity_wmi.py", "entity_wbem.py". urlPathShort = urlNoArgs[idxSurvol + len(survolPrefix):] else: urlPathShort = urlNoArgs[idxHtbin:] # urlPathShort is the actual script to load. urlDirNam = os.path.dirname(urlPathShort) moduNam = urlDirNam.replace("/", ".") urlFilNam = os.path.basename(urlPathShort) sys.stderr.write( "urlPathShort=%s urlDirNam=%s moduNam=%s urlFilNam=%s\n" % (urlPathShort, urlDirNam, moduNam, urlFilNam)) try: # argDir="sources_types.win32" urlFileNam="enumerate_top_level_windows.py" importedMod = lib_util.GetScriptModule(moduNam, urlFilNam) except Exception: errorMsg = sys.exc_info()[1] sys.stderr.write( "Caught %s when loading moduNam=%s urlFilNam=%s\n" % (errorMsg, moduNam, urlFilNam)) continue if not importedMod: cumulatedError = "merge_scripts.py Cannot import complete_url=%s" % ( complete_url) continue try: # The entire URL must be "injected" so the parameters will be properly parsed, # when Main() call lib_util.RequestUri(). urlUnquote = lib_util.urllib_unquote(complete_url) os.environ["REQUEST_URI"] = urlUnquote os.environ['SCRIPT_NAME'] = urlFilNam # "xid=EURO%5CLONL00111310@process:16580" os.environ['QUERY_STRING'] = cgiQueryString lib_common.ErrorMessageEnable(False) importedMod.Main() except Exception: errorMsg = sys.exc_info()[1] sys.stderr.write( "Caught %s when executing Main in moduNam=%s urlFilNam=%s\n" % (errorMsg, moduNam, urlFilNam)) if cumulatedError != "": cumulatedError += " ; " cumulatedError += " url=" + urlNoArgs + " / " + urlFilNam + ":" + str( errorMsg) continue lib_common.ErrorMessageEnable(True) os.environ["REQUEST_URI"] = origReqUri # OutCgiRdf has been called by each script without writing anything, # but the specific parameters per script are stored inside. # TESTER AVEC CA: # http://127.0.0.1:8000/survol/merge_scripts.py?url=aHRiaW4vc291cmNlc190eXBlcy9hZGRyL3NvY2tldF9ob3N0LnB5P3hpZD1hZGRyLklkJTNEMTkyLjE2OC4xLjg4JTNBc3No&url=aHRiaW4vc291cmNlc190eXBlcy9DSU1fQ29tcHV0ZXJTeXN0ZW0vaG9zdG5hbWVfbm1hcC5weT94aWQ9Q0lNX0NvbXB1dGVyU3lzdGVtLk5hbWUlM0RVbmtub3duLTMwLWI1LWMyLTAyLTBjLWI1LTI&url=aHRiaW4vc291cmNlc190eXBlcy9hZGRyL3NvY2tldF9ob3N0LnB5P3hpZD1hZGRyLklkJTNEMTkyLjE2OC4xLjg4JTNBc3ZybG9j&url=aHRiaW4vZW50aXR5LnB5P3hpZD1zbWJzaHIuSWQ9Ly8vL1dETXlDbG91ZE1pcnJvci9yY2hhdGVhdQ&url=aHRiaW4vc291cmNlc190eXBlcy9DSU1fQ29tcHV0ZXJTeXN0ZW0vY29ubmVjdGVkX3NvY2tldHMucHk_eGlkPUNJTV9Db21wdXRlclN5c3RlbS5OYW1lJTNEVW5rbm93bi0zMC1iNS1jMi0wMi0wYy1iNS0y&url=aHRiaW4vc291cmNlc190eXBlcy9DSU1fQ29tcHV0ZXJTeXN0ZW0vaG9zdG5hbWVfbm1hcC5weT94aWQ9Q0lNX0NvbXB1dGVyU3lzdGVtLk5hbWUlM0RVbmtub3duLTMwLWI1LWMyLTAyLTBjLWI1LTI&url=aHRiaW4vc291cmNlc190eXBlcy9ncm91cC9saW51eF91c2VyX2dyb3VwLnB5P3hpZD1ncm91cC5OYW1lJTNEYXBhY2hl&url=aHRiaW4vc291cmNlc190eXBlcy91c2VyL3VzZXJfbGludXhfaWQucHk_eGlkPXVzZXIuRG9tYWluJTNETG9jYWxIb3N0JTJDTmFtZSUzRGFwYWNoZQ&url=aHRiaW4vc291cmNlc190eXBlcy9hZGRyL3NvY2tldF9jb25uZWN0ZWRfcHJvY2Vzc2VzLnB5P3hpZD1hZGRyLklkJTNEMTkyLjE2OC4xLjg4JTNBdGVsbmV0&url=aHRiaW4vc291cmNlc190eXBlcy91c2VyL3VzZXJfcHJvY2Vzc2VzLnB5P3hpZD11c2VyLkRvbWFpbiUzRExvY2FsSG9zdCUyQ05hbWUlM0RhcGFjaGU&url=aHRiaW4vc291cmNlc190eXBlcy9DSU1fUHJvY2Vzcy9wcm9jZXNzX2N3ZC5weT94aWQ9Q0lNX1Byb2Nlc3MuSGFuZGxlJTNEMTQ3MDU&url=aHRiaW4vc291cmNlc190eXBlcy9hZGRyL3NvY2tldF9jb25uZWN0ZWRfcHJvY2Vzc2VzLnB5P3hpZD1hZGRyLklkJTNEMTkyLjE2OC4xLjg4JTNBc3No&url=aHRiaW4vc291cmNlc190eXBlcy9DSU1fUHJvY2Vzcy9wcm9jZXNzX2N3ZC5weT94aWQ9Q0lNX1Byb2Nlc3MuSGFuZGxlJTNEMTQ3MDU&url=aHRiaW4vc291cmNlc190eXBlcy9hZGRyL3NvY2tldF9jb25uZWN0ZWRfcHJvY2Vzc2VzLnB5P3hpZD1hZGRyLklkJTNEMTkyLjE2OC4xLjg4JTNBdGVsbmV0&url=aHRiaW4vc291cmNlc190eXBlcy9hZGRyL3NvY2tldF9jb25uZWN0ZWRfcHJvY2Vzc2VzLnB5P3hpZD1hZGRyLklkJTNEMTkyLjE2OC4xLjg4JTNBdGVsbmV0&url=aHRiaW4vc291cmNlc190eXBlcy9hZGRyL3NvY2tldF9jb25uZWN0ZWRfcHJvY2Vzc2VzLnB5P3hpZD1hZGRyLklkJTNEMTkyLjE2OC4xLjg4JTNBc3No&url=aHR0cDovLzE5Mi4xNjguMS44ODo4MC9TdXJ2b2wvaHRiaW4vZW50aXR5LnB5P3hpZD1DSU1fUHJvY2Vzcy5IYW5kbGU9MjA1MTI lib_common.MergeOutCgiRdf(theMode, cumulatedError)