def test_sparql_grandparent(self): rdflib_graph = CreateGraph() # C:/Windows/temp\\survol_temp_file_12532.tmp' tmp_pathname = _create_temp_file() # Sparql does not accept backslashes. tmp_pathname = lib_util.standardized_file_path(tmp_pathname) sparql_query = """ PREFIX survol: <%s> SELECT ?grandparent_name WHERE { ?url_grandparent a survol:CIM_Directory . ?url_directory a survol:CIM_Directory . ?url_datafile a survol:CIM_DataFile . ?url_grandparent survol:CIM_DirectoryContainsFile ?url_directory . ?url_directory survol:CIM_DirectoryContainsFile ?url_datafile . ?url_grandparent survol:Name ?grandparent_name . ?url_datafile survol:Name "%s" . } """ % (survol_namespace, tmp_pathname) query_result = list(rdflib_graph.query(sparql_query)) print("Result=", query_result) temp_dir_path_dir = lib_util.standardized_file_path( os.path.dirname(TempDirPath)) self.assertEqual(str(query_result[0][0]), temp_dir_path_dir)
def test_sparql_grandchildren_directories(self): rdflib_graph = CreateGraph() sparql_query = """ PREFIX survol: <%s> SELECT ?subdirectory_name WHERE { ?url_grandparent a survol:CIM_Directory . ?url_directory a survol:CIM_Directory . ?url_subdirectory a survol:CIM_Directory . ?url_grandparent survol:CIM_DirectoryContainsFile ?url_directory . ?url_directory survol:CIM_DirectoryContainsFile ?url_subdirectory . ?url_grandparent survol:Name "%s" . ?url_subdirectory survol:Name ?subdirectory_name . } """ % (survol_namespace, TempDirPath) query_result = list(rdflib_graph.query(sparql_query)) expected_dirs = set() for root_dir, dir_lists, files_list in os.walk(TempDirPath): if lib_util.standardized_file_path( os.path.dirname(root_dir)) == TempDirPath: for one_file_name in dir_lists: sub_path_name = lib_util.standardized_file_path( os.path.join(root_dir, one_file_name)) expected_dirs.add(sub_path_name) actual_dirs = set( [str(one_path_url[0]) for one_path_url in query_result]) print("actual_dirs=", actual_dirs) print("expected_dirs=", expected_dirs) self.assertEqual(actual_dirs, expected_dirs)
def test_from_python2(self): """This searches the content of a process memory which contains a SQL memory.""" proc_open = _start_subprocess(sys.executable, sample_python_script) my_source_filenames = lib_client.SourceLocal( "sources_types/CIM_Process/memory_regex_search/search_filenames.py", "CIM_Process", Handle=proc_open.pid) triple_filenames = my_source_filenames.get_triplestore() filenames_set = set() for one_instance in triple_filenames.get_instances(): if type(one_instance).__name__ == 'CIM_DataFile': filenames_set.add(one_instance.Name) for one_filename in sorted(filenames_set): print(" ", one_filename) self.assertTrue( lib_util.standardized_file_path(windows_system32_cmd_exe) in filenames_set) self.assertTrue( lib_util.standardized_file_path(sys.executable) in filenames_set) # This filepath is calculated in the Python script file_name_with_slashes = os.path.join( os.path.dirname(sys.executable), "this_is_a_file_name_with_slashes.cpp").replace("\\", "/") self.assertTrue(file_name_with_slashes in filenames_set) tst_stdout, tst_stderr = proc_open.communicate() self.assertEqual(tst_stderr, None)
def unique_temporary_path(prefix, extension): temp_file = "%s_%d_%d%s" % (prefix, CurrentPid, int( time.time()), extension) # This is done in two stages because the case of the file is OK, but not the directory. # This function does not change non existent files. temp_dir = lib_util.standardized_file_path(tempfile.gettempdir()) temp_path = os.path.join(temp_dir, temp_file) temp_path = lib_util.standardized_file_path(temp_path) return temp_path
def DisplayMappedProcesses(grph, file_name): """This displays all processes mapping a given filename. This simply iterates on processes, then on mapped files of each process. This is not very efficient but there is no other way.""" grph.add((lib_common.nodeMachine, pc.property_hostname, lib_util.NodeLiteral(lib_util.currentHostname))) # This is also a file mapped into memory. uri_mapped_file = lib_uris.gUriGen.FileUri(file_name) uri_mem_map = None try: statinfo = os.stat(file_name) except Exception as exc: grph.add((uri_mapped_file, lib_common.MakeProp("Error"), lib_util.NodeLiteral(str(exc)))) return file_size = lib_util.AddSIUnit(statinfo.st_size, "B") grph.add((uri_mapped_file, pc.property_file_size, lib_util.NodeLiteral(file_size))) prop_memory_rss = lib_common.MakeProp("Resident Set Size") for proc in psutil.process_iter(): pid = proc.pid try: all_maps = proc.memory_maps() except Exception as exc: # Probably psutil.AccessDenied continue for the_map in all_maps: # This, because all Windows paths are "standardized" by us. same_fil = lib_util.standardized_file_path( the_map.path) == lib_util.standardized_file_path(file_name) if same_fil: # Maybe this is the first mapping we have found. if uri_mem_map == None: uri_mem_map = lib_uris.gUriGen.MemMapUri(file_name) grph.add( (uri_mapped_file, pc.property_mapped, uri_mem_map)) node_process = lib_uris.gUriGen.PidUri(pid) # The property is reversed because of display. grph.add((uri_mem_map, pc.property_memmap, node_process)) grph.add( (node_process, pc.property_pid, lib_util.NodeLiteral(pid))) # Displays the RSS only if different from the file size. if the_map.rss != statinfo.st_size: grph.add((node_process, prop_memory_rss, lib_util.NodeLiteral(the_map.rss)))
def unique_temporary_path(prefix, extension): """ "It is a wrapper around temporary file creation, and ensures that the resulting filename can be used everywhere in Survol library. """ temp_file = "%s_%d_%d%s" % (prefix, CurrentPid, int( time.time()), extension) # This is done in two stages because the case of the file is OK, but not the directory. # This function does not change non existent files. temp_dir = lib_util.standardized_file_path(tempfile.gettempdir()) temp_path = os.path.join(temp_dir, temp_file) temp_path = lib_util.standardized_file_path(temp_path) return temp_path
def test_sparql_subdirectory_2(self): """Tests that a second-level directory is detected. """ rdflib_graph = CreateGraph() dir_path = os.path.join(TempDirPath, "survol_temp_dir%s_1" % unique_string, "survol_temp_dir%s_2" % unique_string) os.makedirs(dir_path) dir_path = lib_util.standardized_file_path(dir_path) sparql_query = """ PREFIX survol: <%s> SELECT ?subdirectory_name WHERE { ?url_directory_0 a survol:CIM_Directory . ?url_directory_1 a survol:CIM_Directory . ?url_directory_2 a survol:CIM_Directory . ?url_directory_0 survol:CIM_DirectoryContainsFile ?url_directory_1 . ?url_directory_1 survol:CIM_DirectoryContainsFile ?url_directory_2 . ?url_directory_0 survol:Name "%s" . ?url_directory_2 survol:Name ?subdirectory_name . } """ % (survol_namespace, TempDirPath) query_result = list(rdflib_graph.query(sparql_query)) print("dir_path=", dir_path) actual_files = set( [str(one_path_url[0]) for one_path_url in query_result]) print("actual_files=", actual_files) self.assertTrue(dir_path in actual_files)
def DatabaseEnvParams(process_id): # This is imported here to avoid circular references. from sources_types import CIM_Process logging.debug("process_id=%s", str(process_id)) # Get the list of files open by the process. try: proc_obj = CIM_Process.PsutilGetProcObj(int(process_id)) fillist = proc_obj.open_files() except Exception as exc: lib_common.ErrorMessageHtml("Caught:" + str(exc) + ": process_id=" + str(process_id)) list_args = [] for fil_obj in fillist: fil_nam = fil_obj.path logging.debug("DatabaseEnvParams process_id=%s fil_nam=%s", str(process_id), fil_nam) if IsSqliteDatabase(fil_nam): logging.debug("DatabaseEnvParams ADDING fil_nam=%s", fil_nam) fil_nam_clean = lib_util.standardized_file_path(fil_nam) fil_def = {"File": fil_nam_clean} list_args.append(fil_def) logging.debug("DatabaseEnvParams len=%d", len(list_args)) return "sqlite/query", list_args
def ClassUri(self, class_name, path=""): # The URL should never contain the chars "<" or ">". #class_name = lib_util.Base64Encode(class_name) path = lib_util.standardized_file_path(path) # return self.node_from_dict("class", {"Name": class_name, "File": lib_util.EncodeUri(path)}) assert isinstance(class_name, str) return self.node_from_dict("class", {"Name": class_name, "File": path})
def FillOnePackage(grph,node,good_pckg): # >>> dir(installed_packages[0]) # ['PKG_INFO', '__class__', '__delattr__', '__dict__', '__doc__', '__eq__', '__format__', '__ge__', '__getattr__', '__getattribute__', # '__gt__', '__hash__', '__init__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', # '__setattr__', '__sizeof__', '__str__', '__subclasshook__', '__weakref__', '_dep_map', '_get_metadata', '_key', '_provider', '_relo # ad_version', '_version', '_warn_legacy_version', 'activate', 'as_requirement', 'check_version_conflict', 'clone', 'egg_name', 'extra # s', 'from_filename', 'from_location', 'get_entry_info', 'get_entry_map', 'has_version', 'hashcmp', 'insert_on', 'key', 'load_entry_p # oint', 'location', 'parsed_version', 'platform', 'precedence', 'project_name', 'py_version', 'requires', 'version'] grph.add( (node, propPythonVersion, lib_common.NodeLiteral(good_pckg.version) ) ) grph.add( (node, lib_common.MakeProp("Platform"), lib_common.NodeLiteral(good_pckg.platform) ) ) grph.add( (node, lib_common.MakeProp("project_name"), lib_common.NodeLiteral(good_pckg.project_name) ) ) # >>> pip.get_installed_distributions()[1].requires() # [Requirement.parse('distribute'), Requirement.parse('werkzeug'), Requirement.parse('mako')] # '_Requirement__hash', '__contains__','__doc__', '__eq__', '__hash__','__init__', '__module__', '__ne__', # '__repr__', '__str__', 'extras','hashCmp', 'key', 'marker_fn', 'parse','project_name', 'specifier', 'specs','unsafe_name' # strReq = "+".join( [ str(dir(req)) for req in good_pckg.requires() ]) for subReq in good_pckg.requires(): subNode = MakeUri( subReq.key ) # [('>=', '4.0.0')]+[]+[('>=','4.0')]+[] aSpecs = subReq.specs if aSpecs: grph.add( (subNode, lib_common.MakeProp("Condition"), lib_common.NodeLiteral( str(aSpecs) ) ) ) grph.add( (node, lib_common.MakeProp("requires"), subNode ) ) grph.add( (node, lib_common.MakeProp("py_version"), lib_common.NodeLiteral(good_pckg.py_version) ) ) grph.add( (node, lib_common.MakeProp("precedence"), lib_common.NodeLiteral(good_pckg.precedence) ) ) grph.add( (node, lib_common.MakeProp("egg_name"), lib_common.NodeLiteral(good_pckg.egg_name()) ) ) # This might return location="c:\python27\lib\site-packages" cleanLocaDir = lib_util.standardized_file_path(good_pckg.location) nodeLocation = lib_common.gUriGen.DirectoryUri(cleanLocaDir) grph.add( (node, lib_common.MakeProp("Location"),nodeLocation ) )
def test_sparql_executable_parent_process(self): """Executable of the parent process.""" rdflib_graph = CreateGraph() print("CurrentParentPid=", CurrentParentPid) sparql_query = """ PREFIX survol: <%s> SELECT ?executable_name WHERE { ?url_proc survol:Handle %d . ?url_proc survol:ParentProcessId ?CurrentParentPid . ?url_proc rdf:type survol:CIM_Process . ?url_parent_proc survol:Handle ?CurrentParentPid . ?url_parent_proc survol:CIM_ProcessExecutable ?url_executable_file . ?url_parent_proc rdf:type survol:CIM_Process . ?url_executable_file rdf:type survol:CIM_DataFile . ?url_executable_file survol:Name ?executable_name . } """ % (survol_namespace, CurrentPid) query_result = list(rdflib_graph.query(sparql_query)) print("query_result=", query_result) actual_executable_name = [ str(one_value[0]) for one_value in query_result ][0] print("actual_executable_name=", actual_executable_name) expected_executable_name = lib_util.standardized_file_path( psutil.Process().parent().exe()) print("expected_executable_name=", expected_executable_name) self.assertTrue(expected_executable_name == actual_executable_name)
def test_from_perl(self): """This searches the content of a process memory which contains a SQL memory.""" proc_open = _start_subprocess(_perl_path, sample_perl_script) my_source_filenames = lib_client.SourceLocal( "sources_types/CIM_Process/memory_regex_search/search_filenames.py", "CIM_Process", Handle=proc_open.pid) sys.stdout.flush() triple_filenames = my_source_filenames.get_triplestore() print("len(triple_filenames)=", len(triple_filenames)) filenames_set = set() for one_instance in triple_filenames.get_instances(): if type(one_instance).__name__ == 'CIM_DataFile': filenames_set.add(one_instance.Name) for one_filename in sorted(filenames_set): print(" ", one_filename) self.assertTrue(_perl_path in filenames_set) self.assertTrue( lib_util.standardized_file_path(windows_system32_cmd_exe) in filenames_set) proc_open.communicate()
def DatabaseEnvParams(processId): # This is imported here to avoid circular references. from sources_types import CIM_Process DEBUG("\nDatabaseEnvParams processId=%s", str(processId)) # Get the list of files open by the process. try: proc_obj = CIM_Process.PsutilGetProcObj(int(processId)) fillist = CIM_Process.PsutilProcOpenFiles(proc_obj) except Exception: exc = sys.exc_info()[1] lib_common.ErrorMessageHtml("Caught:" + str(exc) + ": processId=" + str(processId)) listArgs = [] for filObj in fillist: filNam = filObj.path DEBUG("DatabaseEnvParams processId=%s filNam=%s", str(processId), filNam) if IsSqliteDatabase(filNam): DEBUG("DatabaseEnvParams ADDING filNam=%s", filNam) filNamClean = lib_util.standardized_file_path(filNam) filDef = {"File": filNamClean} listArgs.append(filDef) DEBUG("DatabaseEnvParams len=%d\n", len(listArgs)) return ("sqlite/query", listArgs)
def ClassUri(self, class_name, path=""): # The URL should never contain the chars "<" or ">". class_name = lib_util.Base64Encode(class_name) path = lib_util.standardized_file_path(path) return self.UriMakeFromDict("class", { "Name": class_name, "File": lib_util.EncodeUri(path) })
def SymbolUri(self, symbol_name, path=""): # The URL should never contain the chars "<" or ">". # TODO: Move that to linker_symbol/__init__.py and see sources_types.sql.query.MakeUri # TODO: Alphabetical order !!!! path = lib_util.standardized_file_path(path) assert isinstance(symbol_name, str) return self.node_from_dict("linker_symbol", { "Name": symbol_name, "File": path })
def SymbolUri(self, symbol_name, path=""): # The URL should never contain the chars "<" or ">". symbol_name = lib_util.Base64Encode(symbol_name) # TODO: Move that to linker_symbol/__init__.py and see sources_types.sql.query.MakeUri # TODO: Alphabetical order !!!! path = lib_util.standardized_file_path(path) return self.UriMakeFromDict("linker_symbol", { "Name": symbol_name, "File": lib_util.EncodeUri(path) })
def Main(): cgiEnv = lib_common.ScriptEnvironment() grph = cgiEnv.GetGraph() net_share_cmd = ["net", "share"] net_share_pipe = lib_common.SubProcPOpen(net_share_cmd) net_share_last_output, net_share_err = net_share_pipe.communicate() # Converts to string for Python3. as_str = net_share_last_output.decode("utf-8") lines = as_str.split('\n') seen_hyphens = False for lin in lines: if re.match(".*-------.*", lin): seen_hyphens = True continue if re.match(".*The command completed successfully.*", lin): break if not seen_hyphens: continue tst_share = re.match(r'^([A-Za-z0-9_$]+) +([^ ]+).*', lin) if not tst_share: continue shr_nam = tst_share.group(1) # Nasty formatting of "NET SHARE" command. if len(lin) >= 45: # There is a remark or a very long resource. if lin[44] == ' ': # Character just before remark is a space. shr_res = lin[13:44].rstrip() else: shr_res = lin[13:] else: shr_res = lin[13:] share_node = lib_uris.gUriGen.SmbShareUri(shr_nam) grph.add((lib_common.nodeMachine, pc.property_smbshare, share_node)) # mount_node = lib_uris.gUriGen.FileUri( "//" + lib_util.currentHostname + "/" + shr_res ) shr_res = shr_res.strip() shr_res = lib_util.standardized_file_path(shr_res) mount_node = lib_uris.gUriGen.DirectoryUri(shr_res) grph.add((share_node, pc.property_smbmount, mount_node)) cgiEnv.OutCgiRdf()
def Main(): cgiEnv = lib_common.CgiEnv() pid = int(cgiEnv.GetId()) # TODO: These are probably in win32com or a similar module. PROCESS_QUERY_INFORMATION = 0x0400 PROCESS_VM_READ = 0x0010 grph = cgiEnv.GetGraph() node_process = lib_common.gUriGen.PidUri(pid) exec_node = CIM_Process.AddInfo(grph, node_process, [pid]) #Get handle to the process based on PID hProcess = kernel.OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, False, pid) if hProcess: ModuType = c_ulong * 512 hModuleArr = ModuType() rawCntModules = c_ulong() psapi.EnumProcessModules(hProcess, byref(hModuleArr), sizeof(hModuleArr), byref(rawCntModules)) nbModules = int(rawCntModules.value / sizeof(c_ulong())) if nbModules >= 512: raise Exception("Disaster overrun") modname = c_buffer(256) for idx in range(0, nbModules): retLen = psapi.GetModuleFileNameExA(hProcess, hModuleArr[idx], modname, sizeof(modname)) if retLen == 0: # Maybe the string is empty. continue raw_filename_bytes = modname[:retLen] raw_filename_as_str = raw_filename_bytes.decode() filnam = lib_util.standardized_file_path(raw_filename_as_str) #if lib_util.is_py3: # # Truncation because "b'C:/xxx/yyy.zzz'", on Python 3 # # Backslashes are duplicated. # filnam = str(tab).replace('\\','/')[2:-1].replace("//","/") #else: # # Windows "\\" must be replaced by "/", so the URLs are the same for all tools. # filnam = str(tab).replace('\\','/') # The same filename might appear several times. DEBUG("idx=%d retLen=%d filnam=%s", idx, retLen, filnam) libNode = lib_common.gUriGen.SharedLibUri(filnam) grph.add((node_process, pc.property_library_depends, libNode)) kernel.CloseHandle(hProcess) cgiEnv.OutCgiRdf()
def DirectoryUri(self, path): # Normalize a pathname by collapsing redundant separators and up-level references # so that A//B, A/B/, A/./B and A/foo/../B all become A/B. # This string manipulation may change the meaning of a path that contains symbolic links. # On Windows, it converts forward slashes to backward slashes. path = os.path.normpath(path) # Backslashes are a real problem everywhere. # On top of that, escaping the backslash is not enough because strings are sometimes truncated for display. # Better have our own "canonical" notation for filenames, so replace them. # If needed, they can always be replaced by a normal slash. # path = lib_util.standardized_file_path(path) return self.node_from_args("CIM_Directory", path)
def Main(): cgiEnv = lib_common.CgiEnv() filNam = cgiEnv.GetId() filNam = lib_util.standardized_file_path(filNam) DEBUG("filNam=%s", filNam) filNode = lib_common.gUriGen.DirectoryUri(filNam) grph = cgiEnv.GetGraph() info = CIM_DataFile.GetInfoStat(filNam) # st_mode: protection bits. # st_ino: inode number. # st_dev: device. CIM_DataFile.AddDevice(grph, filNode, info) CIM_DataFile.AddStatNode(grph, filNode, info) CIM_DataFile.AddMagic(grph, filNode, filNam) # st_nlink: number of hard links. CIM_DataFile.AffFileOwner(grph, filNode, filNam) # Displays the file and the parent directories/ currFilNam = filNam currNode = filNode while True: dirPath = os.path.dirname(currFilNam) if dirPath == currFilNam: break if dirPath == "": break dirNode = lib_common.gUriGen.DirectoryUri(dirPath) grph.add((dirNode, pc.property_directory, currNode)) DEBUG("dirPath=%s", dirPath) statPath = os.stat(dirPath) CIM_DataFile.AddStatNode(grph, dirNode, statPath) CIM_DataFile.AddFileProperties(grph, currNode, currFilNam) currFilNam = dirPath currNode = dirNode # If windows, print more information: DLL version etc... # http://stackoverflow.com/questions/580924/python-windows-file-version-attribute # cgiEnv.OutCgiRdf() cgiEnv.OutCgiRdf("LAYOUT_TWOPI")
def AddInfo(grph, node, entity_ids_arr): """ This creates a couple of nodes about a file. """ file_name = entity_ids_arr[0] if not file_name: # Faster than comparing to an empty string. return # Cleanup the filename. This function is called without knowledge of the specific case, # therefore the cleanup can only be done in code related to this entity type. file_name = lib_util.standardized_file_path(file_name) AddMagic(grph, node, file_name) AddStat(grph, node, file_name) AddHtml(grph, node, file_name) _add_parent_dir(grph, node, file_name)
def test_sparql_files_in_executable_process_dir(self): """Display the files in the directory of the current process'executable.""" rdflib_graph = CreateGraph() sparql_query = """ PREFIX survol: <%s> SELECT ?datafile_name WHERE { ?url_proc survol:Handle %d . ?url_proc survol:CIM_ProcessExecutable ?url_executable_datafile . ?url_proc rdf:type survol:CIM_Process . ?url_executable_datafile rdf:type survol:CIM_DataFile . ?url_datafile rdf:type survol:CIM_DataFile . ?url_directory rdf:type survol:CIM_Directory . ?url_directory survol:CIM_DirectoryContainsFile ?url_datafile . ?url_directory survol:CIM_DirectoryContainsFile ?url_executable_datafile . ?url_datafile survol:Name ?datafile_name . } """ % (survol_namespace, CurrentPid) query_result = list(rdflib_graph.query(sparql_query)) files_names_result = {str(one_value[0]) for one_value in query_result} # print("files_names_result=", files_names_result) print("files_names_result len:", len(files_names_result)) print("sys_executable_case=", sys_executable_case) self.assertTrue(sys_executable_case in files_names_result) # Compare with the list of the files the directory of the executable. path_names_set = set() for root_dir, dir_lists, files_list in os.walk( os.path.dirname(sys_executable_case)): for one_file_name in files_list: sub_path_name = lib_util.standardized_file_path( os.path.join(root_dir, one_file_name)) path_names_set.add(sub_path_name) break # print("Expected list of files:", path_names_set) print("Expected list of files length:", len(path_names_set)) print("Files differences A:", path_names_set.difference(files_names_result)) print("Files differences B:", files_names_result.difference(path_names_set)) self.assertEqual(path_names_set, files_names_result)
def test_sparql_children_files(self): rdflib_graph = CreateGraph() # C:/Windows/temp\\survol_temp_file_12532.tmp' tmp_pathname = lib_util.standardized_file_path(_create_temp_file()) sparql_query = """ PREFIX survol: <%s> SELECT ?datafile_name WHERE { ?url_directory a survol:CIM_Directory . ?url_datafile a survol:CIM_DataFile . ?url_directory survol:CIM_DirectoryContainsFile ?url_datafile . ?url_datafile survol:Name ?datafile_name . ?url_directory survol:Name "%s" . } """ % (survol_namespace, TempDirPath) query_result = list(rdflib_graph.query(sparql_query)) print("Result=", query_result) self.assertTrue( tmp_pathname in [str(node[0]) for node in query_result])
def Main(): cgiEnv = lib_common.ScriptEnvironment() pid = int(cgiEnv.GetId()) # TODO: These are probably in win32com or a similar module. PROCESS_QUERY_INFORMATION = 0x0400 PROCESS_VM_READ = 0x0010 grph = cgiEnv.GetGraph() node_process = lib_uris.gUriGen.PidUri(pid) exec_node = CIM_Process.AddInfo(grph, node_process, [pid]) #Get handle to the process based on PID hProcess = kernel.OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, False, pid) if hProcess: ModuType = c_ulong * 512 hModuleArr = ModuType() raw_cnt_modules = c_ulong() psapi.EnumProcessModules(hProcess, byref(hModuleArr), sizeof(hModuleArr), byref(raw_cnt_modules)) nb_modules = int(raw_cnt_modules.value/sizeof(c_ulong())) if nb_modules >= 512: raise Exception("Disaster overrun") modname = c_buffer(256) for idx in range(0, nb_modules): ret_len = psapi.GetModuleFileNameExA(hProcess, hModuleArr[idx], modname, sizeof(modname)) if ret_len == 0: # Maybe the string is empty. continue raw_filename_bytes = modname[:ret_len] raw_filename_as_str = raw_filename_bytes.decode() filnam = lib_util.standardized_file_path(raw_filename_as_str) lib_node = lib_uris.gUriGen.SharedLibUri(filnam) grph.add((node_process, pc.property_library_depends, lib_node)) kernel.CloseHandle(hProcess) cgiEnv.OutCgiRdf()
def test_sparql_subdirectory_down_up_4(self): rdflib_graph = CreateGraph() sparql_query = """ PREFIX survol: <%s> SELECT ?directory_name WHERE { ?url_directory_0a a survol:CIM_Directory . ?url_directory_1a a survol:CIM_Directory . ?url_directory_2a a survol:CIM_Directory . ?url_directory_3a a survol:CIM_Directory . ?url_directory_4X a survol:CIM_Directory . ?url_directory_3b a survol:CIM_Directory . ?url_directory_2b a survol:CIM_Directory . ?url_directory_1b a survol:CIM_Directory . ?url_directory_0b a survol:CIM_Directory . ?url_directory_0a survol:CIM_DirectoryContainsFile ?url_directory_1a . ?url_directory_1a survol:CIM_DirectoryContainsFile ?url_directory_2a . ?url_directory_2a survol:CIM_DirectoryContainsFile ?url_directory_3a . ?url_directory_3a survol:CIM_DirectoryContainsFile ?url_directory_4X . ?url_directory_3b survol:CIM_DirectoryContainsFile ?url_directory_4X . ?url_directory_2b survol:CIM_DirectoryContainsFile ?url_directory_3b . ?url_directory_1b survol:CIM_DirectoryContainsFile ?url_directory_2b . ?url_directory_0b survol:CIM_DirectoryContainsFile ?url_directory_1b . ?url_directory_0a survol:Name "%s" . ?url_directory_0b survol:Name ?directory_name . } """ % (survol_namespace, TempDirPath) query_result = list(rdflib_graph.query(sparql_query)) actual_files = [ lib_util.standardized_file_path(str(one_path_url[0])) for one_path_url in query_result ] print("actual_files=", actual_files) self.assertEqual(actual_files[0], TempDirPath) # All directories must be identical. unique_set = set(actual_files) self.assertEqual(len(unique_set), 1)
def _create_our_node(grph, root_node, entity_host, name_space, class_name, entity_id): """This try to find a correct url for an entity type, without an entity id. At the moment, we just expect a file called "enumerate_<entity>.py" """ enumerate_script = "enumerate_" + class_name + ".py" base_dir = lib_util.gblTopScripts + "/sources_types" # TODO: This is absurd !!! Why looping, because the filename is already known !?!? for dirpath, dirnames, filenames in os.walk(base_dir): for filename in [f for f in filenames if f == enumerate_script]: short_dir = dirpath[len(lib_util.gblTopScripts):] full_script_nam = lib_util.standardized_file_path( os.path.join(short_dir, filename)) logging.debug("full_script_nam=%s", full_script_nam) # TODO: Maybe remove the beginning of the file. local_class_url = lib_util.ScriptizeCimom(full_script_nam, class_name, entity_host) local_class_node = lib_common.NodeUrl(local_class_url) grph.add((root_node, pc.property_directory, local_class_node))
CurrentPid = os.getpid() CurrentProcessPath = 'CIM_Process.Handle=%d' % CurrentPid CurrentParentPid = psutil.Process().ppid() # The agent urls point to the Survol adhoc CGI server: "http://myhost-hp:8000" # The tests use different port numbers to avoid interferences between servers, # if port numbers are not freed etc... Problems are easier to find. RemoteGeneralTestServerPort = 8000 RemoteEventsTestServerPort = 8001 RemoteSparqlTestServerPort = 8002 RemoteGraphvizTestServerPort = 8003 RemoteHtmlTestServerPort = 8004 # Several Survol scripts return this executable among their results, so it can be tested. CurrentExecutable = lib_util.standardized_file_path(sys.executable) CurrentExecutablePath = 'CIM_DataFile.Name=%s' % CurrentExecutable # This is called at the end of the execution of a Survol agent created here for tests. # It displays the content of a log file created by this agent. def __dump_server_content(log_filename): sys.stdout.write("Agent log file: %s\n" % log_filename) try: agent_stream = open(log_filename) for line_stream in agent_stream: sys.stdout.write(">>> %s" % line_stream) agent_stream.close() sys.stdout.write("Agent log file end\n") except Exception as exc:
def Main(): cgiEnv = lib_common.ScriptEnvironment() try: the_pid = int(cgiEnv.GetId()) except Exception: lib_common.ErrorMessageHtml("Must provide a pid") # If cannot be the current pid, otherwise it will block. if the_pid == os.getpid(): lib_common.ErrorMessageHtml("Cannot debug current process") if not lib_util.isPlatformWindows: lib_common.ErrorMessageHtml("This works only on Windows platforms") grph = cgiEnv.GetGraph() # Starts a second session cdb_fil = lib_util.TmpFile("CdbCommand", "cdb") cdb_fd = open(cdb_fil.Name, "w") cdb_fd.write("lm\n") # List loaded modules cdb_fd.write("k\n") # Display stack backtrace. cdb_fd.write("qd\n") # Quit and detach. cdb_fd.close() cdb_cmd = "cdb -p " + str(the_pid) + " -cf " + cdb_fil.Name proc_node = lib_uris.gUriGen.PidUri(the_pid) call_node_prev = None modules_map = {} logging.debug("Starting cdb_cmd=%s", cdb_cmd) try: cdb_pipe = lib_common.SubProcPOpen(cdb_cmd) except WindowsError as exc: lib_common.ErrorMessageHtml("cdb not available: Caught:%s" % str(exc)) logging.debug("Started cdb_cmd=%s", cdb_cmd) cdb_output, cdb_err = cdb_pipe.communicate() # Without decode, "TypeError: Type str does not support the buffer API" cdb_str = cdb_output.decode("utf-8") call_depth = 0 for dot_line in cdb_str.split('\n'): err_match = re.match(".*parameter is incorrect.*", dot_line) if err_match: lib_common.ErrorMessageHtml("CDB:"+dot_line) # 76590000 766a0000 kernel32 (export symbols) C:\Windows\syswow64\kernel32.dll match_lm = re.match(r"[0-9a-fA-F]+ [0-9a-fA-F]+ +([^ ]*) +\(export symbols\) +(.*)", dot_line ) if match_lm: module_name = match_lm.group(1) dll_name_raw = match_lm.group(2).strip() dll_name = lib_util.standardized_file_path(dll_name_raw) logging.debug("module_name=%s dll_name=%s", module_name, dll_name) modules_map[module_name] = dll_name continue # 295cfb0c 00000000 ntdll!RtlInitializeExceptionChain+0x36 # Another format, maybe because of a 64 bits machine. # 00000000`02edff90 00000000`00000000 ntdll!RtlUserThreadStart+0x21 match_k = re.match("[`0-9a-fA-F]+ [`0-9a-fA-F]+ ([^!]*)!([^+]*)", dot_line) if match_k: module_name = match_k.group(1) try: dll_name = modules_map[module_name] except KeyError: dll_name = module_name func_name = match_k.group(2).strip() logging.debug("module_name=%s dll_name=%s func_name=%s", module_name, dll_name, func_name) dll_name = CDB.TestIfKnownDll(dll_name) call_node_prev = survol_symbol.AddFunctionCall(grph, call_node_prev, proc_node, func_name, dll_name) grph.add((call_node_prev, lib_properties.MakeProp("Call_depth"), lib_util.NodeLiteral(call_depth))) call_depth += 1 continue logging.debug("dot_line=%s", dot_line) logging.debug("Parsed cdb result") call_node_prev = survol_symbol.AddFunctionCall(grph, call_node_prev, proc_node, None, None) CIM_Process.AddInfo(grph, proc_node, [the_pid]) # http://msdn.microsoft.com/en-us/library/windows/hardware/ff539058(v=vs.85).aspx # # This section describes how to perform basic debugging tasks using # the Microsoft Console Debugger (CDB) and Microsoft NT Symbolic Debugger (NTSD). # CDB and NTSD are identical in every way, except that NTSD spawns # a new text window when it is started, whereas CDB inherits # the Command Prompt window from which it was invoked. # The instructions in this section are given for CDB, # but they work equally well for NTSD. For a discussion # of when to use CDB or NTSD, see Debugging Environments. cgiEnv.OutCgiRdf("LAYOUT_SPLINE")
def Main(): cgiEnv = lib_common.ScriptEnvironment() try: the_pid = int(cgiEnv.GetId()) except Exception: lib_common.ErrorMessageHtml("Must provide a pid") if not lib_util.isPlatformWindows: lib_common.ErrorMessageHtml("This works only on Windows platforms") # If cannot be the current pid, otherwise it will block. if the_pid == os.getpid(): lib_common.ErrorMessageHtml("Cannot debug current process") grph = cgiEnv.GetGraph() # Starts a second session cdb_fil = lib_util.TmpFile("CdbCommand", "cdb") cdb_fd = open(cdb_fil.Name, "w") cdb_fd.write("lmv\n") # List loaded modules, verbose mode. cdb_fd.write("qd\n") # Quit and detach. cdb_fd.close() cdb_cmd = "cdb -p " + str(the_pid) + " -cf " + cdb_fil.Name proc_node = lib_uris.gUriGen.PidUri(the_pid) logging.debug("Starting cdb_cmd=%s", cdb_cmd) try: cdb_pipe = lib_common.SubProcPOpen(cdb_cmd) except WindowsError as exc: lib_common.ErrorMessageHtml("cdb not available: Caught:%s" % str(exc)) logging.debug("Started cdb_cmd=%s", cdb_cmd) cdb_output, cdb_err = cdb_pipe.communicate() # Without decode, "TypeError: Type str does not support the buffer API" cdb_str = cdb_output.decode("utf-8", "ignore") prop_loaded_module = lib_common.MakeProp("Loaded module") for dot_line in cdb_str.split('\n'): # moduleName=uDWM moduleStatus=deferred file_name= # dot_line= Image path: C:\windows\system32\uDWM.dll # dot_line= Image name: uDWM.dll # dot_line= Timestamp: Tue Jul 14 02:33:35 2009 (4A5BE06F) # dot_line= CheckSum: 0005E9A4 # dot_line= ImageSize: 00057000 # dot_line= File version: 6.1.7600.16385 # dot_line= Product version: 6.1.7600.16385 # dot_line= File flags: 0 (Mask 3F) # dot_line= File OS: 40004 NT Win32 # dot_line= File type: 2.0 Dll # dot_line= File date: 00000000.00000000 # dot_line= Translations: 0409.04b0 # dot_line= CompanyName: Microsoft Corporation # dot_line= ProductName: Microsoft Windows Operating System # dot_line= InternalName: udwm.dll # dot_line= OriginalFilename: udwm.dll # dot_line= ProductVersion: 6.1.7600.16385 # dot_line= FileVersion: 6.1.7600.16385 (win7_rtm.090713-1255) # dot_line= FileDescription: Microsoft Desktop Window Manager # dot_line= LegalCopyright: Microsoft Corporation. All rights reserved. match_lin = re.match(" *Image path: *(.*)", dot_line) if match_lin: file_name = match_lin.group(1) file_name = CDB.TestIfKnownDll(file_name) file_name = file_name.strip() file_name = lib_util.standardized_file_path(file_name) file_node = lib_uris.gUriGen.FileUri(file_name) grph.add((proc_node, prop_loaded_module, file_node)) continue match_lin = re.match(" *CompanyName: *(.*)", dot_line) if match_lin: company_name = match_lin.group(1) grph.add((file_node, lib_common.MakeProp("Company Name"), lib_util.NodeLiteral(company_name))) continue match_lin = re.match(" *File OS: *(.*)", dot_line) if match_lin: file_os = match_lin.group(1) grph.add((file_node, lib_common.MakeProp("File OS"), lib_util.NodeLiteral(file_os))) continue match_lin = re.match(" *FileDescription: *(.*)", dot_line) if match_lin: file_description = match_lin.group(1) grph.add((file_node, lib_common.MakeProp("Description"), lib_util.NodeLiteral(file_description))) continue logging.debug("Parsed cdb result") CIM_Process.AddInfo(grph, proc_node, [the_pid]) cgiEnv.OutCgiRdf("LAYOUT_RECT", [prop_loaded_module])
cgiEnv = lib_common.ScriptEnvironment() process_id = cgiEnv.GetId() logging.debug("Snapshot process_id=%s" % process_id) # This just returns one triple. grph = cgiEnv.GetGraph() process_node = lib_uris.gUriGen.PidUri(process_id) grph.add((process_node, pc.property_pid, lib_util.NodeLiteral(process_id))) cgiEnv.OutCgiRdf() # FIXME: Must finish this. if dockit: dockit_dirname = lib_util.standardized_file_path( os.path.dirname(dockit.__file__)) logging.debug("File=" + __file__ + " dockit_dirname=" + dockit_dirname) def _atexit_handler_detach(process_id): """This is called when this CGI script leaves for any reason. Its purpose is to detach from the target process.""" logging.info("_atexit_handler process_id=%d" % process_id) def SendEvents(): """This is called in a subprocess started by the Python module supervisor.""" logging.info("SendEvents") # FIXME: