def test_sparql_grandparent(self):
rdflib_graph = CreateGraph()
# C:/Windows/temp\\survol_temp_file_12532.tmp'
tmp_pathname = _create_temp_file()
# Sparql does not accept backslashes.
tmp_pathname = lib_util.standardized_file_path(tmp_pathname)
sparql_query = """
PREFIX survol: <%s>
SELECT ?grandparent_name WHERE {
?url_grandparent a survol:CIM_Directory .
?url_directory a survol:CIM_Directory .
?url_datafile a survol:CIM_DataFile .
?url_grandparent survol:CIM_DirectoryContainsFile ?url_directory .
?url_directory survol:CIM_DirectoryContainsFile ?url_datafile .
?url_grandparent survol:Name ?grandparent_name .
?url_datafile survol:Name "%s" .
}
""" % (survol_namespace, tmp_pathname)
query_result = list(rdflib_graph.query(sparql_query))
print("Result=", query_result)
temp_dir_path_dir = lib_util.standardized_file_path(
os.path.dirname(TempDirPath))
self.assertEqual(str(query_result[0][0]), temp_dir_path_dir)
def test_sparql_grandchildren_directories(self):
rdflib_graph = CreateGraph()
sparql_query = """
PREFIX survol: <%s>
SELECT ?subdirectory_name WHERE {
?url_grandparent a survol:CIM_Directory .
?url_directory a survol:CIM_Directory .
?url_subdirectory a survol:CIM_Directory .
?url_grandparent survol:CIM_DirectoryContainsFile ?url_directory .
?url_directory survol:CIM_DirectoryContainsFile ?url_subdirectory .
?url_grandparent survol:Name "%s" .
?url_subdirectory survol:Name ?subdirectory_name .
}
""" % (survol_namespace, TempDirPath)
query_result = list(rdflib_graph.query(sparql_query))
expected_dirs = set()
for root_dir, dir_lists, files_list in os.walk(TempDirPath):
if lib_util.standardized_file_path(
os.path.dirname(root_dir)) == TempDirPath:
for one_file_name in dir_lists:
sub_path_name = lib_util.standardized_file_path(
os.path.join(root_dir, one_file_name))
expected_dirs.add(sub_path_name)
actual_dirs = set(
[str(one_path_url[0]) for one_path_url in query_result])
print("actual_dirs=", actual_dirs)
print("expected_dirs=", expected_dirs)
self.assertEqual(actual_dirs, expected_dirs)
def test_from_python2(self):
"""This searches the content of a process memory which contains a SQL memory."""
proc_open = _start_subprocess(sys.executable, sample_python_script)
my_source_filenames = lib_client.SourceLocal(
"sources_types/CIM_Process/memory_regex_search/search_filenames.py",
"CIM_Process",
Handle=proc_open.pid)
triple_filenames = my_source_filenames.get_triplestore()
filenames_set = set()
for one_instance in triple_filenames.get_instances():
if type(one_instance).__name__ == 'CIM_DataFile':
filenames_set.add(one_instance.Name)
for one_filename in sorted(filenames_set):
print(" ", one_filename)
self.assertTrue(
lib_util.standardized_file_path(windows_system32_cmd_exe) in
filenames_set)
self.assertTrue(
lib_util.standardized_file_path(sys.executable) in filenames_set)
# This filepath is calculated in the Python script
file_name_with_slashes = os.path.join(
os.path.dirname(sys.executable),
"this_is_a_file_name_with_slashes.cpp").replace("\\", "/")
self.assertTrue(file_name_with_slashes in filenames_set)
tst_stdout, tst_stderr = proc_open.communicate()
self.assertEqual(tst_stderr, None)
def unique_temporary_path(prefix, extension):
temp_file = "%s_%d_%d%s" % (prefix, CurrentPid, int(
time.time()), extension)
# This is done in two stages because the case of the file is OK, but not the directory.
# This function does not change non existent files.
temp_dir = lib_util.standardized_file_path(tempfile.gettempdir())
temp_path = os.path.join(temp_dir, temp_file)
temp_path = lib_util.standardized_file_path(temp_path)
return temp_path
def DisplayMappedProcesses(grph, file_name):
"""This displays all processes mapping a given filename.
This simply iterates on processes, then on mapped files of each process.
This is not very efficient but there is no other way."""
grph.add((lib_common.nodeMachine, pc.property_hostname,
lib_util.NodeLiteral(lib_util.currentHostname)))
# This is also a file mapped into memory.
uri_mapped_file = lib_uris.gUriGen.FileUri(file_name)
uri_mem_map = None
try:
statinfo = os.stat(file_name)
except Exception as exc:
grph.add((uri_mapped_file, lib_common.MakeProp("Error"),
lib_util.NodeLiteral(str(exc))))
return
file_size = lib_util.AddSIUnit(statinfo.st_size, "B")
grph.add((uri_mapped_file, pc.property_file_size,
lib_util.NodeLiteral(file_size)))
prop_memory_rss = lib_common.MakeProp("Resident Set Size")
for proc in psutil.process_iter():
pid = proc.pid
try:
all_maps = proc.memory_maps()
except Exception as exc:
# Probably psutil.AccessDenied
continue
for the_map in all_maps:
# This, because all Windows paths are "standardized" by us.
same_fil = lib_util.standardized_file_path(
the_map.path) == lib_util.standardized_file_path(file_name)
if same_fil:
# Maybe this is the first mapping we have found.
if uri_mem_map == None:
uri_mem_map = lib_uris.gUriGen.MemMapUri(file_name)
grph.add(
(uri_mapped_file, pc.property_mapped, uri_mem_map))
node_process = lib_uris.gUriGen.PidUri(pid)
# The property is reversed because of display.
grph.add((uri_mem_map, pc.property_memmap, node_process))
grph.add(
(node_process, pc.property_pid, lib_util.NodeLiteral(pid)))
# Displays the RSS only if different from the file size.
if the_map.rss != statinfo.st_size:
grph.add((node_process, prop_memory_rss,
lib_util.NodeLiteral(the_map.rss)))
def unique_temporary_path(prefix, extension):
"""
"It is a wrapper around temporary file creation, and ensures that the resulting filename
can be used everywhere in Survol library.
"""
temp_file = "%s_%d_%d%s" % (prefix, CurrentPid, int(
time.time()), extension)
# This is done in two stages because the case of the file is OK, but not the directory.
# This function does not change non existent files.
temp_dir = lib_util.standardized_file_path(tempfile.gettempdir())
temp_path = os.path.join(temp_dir, temp_file)
temp_path = lib_util.standardized_file_path(temp_path)
return temp_path
def test_sparql_subdirectory_2(self):
"""Tests that a second-level directory is detected. """
rdflib_graph = CreateGraph()
dir_path = os.path.join(TempDirPath,
"survol_temp_dir%s_1" % unique_string,
"survol_temp_dir%s_2" % unique_string)
os.makedirs(dir_path)
dir_path = lib_util.standardized_file_path(dir_path)
sparql_query = """
PREFIX survol: <%s>
SELECT ?subdirectory_name WHERE {
?url_directory_0 a survol:CIM_Directory .
?url_directory_1 a survol:CIM_Directory .
?url_directory_2 a survol:CIM_Directory .
?url_directory_0 survol:CIM_DirectoryContainsFile ?url_directory_1 .
?url_directory_1 survol:CIM_DirectoryContainsFile ?url_directory_2 .
?url_directory_0 survol:Name "%s" .
?url_directory_2 survol:Name ?subdirectory_name .
}
""" % (survol_namespace, TempDirPath)
query_result = list(rdflib_graph.query(sparql_query))
print("dir_path=", dir_path)
actual_files = set(
[str(one_path_url[0]) for one_path_url in query_result])
print("actual_files=", actual_files)
self.assertTrue(dir_path in actual_files)
def DatabaseEnvParams(process_id):
# This is imported here to avoid circular references.
from sources_types import CIM_Process
logging.debug("process_id=%s", str(process_id))
# Get the list of files open by the process.
try:
proc_obj = CIM_Process.PsutilGetProcObj(int(process_id))
fillist = proc_obj.open_files()
except Exception as exc:
lib_common.ErrorMessageHtml("Caught:" + str(exc) + ": process_id=" +
str(process_id))
list_args = []
for fil_obj in fillist:
fil_nam = fil_obj.path
logging.debug("DatabaseEnvParams process_id=%s fil_nam=%s",
str(process_id), fil_nam)
if IsSqliteDatabase(fil_nam):
logging.debug("DatabaseEnvParams ADDING fil_nam=%s", fil_nam)
fil_nam_clean = lib_util.standardized_file_path(fil_nam)
fil_def = {"File": fil_nam_clean}
list_args.append(fil_def)
logging.debug("DatabaseEnvParams len=%d", len(list_args))
return "sqlite/query", list_args
def ClassUri(self, class_name, path=""):
# The URL should never contain the chars "<" or ">".
#class_name = lib_util.Base64Encode(class_name)
path = lib_util.standardized_file_path(path)
# return self.node_from_dict("class", {"Name": class_name, "File": lib_util.EncodeUri(path)})
assert isinstance(class_name, str)
return self.node_from_dict("class", {"Name": class_name, "File": path})
def FillOnePackage(grph,node,good_pckg):
# >>> dir(installed_packages[0])
# ['PKG_INFO', '__class__', '__delattr__', '__dict__', '__doc__', '__eq__', '__format__', '__ge__', '__getattr__', '__getattribute__',
# '__gt__', '__hash__', '__init__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__',
# '__setattr__', '__sizeof__', '__str__', '__subclasshook__', '__weakref__', '_dep_map', '_get_metadata', '_key', '_provider', '_relo
# ad_version', '_version', '_warn_legacy_version', 'activate', 'as_requirement', 'check_version_conflict', 'clone', 'egg_name', 'extra
# s', 'from_filename', 'from_location', 'get_entry_info', 'get_entry_map', 'has_version', 'hashcmp', 'insert_on', 'key', 'load_entry_p
# oint', 'location', 'parsed_version', 'platform', 'precedence', 'project_name', 'py_version', 'requires', 'version']
grph.add( (node, propPythonVersion, lib_common.NodeLiteral(good_pckg.version) ) )
grph.add( (node, lib_common.MakeProp("Platform"), lib_common.NodeLiteral(good_pckg.platform) ) )
grph.add( (node, lib_common.MakeProp("project_name"), lib_common.NodeLiteral(good_pckg.project_name) ) )
# >>> pip.get_installed_distributions()[1].requires()
# [Requirement.parse('distribute'), Requirement.parse('werkzeug'), Requirement.parse('mako')]
# '_Requirement__hash', '__contains__','__doc__', '__eq__', '__hash__','__init__', '__module__', '__ne__',
# '__repr__', '__str__', 'extras','hashCmp', 'key', 'marker_fn', 'parse','project_name', 'specifier', 'specs','unsafe_name'
# strReq = "+".join( [ str(dir(req)) for req in good_pckg.requires() ])
for subReq in good_pckg.requires():
subNode = MakeUri( subReq.key )
# [('>=', '4.0.0')]+[]+[('>=','4.0')]+[]
aSpecs = subReq.specs
if aSpecs:
grph.add( (subNode, lib_common.MakeProp("Condition"), lib_common.NodeLiteral( str(aSpecs) ) ) )
grph.add( (node, lib_common.MakeProp("requires"), subNode ) )
grph.add( (node, lib_common.MakeProp("py_version"), lib_common.NodeLiteral(good_pckg.py_version) ) )
grph.add( (node, lib_common.MakeProp("precedence"), lib_common.NodeLiteral(good_pckg.precedence) ) )
grph.add( (node, lib_common.MakeProp("egg_name"), lib_common.NodeLiteral(good_pckg.egg_name()) ) )
# This might return location="c:\python27\lib\site-packages"
cleanLocaDir = lib_util.standardized_file_path(good_pckg.location)
nodeLocation = lib_common.gUriGen.DirectoryUri(cleanLocaDir)
grph.add( (node, lib_common.MakeProp("Location"),nodeLocation ) )
def test_sparql_executable_parent_process(self):
"""Executable of the parent process."""
rdflib_graph = CreateGraph()
print("CurrentParentPid=", CurrentParentPid)
sparql_query = """
PREFIX survol: <%s>
SELECT ?executable_name
WHERE
{
?url_proc survol:Handle %d .
?url_proc survol:ParentProcessId ?CurrentParentPid .
?url_proc rdf:type survol:CIM_Process .
?url_parent_proc survol:Handle ?CurrentParentPid .
?url_parent_proc survol:CIM_ProcessExecutable ?url_executable_file .
?url_parent_proc rdf:type survol:CIM_Process .
?url_executable_file rdf:type survol:CIM_DataFile .
?url_executable_file survol:Name ?executable_name .
}
""" % (survol_namespace, CurrentPid)
query_result = list(rdflib_graph.query(sparql_query))
print("query_result=", query_result)
actual_executable_name = [
str(one_value[0]) for one_value in query_result
][0]
print("actual_executable_name=", actual_executable_name)
expected_executable_name = lib_util.standardized_file_path(
psutil.Process().parent().exe())
print("expected_executable_name=", expected_executable_name)
self.assertTrue(expected_executable_name == actual_executable_name)
def test_from_perl(self):
"""This searches the content of a process memory which contains a SQL memory."""
proc_open = _start_subprocess(_perl_path, sample_perl_script)
my_source_filenames = lib_client.SourceLocal(
"sources_types/CIM_Process/memory_regex_search/search_filenames.py",
"CIM_Process",
Handle=proc_open.pid)
sys.stdout.flush()
triple_filenames = my_source_filenames.get_triplestore()
print("len(triple_filenames)=", len(triple_filenames))
filenames_set = set()
for one_instance in triple_filenames.get_instances():
if type(one_instance).__name__ == 'CIM_DataFile':
filenames_set.add(one_instance.Name)
for one_filename in sorted(filenames_set):
print(" ", one_filename)
self.assertTrue(_perl_path in filenames_set)
self.assertTrue(
lib_util.standardized_file_path(windows_system32_cmd_exe) in
filenames_set)
proc_open.communicate()
def DatabaseEnvParams(processId):
# This is imported here to avoid circular references.
from sources_types import CIM_Process
DEBUG("\nDatabaseEnvParams processId=%s", str(processId))
# Get the list of files open by the process.
try:
proc_obj = CIM_Process.PsutilGetProcObj(int(processId))
fillist = CIM_Process.PsutilProcOpenFiles(proc_obj)
except Exception:
exc = sys.exc_info()[1]
lib_common.ErrorMessageHtml("Caught:" + str(exc) + ": processId=" +
str(processId))
listArgs = []
for filObj in fillist:
filNam = filObj.path
DEBUG("DatabaseEnvParams processId=%s filNam=%s", str(processId),
filNam)
if IsSqliteDatabase(filNam):
DEBUG("DatabaseEnvParams ADDING filNam=%s", filNam)
filNamClean = lib_util.standardized_file_path(filNam)
filDef = {"File": filNamClean}
listArgs.append(filDef)
DEBUG("DatabaseEnvParams len=%d\n", len(listArgs))
return ("sqlite/query", listArgs)
def ClassUri(self, class_name, path=""):
# The URL should never contain the chars "<" or ">".
class_name = lib_util.Base64Encode(class_name)
path = lib_util.standardized_file_path(path)
return self.UriMakeFromDict("class", {
"Name": class_name,
"File": lib_util.EncodeUri(path)
})
def SymbolUri(self, symbol_name, path=""):
# The URL should never contain the chars "<" or ">".
# TODO: Move that to linker_symbol/__init__.py and see sources_types.sql.query.MakeUri
# TODO: Alphabetical order !!!!
path = lib_util.standardized_file_path(path)
assert isinstance(symbol_name, str)
return self.node_from_dict("linker_symbol", {
"Name": symbol_name,
"File": path
})
def SymbolUri(self, symbol_name, path=""):
# The URL should never contain the chars "<" or ">".
symbol_name = lib_util.Base64Encode(symbol_name)
# TODO: Move that to linker_symbol/__init__.py and see sources_types.sql.query.MakeUri
# TODO: Alphabetical order !!!!
path = lib_util.standardized_file_path(path)
return self.UriMakeFromDict("linker_symbol", {
"Name": symbol_name,
"File": lib_util.EncodeUri(path)
})
def Main():
cgiEnv = lib_common.ScriptEnvironment()
grph = cgiEnv.GetGraph()
net_share_cmd = ["net", "share"]
net_share_pipe = lib_common.SubProcPOpen(net_share_cmd)
net_share_last_output, net_share_err = net_share_pipe.communicate()
# Converts to string for Python3.
as_str = net_share_last_output.decode("utf-8")
lines = as_str.split('\n')
seen_hyphens = False
for lin in lines:
if re.match(".*-------.*", lin):
seen_hyphens = True
continue
if re.match(".*The command completed successfully.*", lin):
break
if not seen_hyphens:
continue
tst_share = re.match(r'^([A-Za-z0-9_$]+) +([^ ]+).*', lin)
if not tst_share:
continue
shr_nam = tst_share.group(1)
# Nasty formatting of "NET SHARE" command.
if len(lin) >= 45:
# There is a remark or a very long resource.
if lin[44] == ' ':
# Character just before remark is a space.
shr_res = lin[13:44].rstrip()
else:
shr_res = lin[13:]
else:
shr_res = lin[13:]
share_node = lib_uris.gUriGen.SmbShareUri(shr_nam)
grph.add((lib_common.nodeMachine, pc.property_smbshare, share_node))
# mount_node = lib_uris.gUriGen.FileUri( "//" + lib_util.currentHostname + "/" + shr_res )
shr_res = shr_res.strip()
shr_res = lib_util.standardized_file_path(shr_res)
mount_node = lib_uris.gUriGen.DirectoryUri(shr_res)
grph.add((share_node, pc.property_smbmount, mount_node))
cgiEnv.OutCgiRdf()
def Main():
cgiEnv = lib_common.CgiEnv()
pid = int(cgiEnv.GetId())
# TODO: These are probably in win32com or a similar module.
PROCESS_QUERY_INFORMATION = 0x0400
PROCESS_VM_READ = 0x0010
grph = cgiEnv.GetGraph()
node_process = lib_common.gUriGen.PidUri(pid)
exec_node = CIM_Process.AddInfo(grph, node_process, [pid])
#Get handle to the process based on PID
hProcess = kernel.OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ,
False, pid)
if hProcess:
ModuType = c_ulong * 512
hModuleArr = ModuType()
rawCntModules = c_ulong()
psapi.EnumProcessModules(hProcess, byref(hModuleArr),
sizeof(hModuleArr), byref(rawCntModules))
nbModules = int(rawCntModules.value / sizeof(c_ulong()))
if nbModules >= 512:
raise Exception("Disaster overrun")
modname = c_buffer(256)
for idx in range(0, nbModules):
retLen = psapi.GetModuleFileNameExA(hProcess, hModuleArr[idx],
modname, sizeof(modname))
if retLen == 0:
# Maybe the string is empty.
continue
raw_filename_bytes = modname[:retLen]
raw_filename_as_str = raw_filename_bytes.decode()
filnam = lib_util.standardized_file_path(raw_filename_as_str)
#if lib_util.is_py3:
# # Truncation because "b'C:/xxx/yyy.zzz'", on Python 3
# # Backslashes are duplicated.
# filnam = str(tab).replace('\\','/')[2:-1].replace("//","/")
#else:
# # Windows "\\" must be replaced by "/", so the URLs are the same for all tools.
# filnam = str(tab).replace('\\','/')
# The same filename might appear several times.
DEBUG("idx=%d retLen=%d filnam=%s", idx, retLen, filnam)
libNode = lib_common.gUriGen.SharedLibUri(filnam)
grph.add((node_process, pc.property_library_depends, libNode))
kernel.CloseHandle(hProcess)
cgiEnv.OutCgiRdf()
def DirectoryUri(self, path):
# Normalize a pathname by collapsing redundant separators and up-level references
# so that A//B, A/B/, A/./B and A/foo/../B all become A/B.
# This string manipulation may change the meaning of a path that contains symbolic links.
# On Windows, it converts forward slashes to backward slashes.
path = os.path.normpath(path)
# Backslashes are a real problem everywhere.
# On top of that, escaping the backslash is not enough because strings are sometimes truncated for display.
# Better have our own "canonical" notation for filenames, so replace them.
# If needed, they can always be replaced by a normal slash.
#
path = lib_util.standardized_file_path(path)
return self.node_from_args("CIM_Directory", path)
def Main():
cgiEnv = lib_common.CgiEnv()
filNam = cgiEnv.GetId()
filNam = lib_util.standardized_file_path(filNam)
DEBUG("filNam=%s", filNam)
filNode = lib_common.gUriGen.DirectoryUri(filNam)
grph = cgiEnv.GetGraph()
info = CIM_DataFile.GetInfoStat(filNam)
# st_mode: protection bits.
# st_ino: inode number.
# st_dev: device.
CIM_DataFile.AddDevice(grph, filNode, info)
CIM_DataFile.AddStatNode(grph, filNode, info)
CIM_DataFile.AddMagic(grph, filNode, filNam)
# st_nlink: number of hard links.
CIM_DataFile.AffFileOwner(grph, filNode, filNam)
# Displays the file and the parent directories/
currFilNam = filNam
currNode = filNode
while True:
dirPath = os.path.dirname(currFilNam)
if dirPath == currFilNam:
break
if dirPath == "":
break
dirNode = lib_common.gUriGen.DirectoryUri(dirPath)
grph.add((dirNode, pc.property_directory, currNode))
DEBUG("dirPath=%s", dirPath)
statPath = os.stat(dirPath)
CIM_DataFile.AddStatNode(grph, dirNode, statPath)
CIM_DataFile.AddFileProperties(grph, currNode, currFilNam)
currFilNam = dirPath
currNode = dirNode
# If windows, print more information: DLL version etc...
# http://stackoverflow.com/questions/580924/python-windows-file-version-attribute
# cgiEnv.OutCgiRdf()
cgiEnv.OutCgiRdf("LAYOUT_TWOPI")
def AddInfo(grph, node, entity_ids_arr):
"""
This creates a couple of nodes about a file.
"""
file_name = entity_ids_arr[0]
if not file_name: # Faster than comparing to an empty string.
return
# Cleanup the filename. This function is called without knowledge of the specific case,
# therefore the cleanup can only be done in code related to this entity type.
file_name = lib_util.standardized_file_path(file_name)
AddMagic(grph, node, file_name)
AddStat(grph, node, file_name)
AddHtml(grph, node, file_name)
_add_parent_dir(grph, node, file_name)
def test_sparql_files_in_executable_process_dir(self):
"""Display the files in the directory of the current process'executable."""
rdflib_graph = CreateGraph()
sparql_query = """
PREFIX survol: <%s>
SELECT ?datafile_name
WHERE
{
?url_proc survol:Handle %d .
?url_proc survol:CIM_ProcessExecutable ?url_executable_datafile .
?url_proc rdf:type survol:CIM_Process .
?url_executable_datafile rdf:type survol:CIM_DataFile .
?url_datafile rdf:type survol:CIM_DataFile .
?url_directory rdf:type survol:CIM_Directory .
?url_directory survol:CIM_DirectoryContainsFile ?url_datafile .
?url_directory survol:CIM_DirectoryContainsFile ?url_executable_datafile .
?url_datafile survol:Name ?datafile_name .
}
""" % (survol_namespace, CurrentPid)
query_result = list(rdflib_graph.query(sparql_query))
files_names_result = {str(one_value[0]) for one_value in query_result}
# print("files_names_result=", files_names_result)
print("files_names_result len:", len(files_names_result))
print("sys_executable_case=", sys_executable_case)
self.assertTrue(sys_executable_case in files_names_result)
# Compare with the list of the files the directory of the executable.
path_names_set = set()
for root_dir, dir_lists, files_list in os.walk(
os.path.dirname(sys_executable_case)):
for one_file_name in files_list:
sub_path_name = lib_util.standardized_file_path(
os.path.join(root_dir, one_file_name))
path_names_set.add(sub_path_name)
break
# print("Expected list of files:", path_names_set)
print("Expected list of files length:", len(path_names_set))
print("Files differences A:",
path_names_set.difference(files_names_result))
print("Files differences B:",
files_names_result.difference(path_names_set))
self.assertEqual(path_names_set, files_names_result)
def test_sparql_children_files(self):
rdflib_graph = CreateGraph()
# C:/Windows/temp\\survol_temp_file_12532.tmp'
tmp_pathname = lib_util.standardized_file_path(_create_temp_file())
sparql_query = """
PREFIX survol: <%s>
SELECT ?datafile_name WHERE {
?url_directory a survol:CIM_Directory .
?url_datafile a survol:CIM_DataFile .
?url_directory survol:CIM_DirectoryContainsFile ?url_datafile .
?url_datafile survol:Name ?datafile_name .
?url_directory survol:Name "%s" .
}
""" % (survol_namespace, TempDirPath)
query_result = list(rdflib_graph.query(sparql_query))
print("Result=", query_result)
self.assertTrue(
tmp_pathname in [str(node[0]) for node in query_result])
def Main():
cgiEnv = lib_common.ScriptEnvironment()
pid = int(cgiEnv.GetId())
# TODO: These are probably in win32com or a similar module.
PROCESS_QUERY_INFORMATION = 0x0400
PROCESS_VM_READ = 0x0010
grph = cgiEnv.GetGraph()
node_process = lib_uris.gUriGen.PidUri(pid)
exec_node = CIM_Process.AddInfo(grph, node_process, [pid])
#Get handle to the process based on PID
hProcess = kernel.OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, False, pid)
if hProcess:
ModuType = c_ulong * 512
hModuleArr = ModuType()
raw_cnt_modules = c_ulong()
psapi.EnumProcessModules(hProcess, byref(hModuleArr), sizeof(hModuleArr), byref(raw_cnt_modules))
nb_modules = int(raw_cnt_modules.value/sizeof(c_ulong()))
if nb_modules >= 512:
raise Exception("Disaster overrun")
modname = c_buffer(256)
for idx in range(0, nb_modules):
ret_len = psapi.GetModuleFileNameExA(hProcess, hModuleArr[idx], modname, sizeof(modname))
if ret_len == 0:
# Maybe the string is empty.
continue
raw_filename_bytes = modname[:ret_len]
raw_filename_as_str = raw_filename_bytes.decode()
filnam = lib_util.standardized_file_path(raw_filename_as_str)
lib_node = lib_uris.gUriGen.SharedLibUri(filnam)
grph.add((node_process, pc.property_library_depends, lib_node))
kernel.CloseHandle(hProcess)
cgiEnv.OutCgiRdf()
def test_sparql_subdirectory_down_up_4(self):
rdflib_graph = CreateGraph()
sparql_query = """
PREFIX survol: <%s>
SELECT ?directory_name WHERE {
?url_directory_0a a survol:CIM_Directory .
?url_directory_1a a survol:CIM_Directory .
?url_directory_2a a survol:CIM_Directory .
?url_directory_3a a survol:CIM_Directory .
?url_directory_4X a survol:CIM_Directory .
?url_directory_3b a survol:CIM_Directory .
?url_directory_2b a survol:CIM_Directory .
?url_directory_1b a survol:CIM_Directory .
?url_directory_0b a survol:CIM_Directory .
?url_directory_0a survol:CIM_DirectoryContainsFile ?url_directory_1a .
?url_directory_1a survol:CIM_DirectoryContainsFile ?url_directory_2a .
?url_directory_2a survol:CIM_DirectoryContainsFile ?url_directory_3a .
?url_directory_3a survol:CIM_DirectoryContainsFile ?url_directory_4X .
?url_directory_3b survol:CIM_DirectoryContainsFile ?url_directory_4X .
?url_directory_2b survol:CIM_DirectoryContainsFile ?url_directory_3b .
?url_directory_1b survol:CIM_DirectoryContainsFile ?url_directory_2b .
?url_directory_0b survol:CIM_DirectoryContainsFile ?url_directory_1b .
?url_directory_0a survol:Name "%s" .
?url_directory_0b survol:Name ?directory_name .
}
""" % (survol_namespace, TempDirPath)
query_result = list(rdflib_graph.query(sparql_query))
actual_files = [
lib_util.standardized_file_path(str(one_path_url[0]))
for one_path_url in query_result
]
print("actual_files=", actual_files)
self.assertEqual(actual_files[0], TempDirPath)
# All directories must be identical.
unique_set = set(actual_files)
self.assertEqual(len(unique_set), 1)
def _create_our_node(grph, root_node, entity_host, name_space, class_name,
entity_id):
"""This try to find a correct url for an entity type, without an entity id.
At the moment, we just expect a file called "enumerate_<entity>.py" """
enumerate_script = "enumerate_" + class_name + ".py"
base_dir = lib_util.gblTopScripts + "/sources_types"
# TODO: This is absurd !!! Why looping, because the filename is already known !?!?
for dirpath, dirnames, filenames in os.walk(base_dir):
for filename in [f for f in filenames if f == enumerate_script]:
short_dir = dirpath[len(lib_util.gblTopScripts):]
full_script_nam = lib_util.standardized_file_path(
os.path.join(short_dir, filename))
logging.debug("full_script_nam=%s", full_script_nam)
# TODO: Maybe remove the beginning of the file.
local_class_url = lib_util.ScriptizeCimom(full_script_nam,
class_name, entity_host)
local_class_node = lib_common.NodeUrl(local_class_url)
grph.add((root_node, pc.property_directory, local_class_node))
CurrentPid = os.getpid()
CurrentProcessPath = 'CIM_Process.Handle=%d' % CurrentPid
CurrentParentPid = psutil.Process().ppid()
# The agent urls point to the Survol adhoc CGI server: "http://myhost-hp:8000"
# The tests use different port numbers to avoid interferences between servers,
# if port numbers are not freed etc... Problems are easier to find.
RemoteGeneralTestServerPort = 8000
RemoteEventsTestServerPort = 8001
RemoteSparqlTestServerPort = 8002
RemoteGraphvizTestServerPort = 8003
RemoteHtmlTestServerPort = 8004
# Several Survol scripts return this executable among their results, so it can be tested.
CurrentExecutable = lib_util.standardized_file_path(sys.executable)
CurrentExecutablePath = 'CIM_DataFile.Name=%s' % CurrentExecutable
# This is called at the end of the execution of a Survol agent created here for tests.
# It displays the content of a log file created by this agent.
def __dump_server_content(log_filename):
sys.stdout.write("Agent log file: %s\n" % log_filename)
try:
agent_stream = open(log_filename)
for line_stream in agent_stream:
sys.stdout.write(">>> %s" % line_stream)
agent_stream.close()
sys.stdout.write("Agent log file end\n")
except Exception as exc:
def Main():
cgiEnv = lib_common.ScriptEnvironment()
try:
the_pid = int(cgiEnv.GetId())
except Exception:
lib_common.ErrorMessageHtml("Must provide a pid")
# If cannot be the current pid, otherwise it will block.
if the_pid == os.getpid():
lib_common.ErrorMessageHtml("Cannot debug current process")
if not lib_util.isPlatformWindows:
lib_common.ErrorMessageHtml("This works only on Windows platforms")
grph = cgiEnv.GetGraph()
# Starts a second session
cdb_fil = lib_util.TmpFile("CdbCommand", "cdb")
cdb_fd = open(cdb_fil.Name, "w")
cdb_fd.write("lm\n") # List loaded modules
cdb_fd.write("k\n") # Display stack backtrace.
cdb_fd.write("qd\n") # Quit and detach.
cdb_fd.close()
cdb_cmd = "cdb -p " + str(the_pid) + " -cf " + cdb_fil.Name
proc_node = lib_uris.gUriGen.PidUri(the_pid)
call_node_prev = None
modules_map = {}
logging.debug("Starting cdb_cmd=%s", cdb_cmd)
try:
cdb_pipe = lib_common.SubProcPOpen(cdb_cmd)
except WindowsError as exc:
lib_common.ErrorMessageHtml("cdb not available: Caught:%s" % str(exc))
logging.debug("Started cdb_cmd=%s", cdb_cmd)
cdb_output, cdb_err = cdb_pipe.communicate()
# Without decode, "TypeError: Type str does not support the buffer API"
cdb_str = cdb_output.decode("utf-8")
call_depth = 0
for dot_line in cdb_str.split('\n'):
err_match = re.match(".*parameter is incorrect.*", dot_line)
if err_match:
lib_common.ErrorMessageHtml("CDB:"+dot_line)
# 76590000 766a0000 kernel32 (export symbols) C:\Windows\syswow64\kernel32.dll
match_lm = re.match(r"[0-9a-fA-F]+ [0-9a-fA-F]+ +([^ ]*) +\(export symbols\) +(.*)", dot_line )
if match_lm:
module_name = match_lm.group(1)
dll_name_raw = match_lm.group(2).strip()
dll_name = lib_util.standardized_file_path(dll_name_raw)
logging.debug("module_name=%s dll_name=%s", module_name, dll_name)
modules_map[module_name] = dll_name
continue
# 295cfb0c 00000000 ntdll!RtlInitializeExceptionChain+0x36
# Another format, maybe because of a 64 bits machine.
# 00000000`02edff90 00000000`00000000 ntdll!RtlUserThreadStart+0x21
match_k = re.match("[`0-9a-fA-F]+ [`0-9a-fA-F]+ ([^!]*)!([^+]*)", dot_line)
if match_k:
module_name = match_k.group(1)
try:
dll_name = modules_map[module_name]
except KeyError:
dll_name = module_name
func_name = match_k.group(2).strip()
logging.debug("module_name=%s dll_name=%s func_name=%s", module_name, dll_name, func_name)
dll_name = CDB.TestIfKnownDll(dll_name)
call_node_prev = survol_symbol.AddFunctionCall(grph, call_node_prev, proc_node, func_name, dll_name)
grph.add((call_node_prev, lib_properties.MakeProp("Call_depth"), lib_util.NodeLiteral(call_depth)))
call_depth += 1
continue
logging.debug("dot_line=%s", dot_line)
logging.debug("Parsed cdb result")
call_node_prev = survol_symbol.AddFunctionCall(grph, call_node_prev, proc_node, None, None)
CIM_Process.AddInfo(grph, proc_node, [the_pid])
# http://msdn.microsoft.com/en-us/library/windows/hardware/ff539058(v=vs.85).aspx
#
# This section describes how to perform basic debugging tasks using
# the Microsoft Console Debugger (CDB) and Microsoft NT Symbolic Debugger (NTSD).
# CDB and NTSD are identical in every way, except that NTSD spawns
# a new text window when it is started, whereas CDB inherits
# the Command Prompt window from which it was invoked.
# The instructions in this section are given for CDB,
# but they work equally well for NTSD. For a discussion
# of when to use CDB or NTSD, see Debugging Environments.
cgiEnv.OutCgiRdf("LAYOUT_SPLINE")
def Main():
cgiEnv = lib_common.ScriptEnvironment()
try:
the_pid = int(cgiEnv.GetId())
except Exception:
lib_common.ErrorMessageHtml("Must provide a pid")
if not lib_util.isPlatformWindows:
lib_common.ErrorMessageHtml("This works only on Windows platforms")
# If cannot be the current pid, otherwise it will block.
if the_pid == os.getpid():
lib_common.ErrorMessageHtml("Cannot debug current process")
grph = cgiEnv.GetGraph()
# Starts a second session
cdb_fil = lib_util.TmpFile("CdbCommand", "cdb")
cdb_fd = open(cdb_fil.Name, "w")
cdb_fd.write("lmv\n") # List loaded modules, verbose mode.
cdb_fd.write("qd\n") # Quit and detach.
cdb_fd.close()
cdb_cmd = "cdb -p " + str(the_pid) + " -cf " + cdb_fil.Name
proc_node = lib_uris.gUriGen.PidUri(the_pid)
logging.debug("Starting cdb_cmd=%s", cdb_cmd)
try:
cdb_pipe = lib_common.SubProcPOpen(cdb_cmd)
except WindowsError as exc:
lib_common.ErrorMessageHtml("cdb not available: Caught:%s" % str(exc))
logging.debug("Started cdb_cmd=%s", cdb_cmd)
cdb_output, cdb_err = cdb_pipe.communicate()
# Without decode, "TypeError: Type str does not support the buffer API"
cdb_str = cdb_output.decode("utf-8", "ignore")
prop_loaded_module = lib_common.MakeProp("Loaded module")
for dot_line in cdb_str.split('\n'):
# moduleName=uDWM moduleStatus=deferred file_name=
# dot_line= Image path: C:\windows\system32\uDWM.dll
# dot_line= Image name: uDWM.dll
# dot_line= Timestamp: Tue Jul 14 02:33:35 2009 (4A5BE06F)
# dot_line= CheckSum: 0005E9A4
# dot_line= ImageSize: 00057000
# dot_line= File version: 6.1.7600.16385
# dot_line= Product version: 6.1.7600.16385
# dot_line= File flags: 0 (Mask 3F)
# dot_line= File OS: 40004 NT Win32
# dot_line= File type: 2.0 Dll
# dot_line= File date: 00000000.00000000
# dot_line= Translations: 0409.04b0
# dot_line= CompanyName: Microsoft Corporation
# dot_line= ProductName: Microsoft Windows Operating System
# dot_line= InternalName: udwm.dll
# dot_line= OriginalFilename: udwm.dll
# dot_line= ProductVersion: 6.1.7600.16385
# dot_line= FileVersion: 6.1.7600.16385 (win7_rtm.090713-1255)
# dot_line= FileDescription: Microsoft Desktop Window Manager
# dot_line= LegalCopyright: Microsoft Corporation. All rights reserved.
match_lin = re.match(" *Image path: *(.*)", dot_line)
if match_lin:
file_name = match_lin.group(1)
file_name = CDB.TestIfKnownDll(file_name)
file_name = file_name.strip()
file_name = lib_util.standardized_file_path(file_name)
file_node = lib_uris.gUriGen.FileUri(file_name)
grph.add((proc_node, prop_loaded_module, file_node))
continue
match_lin = re.match(" *CompanyName: *(.*)", dot_line)
if match_lin:
company_name = match_lin.group(1)
grph.add((file_node, lib_common.MakeProp("Company Name"),
lib_util.NodeLiteral(company_name)))
continue
match_lin = re.match(" *File OS: *(.*)", dot_line)
if match_lin:
file_os = match_lin.group(1)
grph.add((file_node, lib_common.MakeProp("File OS"),
lib_util.NodeLiteral(file_os)))
continue
match_lin = re.match(" *FileDescription: *(.*)", dot_line)
if match_lin:
file_description = match_lin.group(1)
grph.add((file_node, lib_common.MakeProp("Description"),
lib_util.NodeLiteral(file_description)))
continue
logging.debug("Parsed cdb result")
CIM_Process.AddInfo(grph, proc_node, [the_pid])
cgiEnv.OutCgiRdf("LAYOUT_RECT", [prop_loaded_module])
cgiEnv = lib_common.ScriptEnvironment()
process_id = cgiEnv.GetId()
logging.debug("Snapshot process_id=%s" % process_id)
# This just returns one triple.
grph = cgiEnv.GetGraph()
process_node = lib_uris.gUriGen.PidUri(process_id)
grph.add((process_node, pc.property_pid, lib_util.NodeLiteral(process_id)))
cgiEnv.OutCgiRdf()
# FIXME: Must finish this.
if dockit:
dockit_dirname = lib_util.standardized_file_path(
os.path.dirname(dockit.__file__))
logging.debug("File=" + __file__ + " dockit_dirname=" + dockit_dirname)
def _atexit_handler_detach(process_id):
"""This is called when this CGI script leaves for any reason.
Its purpose is to detach from the target process."""
logging.info("_atexit_handler process_id=%d" % process_id)
def SendEvents():
"""This is called in a subprocess started by the Python module supervisor."""
logging.info("SendEvents")
# FIXME:
