def test_plugin(self):
print "\n ----------------------- Testing pluging manager via API -------------------------\n"
self.swirl = Swirl("test", datetime.now())
for i in self.files:
swirlFile = PluginManager.getSwirl(i)
self.swirl.addFile(swirlFile)
p = PluginManager.get_plugins()
print "plugins: ", p
self.assertEqual(len(p), self.availablePlugin,
msg="Plugin manager did not load all the available plugins (available %d, detected %d) "
% (len(p) , self.availablePlugin) )
def check(self):
"""actually perform the check on the system and return True if all
the dependencies can be satisfied on the current system
"""
self.error = []
depList = self.swirl.getDependencies()
returnValue = True
PluginManager.addSystemPaths(self.extraPath)
for dep in depList:
if not PluginManager.isDepsatisfied(dep):
self.error.append(dep.depname)
returnValue = False
return returnValue
def check(self):
"""
It performs the check on the system and verifies that all the
dependencies of this Swirl can be satisfied.
:rtype: bool
:return: True if the check passes False otherwise. The list of
missing dependencies can be retrieved with :meth:`getError`
"""
returnValue = True
# this method of using rpath is not totaly correct but it's faster
# so for the moment we have to live with this
for swF in self.swirl.execedFiles:
rpath = swF.rpaths + self.extraPath + utils.getLDLibraryPath(
swF.env)
for swf_dep in [
swF
] + self.swirl.getListSwirlFilesDependentStatic(swF):
for dep in swf_dep.staticDependencies:
if not PluginManager.getPathToLibrary(dep, rpath=rpath):
self.missingDeps.add(dep)
returnValue = False
for dynamic_dep in swF.dynamicDependencies:
if not os.path.exists(dynamic_dep.path):
self.missingDeps = self.missingDeps.union(
dynamic_dep.provides)
returnValue = False
return returnValue
def searchModules(self):
"""
It searches for missing dependencies using the 'module' command line.
:meth:`check` should be called before this
:rtype: string
:return: with a human readable list of module which can satisfy
missing dependencies
"""
# loop through all the modules
retDict = {}
(output, retval) = utils.getOutputAsList(
["bash", "-c", "module -t avail 2>&1"])
if retval:
print("Unable to run module command, verify it\'s in the path.")
return ""
for module in output:
# in the output there are some paths! remove them e.g. "/opt/module/blabla:"
if ':' in module:
continue
# remove (default)
module = module.split("(default)")[0]
(output, retval) = utils.getOutputAsList(
["bash", "-c", "module show " + module + " 2>&1"])
if retval:
#print "Unable to fetch module information: \'module show " + module + "\'"
# all module which depend on another module return 1
pass
for line in output:
if 'LD_LIBRARY_PATH' in line:
# we found another path to scan
path = line.split('LD_LIBRARY_PATH')[1]
path = [i.strip() for i in path.split(":")] #strip
PluginManager.systemPath = path # update path
for dep in self.missingDeps:
if PluginManager.getPathToLibrary(dep, False):
#we found a candidate for this missing dependency
if module not in retDict:
retDict[module] = []
retDict[module].append(dep.getName())
retStr = ""
for mod in retDict:
retStr += " " + mod + " satisfies "
num_deps = len(retDict[mod])
if num_deps == len(self.missingDeps):
retStr += "all "
retStr += "" + str(num_deps) + " dependencies:\n"
# print the deps
retStr += " " + "\n ".join(retDict[mod]) + "\n"
return retStr
def searchModules(self):
"""
It searches for missing dependencies using the 'module' command line.
:meth:`check` should be called before this
:rtype: string
:return: with a human readable list of module which can satisfy
missing dependencies
"""
# loop through all the modules
retDict = {}
(output, retval) = utils.getOutputAsList(["bash", "-c", "module -t avail 2>&1"])
if retval:
print "Unable to run module command, verify it\'s in the path."
return ""
for module in output :
# in the output there are some paths! remove them e.g. "/opt/module/blabla:"
if ':' in module:
continue
# remove (default)
module = module.split("(default)")[0]
(output, retval) = utils.getOutputAsList(["bash", "-c",
"module show " + module + " 2>&1"])
if retval:
#print "Unable to fetch module information: \'module show " + module + "\'"
# all module which depend on another module return 1
pass
for line in output:
if 'LD_LIBRARY_PATH' in line:
# we found another path to scan
path = line.split('LD_LIBRARY_PATH')[1]
path = [i.strip() for i in path.split(":")] #strip
PluginManager.systemPath = path # update path
for dep in self.missingDeps:
if PluginManager.getPathToLibrary(dep, False):
#we found a candidate for this missing dependency
if module not in retDict:
retDict[module] = []
retDict[module].append(dep.getName())
retStr = ""
for mod in retDict:
retStr += " " + mod + " satisfies "
num_deps = len(retDict[mod])
if num_deps == len(self.missingDeps):
retStr += "all "
retStr += "" + str(num_deps) + " dependencies:\n"
# print the deps
retStr += " " + "\n ".join(retDict[mod]) + "\n"
return retStr
def checkHash(self):
"""check if any dep was modified since the swirl file creation
(using checksuming) """
self.error = []
depList = self.swirl.getDependencies()
returnValue = True
for dep in depList:
path = PluginManager.getPathToLibrary(dep)
if not path:
continue
hash = getHash(path, dep.pluginName)
if not hash in dep.hashList:
self.error.append(dep.depname)
returnValue = False
print dep.depname, " computed ", hash, " originals ", dep.hashList
return returnValue
def checkHash(self, verbose=False):
"""
It checks if any dependency was modified since the swirl file creation
(using checksumming)
:type verbose: bool
:param verbose: if True it will generate more verbose error message
:rtype: bool
:return: True if the check passes False otherwise. The list of
modified dependencies can be retrieved with :meth:getError()
"""
self.error = []
pathCache = []
returnValue = True
for dep in self.swirl.getDependencies():
path = PluginManager.getPathToLibrary(dep)
if not path:
# error `
tmpStr = str(dep)
if verbose:
tmpStr += " unable to find its file"
self.error.append(tmpStr)
returnValue = False
continue
if path in pathCache:
#we already did this file
continue
hash = getHash(path, dep.type)
pathCache.append(path)
swirlProvider = self.swirl.getSwirlFileByProv(dep)
if not swirlProvider:
self.error.append("SwirlFile has unresolved dependency " + str(dep) \
+ " the hash can not be verified")
returnValue = False
if hash != swirlProvider.md5sum :
tmpStr = str(dep)
if verbose:
tmpStr += " wrong hash (computed " + hash + " originals " + \
swirlProvider.md5sum + ")"
self.error.append(tmpStr)
returnValue = False
return returnValue
def checkHash(self, verbose=False):
"""
It checks if any dependency was modified since the swirl file creation
(using checksumming)
:type verbose: bool
:param verbose: if True it will generate more verbose error message
:rtype: bool
:return: True if the check passes False otherwise. The list of
modified dependencies can be retrieved with :meth:getError()
"""
self.error = []
pathCache = []
returnValue = True
for dep in self.swirl.getDependencies():
path = PluginManager.getPathToLibrary(dep)
if not path:
# error `
tmpStr = str(dep)
if verbose:
tmpStr += " unable to find its file"
self.error.append(tmpStr)
returnValue = False
continue
if path in pathCache:
#we already did this file
continue
hash = getHash(path, dep.type)
pathCache.append(path)
swirlProvider = self.swirl.getSwirlFileByProv(dep)
if not swirlProvider:
self.error.append("SwirlFile has unresolved dependency " + str(dep) \
+ " the hash can not be verified")
returnValue = False
if hash != swirlProvider.md5sum:
tmpStr = str(dep)
if verbose:
tmpStr += " wrong hash (computed " + hash + " originals " + \
swirlProvider.md5sum + ")"
self.error.append(tmpStr)
returnValue = False
return returnValue
def check(self):
"""
It performs the check on the system and verifies that all the
dependencies of this Swirl can be satisfied.
:rtype: bool
:return: True if the check passes False otherwise. The list of
missing dependencies can be retrieved with :meth:`getError`
"""
returnValue = True
# this method of using rpath is not totaly correct but it's faster
# so for the moment we have to live with this
for swF in self.swirl.execedFiles:
rpath = swF.rpaths + self.extraPath + utils.getLDLibraryPath(swF.env)
for swf_dep in [swF] + self.swirl.getListSwirlFilesDependentStatic(swF):
for dep in swf_dep.staticDependencies:
if not PluginManager.getPathToLibrary(dep, rpath = rpath):
self.missingDeps.add(dep)
returnValue = False
for dynamic_dep in swF.dynamicDependencies:
if not os.path.exists(dynamic_dep.path):
self.missingDeps = self.missingDeps.union(dynamic_dep.provides)
returnValue = False
return returnValue
def __init__(self, name, fileList, processIDs):
"""give a file list and a name construct a swirl into memory """
self._pathCache = {}
self._detectedPackageManager()
self.swirl = Swirl(name, datetime.now())
#
# let's see if we have proecss ID we might need to scan for dynamic dependecies
# with the help of the /proc FS
#
# synamicDependencies = { 'binarypath' : [list of file it depends to],
# '/bin/bash' : ['/lib/x86_64-linux-gnu/libnss_files-2.15.so',
# '/lib/x86_64-linux-gnu/libnss_nis-2.15.so']}
dynamicDependecies = {}
if processIDs :
if not fileList :
fileList = []
for proc in processIDs.split(','):
proc = proc.strip()
# add the binary
binaryFile = os.readlink('/proc/' + proc + '/exe')
fileList.append( binaryFile )
dynamicDependecies[binaryFile] = []
f=open('/proc/' + proc + '/maps')
maps = f.read()
f.close()
for i in maps.split('\n'):
tokens = i.split()
if len(tokens) > 5 and 'x' in tokens[1] and os.path.isfile(tokens[5]):
# memory mapped area is executable and point to a files
dynamicDependecies[binaryFile].append( tokens[5] )
for i in fileList:
if os.path.islink(i):
swirlFile = SwirlFile( i )
swirlFile.type = 'link'
self.swirl.addFile(swirlFile)
elif os.path.isfile(i):
if i in dynamicDependecies:
swirlFile = PluginManager.getSwirl(i)
else:
swirlFile = PluginManager.getSwirl(i)
#self._hashDependencies(swirlFile)
self.swirl.addFile(swirlFile)
elif os.path.isdir(i):
pass
else:
raise IOError("The file %s cannot be opened." % i)
#
# we might need to add the dynamic dependencies to the swirl
# if they did not get detected already
for fileName in dynamicDependecies.keys():
swirlFile = self.swirl.getSwirlFile(fileName)
listDepFile = swirlFile.getListDependenciesFiles()
for dynamicDepFile in dynamicDependecies[fileName]:
if dynamicDepFile not in listDepFile:
newDeps = PluginManager.getDependeciesFromPath(dynamicDepFile)
for i in newDeps:
swirlFile.addDependency( i )
# now that I have all the dependencies in place (both static and dynamic)
# I can do the hashing
for i in self.swirl.swirlFiles:
self._hashDependencies(i)
def __init__(self, name, fileList, processIDs, execCmd):
"""give a file list and a name construct a swirl into memory """
self._detectedPackageManager()
self.swirl = Swirl(name, datetime.now())
if execCmd:
self.swirl.cmdLine = execCmd
#
# dependencies discovered with dinamic methods
# dynamicDependencies = { 'binarypath' : [list of file it depends to],
# '/bin/bash' : ['/lib/x86_64-linux-gnu/libnss_files-2.15.so',
# '/lib/x86_64-linux-gnu/libnss_nis-2.15.so']}
if execCmd:
self._straceCmd(execCmd)
# let's see if we have proecss ID we might need to scan for dynamic dependecies
# with the help of the /proc FS
elif processIDs:
for proc in processIDs.split(','):
pid = proc.strip()
# add the binary
tcb = FingerPrint.syscalltracer.TracerControlBlock(pid)
tcb.updateSharedLibraries()
dynamicDependecies = FingerPrint.syscalltracer.TracerControlBlock.dependencies
# set up the fileList for the static dependency detection
if not fileList:
fileList = []
# it does not work on python 2.5 aka RHEL 5.X
# fileList = [f if f[0] == '/' else os.path.normpath(os.getcwd() + '/' + f) for f in fileList]
def norm_path(path):
if path[0] == '/':
return os.path.normpath(path)
else:
return os.path.normpath(os.getcwd() + '/' + path)
fileList = [norm_path(f) for f in fileList]
fileList = fileList + list(dynamicDependecies.keys())
# add all the fileList to the swirl and figure out all their static libraries
for binPath in fileList:
if os.path.isfile(binPath):
cmd = binPath
if binPath in FingerPrint.syscalltracer.TracerControlBlock.cmdline:
# the user cmdline could be a symlink so we want to keep track
cmd = FingerPrint.syscalltracer.TracerControlBlock.cmdline[
binPath][0]
cmd = utils.which(cmd)
if not cmd:
cmd = binPath
if cmd and cmd[0] != '/':
# hmm we have a relative path
pwd = FingerPrint.syscalltracer.TracerControlBlock.get_env_variable(
binPath, "PWD")
pwd = pwd + '/' + cmd
if os.path.isfile(pwd) and os.access(pwd, os.X_OK):
cmd = pwd
else:
# TODO this way to resolving relative path is not 100% correct
# it should be done in the syscalltracer
cmd = binPath
# load the env
tmpenv = []
if binPath in FingerPrint.syscalltracer.TracerControlBlock.env:
for var in FingerPrint.syscalltracer.TracerControlBlock.env[
binPath]:
if '=' in var and '=()' not in var:
tmpenv.append(var)
swirlFile = PluginManager.getSwirl(cmd, self.swirl, tmpenv)
self.swirl.execedFiles.append(swirlFile)
elif os.path.isdir(binPath):
pass
else:
raise IOError("The file %s cannot be opened." % binPath)
#
# we might need to add the dynamic dependencies to the swirl
# if they did not get detected already
for fileName in list(dynamicDependecies.keys()):
swirlFile = PluginManager.getSwirl(fileName, self.swirl)
#let's add it to the execed file list
if swirlFile not in self.swirl.execedFiles:
# TODO remove me... this should never be called
self.swirl.execedFiles.append(swirlFile)
for dynamicDepFile in dynamicDependecies[fileName]:
newSwirlFileDependency = PluginManager.getSwirl(
dynamicDepFile, self.swirl, swirlFile.env)
#I need to verify it if is static dep or dynamic dep
#TODO need to optimize this
swirlDependencies = self.swirl.getListSwirlFilesDependentStatic(
swirlFile)
if newSwirlFileDependency.path not in [
x.path for x in swirlDependencies
]:
swirlFile.dynamicDependencies.append(
newSwirlFileDependency)
# let's see if it used some Data files
# files is a double dictionary see TraceContrlBlock for its structure
files = FingerPrint.syscalltracer.TracerControlBlock.files
# excludeFileName: file whcih should be ingored and not added to the swirl
excludeFileName = ['/etc/ld.so.cache']
for lib_swFile in files:
swirlFile = PluginManager.getSwirl(lib_swFile, self.swirl)
if swirlFile.isLoader():
continue
all_dependencies_files = []
#TODO take this for loop out of the for lib_swFile loop
for deps in self.swirl.getListSwirlFilesDependentStaticAndDynamic(
swirlFile):
all_dependencies_files += deps.getPaths()
for execFile in files[lib_swFile]:
for openedFile in files[lib_swFile][execFile]:
# we need to remove some useless files from the opened list
# if not depenedencies will be listed as opned files
if openedFile not in excludeFileName and not os.path.isdir(
openedFile):
swirlOpenedFile = PluginManager.getSwirl(
openedFile, self.swirl)
if swirlOpenedFile.path not in all_dependencies_files:
if execFile not in swirlFile.openedFiles:
swirlFile.openedFiles[execFile] = []
swirlFile.openedFiles[execFile].append(
swirlOpenedFile)
self.swirl.ldconf_paths = self._get_ldconf_paths()
# get hash and package name for each swirlFile
for swf in self.swirl.swirlFiles:
#let's skip relative path
if swf.path[0] != '$' and os.path.exists(swf.path):
swf.md5sum = sergeant.getHash(swf.path, swf.type)
#TODO make this code nicer
if sergeant.is_special_folder(swf.path):
swf.package = None
else:
swf.package = self._getPackage(swf.path)
def __init__(self, name, fileList, processIDs, execCmd):
"""give a file list and a name construct a swirl into memory """
self._detectedPackageManager()
self.swirl = Swirl(name, datetime.now())
if execCmd :
self.swirl.cmdLine = execCmd
#
# dependencies discovered with dinamic methods
# dynamicDependencies = { 'binarypath' : [list of file it depends to],
# '/bin/bash' : ['/lib/x86_64-linux-gnu/libnss_files-2.15.so',
# '/lib/x86_64-linux-gnu/libnss_nis-2.15.so']}
if execCmd :
self._straceCmd(execCmd)
# let's see if we have proecss ID we might need to scan for dynamic dependecies
# with the help of the /proc FS
elif processIDs :
for proc in processIDs.split(','):
pid = proc.strip()
# add the binary
tcb = FingerPrint.syscalltracer.TracerControlBlock(pid)
tcb.updateSharedLibraries()
dynamicDependecies = FingerPrint.syscalltracer.TracerControlBlock.dependencies
# set up the fileList for the static dependency detection
if not fileList :
fileList = []
# it does not work on python 2.5 aka RHEL 5.X
# fileList = [f if f[0] == '/' else os.path.normpath(os.getcwd() + '/' + f) for f in fileList]
def norm_path(path):
if path[0] == '/':
return os.path.normpath(path)
else:
return os.path.normpath(os.getcwd() + '/' + path)
fileList = [ norm_path(f) for f in fileList ]
fileList = fileList + dynamicDependecies.keys()
# add all the fileList to the swirl and figure out all their static libraries
for binPath in fileList:
if os.path.isfile(binPath):
cmd = binPath
if binPath in FingerPrint.syscalltracer.TracerControlBlock.cmdline:
# the user cmdline could be a symlink so we want to keep track
cmd = FingerPrint.syscalltracer.TracerControlBlock.cmdline[binPath][0]
cmd = utils.which(cmd)
if not cmd :
cmd = binPath
if cmd and cmd[0] != '/':
# hmm we have a relative path
pwd = FingerPrint.syscalltracer.TracerControlBlock.get_env_variable(binPath, "PWD")
pwd = pwd + '/' + cmd
if os.path.isfile(pwd) and os.access(pwd, os.X_OK):
cmd = pwd
else:
# TODO this way to resolving relative path is not 100% correct
# it should be done in the syscalltracer
cmd = binPath
# load the env
tmpenv = []
if binPath in FingerPrint.syscalltracer.TracerControlBlock.env:
for var in FingerPrint.syscalltracer.TracerControlBlock.env[binPath]:
if '=' in var and '=()' not in var:
tmpenv.append(var)
swirlFile = PluginManager.getSwirl(cmd, self.swirl, tmpenv)
self.swirl.execedFiles.append(swirlFile)
elif os.path.isdir(binPath):
pass
else:
raise IOError("The file %s cannot be opened." % binPath)
#
# we might need to add the dynamic dependencies to the swirl
# if they did not get detected already
for fileName in dynamicDependecies.keys():
swirlFile = PluginManager.getSwirl(fileName, self.swirl)
#let's add it to the execed file list
if swirlFile not in self.swirl.execedFiles:
# TODO remove me... this should never be called
self.swirl.execedFiles.append(swirlFile)
for dynamicDepFile in dynamicDependecies[fileName]:
newSwirlFileDependency = PluginManager.getSwirl(dynamicDepFile, self.swirl,
swirlFile.env)
#I need to verify it if is static dep or dynamic dep
#TODO need to optimize this
swirlDependencies = self.swirl.getListSwirlFilesDependentStatic( swirlFile )
if newSwirlFileDependency.path not in [x.path for x in swirlDependencies]:
swirlFile.dynamicDependencies.append(newSwirlFileDependency)
# let's see if it used some Data files
# files is a double dictionary see TraceContrlBlock for its structure
files = FingerPrint.syscalltracer.TracerControlBlock.files
# excludeFileName: file whcih should be ingored and not added to the swirl
excludeFileName = ['/etc/ld.so.cache']
for lib_swFile in files:
swirlFile = PluginManager.getSwirl(lib_swFile, self.swirl)
if swirlFile.isLoader() :
continue
all_dependencies_files=[]
#TODO take this for loop out of the for lib_swFile loop
for deps in self.swirl.getListSwirlFilesDependentStaticAndDynamic(swirlFile):
all_dependencies_files += deps.getPaths()
for execFile in files[lib_swFile]:
for openedFile in files[lib_swFile][execFile]:
# we need to remove some useless files from the opened list
# if not depenedencies will be listed as opned files
if openedFile not in excludeFileName and not os.path.isdir(openedFile):
swirlOpenedFile = PluginManager.getSwirl(openedFile, self.swirl)
if swirlOpenedFile.path not in all_dependencies_files:
if execFile not in swirlFile.openedFiles:
swirlFile.openedFiles[execFile] = []
swirlFile.openedFiles[execFile].append(swirlOpenedFile)
self.swirl.ldconf_paths = self._get_ldconf_paths()
# get hash and package name for each swirlFile
for swf in self.swirl.swirlFiles:
#let's skip relative path
if swf.path[0] != '$' and os.path.exists(swf.path):
swf.md5sum = sergeant.getHash(swf.path, swf.type)
#TODO make this code nicer
if sergeant.is_special_folder( swf.path ) :
swf.package = None
else:
swf.package = self._getPackage(swf.path)