def archive(self):
"""
It triggers the creation of the archive.
:rtype: bool
:return: it returns false in case of failure
"""
#we need a valid swirl
if not self.sergeant.check():
logger.error("The fingerprint " + self.sergeant.filename +
" fails:\n " +
"\n ".join(self.sergeant.getError()) +
"\n\nThe archive creation failed.\n")
return False
# prepare the folders for the tar
base_tar = tempfile.mkdtemp()
base_path = os.path.join(base_tar, def_base_dir)
os.mkdir(base_path)
# copy all the files referenced by this swirl
for swf in self.sergeant.swirl.swirlFiles:
if swf.path[0] == '$' or sergeant.is_special_folder(swf.path) or \
is_special_file(swf.path):
#TODO maybe we could keep user data into a special folder?
# this file belongs to the special folders let's skip it
continue
if os.path.exists(swf.path) and swf.md5sum:
# the file was not a temporary file
dest_path_dir = os.path.join(base_path, swf.md5sum)
dest_path_full = os.path.join(dest_path_dir,
os.path.basename(swf.path))
if not os.path.exists(dest_path_full):
# do not copy twice the same file
if not os.path.exists(dest_path_dir):
os.mkdir(dest_path_dir)
shutil.copy2(swf.path, dest_path_dir)
if sergeant.prelink:
utils.getOutputAsList(
[sergeant.prelink, "-u", dest_path_full])
#for i in swf.links:
# new_link = os.path.join(temp_path, os.path.basename(i))
# if not os.path.exists( new_link ):
# os.symlink( os.path.basename(swf.path), new_link)
# copy the swirl itself
shutil.copy2(self.sergeant.filename,
os.path.join(base_tar, def_swirl_path))
# let's do the tar
tar = tarfile.open(self.archive_filename, "w:gz")
cwd = os.getcwd()
os.chdir(base_tar)
tar.add(".")
tar.close()
os.chdir(cwd)
shutil.rmtree(base_tar)
return True
def archive(self):
"""
It triggers the creation of the archive.
:rtype: bool
:return: it returns false in case of failure
"""
#we need a valid swirl
if not self.sergeant.check():
logger.error("The fingerprint " + self.sergeant.filename + " fails:\n " +
"\n ".join(self.sergeant.getError()) + "\n\nThe archive creation failed.\n")
return False
# prepare the folders for the tar
base_tar = tempfile.mkdtemp()
base_path = os.path.join(base_tar, def_base_dir)
os.mkdir(base_path)
# copy all the files referenced by this swirl
for swf in self.sergeant.swirl.swirlFiles:
if swf.path[0] == '$' or sergeant.is_special_folder(swf.path) or \
is_special_file(swf.path):
#TODO maybe we could keep user data into a special folder?
# this file belongs to the special folders let's skip it
continue
if os.path.exists(swf.path) and swf.md5sum:
# the file was not a temporary file
dest_path_dir = os.path.join(base_path, swf.md5sum)
dest_path_full = os.path.join(dest_path_dir, os.path.basename(swf.path))
if not os.path.exists(dest_path_full):
# do not copy twice the same file
if not os.path.exists(dest_path_dir):
os.mkdir(dest_path_dir)
shutil.copy2(swf.path, dest_path_dir)
if sergeant.prelink :
utils.getOutputAsList([sergeant.prelink, "-u", dest_path_full])
#for i in swf.links:
# new_link = os.path.join(temp_path, os.path.basename(i))
# if not os.path.exists( new_link ):
# os.symlink( os.path.basename(swf.path), new_link)
# copy the swirl itself
shutil.copy2(self.sergeant.filename, os.path.join(base_tar, def_swirl_path))
# let's do the tar
tar = tarfile.open(self.archive_filename, "w:gz")
cwd = os.getcwd()
os.chdir(base_tar)
tar.add(".")
tar.close()
os.chdir(cwd)
shutil.rmtree(base_tar)
return True
def _resolve_file(self, swirl_file, use_remapping = False):
""" this function recursively try to resolve the swirlFile
this function will add the package name to self.packages if it can find
an rpm which can sattisfy it, if not, it will add this swirlf_file to the
self.files """
if swirl_file in self.files or swirl_file in self.processed_package:
return
self.processed_package.append(swirl_file)
# if swirl_file.path in yum db add rpm to self.packages
# else add swirl_file to self.files
packages = []
if 'ELF' in swirl_file.type and swirl_file.executable \
and not (use_remapping and swirl_file.isLoader()):
# executable and not a loader if we are using remppaing
packages = self._get_package_from_dep([swirl_file.path])
elif 'ELF' in swirl_file.type and not swirl_file.executable and not use_remapping:
# library
# do not process it if we are using remapping
packages = self._get_package_from_dep(swirl_file.getPaths(), False)
elif swirl_file.path[0] == '$' or sergeant.is_special_folder(swirl_file.path) \
or is_special_file(swirl_file.path):
# this file belongs to the special folders or it's a relative path
return
elif 'ELF' not in swirl_file.type:
# data
# TODO what do we do with this when we use remapping?
packages = self._get_package_from_dep([swirl_file.path])
if packages :
if len(packages) > 1 :
error_message = "The file " + swirl_file.path + " "
error_message += "is provided by more than one RPM: " + ", ".join(packages)
error_message += "\nAdding " + packages[0]
logger.error(error_message)
if swirl_file.package not in self.wanted_pcks and \
packages[0] not in self._excluded_packages:
self.skipped_swfs.add( swirl_file )
self.packages.add( packages[0] )
# so we found a package which can handle this swf but we should still process its
# opened file in case it is an interpreter
self._process_open_file(swirl_file, use_remapping)
logger.debug("Adding package " + packages[0] + " for swirl " + swirl_file.path)
return
else:
self.disable_pcks |= set(packages)
logger.debug("Adding swirl: " + swirl_file.path)
if 'ELF' in swirl_file.type and not use_remapping:
# for ELF swf if we select the libmpi.so.2 we also want to carry all its dynamic libraries
# even if their name matches an available package for this reason we use wanted_pcks
self.wanted_pcks.add(swirl_file.package)
self.files.append(swirl_file)
#
# for each swf in swirlFile.all_dependencies
# if not already in self.files _resolve_file(swf)
deps = self.swirl.getListSwirlFileProvide( swirl_file.staticDependencies )+\
swirl_file.dynamicDependencies
for new_swf in deps:
if new_swf not in self.files:
#this file is already in the included files
self._resolve_file(new_swf, use_remapping)
self._process_open_file(swirl_file, use_remapping)
def _resolve_file(self, swirl_file, use_remapping=False):
""" this function recursively try to resolve the swirlFile
this function will add the package name to self.packages if it can find
an rpm which can sattisfy it, if not, it will add this swirlf_file to the
self.files """
if swirl_file in self.files or swirl_file in self.processed_package:
return
self.processed_package.append(swirl_file)
# if swirl_file.path in yum db add rpm to self.packages
# else add swirl_file to self.files
packages = []
if 'ELF' in swirl_file.type and swirl_file.executable \
and not (use_remapping and swirl_file.isLoader()):
# executable and not a loader if we are using remppaing
packages = self._get_package_from_dep([swirl_file.path])
elif 'ELF' in swirl_file.type and not swirl_file.executable and not use_remapping:
# library
# do not process it if we are using remapping
packages = self._get_package_from_dep(swirl_file.getPaths(), False)
elif swirl_file.path[0] == '$' or sergeant.is_special_folder(swirl_file.path) \
or is_special_file(swirl_file.path):
# this file belongs to the special folders or it's a relative path
return
elif 'ELF' not in swirl_file.type:
# data
# TODO what do we do with this when we use remapping?
packages = self._get_package_from_dep([swirl_file.path])
if packages:
if len(packages) > 1:
error_message = "The file " + swirl_file.path + " "
error_message += "is provided by more than one RPM: " + ", ".join(
packages)
error_message += "\nAdding " + packages[0]
logger.error(error_message)
if swirl_file.package not in self.wanted_pcks and \
packages[0] not in self._excluded_packages:
self.skipped_swfs.add(swirl_file)
self.packages.add(packages[0])
# so we found a package which can handle this swf but we should still process its
# opened file in case it is an interpreter
self._process_open_file(swirl_file, use_remapping)
logger.debug("Adding package " + packages[0] + " for swirl " +
swirl_file.path)
return
else:
self.disable_pcks |= set(packages)
logger.debug("Adding swirl: " + swirl_file.path)
if 'ELF' in swirl_file.type and not use_remapping:
# for ELF swf if we select the libmpi.so.2 we also want to carry all its dynamic libraries
# even if their name matches an available package for this reason we use wanted_pcks
self.wanted_pcks.add(swirl_file.package)
self.files.append(swirl_file)
#
# for each swf in swirlFile.all_dependencies
# if not already in self.files _resolve_file(swf)
deps = self.swirl.getListSwirlFileProvide( swirl_file.staticDependencies )+\
swirl_file.dynamicDependencies
for new_swf in deps:
if new_swf not in self.files:
#this file is already in the included files
self._resolve_file(new_swf, use_remapping)
self._process_open_file(swirl_file, use_remapping)
def __init__(self, name, fileList, processIDs, execCmd):
"""give a file list and a name construct a swirl into memory """
self._detectedPackageManager()
self.swirl = Swirl(name, datetime.now())
if execCmd:
self.swirl.cmdLine = execCmd
#
# dependencies discovered with dinamic methods
# dynamicDependencies = { 'binarypath' : [list of file it depends to],
# '/bin/bash' : ['/lib/x86_64-linux-gnu/libnss_files-2.15.so',
# '/lib/x86_64-linux-gnu/libnss_nis-2.15.so']}
if execCmd:
self._straceCmd(execCmd)
# let's see if we have proecss ID we might need to scan for dynamic dependecies
# with the help of the /proc FS
elif processIDs:
for proc in processIDs.split(','):
pid = proc.strip()
# add the binary
tcb = FingerPrint.syscalltracer.TracerControlBlock(pid)
tcb.updateSharedLibraries()
dynamicDependecies = FingerPrint.syscalltracer.TracerControlBlock.dependencies
# set up the fileList for the static dependency detection
if not fileList:
fileList = []
# it does not work on python 2.5 aka RHEL 5.X
# fileList = [f if f[0] == '/' else os.path.normpath(os.getcwd() + '/' + f) for f in fileList]
def norm_path(path):
if path[0] == '/':
return os.path.normpath(path)
else:
return os.path.normpath(os.getcwd() + '/' + path)
fileList = [norm_path(f) for f in fileList]
fileList = fileList + dynamicDependecies.keys()
# add all the fileList to the swirl and figure out all their static libraries
for binPath in fileList:
if os.path.isfile(binPath):
cmd = binPath
if binPath in FingerPrint.syscalltracer.TracerControlBlock.cmdline:
# the user cmdline could be a symlink so we want to keep track
cmd = FingerPrint.syscalltracer.TracerControlBlock.cmdline[
binPath][0]
cmd = utils.which(cmd)
if not cmd:
cmd = binPath
if cmd and cmd[0] != '/':
# hmm we have a relative path
pwd = FingerPrint.syscalltracer.TracerControlBlock.get_env_variable(
binPath, "PWD")
pwd = pwd + '/' + cmd
if os.path.isfile(pwd) and os.access(pwd, os.X_OK):
cmd = pwd
else:
# TODO this way to resolving relative path is not 100% correct
# it should be done in the syscalltracer
cmd = binPath
# load the env
tmpenv = []
if binPath in FingerPrint.syscalltracer.TracerControlBlock.env:
for var in FingerPrint.syscalltracer.TracerControlBlock.env[
binPath]:
if '=' in var and '=()' not in var:
tmpenv.append(var)
swirlFile = PluginManager.getSwirl(cmd, self.swirl, tmpenv)
self.swirl.execedFiles.append(swirlFile)
elif os.path.isdir(binPath):
pass
else:
raise IOError("The file %s cannot be opened." % binPath)
#
# we might need to add the dynamic dependencies to the swirl
# if they did not get detected already
for fileName in dynamicDependecies.keys():
swirlFile = PluginManager.getSwirl(fileName, self.swirl)
#let's add it to the execed file list
if swirlFile not in self.swirl.execedFiles:
# TODO remove me... this should never be called
self.swirl.execedFiles.append(swirlFile)
for dynamicDepFile in dynamicDependecies[fileName]:
newSwirlFileDependency = PluginManager.getSwirl(
dynamicDepFile, self.swirl, swirlFile.env)
#I need to verify it if is static dep or dynamic dep
#TODO need to optimize this
swirlDependencies = self.swirl.getListSwirlFilesDependentStatic(
swirlFile)
if newSwirlFileDependency.path not in [
x.path for x in swirlDependencies
]:
swirlFile.dynamicDependencies.append(
newSwirlFileDependency)
# let's see if it used some Data files
# files is a double dictionary see TraceContrlBlock for its structure
files = FingerPrint.syscalltracer.TracerControlBlock.files
# excludeFileName: file whcih should be ingored and not added to the swirl
excludeFileName = ['/etc/ld.so.cache']
for lib_swFile in files:
swirlFile = PluginManager.getSwirl(lib_swFile, self.swirl)
if swirlFile.isLoader():
continue
all_dependencies_files = []
#TODO take this for loop out of the for lib_swFile loop
for deps in self.swirl.getListSwirlFilesDependentStaticAndDynamic(
swirlFile):
all_dependencies_files += deps.getPaths()
for execFile in files[lib_swFile]:
for openedFile in files[lib_swFile][execFile]:
# we need to remove some useless files from the opened list
# if not depenedencies will be listed as opned files
if openedFile not in excludeFileName and not os.path.isdir(
openedFile):
swirlOpenedFile = PluginManager.getSwirl(
openedFile, self.swirl)
if swirlOpenedFile.path not in all_dependencies_files:
if execFile not in swirlFile.openedFiles:
swirlFile.openedFiles[execFile] = []
swirlFile.openedFiles[execFile].append(
swirlOpenedFile)
self.swirl.ldconf_paths = self._get_ldconf_paths()
# get hash and package name for each swirlFile
for swf in self.swirl.swirlFiles:
#let's skip relative path
if swf.path[0] != '$' and os.path.exists(swf.path):
swf.md5sum = sergeant.getHash(swf.path, swf.type)
#TODO make this code nicer
if sergeant.is_special_folder(swf.path):
swf.package = None
else:
swf.package = self._getPackage(swf.path)
def __init__(self, name, fileList, processIDs, execCmd):
"""give a file list and a name construct a swirl into memory """
self._detectedPackageManager()
self.swirl = Swirl(name, datetime.now())
if execCmd :
self.swirl.cmdLine = execCmd
#
# dependencies discovered with dinamic methods
# dynamicDependencies = { 'binarypath' : [list of file it depends to],
# '/bin/bash' : ['/lib/x86_64-linux-gnu/libnss_files-2.15.so',
# '/lib/x86_64-linux-gnu/libnss_nis-2.15.so']}
if execCmd :
self._straceCmd(execCmd)
# let's see if we have proecss ID we might need to scan for dynamic dependecies
# with the help of the /proc FS
elif processIDs :
for proc in processIDs.split(','):
pid = proc.strip()
# add the binary
tcb = FingerPrint.syscalltracer.TracerControlBlock(pid)
tcb.updateSharedLibraries()
dynamicDependecies = FingerPrint.syscalltracer.TracerControlBlock.dependencies
# set up the fileList for the static dependency detection
if not fileList :
fileList = []
# it does not work on python 2.5 aka RHEL 5.X
# fileList = [f if f[0] == '/' else os.path.normpath(os.getcwd() + '/' + f) for f in fileList]
def norm_path(path):
if path[0] == '/':
return os.path.normpath(path)
else:
return os.path.normpath(os.getcwd() + '/' + path)
fileList = [ norm_path(f) for f in fileList ]
fileList = fileList + dynamicDependecies.keys()
# add all the fileList to the swirl and figure out all their static libraries
for binPath in fileList:
if os.path.isfile(binPath):
cmd = binPath
if binPath in FingerPrint.syscalltracer.TracerControlBlock.cmdline:
# the user cmdline could be a symlink so we want to keep track
cmd = FingerPrint.syscalltracer.TracerControlBlock.cmdline[binPath][0]
cmd = utils.which(cmd)
if not cmd :
cmd = binPath
if cmd and cmd[0] != '/':
# hmm we have a relative path
pwd = FingerPrint.syscalltracer.TracerControlBlock.get_env_variable(binPath, "PWD")
pwd = pwd + '/' + cmd
if os.path.isfile(pwd) and os.access(pwd, os.X_OK):
cmd = pwd
else:
# TODO this way to resolving relative path is not 100% correct
# it should be done in the syscalltracer
cmd = binPath
# load the env
tmpenv = []
if binPath in FingerPrint.syscalltracer.TracerControlBlock.env:
for var in FingerPrint.syscalltracer.TracerControlBlock.env[binPath]:
if '=' in var and '=()' not in var:
tmpenv.append(var)
swirlFile = PluginManager.getSwirl(cmd, self.swirl, tmpenv)
self.swirl.execedFiles.append(swirlFile)
elif os.path.isdir(binPath):
pass
else:
raise IOError("The file %s cannot be opened." % binPath)
#
# we might need to add the dynamic dependencies to the swirl
# if they did not get detected already
for fileName in dynamicDependecies.keys():
swirlFile = PluginManager.getSwirl(fileName, self.swirl)
#let's add it to the execed file list
if swirlFile not in self.swirl.execedFiles:
# TODO remove me... this should never be called
self.swirl.execedFiles.append(swirlFile)
for dynamicDepFile in dynamicDependecies[fileName]:
newSwirlFileDependency = PluginManager.getSwirl(dynamicDepFile, self.swirl,
swirlFile.env)
#I need to verify it if is static dep or dynamic dep
#TODO need to optimize this
swirlDependencies = self.swirl.getListSwirlFilesDependentStatic( swirlFile )
if newSwirlFileDependency.path not in [x.path for x in swirlDependencies]:
swirlFile.dynamicDependencies.append(newSwirlFileDependency)
# let's see if it used some Data files
# files is a double dictionary see TraceContrlBlock for its structure
files = FingerPrint.syscalltracer.TracerControlBlock.files
# excludeFileName: file whcih should be ingored and not added to the swirl
excludeFileName = ['/etc/ld.so.cache']
for lib_swFile in files:
swirlFile = PluginManager.getSwirl(lib_swFile, self.swirl)
if swirlFile.isLoader() :
continue
all_dependencies_files=[]
#TODO take this for loop out of the for lib_swFile loop
for deps in self.swirl.getListSwirlFilesDependentStaticAndDynamic(swirlFile):
all_dependencies_files += deps.getPaths()
for execFile in files[lib_swFile]:
for openedFile in files[lib_swFile][execFile]:
# we need to remove some useless files from the opened list
# if not depenedencies will be listed as opned files
if openedFile not in excludeFileName and not os.path.isdir(openedFile):
swirlOpenedFile = PluginManager.getSwirl(openedFile, self.swirl)
if swirlOpenedFile.path not in all_dependencies_files:
if execFile not in swirlFile.openedFiles:
swirlFile.openedFiles[execFile] = []
swirlFile.openedFiles[execFile].append(swirlOpenedFile)
self.swirl.ldconf_paths = self._get_ldconf_paths()
# get hash and package name for each swirlFile
for swf in self.swirl.swirlFiles:
#let's skip relative path
if swf.path[0] != '$' and os.path.exists(swf.path):
swf.md5sum = sergeant.getHash(swf.path, swf.type)
#TODO make this code nicer
if sergeant.is_special_folder( swf.path ) :
swf.package = None
else:
swf.package = self._getPackage(swf.path)
