def GetDefinedSymbols(self):
"""Returns text symbols (i.e. defined functions) for packaged ELF objects
To do this we parse output lines from nm similar to the following. "T"s are
the definitions which we are after.
0000104000 D _lib_version
0000986980 D _libiconv_version
0000000000 U abort
0000097616 T aliases_lookup
"""
binaries = self.ListBinaries()
defined_symbols = {}
for binary in binaries:
binary_abspath = os.path.join(self.directory, self.GetFilesDir(), binary)
# Get parsable, ld.so.1 relevant SHT_DYNSYM symbol information
args = ["/usr/ccs/bin/nm", "-p", "-D", binary_abspath]
retcode, stdout, stderr = shell.ShellCommand(args)
if retcode:
logging.error("%s returned an error: %s", args, stderr)
# Should it just skip over an error?
continue
nm_out = stdout.splitlines()
defined_symbols[binary] = []
for line in nm_out:
sym = self._ParseNmSymLine(line)
if not sym:
continue
if sym['type'] not in ("T", "D", "B"):
continue
defined_symbols[binary].append(sym['name'])
return defined_symbols
def _GetIpsPkgcontentStream(self):
args = [
"pkg", "contents", "-H", "-o",
"path,action.name,pkg.name,target,mode,owner,group", "-t",
"dir,file,hardlink,link"
]
ret, stdout, unused_stderr = shell.ShellCommand(args)
return stdout.splitlines()
def _GetUname(self, uname_option=None):
args = ["uname"]
if uname_option:
args.append(uname_option)
# TODO: Don't fork during unit tests
ret, stdout, unused_stderr = shell.ShellCommand(args)
if ret:
raise SubprocessError("Running uname has failed.")
return stdout.strip()
def _GetSrv4PkginfosStream(self):
"""Calls pkginfo if file is not specified."""
if self.infile_pkginfo:
pkginfo_stream = open(self.infile_pkginfo, "r")
else:
args = ["pkginfo"]
ret, stdout, stderr = shell.ShellCommand(args)
pkginfo_stream = stdout.splitlines()
return pkginfo_stream
def GetPkgname(self):
"""It's necessary to figure out the pkgname from the .pkg file.
# nawk 'NR == 2 {print $1; exit;} $f
"""
if not self.pkgname:
gunzipped_path = self.GetGunzippedPath()
args = ["nawk", "NR == 2 {print $1; exit;}", gunzipped_path]
ret_code, stdout, stderr = shell.ShellCommand(args)
self.pkgname = stdout.strip()
logging.debug("GetPkgname(): %s", repr(self.pkgname))
return self.pkgname
def GetLddMinusRlines(self):
"""Returns ldd -r output."""
binaries = self.ListBinaries()
base_dir = self.GetBasedir()
ldd_output = {}
for binary in binaries:
binary_abspath = os.path.join(self.directory, self.GetFilesDir(), binary)
if base_dir:
binary = os.path.join(base_dir, binary)
# this could be potentially moved into the DirectoryFormatPackage class.
# ldd needs the binary to be executable
os.chmod(binary_abspath, 0755)
args = ["ldd", "-Ur", binary_abspath]
# ldd can be stuck while ran on a some binaries, so we define
# a timeout (problem encountered with uconv)
retcode, stdout, stderr = shell.ShellCommand(args, timeout=10)
if retcode:
# There three cases where we will ignore an ldd error
# - if we are trying to analyze a 64 bits binary on a Solaris 9 x86
# solaris 9 exists only in 32 bits, so we can't do this
# We ignore the error as it is likely that the ldd infos will be
# the same on the 32 bits binaries
# - if we are trying to analyze a binary from another architecture
# we ignore this error as it will be caught by another checkpkg test
# - if we are trying to analyze a statically linked binaries
# we care only about dynamic binary so we ignore the error
#
uname_info = os.uname()
if ((uname_info[2] == '5.9' and uname_info[4] == 'i86pc' and
'/amd64/' in binary_abspath and
'has wrong class or data encoding' in stderr) or
re.search(r'ELF machine type: EM_\w+: '
r'is incompatible with system', stderr)
or 'file is not a dynamic executable or shared object' in stderr):
ldd_output[binary] = []
continue
raise package.SystemUtilityError("%s returned an error: %s" % (args, stderr))
ldd_info = []
for line in stdout.splitlines():
result = self._ParseLddDashRline(line, binary_abspath)
if result:
ldd_info.append(result)
ldd_output[binary] = ldd_info
return ldd_output
def SetPkginfoEntry(self, key, value):
pkginfo = self.GetParsedPkginfo()
logging.debug("Setting %s to %s", repr(key), repr(value))
pkginfo[key] = value
self.WritePkginfo(pkginfo)
pkgmap_path = os.path.join(self.directory, "pkgmap")
pkgmap_fd = open(pkgmap_path, "r")
new_pkgmap_lines = []
pkginfo_re = re.compile("1 i pkginfo")
ws_re = re.compile(r"\s+")
for line in pkgmap_fd:
if pkginfo_re.search(line):
fields = ws_re.split(line)
# 3: size
# 4: sum
pkginfo_path = os.path.join(self.directory, "pkginfo")
args = ["cksum", pkginfo_path]
_, stdout, stderr = shell.ShellCommand(args)
size = ws_re.split(stdout)[1]
args = ["sum", pkginfo_path]
_, stdout, stderr = shell.ShellCommand(args)
sum_process.wait()
sum_value = ws_re.split(stdout)[0]
fields[3] = size
fields[4] = sum_value
logging.debug("New pkgmap line: %s", fields)
line = " ".join(fields)
new_pkgmap_lines.append(line.strip())
pkgmap_fd.close()
# Write that back
pkgmap_path_new = pkgmap_path + ".new"
logging.debug("Writing back to %s", pkgmap_path_new)
pkgmap_fd = open(pkgmap_path_new, "w")
pkgmap_fd.write("\n".join(new_pkgmap_lines))
pkgmap_fd.close()
shutil.move(pkgmap_path_new, pkgmap_path)
def Pkgtrans(self, src_file, destdir, pkgname):
"""A proxy for the pkgtrans command.
This requires custom-pkgtrans to be available.
"""
if not os.path.isdir(destdir):
raise PackageError("%s doesn't exist or is not a directory" %
destdir)
args = [
os.path.join(os.path.dirname(__file__), "custom-pkgtrans"),
src_file, destdir, pkgname
]
ret, stdout, stderr = shell.ShellCommand(args)
if ret:
logging.error(stdout)
logging.error(stderr)
logging.error("% has failed" % args)
def testGetLddMinusRlinesReloc(self):
ip = inspective_package.InspectivePackage("/tmp/CSWfake")
self.mox.StubOutWithMock(ip, 'GetBasedir')
self.mox.StubOutWithMock(ip, 'ListBinaries')
self.mox.StubOutWithMock(ip, 'GetFilesDir')
self.mox.StubOutWithMock(os, 'chmod')
self.mox.StubOutWithMock(os, 'uname')
ip.GetBasedir().AndReturn('opt/csw')
os.chmod('/tmp/CSWfake/reloc/bin/foo', 0755)
ip.ListBinaries().AndReturn(['bin/foo'])
ip.GetFilesDir().AndReturn('reloc')
self.mox.StubOutWithMock(shell, 'ShellCommand')
shell.ShellCommand(['ldd', '-Ur', '/tmp/CSWfake/reloc/bin/foo'],
timeout=10).AndReturn((0, "", ""))
self.mox.StubOutWithMock(ip, '_ParseLddDashRline')
self.mox.ReplayAll()
self.assertEqual({'opt/csw/bin/foo': []}, ip.GetLddMinusRlines())
def testGetLddMinusRlinesThrows(self):
ip = inspective_package.InspectivePackage("/tmp/CSWfake")
self.mox.StubOutWithMock(ip, 'GetBasedir')
self.mox.StubOutWithMock(ip, 'ListBinaries')
self.mox.StubOutWithMock(ip, 'GetFilesDir')
self.mox.StubOutWithMock(os, 'chmod')
self.mox.StubOutWithMock(os, 'uname')
ip.GetBasedir().AndReturn('/')
os.chmod('/tmp/CSWfake/root/opt/csw/bin/foo', 0755)
os.uname().AndReturn('i86pc')
ip.GetFilesDir().AndReturn('root')
ip.ListBinaries().AndReturn(['opt/csw/bin/foo'])
self.mox.StubOutWithMock(shell, 'ShellCommand')
shell.ShellCommand(['ldd', '-Ur', '/tmp/CSWfake/root/opt/csw/bin/foo'],
timeout=10).AndReturn((1, "", "boo"))
self.mox.StubOutWithMock(ip, '_ParseLddDashRline')
self.mox.ReplayAll()
self.assertRaises(package.SystemUtilityError, ip.GetLddMinusRlines)
def GetBinaryDumpInfo(self):
# Binaries. This could be split off to a separate function.
# man ld.so.1 for more info on this hack
env = copy.copy(os.environ)
env["LD_NOAUXFLTR"] = "1"
binaries_dump_info = []
basedir = self.GetBasedir()
for binary in self.ListBinaries():
binary_abs_path = os.path.join(self.directory, self.GetFilesDir(), binary)
if basedir:
binary = os.path.join(basedir, binary)
binary_base_name = os.path.basename(binary)
args = [common_constants.DUMP_BIN, "-Lv", binary_abs_path]
retcode, stdout, stderr = shell.ShellCommand(args, env)
binary_data = ldd_emul.ParseDumpOutput(stdout)
binary_data["path"] = binary
binary_data["base_name"] = binary_base_name
binaries_dump_info.append(binary_data)
return binaries_dump_info
def testGetBinaryElfInfoReloc(self):
fake_binary = 'lib/libssl.so.1.0.0'
fake_package_path = '/fake/path/CSWfoo'
ip = inspective_package.InspectivePackage(fake_package_path)
self.mox.StubOutWithMock(ip, 'ListBinaries')
self.mox.StubOutWithMock(ip, 'GetBasedir')
self.mox.StubOutWithMock(ip, 'GetFilesDir')
ip.ListBinaries().AndReturn([fake_binary])
ip.GetBasedir().AndReturn('opt/csw')
ip.GetFilesDir().AndReturn('reloc')
self.mox.StubOutWithMock(shell, 'ShellCommand')
args = [
common_constants.ELFDUMP_BIN, '-svy',
os.path.join(fake_package_path, "reloc", fake_binary)
]
shell.ShellCommand(args).AndReturn((0, ELFDUMP_OUTPUT, ""))
self.mox.ReplayAll()
self.assertEqual(BINARY_ELFINFO, ip.GetBinaryElfInfo())
def testGetBinaryDumpInfoReloc(self):
fake_binary = 'bin/foo'
fake_package_path = '/fake/path/CSWfoo'
ip = inspective_package.InspectivePackage(fake_package_path)
self.mox.StubOutWithMock(ip, 'ListBinaries')
self.mox.StubOutWithMock(ip, 'GetBasedir')
self.mox.StubOutWithMock(ip, 'GetFilesDir')
ip.ListBinaries().AndReturn([fake_binary])
ip.GetBasedir().AndReturn('opt/csw')
ip.GetFilesDir().AndReturn('reloc')
self.mox.StubOutWithMock(shell, 'ShellCommand')
args = [
common_constants.DUMP_BIN, '-Lv',
os.path.join(fake_package_path, "reloc", fake_binary)
]
shell.ShellCommand(args, mox.IgnoreArg()).AndReturn(
(0, DUMP_OUTPUT, ""))
self.mox.ReplayAll()
self.assertEqual([BINARY_DUMP_INFO], ip.GetBinaryDumpInfo())
def SyncFromCatalogTree(self, catrel, base_dir, force_unpack=False):
logging.debug("SyncFromCatalogTree(%s, %s, force_unpack=%s)",
repr(catrel), repr(base_dir), force_unpack)
if not os.path.isdir(base_dir):
raise UsageError("%s is not a diractory" % repr(base_dir))
if catrel not in common_constants.DEFAULT_CATALOG_RELEASES:
logging.warning(
"The catalog release %s is not one of the default releases.",
repr(catrel))
sqo_catrel = m.CatalogRelease.selectBy(name=catrel).getOne()
_, uname_stdout, _ = shell.ShellCommand(["uname", "-p"])
current_host_arch = uname_stdout.strip()
for osrel in common_constants.OS_RELS:
logging.info(" OS release: %s", repr(osrel))
sqo_osrel = m.OsRelease.selectBy(short_name=osrel).getOne()
for arch in common_constants.PHYSICAL_ARCHITECTURES:
if current_host_arch != arch:
logging.warning(
"Cannot process packages for achitecture %r "
"because we're currently running on architecture %r.",
arch, current_host_arch)
continue
logging.info(" Architecture: %s", repr(arch))
sqo_arch = m.Architecture.selectBy(name=arch).getOne()
catalog_file = self.ComposeCatalogFilePath(
base_dir, osrel, arch)
if not os.path.exists(catalog_file):
logging.warning("Could not find %s, skipping.",
repr(catalog_file))
continue
logging.info(" %s", catalog_file)
self.SyncFromCatalogFile(osrel,
arch,
catrel,
catalog_file,
force_unpack=force_unpack)
def GetPkgchkOutput(self):
"""Returns: (exit code, stdout, stderr)."""
args = ["/usr/sbin/pkgchk", "-d", self.GetGunzippedPath(), "all"]
return shell.ShellCommand(args)
def _GetIpsPkginfosStream(self):
args = ["pkg", "list", "-H", "-s"]
ret, stdout, stderr = shell.ShellCommand(args)
pkglist_stream = stdout.splitlines()
return pkglist_stream
def testGetBinaryElfInfoWithIgnoredErrors(self):
fake_binary = 'opt/csw/bin/foo'
fake_package_path = '/fake/path/CSWfoo'
fake_elfdump_output = '''
Version Needed Section: .SUNW_version
index file version
[2] libc.so.1 SUNW_1.1
Symbol Table Section: .dynsym
index value size type bind oth ver shndx name
[1] 0x00000000 0x00000000 FUNC GLOB D 2 UNDEF fopen64
Syminfo Section: .SUNW_syminfo
index flags bound to symbol
[1] DBL [1] libc.so.1 fopen64
'''
fake_elfdump_errors = '''
/opt/csw/bin/foo: .dynsym: index[26]: bad symbol entry: : invalid shndx: 26
/opt/csw/bin/foo: .dynsym: bad symbol entry: : invalid shndx: 23
/opt/csw/bin/foo: .dynsym: index[108]: suspicious local symbol entry: _END_: lies within global symbol range (index >= 27)
/opt/csw/bin/foo: .dynsym: index[4]: bad symbol entry: toto: section[24] size: 0: symbol (address 0x36b7fc, size 0x4) lies outside of containing section
/opt/csw/bin/foo: .dynsym: bad symbol entry: Xt_app_con: section[28] size: 0: is smaller than symbol size: 4
'''
fake_binary_elfinfo = {
'opt/csw/bin/foo': {
'symbol table': [
{
'shndx': 'UNDEF',
'soname': 'libc.so.1',
'bind': 'GLOB',
'symbol': 'fopen64',
'version': 'SUNW_1.1',
'flags': 'DBL',
'type': 'FUNC'
},
],
'version needed': [
{
'version': 'SUNW_1.1',
'soname': 'libc.so.1'
},
],
'version definition': [],
}
}
ip = inspective_package.InspectivePackage(fake_package_path)
self.mox.StubOutWithMock(ip, 'ListBinaries')
self.mox.StubOutWithMock(ip, 'GetBasedir')
self.mox.StubOutWithMock(ip, 'GetFilesDir')
ip.ListBinaries().AndReturn([fake_binary])
ip.GetBasedir().AndReturn('')
ip.GetFilesDir().AndReturn('root')
self.mox.StubOutWithMock(shell, 'ShellCommand')
args = [
common_constants.ELFDUMP_BIN, '-svy',
os.path.join(fake_package_path, "root", fake_binary)
]
shell.ShellCommand(args).AndReturn(
(0, fake_elfdump_output, fake_elfdump_errors))
self.mox.ReplayAll()
self.assertEqual(fake_binary_elfinfo, ip.GetBinaryElfInfo())
def GetBinaryElfInfo(self):
"""Returns various informations symbol and versions present in elf header
To do this we parse output lines from elfdump -syv, it's the
only command that will give us all informations we need on
symbols and versions.
We will analyse 3 sections:
- version section: contains soname needed, version interface required
for each soname, and version definition
- symbol table section: contains list of symbol and soname/version
interface providing it
- syminfo section: contains special linking flags for each symbol
"""
binaries = self.ListBinaries()
binaries_elf_info = {}
base_dir = self.GetBasedir()
for binary in binaries:
binary_abspath = os.path.join(self.directory, self.GetFilesDir(), binary)
if base_dir:
binary = os.path.join(base_dir, binary)
# elfdump is the only tool that give us all informations
args = [common_constants.ELFDUMP_BIN, "-svy", binary_abspath]
retcode, stdout, stderr = shell.ShellCommand(args)
if retcode or stderr:
# we ignore for now these elfdump errors which can be catched
# later by check functions,
ignored_error_re = re.compile(
r"""[^:]+:(\s\.((SUNW_l)?dynsym|symtab):\s
((index\[\d+\]:\s)?
(suspicious\s(local|global)\ssymbol\sentry:\s[^:]+:\slies
\swithin\s(local|global)\ssymbol\srange\s\(index\s[<>=]+\s\d+\)
|bad\ssymbol\sentry:\s[^:]+:\ssection\[\d+\]\ssize:\s0(x[0-9a-f]+)?
:\s(symbol\s\(address\s0x[0-9a-f]+,\ssize\s0x[0-9a-f]+\)
\slies\soutside\sof\scontaining\ssection
|is\ssmaller\sthan\ssymbol\ssize:\s\d+)
|bad\ssymbol\sentry:\s:\sinvalid\sshndx:\s\d+
|)
|invalid\ssh_link:\s0)
|\smemory\soverlap\sbetween\ssection\[\d+\]:\s[^:]+:\s
[0-9a-f]+:[0-9a-f]+\sand\ssection\[\d+\]:\s[^:]+:
\s[0-9a-f]+:[0-9a-f]+)
\n""",
re.VERBOSE)
stderr = re.sub(ignored_error_re, "", stderr)
if stderr:
with open("/tmp/elfdump_stdout.log", "w") as fd:
fd.write(stdout)
with open("/tmp/elfdump_stderr.log", "w") as fd:
fd.write(stderr)
msg = ("%s returned one or more errors: %s" % (args, stderr) +
"\n\n" +
"ERROR: elfdump invocation failed. Please copy this message " +
"and the above messages into your report and send " +
"as path of the error report. Logs are saved in " +
"/tmp/elfdump_std(out|err).log for your inspection.")
raise package.Error(msg)
elfdump_out = stdout.splitlines()
symbols = {}
binary_info = {'version definition': [],
'version needed': []}
cur_section = None
for line in elfdump_out:
try:
elf_info, cur_section = self._ParseElfdumpLine(line, cur_section)
except package.StdoutSyntaxError as e:
sys.stderr.write("elfdump out:\n")
sys.stderr.write(stdout)
raise
# header or blank line contains no information
if not elf_info:
continue
# symbol table and syminfo section store various informations
# about the same symbols, so we merge them in a dict
if cur_section in ('symbol table', 'syminfo'):
symbols.setdefault(elf_info['symbol'], {}).update(elf_info)
else:
binary_info[cur_section].append(elf_info)
# elfdump doesn't repeat the name of the soname in the version section
# if it's the same on two contiguous line, e.g.:
# libc.so.1 SUNW_1.1
# SUNWprivate_1.1
# so we have to make sure the information is present in each entry
for i, version in enumerate(binary_info['version needed'][1:]):
if not version['soname']:
version['soname'] = binary_info['version needed'][i]['soname']
# soname version needed are usually displayed sorted by index ...
# but that's not always the case :( so we have to reorder
# the list by index if they are present
if any ( v['index'] for v in binary_info['version needed'] ):
binary_info['version needed'].sort(key=lambda m: int(m['index']))
for version in binary_info['version needed']:
del version['index']
# if it exists, the first "version definition" entry is the base soname
# we don't need this information
if binary_info['version definition']:
binary_info['version definition'].pop(0)
binary_info['symbol table'] = symbols.values()
binary_info['symbol table'].sort(key=lambda m: m['symbol'])
# To not rely of the section order output of elfdump, we resolve
# symbol version informations here after having parsed all output
self._ResolveSymbolsVersionInfo(binary_info)
binaries_elf_info[binary] = binary_info
return binaries_elf_info
