def PackageInfo(jmgr, os_target, sql, args):
sql.connect_table(tables['package_info'])
sql.connect_table(tables['package_dependency'])
pkgname = args[0]
pkg_id = get_package_id(sql, pkgname)
pkg_info = os_target.get_package_info(pkgname)
pkg_deps = os_target.get_package_dependency(pkgname)
# Clean up records
sql.delete_record(tables['package_info'],
'pkg_id=\'' + Table.stringify(pkg_id) + '\'')
sql.delete_record(tables['package_dependency'],
'pkg_id=\'' + Table.stringify(pkg_id) + '\'')
pkg_info['pkg_id'] = pkg_id
sql.append_record(tables['package_info'], pkg_info)
for dep in pkg_deps:
dep_id = get_package_id(sql, dep)
values = dict()
values['pkg_id'] = pkg_id
values['dependency'] = dep_id
sql.append_record(tables['package_dependency'], values)
sql.commit()
def BinarySymbol(jmgr, os_target, sql, args):
sql.connect_table(tables['binary_list'])
sql.connect_table(tables['binary_symbol'])
pkgname = args[0]
bin = args[1]
dir = args[2]
if len(args) > 3:
ref = args[3]
if not package.reference_exists(dir, ref):
dir = None
ref = None
else:
ref = None
unpacked = False
if not dir:
(dir, pkgname, _) = package.unpack_package(os_target, args[0])
if not dir:
return
unpacked = True
exc = None
try:
path = dir + '/' + bin
if not os.path.exists(path):
raise Exception('path ' + path + ' does not exist')
symbols = os_target.get_binary_symbols(dir, bin)
pkg_id = get_package_id(sql, pkgname)
bin_id = get_binary_id(sql, bin)
condition = 'pkg_id=' + Table.stringify(
pkg_id) + ' and bin_id=' + Table.stringify(bin_id)
sql.delete_record(tables['binary_symbol'], condition)
for sym in symbols:
values = dict()
values['pkg_id'] = pkg_id
values['bin_id'] = bin_id
values['symbol_name'] = sym.name
values['version'] = sym.version
values['func_addr'] = sym.addr
sql.append_record(tables['binary_symbol'], values)
sql.update_record(tables['binary_list'], {'callgraph': False},
condition)
sql.commit()
except Exception as err:
exc = sys.exc_info()
if (ref and package.dereference_dir(dir, ref)) or unpacked:
package.remove_dir(dir)
if exc:
raise exc[1], None, exc[2]
def analysis_binary_instr_linear(sql, binary, pkg_id, bin_id):
condition = 'pkg_id=' + Table.stringify(
pkg_id) + ' and bin_id=' + Table.stringify(bin_id)
condition_unknown = condition + ' and known=False'
sql.delete_record(tables['binary_call'], condition)
sql.delete_record(tables['binary_call_unknown'], condition_unknown)
sql.delete_record(tables['binary_opcode_usage'], condition)
sql.delete_record(tables['binary_call_missrate'], condition)
get_callgraph(binary, False, True, False, sql, pkg_id, bin_id)
def get_packages_by_ranks(os_target, sql, min, max):
sql.connect_table(tables['package_popularity'])
packages = os_target.get_packages()
packages_by_ranks = sql.search_record(
tables['package_popularity'], 'rank >= ' + Table.stringify(min) +
' AND rank <= ' + Table.stringify(max), ['package_name'])
result = []
for (name, ) in packages_by_ranks:
if name in packages:
result.append(name)
return result
def analysis_binary_instr(sql, binary, pkg_id, bin_id):
codes = get_callgraph(binary)
condition = 'pkg_id=' + Table.stringify(
pkg_id) + ' and bin_id=' + Table.stringify(bin_id)
condition_unknown = condition + ' and known=False'
sql.delete_record(tables['binary_call'], condition)
sql.delete_record(tables['binary_call_unknown'], condition_unknown)
sql.delete_record(tables['binary_instr_usage'], condition)
for func in codes.funcs:
instrs = dict()
calls = []
for bb in func.bblocks:
for instr in bb.instrs:
if isinstance(instr, InstrCall):
if isinstance(instr.target, int) or isinstance(
instr.target, long):
if not instr.target in calls:
calls.append(instr.target)
elif isinstance(instr.target, Op) and instr.target.val:
if not instr.target.val in calls:
calls.append(instr.target.val)
instr_name = instr.get_instr()
if instr_name in instrs:
instrs[instr_name] = instrs[instr_name] + 1
else:
instrs[instr_name] = 1
for call in calls:
values = dict()
values['pkg_id'] = pkg_id
values['bin_id'] = bin_id
values['func_addr'] = func.entry
if isinstance(call, int) or isinstance(call, long):
values['call_addr'] = call
else:
for addr, sym in codes.dynsyms.items():
if sym.name == call:
values['call_addr'] = sym.value
break
values['call_name'] = call
sql.append_record(tables['binary_call'], values)
for instr, count in instrs.items():
values = dict()
values['pkg_id'] = pkg_id
values['bin_id'] = bin_id
values['func_addr'] = func.entry
values['instr'] = instr
values['count'] = count
sql.append_record(tables['binary_instr_usage'], values)
def BinaryInstr(jmgr, os_target, sql, args):
sql.connect_table(tables['binary_call'])
sql.connect_table(tables['binary_call_unknown'])
sql.connect_table(tables['binary_opcode_usage'])
sql.connect_table(tables['binary_call_missrate'])
# sql.connect_table(tables['instr_list'])
# sql.connect_table(tables['prefix_counts'])
pkgname = args[0]
bin = args[1]
dir = args[2]
pkg_id = get_package_id(sql, pkgname)
if len(args) > 3:
ref = args[3]
if not package.reference_exists(dir, ref):
dir = None
ref = None
else:
ref = None
unpacked = False
if not dir:
(dir, pkgname, _) = package.unpack_package(os_target, args[0])
if not dir:
return
unpacked = True
exc = None
try:
if not os.path.exists(dir + bin):
raise Exception('path ' + dir + bin + ' does not exist')
bin_id = get_binary_id(sql, bin)
os_target.analysis_binary_instr_linear(sql, dir, bin, pkg_id, bin_id)
condition = 'pkg_id=' + Table.stringify(
pkg_id) + ' and bin_id=' + Table.stringify(bin_id)
sql.update_record(tables['binary_list'], {'callgraph': False},
condition)
sql.commit()
except Exception as err:
exc = sys.exc_info()
if (ref and package.dereference_dir(dir, ref)) or unpacked:
package.remove_dir(dir)
if exc:
raise exc[1], None, exc[2]
def get_package_name(sql, id):
sql.connect_table(tables['package_id'])
res = sql.search_record(tables['package_id'], 'id=' + Table.stringify(id),
['package_name'])
if len(res) > 0 and res[0][0]:
return res[0][0]
return None
def get_package_id(sql, pkgname):
sql.connect_table(tables['package_id'])
retry = True
while retry:
res = sql.search_record(tables['package_id'],
'package_name=' + Table.stringify(pkgname),
['id'])
if len(res) > 0 and res[0][0]:
id = res[0][0]
break
res = sql.search_record(tables['package_id'], None, ['MAX(id)'])
if res[0][0] == None:
id = 1
else:
id = int(res[0][0]) + 1
values = dict()
values['id'] = id
values['package_name'] = pkgname
retry = False
try:
sql.append_record(tables['package_id'], values)
sql.commit()
except:
retry = True
pass
return id
def get_binary_id(sql, binary_name):
sql.connect_table(tables['binary_id'])
retry = True
while retry:
res = sql.search_record(tables['binary_id'],
'binary_name=' + Table.stringify(binary_name),
['id'])
if len(res) > 0 and res[0][0]:
id = res[0][0]
break
res = sql.search_record(tables['binary_id'], None, ['MAX(id)'])
if res[0][0] == None:
id = 1
else:
id = int(res[0][0]) + 1
values = dict()
values['id'] = id
values['binary_name'] = binary_name
values['file_name'] = os.path.basename(binary_name)
retry = False
try:
sql.append_record(tables['binary_id'], values)
sql.commit()
except:
retry = True
pass
return id
def append_api_list(sql, type, id, name):
sql.connect_table(tables['api_list'])
retry = True
while retry:
res = sql.search_record(tables['api_list'], 'type=' + Table.stringify(type) + ' and id=' + Table.stringify(id), ['name'])
if len(res) > 0 and res[0][0]:
if res[0][0] != name:
raise Error('duplicate key value (' + Table.stringify(type) + ',' + Table.stringify(id) + ')')
return
values = dict()
values['type'] = type
values['id'] = id
values['name'] = name
retry = False
try:
sql.append_record(tables['api_list'], values)
sql.commit()
except:
retry = True
pass
def append_binary_list(sql, pkgname, dir, binaries):
sql.connect_table(tables['binary_list'])
sql.connect_table(tables['binary_link'])
sql.connect_table(tables['binary_interp'])
pkg_id = get_package_id(sql, pkgname)
insert_values = []
for (bin, type, interpreter) in binaries:
bin_id = get_binary_id(sql, bin)
values = dict()
values['type'] = type
if type == 'lnk':
link = os.readlink(dir + bin)
target = os.path.join(os.path.dirname(bin), link)
target_id = get_binary_id(sql, target)
values['pkg_id'] = pkg_id
values['lnk_id'] = bin_id
values['target'] = target_id
values['linking'] = False
else:
values['pkg_id'] = pkg_id
values['bin_id'] = bin_id
if type == 'scr':
interp_id = get_binary_id(sql, interpreter)
values['interp'] = interp_id
values['callgraph'] = True
else:
values['callgraph'] = False
values['linking'] = False
insert_values.append(values)
condition = 'pkg_id=' + Table.stringify(pkg_id)
sql.delete_record(tables['binary_list'], condition)
sql.delete_record(tables['binary_link'], condition)
sql.delete_record(tables['binary_interp'], condition)
for values in insert_values:
if values['type'] == 'lnk':
sql.append_record(tables['binary_link'], values)
else:
sql.append_record(tables['binary_list'], values)
if values['type'] == 'scr':
sql.append_record(tables['binary_interp'], values)
def BinaryDependency(jmgr, os_target, sql, args):
sql.connect_table(tables['binary_dependency'])
sql.connect_table(tables['binary_interp'])
pkgname = args[0]
bin = args[1]
dir = args[2]
if len(args) > 3:
ref = args[3]
if not package.reference_exists(dir, ref):
dir = None
ref = None
else:
ref = None
unpacked = False
if not dir:
(dir, pkgname, _) = package.unpack_package(os_target, args[0])
if not dir:
return
unpacked = True
exc = None
try:
if not os.path.exists(dir + bin):
raise Exception('path ' + dir + bin + ' does not exist')
dependencies = os_target.get_binary_dependencies(dir, bin)
interp = os_target.get_binary_interpreter(dir, bin)
pkg_id = get_package_id(sql, pkgname)
bin_id = get_binary_id(sql, bin)
if interp:
interp = get_binary_id(sql, interp)
condition = 'pkg_id=' + Table.stringify(
pkg_id) + ' and bin_id=' + Table.stringify(bin_id)
sql.delete_record(tables['binary_dependency'], condition)
sql.delete_record(tables['binary_interp'], condition)
for dep in dependencies:
values = dict()
values['pkg_id'] = pkg_id
values['bin_id'] = bin_id
values['dependency'] = dep
sql.append_record(tables['binary_dependency'], values)
if interp:
values = dict()
values['pkg_id'] = pkg_id
values['bin_id'] = bin_id
values['interp'] = interp
sql.append_record(tables['binary_interp'], values)
sql.update_record(tables['binary_list'], {'linking': False}, condition)
sql.commit()
except Exception as err:
exc = sys.exc_info()
if (ref and package.dereference_dir(dir, ref)) or unpacked:
package.remove_dir(dir)
if exc:
raise exc[1], None, exc[2]
def analysis_binary_call(sql, binary, pkg_id, bin_id):
callers = get_callgraph(binary)
calls = []
unknown_calls = []
apis = []
unknown_apis = []
for caller in callers:
for callee in caller.callees:
values = dict()
values['func_addr'] = caller.func_addr
if isinstance(callee, str) and callee.startswith('<'):
values['target'] = callee[1:-1]
unknown_calls.append(values)
continue
if isinstance(callee, str):
values['call_name'] = callee
else:
values['call_addr'] = callee
calls.append(values)
for syscall in caller.syscalls:
values = dict()
values['func_addr'] = caller.func_addr
if isinstance(syscall, int):
values['api_type'] = linux_defs.SYSCALL
values['api_id'] = syscall
apis.append(values)
continue
if isinstance(syscall, str):
values['target'] = syscall[1:-1]
else:
values['target'] = ''
unknown_apis.append(values)
for syscall in caller.vecsyscalls:
for request in caller.vecsyscalls[syscall]:
values = dict()
values['func_addr'] = caller.func_addr
if syscall == linux_defs.FCNTL_SYSCALL:
values['api_type'] = linux_defs.FCNTL
if syscall == linux_defs.IOCTL_SYSCALL:
values['api_type'] = linux_defs.IOCTL
if syscall == linux_defs.PRCTL_SYSCALL:
values['api_type'] = linux_defs.PRCTL
if isinstance(request, int) or isinstance(request, long):
values['api_id'] = request
apis.append(values)
continue
if isinstance(request, str):
values['target'] = request[1:-1]
else:
values['target'] = ''
unknown_apis.append(values)
for file in caller.files:
values = dict()
values['func_addr'] = caller.func_addr
values['api_type'] = linux_defs.PSEUDOFILE
values['api_id'] = sql.hash_text(file)
values['api_name'] = file
apis.append(values)
for values in apis:
if 'api_name' in values:
os_target.append_api_list(sql, values['api_type'],
values['api_id'], values['api_name'])
condition = 'pkg_id=' + Table.stringify(
pkg_id) + ' and bin_id=' + Table.stringify(bin_id)
condition_unknown = condition + ' and known=False'
sql.delete_record(tables['binary_call'], condition)
sql.delete_record(tables['binary_call_unknown'], condition_unknown)
sql.delete_record(tables['binary_api_usage'], condition)
sql.delete_record(tables['binary_api_usage_unknown'], condition_unknown)
for values in calls:
values['pkg_id'] = pkg_id
values['bin_id'] = bin_id
sql.append_record(tables['binary_call'], values)
for values in unknown_calls:
values['pkg_id'] = pkg_id
values['bin_id'] = bin_id
sql.append_record(tables['binary_call_unknown'], values)
for values in apis:
values['pkg_id'] = pkg_id
values['bin_id'] = bin_id
sql.append_record(tables['binary_api_usage'], values)
for values in unknown_apis:
values['pkg_id'] = pkg_id
values['bin_id'] = bin_id
sql.append_record(tables['binary_api_usage_unknown'], values)
def analysis_binary_instr(sql, binary, pkg_id, bin_id):
codes = get_callgraph(binary)
condition = 'pkg_id=' + Table.stringify(
pkg_id) + ' and bin_id=' + Table.stringify(bin_id)
condition_unknown = condition + ' and known=False'
sql.delete_record(tables['binary_call'], condition)
sql.delete_record(tables['binary_call_unknown'], condition_unknown)
sql.delete_record(tables['binary_opcode_usage'], condition)
for func in codes.funcs:
opcodes = dict()
calls = []
for bb in func.bblocks:
for instr in bb.instrs:
if isinstance(instr, InstrCall):
if isinstance(instr.target, int) or isinstance(
instr.target, long):
if not instr.target in calls:
calls.append(instr.target)
elif isinstance(instr.target, Op) and instr.target.val:
if not instr.target.val in calls:
calls.append(instr.target.val)
opcode = instr.opcode
size = instr.size
prefix = instr.prefixes
mnem = instr.get_instr()
if opcode == '':
continue
if prefix == '':
prefix = chr(0x0)
if (prefix, opcode, size, mnem) in opcodes:
opcodes[(prefix, opcode, size, mnem)] += 1
else:
opcodes[(prefix, opcode, size, mnem)] = 1
for call in calls:
values = dict()
values['pkg_id'] = pkg_id
values['bin_id'] = bin_id
values['func_addr'] = func.entry
if isinstance(call, int) or isinstance(call, long):
values['call_addr'] = call
else:
for addr, sym in codes.dynsyms.items():
if sym.name == call:
values['call_addr'] = sym.value
break
values['call_name'] = call
sql.append_record(tables['binary_call'], values)
for (prefix, opcode, size, mnem), count in opcodes.items():
values = dict()
values['pkg_id'] = pkg_id
values['bin_id'] = bin_id
values['func_addr'] = func.entry
values['prefix'] = int(prefix.encode('hex'), 16)
values['opcode'] = int(opcode.encode('hex'), 16)
values['size'] = size
values['mnem'] = mnem
values['count'] = count
try:
sql.append_record(tables['binary_opcode_usage'], values)
except Exception as e:
logging.info(e)
logging.info(prefix.encode('hex'))
logging.info(opcode)
logging.info(int(opcode.encode('hex'), 16))
logging.info(size)
logging.info(mnem)
logging.info(count)
continue
