def PackageInfo(jmgr, os_target, sql, args): sql.connect_table(tables['package_info']) sql.connect_table(tables['package_dependency']) pkgname = args[0] pkg_id = get_package_id(sql, pkgname) pkg_info = os_target.get_package_info(pkgname) pkg_deps = os_target.get_package_dependency(pkgname) # Clean up records sql.delete_record(tables['package_info'], 'pkg_id=\'' + Table.stringify(pkg_id) + '\'') sql.delete_record(tables['package_dependency'], 'pkg_id=\'' + Table.stringify(pkg_id) + '\'') pkg_info['pkg_id'] = pkg_id sql.append_record(tables['package_info'], pkg_info) for dep in pkg_deps: dep_id = get_package_id(sql, dep) values = dict() values['pkg_id'] = pkg_id values['dependency'] = dep_id sql.append_record(tables['package_dependency'], values) sql.commit()
def BinarySymbol(jmgr, os_target, sql, args): sql.connect_table(tables['binary_list']) sql.connect_table(tables['binary_symbol']) pkgname = args[0] bin = args[1] dir = args[2] if len(args) > 3: ref = args[3] if not package.reference_exists(dir, ref): dir = None ref = None else: ref = None unpacked = False if not dir: (dir, pkgname, _) = package.unpack_package(os_target, args[0]) if not dir: return unpacked = True exc = None try: path = dir + '/' + bin if not os.path.exists(path): raise Exception('path ' + path + ' does not exist') symbols = os_target.get_binary_symbols(dir, bin) pkg_id = get_package_id(sql, pkgname) bin_id = get_binary_id(sql, bin) condition = 'pkg_id=' + Table.stringify( pkg_id) + ' and bin_id=' + Table.stringify(bin_id) sql.delete_record(tables['binary_symbol'], condition) for sym in symbols: values = dict() values['pkg_id'] = pkg_id values['bin_id'] = bin_id values['symbol_name'] = sym.name values['version'] = sym.version values['func_addr'] = sym.addr sql.append_record(tables['binary_symbol'], values) sql.update_record(tables['binary_list'], {'callgraph': False}, condition) sql.commit() except Exception as err: exc = sys.exc_info() if (ref and package.dereference_dir(dir, ref)) or unpacked: package.remove_dir(dir) if exc: raise exc[1], None, exc[2]
def analysis_binary_instr_linear(sql, binary, pkg_id, bin_id): condition = 'pkg_id=' + Table.stringify( pkg_id) + ' and bin_id=' + Table.stringify(bin_id) condition_unknown = condition + ' and known=False' sql.delete_record(tables['binary_call'], condition) sql.delete_record(tables['binary_call_unknown'], condition_unknown) sql.delete_record(tables['binary_opcode_usage'], condition) sql.delete_record(tables['binary_call_missrate'], condition) get_callgraph(binary, False, True, False, sql, pkg_id, bin_id)
def get_packages_by_ranks(os_target, sql, min, max): sql.connect_table(tables['package_popularity']) packages = os_target.get_packages() packages_by_ranks = sql.search_record( tables['package_popularity'], 'rank >= ' + Table.stringify(min) + ' AND rank <= ' + Table.stringify(max), ['package_name']) result = [] for (name, ) in packages_by_ranks: if name in packages: result.append(name) return result
def analysis_binary_instr(sql, binary, pkg_id, bin_id): codes = get_callgraph(binary) condition = 'pkg_id=' + Table.stringify( pkg_id) + ' and bin_id=' + Table.stringify(bin_id) condition_unknown = condition + ' and known=False' sql.delete_record(tables['binary_call'], condition) sql.delete_record(tables['binary_call_unknown'], condition_unknown) sql.delete_record(tables['binary_instr_usage'], condition) for func in codes.funcs: instrs = dict() calls = [] for bb in func.bblocks: for instr in bb.instrs: if isinstance(instr, InstrCall): if isinstance(instr.target, int) or isinstance( instr.target, long): if not instr.target in calls: calls.append(instr.target) elif isinstance(instr.target, Op) and instr.target.val: if not instr.target.val in calls: calls.append(instr.target.val) instr_name = instr.get_instr() if instr_name in instrs: instrs[instr_name] = instrs[instr_name] + 1 else: instrs[instr_name] = 1 for call in calls: values = dict() values['pkg_id'] = pkg_id values['bin_id'] = bin_id values['func_addr'] = func.entry if isinstance(call, int) or isinstance(call, long): values['call_addr'] = call else: for addr, sym in codes.dynsyms.items(): if sym.name == call: values['call_addr'] = sym.value break values['call_name'] = call sql.append_record(tables['binary_call'], values) for instr, count in instrs.items(): values = dict() values['pkg_id'] = pkg_id values['bin_id'] = bin_id values['func_addr'] = func.entry values['instr'] = instr values['count'] = count sql.append_record(tables['binary_instr_usage'], values)
def BinaryInstr(jmgr, os_target, sql, args): sql.connect_table(tables['binary_call']) sql.connect_table(tables['binary_call_unknown']) sql.connect_table(tables['binary_opcode_usage']) sql.connect_table(tables['binary_call_missrate']) # sql.connect_table(tables['instr_list']) # sql.connect_table(tables['prefix_counts']) pkgname = args[0] bin = args[1] dir = args[2] pkg_id = get_package_id(sql, pkgname) if len(args) > 3: ref = args[3] if not package.reference_exists(dir, ref): dir = None ref = None else: ref = None unpacked = False if not dir: (dir, pkgname, _) = package.unpack_package(os_target, args[0]) if not dir: return unpacked = True exc = None try: if not os.path.exists(dir + bin): raise Exception('path ' + dir + bin + ' does not exist') bin_id = get_binary_id(sql, bin) os_target.analysis_binary_instr_linear(sql, dir, bin, pkg_id, bin_id) condition = 'pkg_id=' + Table.stringify( pkg_id) + ' and bin_id=' + Table.stringify(bin_id) sql.update_record(tables['binary_list'], {'callgraph': False}, condition) sql.commit() except Exception as err: exc = sys.exc_info() if (ref and package.dereference_dir(dir, ref)) or unpacked: package.remove_dir(dir) if exc: raise exc[1], None, exc[2]
def get_package_name(sql, id): sql.connect_table(tables['package_id']) res = sql.search_record(tables['package_id'], 'id=' + Table.stringify(id), ['package_name']) if len(res) > 0 and res[0][0]: return res[0][0] return None
def get_package_id(sql, pkgname): sql.connect_table(tables['package_id']) retry = True while retry: res = sql.search_record(tables['package_id'], 'package_name=' + Table.stringify(pkgname), ['id']) if len(res) > 0 and res[0][0]: id = res[0][0] break res = sql.search_record(tables['package_id'], None, ['MAX(id)']) if res[0][0] == None: id = 1 else: id = int(res[0][0]) + 1 values = dict() values['id'] = id values['package_name'] = pkgname retry = False try: sql.append_record(tables['package_id'], values) sql.commit() except: retry = True pass return id
def get_binary_id(sql, binary_name): sql.connect_table(tables['binary_id']) retry = True while retry: res = sql.search_record(tables['binary_id'], 'binary_name=' + Table.stringify(binary_name), ['id']) if len(res) > 0 and res[0][0]: id = res[0][0] break res = sql.search_record(tables['binary_id'], None, ['MAX(id)']) if res[0][0] == None: id = 1 else: id = int(res[0][0]) + 1 values = dict() values['id'] = id values['binary_name'] = binary_name values['file_name'] = os.path.basename(binary_name) retry = False try: sql.append_record(tables['binary_id'], values) sql.commit() except: retry = True pass return id
def append_api_list(sql, type, id, name): sql.connect_table(tables['api_list']) retry = True while retry: res = sql.search_record(tables['api_list'], 'type=' + Table.stringify(type) + ' and id=' + Table.stringify(id), ['name']) if len(res) > 0 and res[0][0]: if res[0][0] != name: raise Error('duplicate key value (' + Table.stringify(type) + ',' + Table.stringify(id) + ')') return values = dict() values['type'] = type values['id'] = id values['name'] = name retry = False try: sql.append_record(tables['api_list'], values) sql.commit() except: retry = True pass
def append_binary_list(sql, pkgname, dir, binaries): sql.connect_table(tables['binary_list']) sql.connect_table(tables['binary_link']) sql.connect_table(tables['binary_interp']) pkg_id = get_package_id(sql, pkgname) insert_values = [] for (bin, type, interpreter) in binaries: bin_id = get_binary_id(sql, bin) values = dict() values['type'] = type if type == 'lnk': link = os.readlink(dir + bin) target = os.path.join(os.path.dirname(bin), link) target_id = get_binary_id(sql, target) values['pkg_id'] = pkg_id values['lnk_id'] = bin_id values['target'] = target_id values['linking'] = False else: values['pkg_id'] = pkg_id values['bin_id'] = bin_id if type == 'scr': interp_id = get_binary_id(sql, interpreter) values['interp'] = interp_id values['callgraph'] = True else: values['callgraph'] = False values['linking'] = False insert_values.append(values) condition = 'pkg_id=' + Table.stringify(pkg_id) sql.delete_record(tables['binary_list'], condition) sql.delete_record(tables['binary_link'], condition) sql.delete_record(tables['binary_interp'], condition) for values in insert_values: if values['type'] == 'lnk': sql.append_record(tables['binary_link'], values) else: sql.append_record(tables['binary_list'], values) if values['type'] == 'scr': sql.append_record(tables['binary_interp'], values)
def BinaryDependency(jmgr, os_target, sql, args): sql.connect_table(tables['binary_dependency']) sql.connect_table(tables['binary_interp']) pkgname = args[0] bin = args[1] dir = args[2] if len(args) > 3: ref = args[3] if not package.reference_exists(dir, ref): dir = None ref = None else: ref = None unpacked = False if not dir: (dir, pkgname, _) = package.unpack_package(os_target, args[0]) if not dir: return unpacked = True exc = None try: if not os.path.exists(dir + bin): raise Exception('path ' + dir + bin + ' does not exist') dependencies = os_target.get_binary_dependencies(dir, bin) interp = os_target.get_binary_interpreter(dir, bin) pkg_id = get_package_id(sql, pkgname) bin_id = get_binary_id(sql, bin) if interp: interp = get_binary_id(sql, interp) condition = 'pkg_id=' + Table.stringify( pkg_id) + ' and bin_id=' + Table.stringify(bin_id) sql.delete_record(tables['binary_dependency'], condition) sql.delete_record(tables['binary_interp'], condition) for dep in dependencies: values = dict() values['pkg_id'] = pkg_id values['bin_id'] = bin_id values['dependency'] = dep sql.append_record(tables['binary_dependency'], values) if interp: values = dict() values['pkg_id'] = pkg_id values['bin_id'] = bin_id values['interp'] = interp sql.append_record(tables['binary_interp'], values) sql.update_record(tables['binary_list'], {'linking': False}, condition) sql.commit() except Exception as err: exc = sys.exc_info() if (ref and package.dereference_dir(dir, ref)) or unpacked: package.remove_dir(dir) if exc: raise exc[1], None, exc[2]
def analysis_binary_call(sql, binary, pkg_id, bin_id): callers = get_callgraph(binary) calls = [] unknown_calls = [] apis = [] unknown_apis = [] for caller in callers: for callee in caller.callees: values = dict() values['func_addr'] = caller.func_addr if isinstance(callee, str) and callee.startswith('<'): values['target'] = callee[1:-1] unknown_calls.append(values) continue if isinstance(callee, str): values['call_name'] = callee else: values['call_addr'] = callee calls.append(values) for syscall in caller.syscalls: values = dict() values['func_addr'] = caller.func_addr if isinstance(syscall, int): values['api_type'] = linux_defs.SYSCALL values['api_id'] = syscall apis.append(values) continue if isinstance(syscall, str): values['target'] = syscall[1:-1] else: values['target'] = '' unknown_apis.append(values) for syscall in caller.vecsyscalls: for request in caller.vecsyscalls[syscall]: values = dict() values['func_addr'] = caller.func_addr if syscall == linux_defs.FCNTL_SYSCALL: values['api_type'] = linux_defs.FCNTL if syscall == linux_defs.IOCTL_SYSCALL: values['api_type'] = linux_defs.IOCTL if syscall == linux_defs.PRCTL_SYSCALL: values['api_type'] = linux_defs.PRCTL if isinstance(request, int) or isinstance(request, long): values['api_id'] = request apis.append(values) continue if isinstance(request, str): values['target'] = request[1:-1] else: values['target'] = '' unknown_apis.append(values) for file in caller.files: values = dict() values['func_addr'] = caller.func_addr values['api_type'] = linux_defs.PSEUDOFILE values['api_id'] = sql.hash_text(file) values['api_name'] = file apis.append(values) for values in apis: if 'api_name' in values: os_target.append_api_list(sql, values['api_type'], values['api_id'], values['api_name']) condition = 'pkg_id=' + Table.stringify( pkg_id) + ' and bin_id=' + Table.stringify(bin_id) condition_unknown = condition + ' and known=False' sql.delete_record(tables['binary_call'], condition) sql.delete_record(tables['binary_call_unknown'], condition_unknown) sql.delete_record(tables['binary_api_usage'], condition) sql.delete_record(tables['binary_api_usage_unknown'], condition_unknown) for values in calls: values['pkg_id'] = pkg_id values['bin_id'] = bin_id sql.append_record(tables['binary_call'], values) for values in unknown_calls: values['pkg_id'] = pkg_id values['bin_id'] = bin_id sql.append_record(tables['binary_call_unknown'], values) for values in apis: values['pkg_id'] = pkg_id values['bin_id'] = bin_id sql.append_record(tables['binary_api_usage'], values) for values in unknown_apis: values['pkg_id'] = pkg_id values['bin_id'] = bin_id sql.append_record(tables['binary_api_usage_unknown'], values)
def analysis_binary_instr(sql, binary, pkg_id, bin_id): codes = get_callgraph(binary) condition = 'pkg_id=' + Table.stringify( pkg_id) + ' and bin_id=' + Table.stringify(bin_id) condition_unknown = condition + ' and known=False' sql.delete_record(tables['binary_call'], condition) sql.delete_record(tables['binary_call_unknown'], condition_unknown) sql.delete_record(tables['binary_opcode_usage'], condition) for func in codes.funcs: opcodes = dict() calls = [] for bb in func.bblocks: for instr in bb.instrs: if isinstance(instr, InstrCall): if isinstance(instr.target, int) or isinstance( instr.target, long): if not instr.target in calls: calls.append(instr.target) elif isinstance(instr.target, Op) and instr.target.val: if not instr.target.val in calls: calls.append(instr.target.val) opcode = instr.opcode size = instr.size prefix = instr.prefixes mnem = instr.get_instr() if opcode == '': continue if prefix == '': prefix = chr(0x0) if (prefix, opcode, size, mnem) in opcodes: opcodes[(prefix, opcode, size, mnem)] += 1 else: opcodes[(prefix, opcode, size, mnem)] = 1 for call in calls: values = dict() values['pkg_id'] = pkg_id values['bin_id'] = bin_id values['func_addr'] = func.entry if isinstance(call, int) or isinstance(call, long): values['call_addr'] = call else: for addr, sym in codes.dynsyms.items(): if sym.name == call: values['call_addr'] = sym.value break values['call_name'] = call sql.append_record(tables['binary_call'], values) for (prefix, opcode, size, mnem), count in opcodes.items(): values = dict() values['pkg_id'] = pkg_id values['bin_id'] = bin_id values['func_addr'] = func.entry values['prefix'] = int(prefix.encode('hex'), 16) values['opcode'] = int(opcode.encode('hex'), 16) values['size'] = size values['mnem'] = mnem values['count'] = count try: sql.append_record(tables['binary_opcode_usage'], values) except Exception as e: logging.info(e) logging.info(prefix.encode('hex')) logging.info(opcode) logging.info(int(opcode.encode('hex'), 16)) logging.info(size) logging.info(mnem) logging.info(count) continue