def graph_stats(args, opts): """ Plugin: Prints a set of statistics about the call and control flow graphs Syntax: graph_stats <oid_1> <oid_2> ... <oid_n> """ valid, invalid = api.valid_oids(args) if not valid: raise ShellSyntaxError("No valid oids found") args = api.expand_oids(valid) for o in args: cg = api.get_field("call_graph", o, "graph") if not cg: continue size = len(cg.nodes()) in_histo = build_freq_histo(cg.in_degree()) out_histo = build_freq_histo(cg.out_degree()) cfgs = api.retrieve("cfg", o) size_dict = {i: cfgs[i].size() for i in cfgs if cfgs[i]} size_histo = build_freq_histo(size_dict) bbs = 0 for s in size_histo: bbs += s * size_histo[s] name = api.get_field("file_meta", o, "names").pop() print "-----------------" print " Graph stats for ", name print print " Functions = ", size print " Basic Blocks = ", bbs print " Call graph in-degree: " pretty_print_dicts(in_histo) print " Call graph out-degree: " pretty_print_dicts(out_histo) print " CFG sizes: " pretty_print_dicts(size_histo) return []
def header(args, opts): """ Displays header info Syntax: header <oid> ... [--verbose] """ args, invalid = api.valid_oids(args) args = api.expand_oids(args) if not args: if current_file: args = [current_file] else: raise ShellSyntaxError("Must provide an oid") headers = [] for oid in args: src_type = api.get_field("src_type", oid, "type") if src_type == "PE": pe_header(oid, opts) elif src_type == "ELF": elf_header(oid, opts) elif src_type == "MACHO": macho_header(oid, opts) elif src_type == "OSX Universal Binary": osx_header(oid, opts) else: print " - Source type %s is unsupported" % (src_type) return []
def import_table(args, opts): """ Displays the import table Syntax: import_table <oid> """ args, invalid = api.valid_oids(args) args = api.expand_oids(args) if not args: if current_file: args = [current_file] else: raise ShellSyntaxError("Must provide an oid") for oid in args: print " - Import Table for %s %s " % (name(oid), oid) header = api.get_field("object_header", [oid], "header") if not header: print " --------------------------" print " <EMPTY HEADER>" print " --------------------------" continue src_type = api.get_field("src_type", oid, "type") if src_type == "PE": pe_import_table(header, opts) elif src_type == "ELF": elf_import_table(header, opts) elif src_type == "MACHO": macho_import_table(header, opts) else: print " - Source type %s is unsupported" % (src_type)
def membership(args, opts): """ Prints the set of collections to which a file belongs. If a collection is passed its membership will not be printed Syntax: membership %<oid> ... """ valid, invalid = api.valid_oids(args) if not valid: raise ShellSyntaxError("No valid oids found") exclude_cids = [oid for oid in valid if api.exists("collections", oid)] main_oids = set(api.expand_oids(valid)) membership_cids = {} cids = [cid for cid in api.collection_cids() if cid not in exclude_cids] for cid in cids: this_oids = set(api.expand_oids(cid)) this_intersection = list(main_oids.intersection(this_oids)) if this_intersection: membership_cids[cid] = this_intersection if "noprint" not in opts: print_membership(membership_cids) return membership_cids
def name_filter(args, opts): """ Use without args to find files with that name, use with args to filter Syntax: name_filter %<oid> --name=<file_name> """ if not "name" in opts: raise ShellSyntaxError("name_filter requires a --name=<file_name> option") oids = [] valid, invalid = api.valid_oids(args) valid = api.expand_oids(valid) name = opts["name"] terms = name.split("*") if not args: if len(terms) == 1: return api.get_oids_with_name(opts["name"]).keys() else: valid = api.retrieve_all_keys("file_meta") if len(terms) == 1: for oid in valid: names = api.get_field("file_meta", oid, "names") if names and opts["name"] in names: oids.append(oid) else: for oid in valid: names = api.get_field("file_meta", oid, "names") if names: for name in names: if name.startswith(terms[0]) and name.endswith(terms[1]): oids.append(oid) return oids
def offset_rva(args, opts): """ Convert from an offset to an RVA Syntax: rva_offset <offset> """ args, invalid = api.valid_oids(args) args = api.expand_oids(args) if not args: if current_file: args = [current_file] else: raise ShellSyntaxError("Must provide an oid or run re_init to set a file") oid = args[0] rvas = [] for val in invalid: if val.startswith("0x"): try: offset = int(val, 16) except: raise ShellSyntaxError("Unrecognized address %s" % val) else: try: offset = int(val) except: raise ShellSyntaxError("Unrecognized address %s" % val) rva = convert_offset_rva(oid, offset) if rva is None: raise ShellSyntaxError("Unrecognized address %s" % val) rvas.append(rva) print " %s (%s) : %s (%s)"%(hex(offset), offset, hex(rva), rva) return rvas
def size_filter(args, opts): """ Filter files by size in bytes Syntax: size_filter %<oid> --min=<size> --max=<size> """ if not args: raise ShellSyntaxError("File name not specified") min_size = 0 max_size = None if "min" in opts: min_size = int(opts["min"]) if "max" in opts: max_size = int(opts["max"]) valid, invalid = api.valid_oids(args) oids = api.expand_oids(valid) filtered_oids = [] for oid in oids: meta = api.retrieve("file_meta", oid) size = meta["size"] if size > min_size and ((not max_size) or size < max_size): filtered_oids.append(oid) return filtered_oids
def random_sample(args, opts): """ Given a list of oids passes a random subset of them syntax: random_sample <oid1> <oid2> ... <oidn> [--n=<size> | --p=<percent>] (default is 10%) """ valid, invalid = api.valid_oids(args) if not valid: raise ShellSyntaxError("No valid oids found") args = api.expand_oids(valid) if not args: return [] nargs = len(args) n = int( round( nargs / float(10) ) ) if "n" in opts: try: n = int(opts["n"]) if n < 0: raise ValueError if n > nargs: n = nargs except ValueError: raise ShellSyntaxError("Invalid integer value for n: %s" % opts["n"]) elif "p" in opts: try: p = float(opts["p"]) if p <= 0 or p > 100: raise ValueError except ValueError: raise ShellSyntaxError("Invalid float value for p: %s" % opts["p"]) n = int( round( len(args) / (100/p) ) ) if n == 0: n = 1 return random.sample(args, n)
def extract_osx(args, opts): """ Imports objects from an OSX Universal Binary Syntax: """ valid, invalid = api.valid_oids(args) if not valid: raise ShellSyntaxError("No valid oids found") args = api.expand_oids(valid) for oid in args: meta = api.retrieve("file_meta", oid) name = meta["names"].pop() src_type = api.retrieve("src_type", oid) if src_type["type"] != "OSX Universal Binary": print " - %s (%s) is not an OSX Universal binary file, skipping" % (name, oid) continue data = api.retrieve("files", oid)["data"] if not data: print " - No data for this file %s (%s) " % (name, oid) continue oh = api.retrieve("object_header", oid) num = oh["header"].num_embedded print " - Found %s files embedded in file %s (%s)" % (num, name, oid) oids = [] newfiles = 0 for f in oh["header"].embedded: beg = f.header_offset end = f.file_end print " + Extracting bytes %s:%s of file type %s" % (beg, end, f.machine) fname = name + "_" + f.machine fpath = os.path.join(api.scratch_dir, fname) print " + Writing temp file to %s" % (fpath) fd = file(fpath, 'wb') fd.write(data[beg:end]) fd.close() print " + Importing file %s" % (fpath) oid, newfile = api.import_file(fpath) oids.append(oid) if newfile: newfiles += 1 print " + Removing temp file from the scratch directory" os.remove(fpath) print print " - Extracted and imported %s files, %s were new" % (len(oids), newfiles) # Return a list of the oids corresponding to the files extracted return oids
def expand(args, opts): """ Passes a list where any cids passed are expanded to the oids in that collection Syntax: &<my_collection> | expand | ... """ valid, invalid = api.valid_oids(args) if not valid: raise ShellSyntaxError("No valid oids found") return api.expand_oids(valid)
def summarize(args, opts): """ Gives a summary of a set of files, including types, extensions, etc. If no argument is passed, gives a summary for the entire datastore (may be very slow). Syntax: summarize %<oid> """ valid, invalid = api.valid_oids(args) valid = set(api.expand_oids(valid)) types = defaultdict(int) extensions = defaultdict(int) sizes = [0,0,0,0,0,0] if not args: valid = set(api.retrieve_all_keys("file_meta")) for oid in valid: meta = api.retrieve("file_meta", oid) names = meta["names"] if names: for name in names: parts = name.split(".") if len(parts) > 1: extensions[parts[-1]] += 1 else: extensions["None"] += 1 t = api.get_field("src_type", oid, "type") if t: types[t] += 1 size = meta["size"] if size < 1024: sizes[0] += 1 elif size < 10*1024: sizes[1] += 1 elif size < 100*1024: sizes[2] += 1 elif size < 1024*1024: sizes[3] += 1 elif size < 10*1024*1024: sizes[4] += 1 else: sizes[5] += 1 print "\nTotal files in set: ", len(valid) print "\nExtensions (files with multiple names counted more than once):" exts = extensions.keys() exts = sorted(exts, key=lambda val: extensions[val], reverse=True) for e in exts: print " ", e, " \t\t :\t\t ", extensions[e] print "\nTypes:" ts = types.keys() ts = sorted(ts, key=lambda val: types[val], reverse=True) for t in ts: print " ", t, " \t\t :\t\t ", types[t] print "\nSizes: " print " Under 1k :", sizes[0] print " 1k - 10k :", sizes[1] print " 10k - 100k :", sizes[2] print " 100k - 1MB :", sizes[3] print " 1MB - 10MB :", sizes[4] print " over 10 MB :", sizes[5] return None
def disassembly(args, opts): """ Displays the disassembly for a file Syntax: disassembly <oid> [--slice=<beg>:<end>] """ args, invalid = api.valid_oids(args) args = api.expand_oids(args) if not args: if current_file: args = [current_file] else: raise ShellSyntaxError("Must provide an oid") start = stop = 0 height = default_height if "slice" in opts: start, stop = get_slice(opts) if "height" in opts: try: width = int(opts["height"]) except ValueError: raise ShellSyntaxError("Invalid height") mod_opts = {} if "module" in opts: mod_opts["module"] = opts["module"] for oid in args: disasm = api.get_field("disassembly", [oid], "insns", mod_opts) #comments = api.get_field("disassembly", [oid], "comments", mod_opts) functions = api.retrieve("function_extract", oid) if not functions: print " No functions found for %s %s" % (name(oid), oid) continue fbreaks = get_fbreaks(functions) system_calls = api.get_field("map_calls", oid, "system_calls") internal_functions = api.get_field("map_calls", oid, "internal_functions") function_calls = dict() if system_calls is None: print " System calls could not be determined for %s %s" % (name(oid), oid) else: function_calls.update(system_calls.items()) if internal_functions is None: print " Internal functions could not be determined for %s %s" % (name(oid), oid) else: function_calls.update(internal_functions.items()) if disasm: print " Disassembly for %s %s" % (name(oid), oid) print " -------------------------------------" print_disassembly(oid, disasm, function_calls, fbreaks, start, stop, height) print " -------------------------------------" else: print " %s could not be disassembled." % name(oid)
def random_shuffle(args, opts): """ Passes a randomized list of oids syntax: random_shuffle <oid1> <oid2> ... <oidn> """ valid, invalid = api.valid_oids(args) if not valid: raise ShellSyntaxError("No valid oids found") args = api.expand_oids(valid) random.shuffle(args) return args
def dex_header(args, opts): """ Plugin: Prints the header for a dex file Syntax: dex_header %<oid> ... """ valid, invalid = api.valid_oids(args) if not valid: raise ShellSyntaxError("No valid oids found") oids = api.expand_oids(valid) for oid in oids: print_dex_header(oid) return oids
def intersection(args, opts): """ Returns the intersection of the collections passed in, non-collection IDs will be ignored Syntax: intersection &col1 &col2 ... """ valid, invalid = api.valid_oids(args) if not valid: raise ShellSyntaxError("No valid oids found") cids = [oid for oid in valid if api.exists("collections", oid)] if not cids: raise ShellSyntaxError("No valid collections found") oids = set(api.expand_oids(cids[0])) for c in cids[1:]: oids = oids.intersection(api.expand_oids(c)) return oids
def strings(args, opts): """ A variation on the typical Unix strings command. Looks for all sequences of 4 or more Asccii printable characters. Prints out the offsets followed by the matched string. Does not require that the strings end with a Null character, \\x00. The file option allows you to specify an output file that the strings will be written to. Returns None. Syntax: strings <oid_1> <oid_2> ... <oid_n> [--file=<outputfile>] """ args, invalid = api.valid_oids(args) args = api.expand_oids(args) if not args: if current_file: args = [current_file] else: raise ShellSyntaxError("Must provide an oid or run re_init to set a file") for oid in args: strings = api.retrieve("strings", oid, opts) if not strings: print " - No strings for", oid else: print " - Strings for", oid write_file = False if "file" in opts: fd = open(opts["file"], "w") write_file = True print " - Writting strings to file %s " % opts["file"] i = 0 s = "" offsets = strings.keys() offsets.sort() for offset in offsets: if (offset - i) < 4: s += strings[offset] else: if len(s) > 1: if write_file: fd.write("Offset:%s\t%s\n" % (i, s)) else: print "Offset:%s\t%s" % (i, s) s = strings[offset] i = offset print
def export_file(args, opts): """ Given a list of oid's exports the files. If tar or zip option used with multiple input files a single file will be exported. Syntax: export_file <oid1> <oid2> ... <oidn> [--zip | --tar [--bz2 | --gz] --name=<export_name> ] """ valid, invalid = api.valid_oids(args) if not valid: raise ShellSyntaxError("No valid oids found") oids = api.expand_oids(valid) if "zip" in opts: export_tar_zip(oids, opts, type="zip") elif "tar" in opts: export_tar_zip(oids, opts, type="tar") else: export_files(oids, opts)
def unarchive(args, opts): """ Try in unarchive (unzip and untar), passes a list of ununarchived oids Syntax: unarchive <oid_1> <oid_2> ... <oid_n> """ valid, invalid = api.valid_oids(args) if not valid: raise ShellSyntaxError("No valid oids found") oids = api.expand_oids(valid) unarchived = [] newfiles = [] print " - Attempting to unarchive (zip, tar) %d files" % len(oids) p = progress.progress(len(oids)) for oid in oids: data = api.get_field(api.source(oid), oid, "data") if not data: print "Not able to process %s" % (oid) p.tick() continue tmp = tmp_file(oid, data) if not tmp: continue aoids = [] noids = [] if tarfile.is_tarfile(tmp): # tar print " - Unpacking a tar file" aoids, noids = import_tarfile(tmp, parent_oid=oid) elif zipfile.is_zipfile(tmp): # zip print " - Unpacking a zip file" aoids, noids = import_zipfile(tmp, parent_oid=oid) unarchived.extend(aoids) newfiles.extend(noids) os.remove(tmp) p.tick() if unarchived: unarchived.extend(unarchive(unarchived, opts)) # Unpacked children print " - Extracted %d files %d are new" % (len(unarchived), len(newfiles)) return unarchived
def relationships(args, opt): """ Plugin: Pring zip and upx file relationships including parents' childrens' parents' childrens' ... Syntax: relationships <oid_1> <oid_2> ... <oid_n> """ valid, invalid = api.valid_oids(args) if not valid: raise ShellSyntaxError("No valid oids found") oids = api.expand_oids(valid) global family for oid in oids: family = {} no_parents = set() build_ancesstors([oid]) build_decendants(family.keys()) name = api.get_field("file_meta", oid, "names").pop() print print " -- File Relationships for %s --" % name print_relationships()
def crypto(args, opts): """ Plugin: Looks for cryptographic methods inside binary files Synatax: crypto %oid --<cryptographic functions>|show Example: crypto %oid --sha1 --sha256 --aes """ args, invalid = api.valid_oids(args) if not args: raise ShellSyntaxError("Must provide an oid") args = api.expand_oids(args) allData = {} if not opts: opts['sha1'] = "" opts['sha256'] = "" opts['aes'] = "" for oid in args: src = api.source(oid) file_data = api.get_field(src, oid, "data") asm = api.retrieve('function_extract', [oid]) if not asm: continue #print asm retVal = {} if 'sha1' in opts: sha1Percent = findSha1(asm,file_data,retVal) retVal['SHA1']['Percent']=sha1Percent*100 if 'sha256'in opts: sha256Percent = findSha256(asm,file_data,retVal) retVal['SHA256']['Percent']=sha256Percent*100 if 'aes' in opts: aesPercent = findAES(asm,file_data,retVal) retVal['AES']['Percent']=aesPercent*100 allData[oid] = retVal return allData
def type_filter(args, opts): """ Use without args to find all files with that type, use with args to filter Syntax: type_filter %<oid> --type=[ PE | ELF | PDF | etc...] """ if not "type" in opts: raise ShellSyntaxError("type_filter requires a --type=[ PE | ELF | PDF | etc...] option") oids = [] valid, invalid = api.valid_oids(args) valid = api.expand_oids(valid) if not args: valid = api.retrieve_all_keys("files") for oid in valid: data = api.retrieve("src_type", oid) if data and data["type"].lower() == opts["type"].lower(): oids.append(oid) return oids
def cat(args, opts): """ Given an oid, displays the text of the file. Should only be run on plain-text files. Will give an error if file appears to not be plain-text. Override with --force. Syntax: cat <oid> """ import string valid, invalid = api.valid_oids(args) if not valid: raise ShellSyntaxError("No valid oids found") oids = api.expand_oids(valid) for o in oids: data = api.get_field("files", o, "data") printable = set(data).issubset(set(string.printable)) if not printable and not "force" in opts: raise ShellSyntaxError("File contains non-printable characters. Use --force to override.") else: print data
def byte_filter(args, opts): """ Use without args to find files with that byte_string, use with args to filter Syntax: byte_filter %<oid> --bytes=<byte_string> """ if not "bytes" in opts: raise ShellSyntaxError("byte_filter requires a --bytes=<byte_string> option") oids = [] valid, invalid = api.valid_oids(args) valid = api.expand_oids(valid) bytes = str(opts["bytes"]) if not args: valid = api.retrieve_all_keys("files") for o in valid: data = api.get_field("files", o, "data") if data.find(bytes) != -1: oids.append(o) return oids
def key_filter(args, opts): """ Use to match the results of a module (module name required). Specify key and optionally value. Syntax: key_filter %<oid> --module=<mod_name> --key=<key> [--value=<value>] """ if not "module" in opts or not "key" in opts: raise ShellSyntaxError("key_filter requires a --module=<mod_name> and a --key=<key> option") oids = [] valid, invalid = api.valid_oids(args) valid = api.expand_oids(valid) if not args: valid = api.retrieve_all_keys("files") if "key" in opts and "value" in opts: oids = api.retrieve("substring_search", valid, {"mod":opts["module"], "key":opts["key"], "value":opts["value"]}) elif "key" in opts: oids = api.retrieve("key_search", valid, {"mod":opts["module"], "key":opts["key"]}) return oids
def dump_js(args, opts): valid, invalid = api.valid_oids(args) oids = api.expand_oids(valid) if not oids: raise ShellSyntaxError("No valid oids") for o in oids: type = api.get_field("src_type", o, "type") if type != "PDF": continue src = api.source(o) data = api.get_field(src, o, "data") i = data.find("<script") while i > -1: j = data.find("</script>") if j > -1: print "Found script:" print data[i:j+9] data = data[j:] i = data.find("<script")
def sections(args, opts): """ Displays sections Syntax: sections <oid> ... [--verbose] """ args, invalid = api.valid_oids(args) args = api.expand_oids(args) if not args: if current_file: args = [current_file] else: raise ShellSyntaxError("Must provide an oid") for oid in args: header = api.get_field("object_header", [oid], "header") if not header: print " <EMPTY HEADER>" print " --------------------------" continue print " - Sections for %s %s" % (name(oid), oid) print_sections(header, opts)
def untar(args, opts): """ Try to untar items passed, passes a list of untarred oids Syntax: untar <oid_1> <oid_2> ... <oid_n> """ valid, invalid = api.valid_oids(args) if not valid: raise ShellSyntaxError("No valid oids found") oids = api.expand_oids(valid) untarred = [] newfiles = [] p = progress.progress(len(oids)) print " - Attempting to untar %d files" % len(oids) for oid in oids: src = api.source(oid) data = api.get_field(src, oid, "data") if not data: print "No data found for %s" % (oid) p.tick() continue tmpname = oid + ".tar.tmp" tmp = tmp_file(tmpname, data) if not tmp: continue if tarfile.is_tarfile(tmp): toids, nfiles = import_tarfile(tmp, parent_oid=oid) untarred.extend(toids) newfiles.extend(nfiles) os.remove(tmp) p.tick() if untarred: untarred.extend(untar(untarred, opts)) # Untar children print " - %d files extracted, %d files are new" % (len(untarred), len(newfiles)) return untarred
def calls(args, opts): """ Displays the function calls for a file Syntax: calls <oid> ... """ args, invalid = api.valid_oids(args) args = api.expand_oids(args) if not args: if current_file: args = [current_file] else: raise ShellSyntaxError("Must provide an oid") for oid in args: internal_functions = api.get_field("map_calls", oid, "internal_functions") if not internal_functions: print " No internal functions found for %s %s" % (name(oid), oid) else: names = list(set(internal_functions.values())) names.sort() print " Internal functions calls for %s %s" % (name(oid), oid) print " -------------------------------------" for f in names: print " -",f print " -------------------------------------" system_calls = api.get_field("map_calls", oid, "system_calls") if not system_calls: print " No system calls found for %s %s" % (name(oid), oid) else: names = list(set(system_calls.values())) names.sort() print " System functions calls for %s %s" % (name(oid), oid) print " -------------------------------------" for f in names: print " -",f print " -------------------------------------"
def entry_point(args, opts): """ Displays the entry point Syntax: pe_header <oid> ... """ args, invalid = api.valid_oids(args) args = api.expand_oids(args) if not args: if current_file: args = [current_file] else: raise ShellSyntaxError("Must provide an oid") for oid in args: disasm = api.get_field("disasm_first_run", [oid], "insns") if disasm: print " Entry point disassembly for %s %s" % (name(oid), oid) print " -------------------------------------" print_disassembly(oid, disasm, {}, {}, 0 , 0, default_height) print " -------------------------------------" else: print " %s could not be disassembled." % name(oid) return []
def match_instr_cfg(args, opts): """ Plugin: Attaches the instructions associated with the control flow returned by the cfg module Syntax: match_instr_cfg %<oid> """ args, invalid = api.valid_oids(args) args = api.expand_oids(args) if not args: raise ShellSyntaxError("Must provide an oid") for oid in args: features = api.retrieve('asm_semantics', [oid]) asm = api.retrieve('function_extract', [oid]) cfg = api.retrieve('cfg', [oid]) basic_blocks = api.retrieve('basic_blocks', [oid]) for func_address in sorted(cfg.keys()): if cfg[func_address]: for node in sorted(cfg[func_address]): line_num = (i for i in xrange(len(basic_blocks[func_address])) if basic_blocks[func_address][i]['first_insn'] == node).next() num_insns = basic_blocks[func_address][line_num]['num_insns'] line_num = (i for i in xrange(len(asm[func_address]['insns'])) if asm[func_address]['insns'][i]['addr'] == node).next() insns_set = asm[func_address]['insns'][line_num:line_num+num_insns] cfg[func_address][node]['insns'] = insns_set return cfg