def membership(args, opts):
"""
Prints the set of collections to which a file belongs.
If a collection is passed its membership will not be printed
Syntax: membership %<oid> ...
"""
valid, invalid = api.valid_oids(args)
if not valid:
raise ShellSyntaxError("No valid oids found")
exclude_cids = [oid for oid in valid if api.exists("collections", oid)]
main_oids = set(api.expand_oids(valid))
membership_cids = {}
cids = [cid for cid in api.collection_cids() if cid not in exclude_cids]
for cid in cids:
this_oids = set(api.expand_oids(cid))
this_intersection = list(main_oids.intersection(this_oids))
if this_intersection:
membership_cids[cid] = this_intersection
if "noprint" not in opts:
print_membership(membership_cids)
return membership_cids
def intersection(args, opts):
"""
Returns the intersection of the collections passed in, non-collection IDs will be ignored
Syntax: intersection &col1 &col2 ...
"""
valid, invalid = api.valid_oids(args)
if not valid:
raise ShellSyntaxError("No valid oids found")
cids = [oid for oid in valid if api.exists("collections", oid)]
if not cids:
raise ShellSyntaxError("No valid collections found")
oids = set(api.expand_oids(cids[0]))
for c in cids[1:]:
oids = oids.intersection(api.expand_oids(c))
return oids
def random_sample(args, opts):
"""
Given a list of oids passes a random subset of them
syntax: random_sample <oid1> <oid2> ... <oidn> [--n=<size> | --p=<percent>]
(default is 10%)
"""
valid, invalid = api.valid_oids(args)
if not valid:
raise ShellSyntaxError("No valid oids found")
args = api.expand_oids(valid)
if not args:
return []
nargs = len(args)
n = int( round( nargs / float(10) ) )
if "n" in opts:
try:
n = int(opts["n"])
if n < 0: raise ValueError
if n > nargs: n = nargs
except ValueError:
raise ShellSyntaxError("Invalid integer value for n: %s" % opts["n"])
elif "p" in opts:
try:
p = float(opts["p"])
if p <= 0 or p > 100: raise ValueError
except ValueError:
raise ShellSyntaxError("Invalid float value for p: %s" % opts["p"])
n = int( round( len(args) / (100/p) ) )
if n == 0: n = 1
return random.sample(args, n)
def graph_stats(args, opts):
""" Plugin: Prints a set of statistics about the call and control flow graphs
Syntax: graph_stats <oid_1> <oid_2> ... <oid_n>
"""
valid, invalid = api.valid_oids(args)
if not valid:
raise ShellSyntaxError("No valid oids found")
args = api.expand_oids(valid)
for o in args:
cg = api.get_field("call_graph", o, "graph")
if not cg:
continue
size = len(cg.nodes())
in_histo = build_freq_histo(cg.in_degree())
out_histo = build_freq_histo(cg.out_degree())
cfgs = api.retrieve("cfg", o)
size_dict = {i: cfgs[i].size() for i in cfgs if cfgs[i]}
size_histo = build_freq_histo(size_dict)
bbs = 0
for s in size_histo:
bbs += s * size_histo[s]
name = api.get_field("file_meta", o, "names").pop()
print "-----------------"
print " Graph stats for ", name
print
print " Functions = ", size
print " Basic Blocks = ", bbs
print " Call graph in-degree: "
pretty_print_dicts(in_histo)
print " Call graph out-degree: "
pretty_print_dicts(out_histo)
print " CFG sizes: "
pretty_print_dicts(size_histo)
return []
def header(args, opts):
"""
Displays header info
Syntax: header <oid> ... [--verbose]
"""
args, invalid = api.valid_oids(args)
args = api.expand_oids(args)
if not args:
if current_file:
args = [current_file]
else:
raise ShellSyntaxError("Must provide an oid")
headers = []
for oid in args:
src_type = api.get_field("src_type", oid, "type")
if src_type == "PE":
pe_header(oid, opts)
elif src_type == "ELF":
elf_header(oid, opts)
elif src_type == "MACHO":
macho_header(oid, opts)
elif src_type == "OSX Universal Binary":
osx_header(oid, opts)
else:
print " - Source type %s is unsupported" % (src_type)
return []
def import_table(args, opts):
"""
Displays the import table
Syntax: import_table <oid>
"""
args, invalid = api.valid_oids(args)
args = api.expand_oids(args)
if not args:
if current_file:
args = [current_file]
else:
raise ShellSyntaxError("Must provide an oid")
for oid in args:
print " - Import Table for %s %s " % (name(oid), oid)
header = api.get_field("object_header", [oid], "header")
if not header:
print " --------------------------"
print " <EMPTY HEADER>"
print " --------------------------"
continue
src_type = api.get_field("src_type", oid, "type")
if src_type == "PE":
pe_import_table(header, opts)
elif src_type == "ELF":
elf_import_table(header, opts)
elif src_type == "MACHO":
macho_import_table(header, opts)
else:
print " - Source type %s is unsupported" % (src_type)
def name_filter(args, opts):
"""
Use without args to find files with that name, use with args to filter
Syntax: name_filter %<oid> --name=<file_name>
"""
if not "name" in opts:
raise ShellSyntaxError("name_filter requires a --name=<file_name> option")
oids = []
valid, invalid = api.valid_oids(args)
valid = api.expand_oids(valid)
name = opts["name"]
terms = name.split("*")
if not args:
if len(terms) == 1:
return api.get_oids_with_name(opts["name"]).keys()
else:
valid = api.retrieve_all_keys("file_meta")
if len(terms) == 1:
for oid in valid:
names = api.get_field("file_meta", oid, "names")
if names and opts["name"] in names:
oids.append(oid)
else:
for oid in valid:
names = api.get_field("file_meta", oid, "names")
if names:
for name in names:
if name.startswith(terms[0]) and name.endswith(terms[1]):
oids.append(oid)
return oids
def size_filter(args, opts):
"""
Filter files by size in bytes
Syntax: size_filter %<oid> --min=<size> --max=<size>
"""
if not args:
raise ShellSyntaxError("File name not specified")
min_size = 0
max_size = None
if "min" in opts:
min_size = int(opts["min"])
if "max" in opts:
max_size = int(opts["max"])
valid, invalid = api.valid_oids(args)
oids = api.expand_oids(valid)
filtered_oids = []
for oid in oids:
meta = api.retrieve("file_meta", oid)
size = meta["size"]
if size > min_size and ((not max_size) or size < max_size):
filtered_oids.append(oid)
return filtered_oids
def offset_rva(args, opts):
"""
Convert from an offset to an RVA
Syntax: rva_offset <offset>
"""
args, invalid = api.valid_oids(args)
args = api.expand_oids(args)
if not args:
if current_file:
args = [current_file]
else:
raise ShellSyntaxError("Must provide an oid or run re_init to set a file")
oid = args[0]
rvas = []
for val in invalid:
if val.startswith("0x"):
try:
offset = int(val, 16)
except:
raise ShellSyntaxError("Unrecognized address %s" % val)
else:
try:
offset = int(val)
except:
raise ShellSyntaxError("Unrecognized address %s" % val)
rva = convert_offset_rva(oid, offset)
if rva is None:
raise ShellSyntaxError("Unrecognized address %s" % val)
rvas.append(rva)
print " %s (%s) : %s (%s)"%(hex(offset), offset, hex(rva), rva)
return rvas
def process(oid, opts):
if len(opts['oid_list']) == 0:
return True
logger.debug("Processing collection " + str(oid))
oid_list = list(set(api.expand_oids(opts["oid_list"])))
data = {"oid_list":oid_list}
api.store(name, oid, data, opts)
return True
def extract_osx(args, opts):
"""
Imports objects from an OSX Universal Binary
Syntax:
"""
valid, invalid = api.valid_oids(args)
if not valid:
raise ShellSyntaxError("No valid oids found")
args = api.expand_oids(valid)
for oid in args:
meta = api.retrieve("file_meta", oid)
name = meta["names"].pop()
src_type = api.retrieve("src_type", oid)
if src_type["type"] != "OSX Universal Binary":
print " - %s (%s) is not an OSX Universal binary file, skipping" % (name, oid)
continue
data = api.retrieve("files", oid)["data"]
if not data:
print " - No data for this file %s (%s) " % (name, oid)
continue
oh = api.retrieve("object_header", oid)
num = oh["header"].num_embedded
print " - Found %s files embedded in file %s (%s)" % (num, name, oid)
oids = []
newfiles = 0
for f in oh["header"].embedded:
beg = f.header_offset
end = f.file_end
print " + Extracting bytes %s:%s of file type %s" % (beg, end, f.machine)
fname = name + "_" + f.machine
fpath = os.path.join(api.scratch_dir, fname)
print " + Writing temp file to %s" % (fpath)
fd = file(fpath, 'wb')
fd.write(data[beg:end])
fd.close()
print " + Importing file %s" % (fpath)
oid, newfile = api.import_file(fpath)
oids.append(oid)
if newfile: newfiles += 1
print " + Removing temp file from the scratch directory"
os.remove(fpath)
print
print " - Extracted and imported %s files, %s were new" % (len(oids), newfiles)
# Return a list of the oids corresponding to the files extracted
return oids
def summarize(args, opts):
"""
Gives a summary of a set of files, including types, extensions, etc. If no argument
is passed, gives a summary for the entire datastore (may be very slow).
Syntax: summarize %<oid>
"""
valid, invalid = api.valid_oids(args)
valid = set(api.expand_oids(valid))
types = defaultdict(int)
extensions = defaultdict(int)
sizes = [0,0,0,0,0,0]
if not args:
valid = set(api.retrieve_all_keys("file_meta"))
for oid in valid:
meta = api.retrieve("file_meta", oid)
names = meta["names"]
if names:
for name in names:
parts = name.split(".")
if len(parts) > 1:
extensions[parts[-1]] += 1
else:
extensions["None"] += 1
t = api.get_field("src_type", oid, "type")
if t: types[t] += 1
size = meta["size"]
if size < 1024: sizes[0] += 1
elif size < 10*1024: sizes[1] += 1
elif size < 100*1024: sizes[2] += 1
elif size < 1024*1024: sizes[3] += 1
elif size < 10*1024*1024: sizes[4] += 1
else: sizes[5] += 1
print "\nTotal files in set: ", len(valid)
print "\nExtensions (files with multiple names counted more than once):"
exts = extensions.keys()
exts = sorted(exts, key=lambda val: extensions[val], reverse=True)
for e in exts:
print " ", e, " \t\t :\t\t ", extensions[e]
print "\nTypes:"
ts = types.keys()
ts = sorted(ts, key=lambda val: types[val], reverse=True)
for t in ts:
print " ", t, " \t\t :\t\t ", types[t]
print "\nSizes: "
print " Under 1k :", sizes[0]
print " 1k - 10k :", sizes[1]
print " 10k - 100k :", sizes[2]
print " 100k - 1MB :", sizes[3]
print " 1MB - 10MB :", sizes[4]
print " over 10 MB :", sizes[5]
return None
def expand(args, opts):
"""
Passes a list where any cids passed are expanded to the oids in that collection
Syntax: &<my_collection> | expand | ...
"""
valid, invalid = api.valid_oids(args)
if not valid:
raise ShellSyntaxError("No valid oids found")
return api.expand_oids(valid)
def disassembly(args, opts):
"""
Displays the disassembly for a file
Syntax: disassembly <oid> [--slice=<beg>:<end>]
"""
args, invalid = api.valid_oids(args)
args = api.expand_oids(args)
if not args:
if current_file:
args = [current_file]
else:
raise ShellSyntaxError("Must provide an oid")
start = stop = 0
height = default_height
if "slice" in opts:
start, stop = get_slice(opts)
if "height" in opts:
try:
width = int(opts["height"])
except ValueError:
raise ShellSyntaxError("Invalid height")
mod_opts = {}
if "module" in opts:
mod_opts["module"] = opts["module"]
for oid in args:
disasm = api.get_field("disassembly", [oid], "insns", mod_opts)
#comments = api.get_field("disassembly", [oid], "comments", mod_opts)
functions = api.retrieve("function_extract", oid)
if not functions:
print " No functions found for %s %s" % (name(oid), oid)
continue
fbreaks = get_fbreaks(functions)
system_calls = api.get_field("map_calls", oid, "system_calls")
internal_functions = api.get_field("map_calls", oid, "internal_functions")
function_calls = dict()
if system_calls is None:
print " System calls could not be determined for %s %s" % (name(oid), oid)
else:
function_calls.update(system_calls.items())
if internal_functions is None:
print " Internal functions could not be determined for %s %s" % (name(oid), oid)
else:
function_calls.update(internal_functions.items())
if disasm:
print " Disassembly for %s %s" % (name(oid), oid)
print " -------------------------------------"
print_disassembly(oid, disasm, function_calls, fbreaks, start, stop, height)
print " -------------------------------------"
else:
print " %s could not be disassembled." % name(oid)
def random_shuffle(args, opts):
"""
Passes a randomized list of oids
syntax: random_shuffle <oid1> <oid2> ... <oidn>
"""
valid, invalid = api.valid_oids(args)
if not valid:
raise ShellSyntaxError("No valid oids found")
args = api.expand_oids(valid)
random.shuffle(args)
return args
def dex_header(args, opts):
""" Plugin: Prints the header for a dex file
Syntax: dex_header %<oid> ...
"""
valid, invalid = api.valid_oids(args)
if not valid:
raise ShellSyntaxError("No valid oids found")
oids = api.expand_oids(valid)
for oid in oids:
print_dex_header(oid)
return oids
def export_file(args, opts):
"""
Given a list of oid's exports the files. If tar or zip option used with
multiple input files a single file will be exported.
Syntax: export_file <oid1> <oid2> ... <oidn> [--zip | --tar [--bz2 | --gz] --name=<export_name> ]
"""
valid, invalid = api.valid_oids(args)
if not valid:
raise ShellSyntaxError("No valid oids found")
oids = api.expand_oids(valid)
if "zip" in opts:
export_tar_zip(oids, opts, type="zip")
elif "tar" in opts:
export_tar_zip(oids, opts, type="tar")
else:
export_files(oids, opts)
def strings(args, opts):
"""
A variation on the typical Unix strings command. Looks for
all sequences of 4 or more Asccii printable characters.
Prints out the offsets followed by the matched string.
Does not require that the strings end with a Null character,
\\x00. The file option allows you to specify an output file
that the strings will be written to.
Returns None.
Syntax: strings <oid_1> <oid_2> ... <oid_n> [--file=<outputfile>]
"""
args, invalid = api.valid_oids(args)
args = api.expand_oids(args)
if not args:
if current_file:
args = [current_file]
else:
raise ShellSyntaxError("Must provide an oid or run re_init to set a file")
for oid in args:
strings = api.retrieve("strings", oid, opts)
if not strings:
print " - No strings for", oid
else:
print " - Strings for", oid
write_file = False
if "file" in opts:
fd = open(opts["file"], "w")
write_file = True
print " - Writting strings to file %s " % opts["file"]
i = 0
s = ""
offsets = strings.keys()
offsets.sort()
for offset in offsets:
if (offset - i) < 4:
s += strings[offset]
else:
if len(s) > 1:
if write_file:
fd.write("Offset:%s\t%s\n" % (i, s))
else:
print "Offset:%s\t%s" % (i, s)
s = strings[offset]
i = offset
print
def unarchive(args, opts):
"""
Try in unarchive (unzip and untar), passes a list of ununarchived oids
Syntax: unarchive <oid_1> <oid_2> ... <oid_n>
"""
valid, invalid = api.valid_oids(args)
if not valid:
raise ShellSyntaxError("No valid oids found")
oids = api.expand_oids(valid)
unarchived = []
newfiles = []
print " - Attempting to unarchive (zip, tar) %d files" % len(oids)
p = progress.progress(len(oids))
for oid in oids:
data = api.get_field(api.source(oid), oid, "data")
if not data:
print "Not able to process %s" % (oid)
p.tick()
continue
tmp = tmp_file(oid, data)
if not tmp: continue
aoids = []
noids = []
if tarfile.is_tarfile(tmp): # tar
print " - Unpacking a tar file"
aoids, noids = import_tarfile(tmp, parent_oid=oid)
elif zipfile.is_zipfile(tmp): # zip
print " - Unpacking a zip file"
aoids, noids = import_zipfile(tmp, parent_oid=oid)
unarchived.extend(aoids)
newfiles.extend(noids)
os.remove(tmp)
p.tick()
if unarchived:
unarchived.extend(unarchive(unarchived, opts)) # Unpacked children
print " - Extracted %d files %d are new" % (len(unarchived), len(newfiles))
return unarchived
def relationships(args, opt):
""" Plugin: Pring zip and upx file relationships including parents' childrens' parents' childrens' ...
Syntax: relationships <oid_1> <oid_2> ... <oid_n>
"""
valid, invalid = api.valid_oids(args)
if not valid:
raise ShellSyntaxError("No valid oids found")
oids = api.expand_oids(valid)
global family
for oid in oids:
family = {}
no_parents = set()
build_ancesstors([oid])
build_decendants(family.keys())
name = api.get_field("file_meta", oid, "names").pop()
print
print " -- File Relationships for %s --" % name
print_relationships()
def crypto(args, opts):
"""
Plugin: Looks for cryptographic methods inside binary files
Synatax:
crypto %oid --<cryptographic functions>|show
Example:
crypto %oid --sha1 --sha256 --aes
"""
args, invalid = api.valid_oids(args)
if not args:
raise ShellSyntaxError("Must provide an oid")
args = api.expand_oids(args)
allData = {}
if not opts:
opts['sha1'] = ""
opts['sha256'] = ""
opts['aes'] = ""
for oid in args:
src = api.source(oid)
file_data = api.get_field(src, oid, "data")
asm = api.retrieve('function_extract', [oid])
if not asm: continue
#print asm
retVal = {}
if 'sha1' in opts:
sha1Percent = findSha1(asm,file_data,retVal)
retVal['SHA1']['Percent']=sha1Percent*100
if 'sha256'in opts:
sha256Percent = findSha256(asm,file_data,retVal)
retVal['SHA256']['Percent']=sha256Percent*100
if 'aes' in opts:
aesPercent = findAES(asm,file_data,retVal)
retVal['AES']['Percent']=aesPercent*100
allData[oid] = retVal
return allData
def type_filter(args, opts):
"""
Use without args to find all files with that type, use with args to filter
Syntax: type_filter %<oid> --type=[ PE | ELF | PDF | etc...]
"""
if not "type" in opts:
raise ShellSyntaxError("type_filter requires a --type=[ PE | ELF | PDF | etc...] option")
oids = []
valid, invalid = api.valid_oids(args)
valid = api.expand_oids(valid)
if not args:
valid = api.retrieve_all_keys("files")
for oid in valid:
data = api.retrieve("src_type", oid)
if data and data["type"].lower() == opts["type"].lower():
oids.append(oid)
return oids
def cat(args, opts):
"""
Given an oid, displays the text of the file. Should only be run on plain-text
files. Will give an error if file appears to not be plain-text. Override with --force.
Syntax: cat <oid>
"""
import string
valid, invalid = api.valid_oids(args)
if not valid:
raise ShellSyntaxError("No valid oids found")
oids = api.expand_oids(valid)
for o in oids:
data = api.get_field("files", o, "data")
printable = set(data).issubset(set(string.printable))
if not printable and not "force" in opts:
raise ShellSyntaxError("File contains non-printable characters. Use --force to override.")
else:
print data
def byte_filter(args, opts):
"""
Use without args to find files with that byte_string, use with args to filter
Syntax: byte_filter %<oid> --bytes=<byte_string>
"""
if not "bytes" in opts:
raise ShellSyntaxError("byte_filter requires a --bytes=<byte_string> option")
oids = []
valid, invalid = api.valid_oids(args)
valid = api.expand_oids(valid)
bytes = str(opts["bytes"])
if not args:
valid = api.retrieve_all_keys("files")
for o in valid:
data = api.get_field("files", o, "data")
if data.find(bytes) != -1:
oids.append(o)
return oids
def key_filter(args, opts):
"""
Use to match the results of a module (module name required). Specify key and optionally value.
Syntax: key_filter %<oid> --module=<mod_name> --key=<key> [--value=<value>]
"""
if not "module" in opts or not "key" in opts:
raise ShellSyntaxError("key_filter requires a --module=<mod_name> and a --key=<key> option")
oids = []
valid, invalid = api.valid_oids(args)
valid = api.expand_oids(valid)
if not args:
valid = api.retrieve_all_keys("files")
if "key" in opts and "value" in opts:
oids = api.retrieve("substring_search", valid,
{"mod":opts["module"], "key":opts["key"], "value":opts["value"]})
elif "key" in opts:
oids = api.retrieve("key_search", valid,
{"mod":opts["module"], "key":opts["key"]})
return oids
def dump_js(args, opts):
valid, invalid = api.valid_oids(args)
oids = api.expand_oids(valid)
if not oids:
raise ShellSyntaxError("No valid oids")
for o in oids:
type = api.get_field("src_type", o, "type")
if type != "PDF":
continue
src = api.source(o)
data = api.get_field(src, o, "data")
i = data.find("<script")
while i > -1:
j = data.find("</script>")
if j > -1:
print "Found script:"
print data[i:j+9]
data = data[j:]
i = data.find("<script")
def sections(args, opts):
"""
Displays sections
Syntax: sections <oid> ... [--verbose]
"""
args, invalid = api.valid_oids(args)
args = api.expand_oids(args)
if not args:
if current_file:
args = [current_file]
else:
raise ShellSyntaxError("Must provide an oid")
for oid in args:
header = api.get_field("object_header", [oid], "header")
if not header:
print " <EMPTY HEADER>"
print " --------------------------"
continue
print " - Sections for %s %s" % (name(oid), oid)
print_sections(header, opts)
def untar(args, opts):
"""
Try to untar items passed, passes a list of untarred oids
Syntax: untar <oid_1> <oid_2> ... <oid_n>
"""
valid, invalid = api.valid_oids(args)
if not valid:
raise ShellSyntaxError("No valid oids found")
oids = api.expand_oids(valid)
untarred = []
newfiles = []
p = progress.progress(len(oids))
print " - Attempting to untar %d files" % len(oids)
for oid in oids:
src = api.source(oid)
data = api.get_field(src, oid, "data")
if not data:
print "No data found for %s" % (oid)
p.tick()
continue
tmpname = oid + ".tar.tmp"
tmp = tmp_file(tmpname, data)
if not tmp: continue
if tarfile.is_tarfile(tmp):
toids, nfiles = import_tarfile(tmp, parent_oid=oid)
untarred.extend(toids)
newfiles.extend(nfiles)
os.remove(tmp)
p.tick()
if untarred:
untarred.extend(untar(untarred, opts)) # Untar children
print " - %d files extracted, %d files are new" % (len(untarred), len(newfiles))
return untarred
def calls(args, opts):
"""
Displays the function calls for a file
Syntax: calls <oid> ...
"""
args, invalid = api.valid_oids(args)
args = api.expand_oids(args)
if not args:
if current_file:
args = [current_file]
else:
raise ShellSyntaxError("Must provide an oid")
for oid in args:
internal_functions = api.get_field("map_calls", oid, "internal_functions")
if not internal_functions:
print " No internal functions found for %s %s" % (name(oid), oid)
else:
names = list(set(internal_functions.values()))
names.sort()
print " Internal functions calls for %s %s" % (name(oid), oid)
print " -------------------------------------"
for f in names:
print " -",f
print " -------------------------------------"
system_calls = api.get_field("map_calls", oid, "system_calls")
if not system_calls:
print " No system calls found for %s %s" % (name(oid), oid)
else:
names = list(set(system_calls.values()))
names.sort()
print " System functions calls for %s %s" % (name(oid), oid)
print " -------------------------------------"
for f in names:
print " -",f
print " -------------------------------------"
def entry_point(args, opts):
"""
Displays the entry point
Syntax: pe_header <oid> ...
"""
args, invalid = api.valid_oids(args)
args = api.expand_oids(args)
if not args:
if current_file:
args = [current_file]
else:
raise ShellSyntaxError("Must provide an oid")
for oid in args:
disasm = api.get_field("disasm_first_run", [oid], "insns")
if disasm:
print " Entry point disassembly for %s %s" % (name(oid), oid)
print " -------------------------------------"
print_disassembly(oid, disasm, {}, {}, 0 , 0, default_height)
print " -------------------------------------"
else:
print " %s could not be disassembled." % name(oid)
return []
