def name_filter(args, opts):
"""
Use without args to find files with that name, use with args to filter
Syntax: name_filter %<oid> --name=<file_name>
"""
if not "name" in opts:
raise ShellSyntaxError("name_filter requires a --name=<file_name> option")
oids = []
valid, invalid = api.valid_oids(args)
valid = api.expand_oids(valid)
name = opts["name"]
terms = name.split("*")
if not args:
if len(terms) == 1:
return api.get_oids_with_name(opts["name"]).keys()
else:
valid = api.retrieve_all_keys("file_meta")
if len(terms) == 1:
for oid in valid:
names = api.get_field("file_meta", oid, "names")
if names and opts["name"] in names:
oids.append(oid)
else:
for oid in valid:
names = api.get_field("file_meta", oid, "names")
if names:
for name in names:
if name.startswith(terms[0]) and name.endswith(terms[1]):
oids.append(oid)
return oids
def tag_filter(oid_list, tag, value="<empty>"):
filtered_oids = []
if not oid_list:
oid_list = api.retrieve_all_keys("files")
if not oid_list:
logger.error("No files exist")
return None
cids = api.retrieve_all_keys("collections")
if cids:
oid_list.extend(cids)
for oid in oid_list:
t = get_tags(oid)
if t and tag in t:
if t[tag] == "<empty>" or value == "<empty>" or value == t[tag] or value in t[tag]:
filtered_oids.append(oid)
return filtered_oids
def summarize(args, opts):
"""
Gives a summary of a set of files, including types, extensions, etc. If no argument
is passed, gives a summary for the entire datastore (may be very slow).
Syntax: summarize %<oid>
"""
valid, invalid = api.valid_oids(args)
valid = set(api.expand_oids(valid))
types = defaultdict(int)
extensions = defaultdict(int)
sizes = [0,0,0,0,0,0]
if not args:
valid = set(api.retrieve_all_keys("file_meta"))
for oid in valid:
meta = api.retrieve("file_meta", oid)
names = meta["names"]
if names:
for name in names:
parts = name.split(".")
if len(parts) > 1:
extensions[parts[-1]] += 1
else:
extensions["None"] += 1
t = api.get_field("src_type", oid, "type")
if t: types[t] += 1
size = meta["size"]
if size < 1024: sizes[0] += 1
elif size < 10*1024: sizes[1] += 1
elif size < 100*1024: sizes[2] += 1
elif size < 1024*1024: sizes[3] += 1
elif size < 10*1024*1024: sizes[4] += 1
else: sizes[5] += 1
print "\nTotal files in set: ", len(valid)
print "\nExtensions (files with multiple names counted more than once):"
exts = extensions.keys()
exts = sorted(exts, key=lambda val: extensions[val], reverse=True)
for e in exts:
print " ", e, " \t\t :\t\t ", extensions[e]
print "\nTypes:"
ts = types.keys()
ts = sorted(ts, key=lambda val: types[val], reverse=True)
for t in ts:
print " ", t, " \t\t :\t\t ", types[t]
print "\nSizes: "
print " Under 1k :", sizes[0]
print " 1k - 10k :", sizes[1]
print " 10k - 100k :", sizes[2]
print " 100k - 1MB :", sizes[3]
print " 1MB - 10MB :", sizes[4]
print " over 10 MB :", sizes[5]
return None
def type_filter(args, opts):
"""
Use without args to find all files with that type, use with args to filter
Syntax: type_filter %<oid> --type=[ PE | ELF | PDF | etc...]
"""
if not "type" in opts:
raise ShellSyntaxError("type_filter requires a --type=[ PE | ELF | PDF | etc...] option")
oids = []
valid, invalid = api.valid_oids(args)
valid = api.expand_oids(valid)
if not args:
valid = api.retrieve_all_keys("files")
for oid in valid:
data = api.retrieve("src_type", oid)
if data and data["type"].lower() == opts["type"].lower():
oids.append(oid)
return oids
def key_filter(args, opts):
"""
Use to match the results of a module (module name required). Specify key and optionally value.
Syntax: key_filter %<oid> --module=<mod_name> --key=<key> [--value=<value>]
"""
if not "module" in opts or not "key" in opts:
raise ShellSyntaxError("key_filter requires a --module=<mod_name> and a --key=<key> option")
oids = []
valid, invalid = api.valid_oids(args)
valid = api.expand_oids(valid)
if not args:
valid = api.retrieve_all_keys("files")
if "key" in opts and "value" in opts:
oids = api.retrieve("substring_search", valid,
{"mod":opts["module"], "key":opts["key"], "value":opts["value"]})
elif "key" in opts:
oids = api.retrieve("key_search", valid,
{"mod":opts["module"], "key":opts["key"]})
return oids
def byte_filter(args, opts):
"""
Use without args to find files with that byte_string, use with args to filter
Syntax: byte_filter %<oid> --bytes=<byte_string>
"""
if not "bytes" in opts:
raise ShellSyntaxError("byte_filter requires a --bytes=<byte_string> option")
oids = []
valid, invalid = api.valid_oids(args)
valid = api.expand_oids(valid)
bytes = str(opts["bytes"])
if not args:
valid = api.retrieve_all_keys("files")
for o in valid:
data = api.get_field("files", o, "data")
if data.find(bytes) != -1:
oids.append(o)
return oids
def extension_filter(args, opts):
"""
Use without args to find files with that extension, use with args to filter
Syntax: extension_filter %<oid> --ext=<extension>
"""
if not "ext" in opts:
raise ShellSyntaxError("extension_filter requires a --ext=<extension> option")
oids = set()
valid, invalid = api.valid_oids(args)
valid = api.expand_oids(valid)
ext = opts["ext"]
if not args:
valid = api.retrieve_all_keys("file_meta")
for oid in valid:
names = api.get_field("file_meta", oid, "names")
if names:
for name in names:
parts = name.split(".")
if len(parts) > 1 and parts[-1].lower() == ext.lower():
oids.add(oid)
return list(oids)