def cmd_tags(self, *args):
parser = argparse.ArgumentParser(
prog='tags', description="Modify tags of the opened file")
parser.add_argument(
'-a',
'--add',
metavar='TAG',
help="Add tags to the opened file (comma separated)")
parser.add_argument('-d',
'--delete',
metavar='TAG',
help="Delete a tag from the opened file")
try:
args = parser.parse_args(args)
except:
return
# This command requires a session to be opened.
if not __sessions__.is_set():
self.log('error', "No open session")
parser.print_usage()
return
# If no arguments are specified, there's not much to do.
# However, it could make sense to also retrieve a list of existing
# tags from this command, and not just from the "find" command alone.
if args.add is None and args.delete is None:
parser.print_usage()
return
# TODO: handle situation where addition or deletion of a tag fail.
db = Database()
if not db.find(key='sha256', value=__sessions__.current.file.sha256):
self.log(
'error', "The opened file is not stored in the database. "
"If you want to add it use the `store` command.")
return
if args.add:
# Add specified tags to the database's entry belonging to
# the opened file.
db.add_tags(__sessions__.current.file.sha256, args.add)
self.log('info', "Tags added to the currently opened file")
# We refresh the opened session to update the attributes.
# Namely, the list of tags returned by the 'info' command
# needs to be re-generated, or it wouldn't show the new tags
# until the existing session is closed a new one is opened.
self.log('info', "Refreshing session to update attributes...")
__sessions__.new(__sessions__.current.file.path)
if args.delete:
# Delete the tag from the database.
db.delete_tag(args.delete, __sessions__.current.file.sha256)
# Refresh the session so that the attributes of the file are
# updated.
self.log('info', "Refreshing session to update attributes...")
__sessions__.new(__sessions__.current.file.path)
def cmd_tags(self, *args):
parser = argparse.ArgumentParser(prog="tags", description="Modify tags of the opened file")
parser.add_argument("-a", "--add", metavar="TAG", help="Add tags to the opened file (comma separated)")
parser.add_argument("-d", "--delete", metavar="TAG", help="Delete a tag from the opened file")
try:
args = parser.parse_args(args)
except:
return
# This command requires a session to be opened.
if not __sessions__.is_set():
self.log("error", "No session opened")
parser.print_usage()
return
# If no arguments are specified, there's not much to do.
# However, it could make sense to also retrieve a list of existing
# tags from this command, and not just from the "find" command alone.
if args.add is None and args.delete is None:
parser.print_usage()
return
# TODO: handle situation where addition or deletion of a tag fail.
db = Database()
if not db.find(key="sha256", value=__sessions__.current.file.sha256):
self.log(
"error",
"The opened file is not stored in the database. " "If you want to add it use the `store` command.",
)
return
if args.add:
# Add specified tags to the database's entry belonging to
# the opened file.
db.add_tags(__sessions__.current.file.sha256, args.add)
self.log("info", "Tags added to the currently opened file")
# We refresh the opened session to update the attributes.
# Namely, the list of tags returned by the 'info' command
# needs to be re-generated, or it wouldn't show the new tags
# until the existing session is closed a new one is opened.
self.log("info", "Refreshing session to update attributes...")
__sessions__.new(__sessions__.current.file.path)
if args.delete:
# Delete the tag from the database.
db.delete_tag(args.delete, __sessions__.current.file.sha256)
# Refresh the session so that the attributes of the file are
# updated.
self.log("info", "Refreshing session to update attributes...")
__sessions__.new(__sessions__.current.file.path)
def tags(tag_action=False):
# Set DB
db = Database()
# Search or Delete
if request.method == 'GET':
action = request.query.action
value = request.query.value.strip()
if value:
if action == 'search':
# This will search all projects
# Get project list
projects = project_list()
# Add Main db to list.
projects.append('../')
# Search All projects
p_list = []
results = {}
for project in projects:
__project__.open(project)
# Init DB
db = Database()
#get results
proj_results = []
rows = db.find(key='tag', value=value)
for row in rows:
if project == '../':
project = 'Main'
proj_results.append([row.name, row.sha256])
results[project] = proj_results
p_list.append(project)
# Return the search template
return template('search.tpl', projects=p_list, results=results)
else:
return template(
'error.tpl',
error="'{0}' Is not a valid tag action".format(action))
# Add / Delete
if request.method == 'POST':
file_hash = request.forms.get('sha256')
project = request.forms.get('project')
tag_name = request.forms.get('tag')
if tag_action == 'add':
if file_hash and project:
tags = request.forms.get('tags')
db.add_tags(file_hash, tags)
if tag_action == 'del':
if file_hash and tag_name:
db.delete_tag(tag_name, file_hash)
redirect('/file/{0}/{1}'.format(project, file_hash))
def tags(tag_action=False):
# Set DB
db = Database()
# Search or Delete
if request.method == 'GET':
action = request.query.action
value = request.query.value.strip()
if value:
if action == 'search':
# This will search all projects
# Get project list
projects = project_list()
# Add Main db to list.
projects.append('../')
# Search All projects
p_list = []
results = {}
for project in projects:
__project__.open(project)
# Init DB
db = Database()
#get results
proj_results = []
rows = db.find(key='tag', value=value)
for row in rows:
if project == '../':
project = 'Main'
proj_results.append([row.name, row.sha256])
results[project] = proj_results
p_list.append(project)
# Return the search template
return template('search.tpl', projects=p_list, results=results)
else:
return template('error.tpl', error="'{0}' Is not a valid tag action".format(action))
# Add / Delete
if request.method == 'POST':
file_hash = request.forms.get('sha256')
project = request.forms.get('project')
tag_name = request.forms.get('tag')
if tag_action == 'add':
if file_hash and project:
tags = request.forms.get('tags')
db.add_tags(file_hash, tags)
if tag_action == 'del':
if file_hash and tag_name:
db.delete_tag(tag_name, file_hash)
redirect('/file/{0}/{1}'.format(project, file_hash))
def tags(tag_action=False):
# Set DB
db = Database()
# Search or Delete
if request.method == "GET":
action = request.query.action
value = request.query.value.strip()
if value:
if action == "search":
# This will search all projects
# Get project list
projects = project_list()
# Add Main db to list.
projects.append("../")
# Search All projects
p_list = []
results = {}
for project in projects:
__project__.open(project)
# Init DB
db = Database()
# get results
proj_results = []
rows = db.find(key="tag", value=value)
for row in rows:
if project == "../":
project = "Main"
proj_results.append([row.name, row.sha256])
results[project] = proj_results
p_list.append(project)
# Return the search template
return template("search.tpl", projects=p_list, results=results)
else:
return template("error.tpl", error="'{0}' Is not a valid tag action".format(action))
# Add / Delete
if request.method == "POST":
file_hash = request.forms.get("sha256")
project = request.forms.get("project")
tag_name = request.forms.get("tag")
if tag_action == "add":
if file_hash and project:
tags = request.forms.get("tags")
db.add_tags(file_hash, tags)
if tag_action == "del":
if file_hash and tag_name:
db.delete_tag(tag_name, file_hash)
redirect("/file/{0}/{1}".format(project, file_hash))
def add_tags():
tags = request.forms.get('tags')
for entry in ['md5', 'sha256', 'ssdeep', 'tag', 'name', 'all']:
value = request.forms.get(entry)
if value:
key = entry
break
db = Database()
rows = db.find(key=key, value=value)
if not rows:
raise HTTPError(404, 'File not found in the database')
for row in rows:
malware_sha256=row.sha256
db.add_tags(malware_sha256, tags)
return jsonize({'message' : 'added'})
def add_tags():
tags = request.forms.get('tags')
for entry in ['md5', 'sha256', 'ssdeep', 'tag', 'name', 'all']:
value = request.forms.get(entry)
if value:
key = entry
break
db = Database()
rows = db.find(key=key, value=value)
if not rows:
raise HTTPError(404, 'File not found in the database')
for row in rows:
malware_sha256 = row.sha256
db.add_tags(malware_sha256, tags)
return jsonize({'message': 'added'})
def run(self):
super(Triage, self).run()
db = Database()
if self.args and self.args.all:
samples = db.find(key='all')
for sample in samples:
tags = []
tags.extend(self._triage_file_type(sample))
db.add_tags(sample.sha256, tags)
# We're running against the already opened file.
else:
if not __sessions__.is_set():
self.log('error', "No open session. This command expects a file to be open.")
return
tags = []
tags.extend(self._triage_file_type(__sessions__.current.file))
db.add_tags(__sessions__.current.file.sha256, tags)
def run(self):
super(Triage, self).run()
db = Database()
if self.args and self.args.all:
samples = db.find(key='all')
for sample in samples:
tags = []
tags.extend(self._triage_file_type(sample))
db.add_tags(sample.sha256, tags)
# We're running against the already opened file.
else:
if not __sessions__.is_set():
self.log('error', "No open session")
return
tags = []
tags.extend(self._triage_file_type(__sessions__.current.file))
db.add_tags(__sessions__.current.file.sha256, tags)
def scan(self):
arg_rule = self.args.rule
arg_scan_all = self.args.all
arg_tag = self.args.tag
arg_verbose = self.args.verbose
externals = {'filename': '', 'filepath': '', 'extension': '', 'filetype': ''}
# If a rule file is specified we compile that, otherwise all
# the rules we have stored locally.
if arg_rule:
# Check if the selected ruleset actually exists.
if not os.path.exists(arg_rule):
self.log('error', "The specified file does not exist at path {0}".format(arg_rule))
return
rules = yara.compile(arg_rule, externals=externals)
# Otherwise, we get all the rules that are stored locally and we
# load them in different namespaces.
else:
filepaths = dict()
for rule in self._get_rules():
filepaths['namespace' + str(rule[0])] = rule[1]
rules = yara.compile(filepaths=filepaths, externals=externals, includes=False)
# Files to scan.
files = []
# If there is a session open and the user didn't specifically
# request to scan the full repository, we just add the currently
# opened file's path.
if __sessions__.is_set() and not arg_scan_all:
files.append(__sessions__.current.file)
# Otherwise we loop through all files in the repository and queue
# them up for scan.
else:
self.log('info', "Scanning all stored files (in the current project)...")
db = Database()
samples = db.find(key='all')
for sample in samples:
files.append(sample)
# Loop through all files to be scanned.
for entry in files:
if entry.size == 0:
continue
self.log('info', "Scanning {0} ({1})".format(entry.name, entry.sha256))
# Check if the entry has a path attribute. This happens when
# there is a session open. We need to distinguish this just for
# the cases where we're scanning an opened file which has not been
# stored yet.
if hasattr(entry, 'path'):
entry_path = entry.path
# This should be triggered only when scanning the full repository.
else:
entry_path = get_sample_path(entry.sha256)
# Check if the file exists before running the yara scan.
if not os.path.exists(entry_path):
self.log('error', "The file does not exist at path {0}".format(entry_path))
return
rows = []
tag_list = []
found = False
# We need this just for some Yara rules.
try:
ext = os.path.splitext(entry.name)[1]
except:
ext = ''
for match in rules.match(entry_path, externals={'filename': entry.name, 'filepath': entry_path, 'extension': ext, 'filetype': entry.type}):
found = True
# Add a row for each string matched by the rule.
if arg_verbose:
for match_string in match.strings:
rows.append([
match.rule,
string_printable(match_string[1]),
string_printable(match_string[0]),
string_printable(match_string[2])]
)
else:
self.log('item', match.rule)
# Add matching rules to our list of tags.
# First it checks if there are tags specified in the metadata
# of the Yara rule.
match_tags = match.meta.get('tags')
# If not, use the rule name.
# TODO: as we add more and more yara rules, we might remove
# this option and only tag the file with rules that had
# tags specified in them.
if not match_tags:
match_tags = match.rule
# Add the tags to the list.
tag_list.append([entry.sha256, match_tags])
if arg_verbose and rows:
header = [
'Rule',
'String',
'Offset',
'Content'
]
self.log('table', dict(header=header, rows=rows))
# If we selected to add tags do that now.
if found and arg_tag:
db = Database()
for tag in tag_list:
db.add_tags(tag[0], tag[1])
# If in a session reset the session to see tags.
if __sessions__.is_set() and not arg_scan_all:
self.log('info', "Refreshing session to update attributes...")
__sessions__.new(__sessions__.current.file.path)
def cmd_tags(self, *args):
def usage():
print("usage: tags [-h] [-a=tags] [-d=tag]")
def help():
usage()
print("")
print("Options:")
print("\t--help (-h)\tShow this help message")
print(
"\t--add (-a)\tAdd tags to the opened file (comma separated)")
print("\t--delete (-d)\tDelete a tag from the opened file")
print("")
try:
opts, argv = getopt.getopt(args, 'ha:d:',
['help', 'add=', 'delete='])
except getopt.GetoptError as e:
print(e)
usage()
return
arg_add = None
arg_delete = None
for opt, value in opts:
if opt in ('-h', '--help'):
help()
return
elif opt in ('-a', '--add'):
arg_add = value
elif opt in ('-d', '--delete'):
arg_delete = value
# This command requires a session to be opened.
if not __sessions__.is_set():
print_error("No session opened")
return
# If no arguments are specified, there's not much to do.
# However, it could make sense to also retrieve a list of existing
# tags from this command, and not just from the "find" command alone.
if not arg_add and not arg_delete:
usage()
return
# TODO: handle situation where addition or deletion of a tag fail.
if arg_add:
# Add specified tags to the database's entry belonging to
# the opened file.
db = Database()
db.add_tags(__sessions__.current.file.sha256, arg_add)
print_info("Tags added to the currently opened file")
# We refresh the opened session to update the attributes.
# Namely, the list of tags returned by the "info" command
# needs to be re-generated, or it wouldn't show the new tags
# until the existing session is closed a new one is opened.
print_info("Refreshing session to update attributes...")
__sessions__.new(__sessions__.current.file.path)
if arg_delete:
# Delete the tag from the database.
Database().delete_tag(arg_delete)
# Refresh the session so that the attributes of the file are
# updated.
print_info("Refreshing session to update attributes...")
__sessions__.new(__sessions__.current.file.path)
def scan(self):
def string_printable(line):
line = str(line)
new_line = ''
for c in line:
if c in printstring.printable:
new_line += c
else:
new_line += '\\x' + c.encode('hex')
return new_line
# This means users can just drop or remove rule files without
# having to worry about maintaining the index.
# TODO: make paths absolute.
# TODO: this regenerates the file at every run, perhaps we
# could find a way to optimize this.
def rule_index():
tmp_path = os.path.join(tempfile.gettempdir(), 'index.yara')
with open(tmp_path, 'w') as rules_index:
for rule_file in os.listdir(self.rule_path):
# Skip if the extension is not right, could cause problems.
if not rule_file.endswith(
'.yar') and not rule_file.endswith('.yara'):
continue
# Skip if it's the index itself.
if rule_file == 'index.yara':
continue
# Add the rule to the index.
line = 'include "{0}"\n'.format(
os.path.join(self.rule_path, rule_file))
rules_index.write(line)
return tmp_path
arg_rule = self.args.rule
arg_scan_all = self.args.all
arg_tag = self.args.tag
arg_verbose = self.args.verbose
# If no custom ruleset is specified, we use the default one.
if not arg_rule:
arg_rule = rule_index()
# Check if the selected ruleset actually exists.
if not os.path.exists(arg_rule):
self.log('error', "No valid Yara ruleset at {0}".format(arg_rule))
return
# Compile all rules from given ruleset.
rules = yara.compile(arg_rule)
files = []
# If there is a session open and the user didn't specifically
# request to scan the full repository, we just add the currently
# opened file's path.
if __sessions__.is_set() and not arg_scan_all:
files.append(__sessions__.current.file)
# Otherwise we loop through all files in the repository and queue
# them up for scan.
else:
self.log('info', "Scanning all stored files...")
db = Database()
samples = db.find(key='all')
for sample in samples:
files.append(sample)
for entry in files:
if entry.size == 0:
continue
self.log('info',
"Scanning {0} ({1})".format(entry.name, entry.sha256))
# Check if the entry has a path attribute. This happens when
# there is a session open. We need to distinguish this just for
# the cases where we're scanning an opened file which has not been
# stored yet.
if hasattr(entry, 'path'):
entry_path = entry.path
# This should be triggered only when scanning the full repository.
else:
entry_path = get_sample_path(entry.sha256)
# Check if the file exists before running the yara scan.
if not os.path.exists(entry_path):
self.log(
'error',
"The file does not exist at path {0}".format(entry_path))
return
rows = []
tag_list = []
found = False
for match in rules.match(entry_path):
found = True
# Add a row for each string matched by the rule.
if arg_verbose:
for string in match.strings:
rows.append([
match.rule,
string_printable(string[1]),
string_printable(string[0]),
string_printable(string[2])
])
else:
self.log('item', match.rule)
# Add matching rules to our list of tags.
# First it checks if there are tags specified in the metadata
# of the Yara rule.
match_tags = match.meta.get('tags')
# If not, use the rule name.
# TODO: as we add more and more yara rules, we might remove
# this option and only tag the file with rules that had
# tags specified in them.
if not match_tags:
match_tags = match.rule
# Add the tags to the list.
tag_list.append([entry.sha256, match_tags])
if arg_verbose and rows:
header = ['Rule', 'String', 'Offset', 'Content']
self.log('table', dict(header=header, rows=rows))
# If we selected to add tags do that now.
if found and arg_tag:
db = Database()
for tag in tag_list:
db.add_tags(tag[0], tag[1])
# If in a session reset the session to see tags.
if __sessions__.is_set() and not arg_scan_all:
self.log('info',
"Refreshing session to update attributes...")
__sessions__.new(__sessions__.current.file.path)
def scan(self):
def usage():
print("usage: yara scan [-a]")
def help():
usage()
print("")
print("Options:")
print("\t--help (-h)\tShow this help message")
print("\t--rule (-r)\tSpecify a ruleset file path (default will run data/yara/index.yara)")
print("\t--all (-a)\tScan all stored files (default if no session is open)")
print("\t--tag (-t)\tTag Files with Rule Name (default is not to)")
print("")
def string_printable(line):
line = str(line)
new_line = ''
for c in line:
if c in printstring.printable:
new_line += c
else:
new_line += '\\x'+c.encode('hex')
return new_line
arg_rule = ''
arg_scan_all = False
arg_tag = False
try:
opts, argv = getopt.getopt(self.args[1:], 'hr:at', ['help', 'rule=', 'all', 'tag'])
except getopt.GetoptError as e:
print(e)
return
for opt, value in opts:
if opt in ('-h', '--help'):
help()
return
if opt in ('-t', '--tag'):
arg_tag = True
elif opt in ('-r', '--rule'):
arg_rule = value
elif opt in ('-a', '--all'):
arg_scan_all = True
# If no custom ruleset is specified, we use the default one.
if not arg_rule:
arg_rule = 'data/yara/index.yara'
# Check if the selected ruleset actually exists.
if not os.path.exists(arg_rule):
print_error("No valid Yara ruleset at {0}".format(arg_rule))
return
# Compile all rules from given ruleset.
rules = yara.compile(arg_rule)
files = []
# If there is a session open and the user didn't specifically
# request to scan the full repository, we just add the currently
# opened file's path.
if __sessions__.is_set() and not arg_scan_all:
files.append(__sessions__.current.file)
# Otherwise we loop through all files in the repository and queue
# them up for scan.
else:
print_info("Scanning all stored files...")
db = Database()
samples = db.find(key='all')
for sample in samples:
files.append(sample)
for entry in files:
print_info("Scanning {0} ({1})".format(entry.name, entry.sha256))
# Check if the entry has a path attribute. This happens when
# there is a session open. We need to distinguish this just for
# the cases where we're scanning an opened file which has not been
# stored yet.
if hasattr(entry, 'path'):
entry_path = entry.path
# This should be triggered only when scanning the full repository.
else:
entry_path = get_sample_path(entry.sha256)
rows = []
tag_list = []
for match in rules.match(entry_path):
# Add a row for each string matched by the rule.
for string in match.strings:
rows.append([match.rule, string_printable(string[1]), string_printable(string[0]), string_printable(string[2])])
# Add matching rules to our list of tags.
# First it checks if there are tags specified in the metadata
# of the Yara rule.
match_tags = match.meta.get('tags')
# If not, use the rule name.
if not match_tags:
match_tags = match.rule
# Add the tags to the list.
tag_list.append([entry.sha256, match_tags])
if rows:
header = [
'Rule',
'String',
'Offset',
'Content'
]
print(table(header=header, rows=rows))
# If we selected to add tags do that now.
if rows and arg_tag:
db = Database()
for tag in tag_list:
db.add_tags(tag[0], tag[1])
# If in a session reset the session to see tags.
if __sessions__.is_set() and not arg_scan_all:
print_info("Refreshing session to update attributes...")
__sessions__.new(__sessions__.current.file.path)
def scan(self):
def string_printable(line):
line = str(line)
new_line = ""
for c in line:
if c in printstring.printable:
new_line += c
else:
new_line += "\\x" + c.encode("hex")
return new_line
# This means users can just drop or remove rule files without
# having to worry about maintaining the index.
# TODO: make paths absolute.
# TODO: this regenerates the file at every run, perhaps we
# could find a way to optimize this.
def rule_index():
tmp_path = os.path.join(tempfile.gettempdir(), "index.yara")
with open(tmp_path, "w") as rules_index:
for rule_file in os.listdir(self.rule_path):
# Skip if the extension is not right, could cause problems.
if not rule_file.endswith(".yar") and not rule_file.endswith(".yara"):
continue
# Skip if it's the index itself.
if rule_file == "index.yara":
continue
# Add the rule to the index.
line = 'include "{0}"\n'.format(os.path.join(self.rule_path, rule_file))
rules_index.write(line)
return tmp_path
arg_rule = self.args.rule
arg_scan_all = self.args.all
arg_tag = self.args.tag
# If no custom ruleset is specified, we use the default one.
if not arg_rule:
arg_rule = rule_index()
# Check if the selected ruleset actually exists.
if not os.path.exists(arg_rule):
self.log("error", "No valid Yara ruleset at {0}".format(arg_rule))
return
# Compile all rules from given ruleset.
rules = yara.compile(arg_rule)
files = []
# If there is a session open and the user didn't specifically
# request to scan the full repository, we just add the currently
# opened file's path.
if __sessions__.is_set() and not arg_scan_all:
files.append(__sessions__.current.file)
# Otherwise we loop through all files in the repository and queue
# them up for scan.
else:
self.log("info", "Scanning all stored files...")
db = Database()
samples = db.find(key="all")
for sample in samples:
files.append(sample)
for entry in files:
if entry.size == 0:
continue
self.log("info", "Scanning {0} ({1})".format(entry.name, entry.sha256))
# Check if the entry has a path attribute. This happens when
# there is a session open. We need to distinguish this just for
# the cases where we're scanning an opened file which has not been
# stored yet.
if hasattr(entry, "path"):
entry_path = entry.path
# This should be triggered only when scanning the full repository.
else:
entry_path = get_sample_path(entry.sha256)
# Check if the file exists before running the yara scan.
if not os.path.exists(entry_path):
self.log("error", "The file does not exist at path {0}".format(entry_path))
return
rows = []
tag_list = []
for match in rules.match(entry_path):
# Add a row for each string matched by the rule.
for string in match.strings:
rows.append(
[
match.rule,
string_printable(string[1]),
string_printable(string[0]),
string_printable(string[2]),
]
)
# Add matching rules to our list of tags.
# First it checks if there are tags specified in the metadata
# of the Yara rule.
match_tags = match.meta.get("tags")
# If not, use the rule name.
# TODO: as we add more and more yara rules, we might remove
# this option and only tag the file with rules that had
# tags specified in them.
if not match_tags:
match_tags = match.rule
# Add the tags to the list.
tag_list.append([entry.sha256, match_tags])
if rows:
header = ["Rule", "String", "Offset", "Content"]
self.log("table", dict(header=header, rows=rows))
# If we selected to add tags do that now.
if rows and arg_tag:
db = Database()
for tag in tag_list:
db.add_tags(tag[0], tag[1])
# If in a session reset the session to see tags.
if __sessions__.is_set() and not arg_scan_all:
self.log("info", "Refreshing session to update attributes...")
__sessions__.new(__sessions__.current.file.path)
class ClamAV(Module):
cmd = 'clamav'
description = 'Scan file from local ClamAV daemon'
authors = ['neriberto']
def __init__(self):
super(ClamAV, self).__init__()
self.parser.add_argument(
'-s',
'--socket',
help='Specify an unix socket (default: Clamd Unix Socket)')
self.parser.add_argument('-a',
'--all',
action='store_true',
help='Scan all files')
self.parser.add_argument(
'-t',
'--tag',
action='store_true',
help='Tag file(s) with the signature name when detect as malware')
self.db = Database()
def run(self):
super(ClamAV, self).run()
if self.args is None:
return
if not HAVE_CLAMD:
self.log(
'error',
"Missing dependency, install pyclamd (`pip install pyclamd`)")
return
if not self.Connect():
self.log('error', 'Daemon is not responding!')
return
if self.args.all:
self.ScanAll()
elif __sessions__.is_set():
self.ScanFile(__sessions__.current.file)
else:
self.log('error', 'No open session')
def ScanAll(self):
samples = self.db.find(key='all')
for sample in samples:
if sample.size == 0:
continue
self.ScanFile(sample)
def Connect(self):
self.daemon = None
self.socket = self.args.socket
try:
if self.socket is not None:
self.daemon = pyclamd.ClamdUnixSocket(self.socket)
self.log('info',
'Using socket {0} to scan'.format(self.socket))
else:
self.daemon = pyclamd.ClamdUnixSocket()
self.socket = 'Clamav'
return self.daemon.ping()
except Exception as ex:
msg = 'Daemon connection failure, {0}'.format(ex)
self.log('error,', msg)
return False
def ScanFile(self, sample):
if isinstance(sample, Malware):
sample_path = get_sample_path(sample.sha256)
else:
sample_path = sample.path
if not os.path.exists(sample_path):
self.log(
'error',
'The file does not exists at path {0}'.format(sample_path))
return
try:
if self.daemon.ping():
with open(sample_path, 'rb') as fd:
results = self.daemon.scan_stream(fd.read())
else:
self.log('error', "Unable to connect to the daemon")
except Exception as ex:
msg = 'Unable to scan file {0} with antivirus daemon, {1}'.format(
sample.sha256, ex)
self.log('error', msg)
return
found = None
name = None
if results:
for item in results:
found = results[item][0]
name = results[item][1]
if found == 'ERROR':
self.log(
'error',
"Check permissions of the binary folder, {0}".format(name))
else:
if name is not None:
if self.args.tag:
self.db.add_tags(sample.sha256, name)
else:
name = 'Threat not found!'
self.log(
'info',
"{0} identify {1} as : {2}".format(self.socket, sample.sha256,
name))
class ClamAV(Module):
cmd = 'clamav'
description = 'Scan file from local ClamAV daemon'
authors = ['neriberto']
def __init__(self):
super(ClamAV, self).__init__()
self.parser.add_argument('-s', '--socket', help='Specify an unix socket (default: Clamd Unix Socket)')
self.parser.add_argument('-a', '--all', action='store_true', help='Scan all files')
self.parser.add_argument('-t', '--tag', action='store_true', help='Tag file(s) with the signature name when detect as malware')
self.db = Database()
def run(self):
super(ClamAV, self).run()
if self.args is None:
return
if not HAVE_CLAMD:
self.log('error', "Missing dependency, install pyclamd (`pip install pyclamd`)")
return
if not self.Connect():
self.log('error', 'Daemon is not responding!')
return
if not __sessions__.is_set():
if self.args.all:
self.ScanAll()
else:
self.log('error', 'No open session')
else:
self.ScanFile(__sessions__.current.file.sha256)
def ScanAll(self):
samples = self.db.find(key='all')
for sample in samples:
if sample.size == 0:
continue
self.ScanFile(sample.sha256)
def Connect(self):
self.daemon = None
self.socket = self.args.socket
try:
if self.socket is not None:
self.daemon = pyclamd.ClamdUnixSocket(self.socket)
self.log('info', 'Using socket {0} to scan'.format(self.socket))
else:
self.daemon = pyclamd.ClamdUnixSocket()
self.socket = 'Clamav'
return self.daemon.ping()
except Exception as ex:
msg = 'Daemon connection failure, {0}'.format(ex)
self.log('error,', msg)
return False
def ScanFile(self, sha256):
if sha256 is not None:
filepath = get_sample_path(sha256)
if not os.path.exists(filepath):
self.log('error', 'The file does not exists at path {0}'.format(filepath))
return
try:
if self.daemon.ping():
with open(filepath, 'r') as fd:
results = self.daemon.scan_stream(fd.read())
else:
self.log('error', "Unable to connect to the daemon")
except Exception as ex:
msg = 'Unable to scan file {0} with antivirus daemon, {1}'.format(sha256, ex)
self.log('error', msg)
return
found = None
name = None
if results:
for item in results:
found = results[item][0]
name = results[item][1]
if found == 'ERROR':
self.log('error', "Check permissions of the binary folder, {0}".format(name))
else:
if name is not None:
if self.args.tag:
self.db.add_tags(sha256, name)
else:
name = 'Threat not found!'
self.log('info', "{0} identify {1} as : {2}".format(self.socket, sha256, name))
def scan(self):
def usage():
print("usage: yara scan [-a]")
def help():
usage()
print("")
print("Options:")
print("\t--help (-h)\tShow this help message")
print(
"\t--rule (-r)\tSpecify a ruleset file path (default will run data/yara/index.yara)"
)
print(
"\t--all (-a)\tScan all stored files (default if no session is open)"
)
print("\t--tag (-t)\tTag Files with Rule Name (default is not to)")
print("")
def string_printable(line):
line = str(line)
new_line = ''
for c in line:
if c in printstring.printable:
new_line += c
else:
new_line += '\\x' + c.encode('hex')
return new_line
# This means users can just drop or remove rule files without
# having to worry about maintaining the index.
# TODO: make paths absolute.
# TODO: this regenerates the file at every run, perhaps we
# could find a way to optimize this.
def rule_index():
with open('data/yara/index.yara', 'w') as rules_index:
for rule_file in os.listdir('data/yara'):
# Skip if the extension is not right, could cause problems.
if not rule_file.endswith(
'.yar') and not rule_file.endswith('.yara'):
continue
# Skip if it's the index itself.
if rule_file == 'index.yara':
continue
# Add the rule to the index.
line = 'include "{0}"\n'.format(rule_file)
rules_index.write(line)
return 'data/yara/index.yara'
arg_rule = ''
arg_scan_all = False
arg_tag = False
try:
opts, argv = getopt.getopt(self.args[1:], 'hr:at',
['help', 'rule=', 'all', 'tag'])
except getopt.GetoptError as e:
print(e)
return
for opt, value in opts:
if opt in ('-h', '--help'):
help()
return
if opt in ('-t', '--tag'):
arg_tag = True
elif opt in ('-r', '--rule'):
arg_rule = value
elif opt in ('-a', '--all'):
arg_scan_all = True
# If no custom ruleset is specified, we use the default one.
if not arg_rule:
arg_rule = rule_index()
# Check if the selected ruleset actually exists.
if not os.path.exists(arg_rule):
print_error("No valid Yara ruleset at {0}".format(arg_rule))
return
# Compile all rules from given ruleset.
rules = yara.compile(arg_rule)
files = []
# If there is a session open and the user didn't specifically
# request to scan the full repository, we just add the currently
# opened file's path.
if __sessions__.is_set() and not arg_scan_all:
files.append(__sessions__.current.file)
# Otherwise we loop through all files in the repository and queue
# them up for scan.
else:
print_info("Scanning all stored files...")
db = Database()
samples = db.find(key='all')
for sample in samples:
files.append(sample)
for entry in files:
if entry.size == 0: continue
print_info("Scanning {0} ({1})".format(entry.name, entry.sha256))
# Check if the entry has a path attribute. This happens when
# there is a session open. We need to distinguish this just for
# the cases where we're scanning an opened file which has not been
# stored yet.
if hasattr(entry, 'path'):
entry_path = entry.path
# This should be triggered only when scanning the full repository.
else:
entry_path = get_sample_path(entry.sha256)
rows = []
tag_list = []
for match in rules.match(entry_path):
# Add a row for each string matched by the rule.
for string in match.strings:
rows.append([
match.rule,
string_printable(string[1]),
string_printable(string[0]),
string_printable(string[2])
])
# Add matching rules to our list of tags.
# First it checks if there are tags specified in the metadata
# of the Yara rule.
match_tags = match.meta.get('tags')
# If not, use the rule name.
# TODO: as we add more and more yara rules, we might remove
# this option and only tag the file with rules that had
# tags specified in them.
if not match_tags:
match_tags = match.rule
# Add the tags to the list.
tag_list.append([entry.sha256, match_tags])
if rows:
header = ['Rule', 'String', 'Offset', 'Content']
print(table(header=header, rows=rows))
# If we selected to add tags do that now.
if rows and arg_tag:
db = Database()
for tag in tag_list:
db.add_tags(tag[0], tag[1])
# If in a session reset the session to see tags.
if __sessions__.is_set() and not arg_scan_all:
print_info("Refreshing session to update attributes...")
__sessions__.new(__sessions__.current.file.path)
def cmd_tags(self, *args):
def usage():
print("usage: tags [-h] [-a=tags] [-d=tag]")
def help():
usage()
print("")
print("Options:")
print("\t--help (-h)\tShow this help message")
print("\t--add (-a)\tAdd tags to the opened file (comma separated)")
print("\t--delete (-d)\tDelete a tag from the opened file")
print("")
try:
opts, argv = getopt.getopt(args, "ha:d:", ["help", "add=", "delete="])
except getopt.GetoptError as e:
print(e)
usage()
return
arg_add = None
arg_delete = None
for opt, value in opts:
if opt in ("-h", "--help"):
help()
return
elif opt in ("-a", "--add"):
arg_add = value
elif opt in ("-d", "--delete"):
arg_delete = value
# This command requires a session to be opened.
if not __sessions__.is_set():
print_error("No session opened")
return
# If no arguments are specified, there's not much to do.
# However, it could make sense to also retrieve a list of existing
# tags from this command, and not just from the "find" command alone.
if not arg_add and not arg_delete:
usage()
return
# TODO: handle situation where addition or deletion of a tag fail.
if arg_add:
# Add specified tags to the database's entry belonging to
# the opened file.
db = Database()
db.add_tags(__sessions__.current.file.sha256, arg_add)
print_info("Tags added to the currently opened file")
# We refresh the opened session to update the attributes.
# Namely, the list of tags returned by the "info" command
# needs to be re-generated, or it wouldn't show the new tags
# until the existing session is closed a new one is opened.
print_info("Refreshing session to update attributes...")
__sessions__.new(__sessions__.current.file.path)
if arg_delete:
# Delete the tag from the database.
Database().delete_tag(arg_delete)
# Refresh the session so that the attributes of the file are
# updated.
print_info("Refreshing session to update attributes...")
__sessions__.new(__sessions__.current.file.path)
def scan(self):
def usage():
self.log('', "usage: yara scan [-a]")
def help():
usage()
self.log('', "")
self.log('', "Options:")
self.log('', "\t--help (-h)\tShow this help message")
self.log('', "\t--rule (-r)\tSpecify a ruleset file path (default will run data/yara/index.yara)")
self.log('', "\t--all (-a)\tScan all stored files (default if no session is open)")
self.log('', "\t--tag (-t)\tTag Files with Rule Name (default is not to)")
self.log('', "")
def string_printable(line):
line = str(line)
new_line = ''
for c in line:
if c in printstring.printable:
new_line += c
else:
new_line += '\\x'+c.encode('hex')
return new_line
# This means users can just drop or remove rule files without
# having to worry about maintaining the index.
# TODO: make paths absolute.
# TODO: this regenerates the file at every run, perhaps we
# could find a way to optimize this.
def rule_index():
with open('data/yara/index.yara', 'w') as rules_index:
for rule_file in os.listdir('data/yara'):
# Skip if the extension is not right, could cause problems.
if not rule_file.endswith('.yar') and not rule_file.endswith('.yara'):
continue
# Skip if it's the index itself.
if rule_file == 'index.yara':
continue
# Add the rule to the index.
line = 'include "{0}"\n'.format(rule_file)
rules_index.write(line)
return 'data/yara/index.yara'
arg_rule = ''
arg_scan_all = False
arg_tag = False
try:
opts, argv = getopt.getopt(self.args[1:], 'hr:at', ['help', 'rule=', 'all', 'tag'])
except getopt.GetoptError as e:
self.log('', e)
return
for opt, value in opts:
if opt in ('-h', '--help'):
help()
return
if opt in ('-t', '--tag'):
arg_tag = True
elif opt in ('-r', '--rule'):
arg_rule = value
elif opt in ('-a', '--all'):
arg_scan_all = True
# If no custom ruleset is specified, we use the default one.
if not arg_rule:
arg_rule = rule_index()
# Check if the selected ruleset actually exists.
if not os.path.exists(arg_rule):
self.log('error', "No valid Yara ruleset at {0}".format(arg_rule))
return
# Compile all rules from given ruleset.
rules = yara.compile(arg_rule)
files = []
# If there is a session open and the user didn't specifically
# request to scan the full repository, we just add the currently
# opened file's path.
if __sessions__.is_set() and not arg_scan_all:
files.append(__sessions__.current.file)
# Otherwise we loop through all files in the repository and queue
# them up for scan.
else:
self.log('info', "Scanning all stored files...")
db = Database()
samples = db.find(key='all')
for sample in samples:
files.append(sample)
for entry in files:
if entry.size == 0: continue
self.log('info', "Scanning {0} ({1})".format(entry.name, entry.sha256))
# Check if the entry has a path attribute. This happens when
# there is a session open. We need to distinguish this just for
# the cases where we're scanning an opened file which has not been
# stored yet.
if hasattr(entry, 'path'):
entry_path = entry.path
# This should be triggered only when scanning the full repository.
else:
entry_path = get_sample_path(entry.sha256)
rows = []
tag_list = []
for match in rules.match(entry_path):
# Add a row for each string matched by the rule.
for string in match.strings:
rows.append([match.rule, string_printable(string[1]), string_printable(string[0]), string_printable(string[2])])
# Add matching rules to our list of tags.
# First it checks if there are tags specified in the metadata
# of the Yara rule.
match_tags = match.meta.get('tags')
# If not, use the rule name.
# TODO: as we add more and more yara rules, we might remove
# this option and only tag the file with rules that had
# tags specified in them.
if not match_tags:
match_tags = match.rule
# Add the tags to the list.
tag_list.append([entry.sha256, match_tags])
if rows:
header = [
'Rule',
'String',
'Offset',
'Content'
]
self.log('table', dict(header=header, rows=rows))
# If we selected to add tags do that now.
if rows and arg_tag:
db = Database()
for tag in tag_list:
db.add_tags(tag[0], tag[1])
# If in a session reset the session to see tags.
if __sessions__.is_set() and not arg_scan_all:
self.log('info', "Refreshing session to update attributes...")
__sessions__.new(__sessions__.current.file.path)
def scan(self):
arg_rule = self.args.rule
arg_scan_all = self.args.all
arg_tag = self.args.tag
arg_verbose = self.args.verbose
externals = {'filename': '', 'filepath': '', 'extension': '', 'filetype': ''}
# If a rule file is specified we compile that, otherwise all
# the rules we have stored locally.
if arg_rule:
# Check if the selected ruleset actually exists.
if not os.path.exists(arg_rule):
self.log('error', "The specified file does not exist at path {0}".format(arg_rule))
return
rules = yara.compile(arg_rule, externals=externals)
# Otherwise, we get all the rules that are stored locally and we
# load them in different namespaces.
else:
filepaths = dict()
for rule in self._get_rules():
# TODO: We pre-compile all rules individually to check whether they are
# loadable or not. This is pretty hacky, there must be a better way.
try:
yara.compile(rule[1], externals=externals)
except yara.SyntaxError as e:
self.log('warning', "Unable to compile rule {0}: {1}".format(rule[1], e))
continue
filepaths['namespace' + str(rule[0])] = rule[1]
rules = yara.compile(filepaths=filepaths, externals=externals, includes=False)
# Files to scan.
files = []
# If there is a session open and the user didn't specifically
# request to scan the full repository, we just add the currently
# opened file's path.
if __sessions__.is_set() and not arg_scan_all:
files.append(__sessions__.current.file)
# Otherwise we loop through all files in the repository and queue
# them up for scan.
else:
self.log('info', "Scanning all stored files (in the current project)...")
db = Database()
samples = db.find(key='all')
for sample in samples:
files.append(sample)
# Loop through all files to be scanned.
for entry in files:
if entry.size == 0:
continue
self.log('info', "Scanning {0} ({1})".format(entry.name, entry.sha256))
# Check if the entry has a path attribute. This happens when
# there is a session open. We need to distinguish this just for
# the cases where we're scanning an opened file which has not been
# stored yet.
if hasattr(entry, 'path'):
entry_path = entry.path
# This should be triggered only when scanning the full repository.
else:
entry_path = get_sample_path(entry.sha256)
# Check if the file exists before running the yara scan.
if not os.path.exists(entry_path):
self.log('error', "The file does not exist at path {0}".format(entry_path))
continue
rows = []
tag_list = []
found = False
# We need this just for some Yara rules.
try:
ext = os.path.splitext(entry.name)[1]
except Exception:
ext = ''
try:
matches = rules.match(entry_path, externals={'filename': entry.name, 'filepath': entry_path, 'extension': ext, 'filetype': entry.type})
except yara.Error as e:
self.log('error', "Yara scan for file {} ({}) failed: {}".format(entry.name, entry.sha256, e))
continue
for match in matches:
found = True
# Add a row for each string matched by the rule.
if arg_verbose:
for match_string in match.strings:
rows.append([
match.rule,
string_printable(match_string[1]),
string_printable(match_string[0]),
string_printable(match_string[2])]
)
else:
self.log('item', match.rule)
# Add matching rules to our list of tags.
# First it checks if there are tags specified in the metadata
# of the Yara rule.
match_tags = match.meta.get('tags')
# If not, use the rule name.
# TODO: as we add more and more yara rules, we might remove
# this option and only tag the file with rules that had
# tags specified in them.
if not match_tags:
match_tags = match.rule
# Add the tags to the list.
tag_list.append([entry.sha256, match_tags])
if arg_verbose and rows:
header = [
'Rule',
'String',
'Offset',
'Content'
]
self.log('table', dict(header=header, rows=rows))
# If we selected to add tags do that now.
if found and arg_tag:
db = Database()
for tag in tag_list:
db.add_tags(tag[0], tag[1])
# If in a session reset the session to see tags.
if __sessions__.is_set() and not arg_scan_all:
self.log('info', "Refreshing session to update attributes...")
__sessions__.new(__sessions__.current.file.path)
def cmd_tags(self, *args):
def usage():
print("usage: tags [-h] [-a=tags] [-d=tag]")
def help():
usage()
print("")
print("Options:")
print("\t--help (-h)\tShow this help message")
print("\t--add (-a)\tAdd tags to the opened file (comma separated)")
print("\t--delete (-d)\tDelete a tag from the opened file")
print("")
try:
opts, argv = getopt.getopt(args, 'ha:d:', ['help', 'add=', 'delete='])
except getopt.GetoptError as e:
print(e)
usage()
return
arg_add = None
arg_delete = None
for opt, value in opts:
if opt in ('-h', '--help'):
help()
return
elif opt in ('-a', '--add'):
arg_add = value
elif opt in ('-d', '--delete'):
arg_delete = value
# This command requires a session to be opened.
if not __session__.is_set():
print_error("No session opened")
return
# If no arguments are specified, there's not much to do.
# However, it could make sense to also retrieve a list of existing
# tags from this command, and not just from the "find" command alone.
if not arg_add and not arg_delete:
print_error("You need to specify an option, either add or delete")
return
if arg_add:
# Add specified tags to the database's entry belonging to
# the opened file.
db = Database()
db.add_tags(__session__.file.sha256, arg_add)
print_info("Tags added to the currently opened file")
# We refresh the opened session to update the attributes.
# Namely, the list of tags returned by the "info" command
# needs to be re-generated, or it wouldn't show the new tags
# until the existing session is closed a new one is opened.
print_info("Refreshing session to update attributes...")
__session__.set(__session__.file.path)
if arg_delete:
# TODO
pass
