def parse_ida_cmds(self, sid, functions):
"""
Parse and add IDA commands dumped by AnalyzeIt,
and updates the functions names if needed
"""
idac = IDAActionsController()
funcs = dict.copy(functions)
fname = self.storage_file + '.idacmd'
act = None
if not os.path.exists(fname):
return funcs
with open(fname) as fdata:
for line in fdata:
if line.startswith('idc.MakeName'):
addr, name = self.get_addr_data(line)
try:
# update functions list with idc.MakeName() information
funcs[addr]['name'] = name
except KeyError:
app.logger.debug("No function found for %x" % (addr))
act = idac.add_name(addr, name)
elif line.startswith('idc.MakeRptCmt'):
addr, cmt = self.get_addr_data(line)
act = idac.add_comment(addr, cmt)
else:
app.logger.debug("Unknown IDA command %s" % (line))
continue
SampleController.add_idaaction(sid, act)
return funcs
def parse_ida_cmds(self, sid, functions):
"""
Parse and add IDA commands dumped by AnalyzeIt,
and updates the functions names if needed
"""
idac = IDAActionsController()
funcs = dict.copy(functions)
fname = self.storage_file + '.idacmd'
act = None
if not os.path.exists(fname):
return funcs
with open(fname) as fdata:
for line in fdata:
if line.startswith('idc.MakeName'):
addr, name = self.get_addr_data(line)
try:
# update functions list with idc.MakeName() information
funcs[addr]['name'] = name
except KeyError:
app.logger.debug("No function found for %x" % (addr))
act = idac.add_name(addr, name)
elif line.startswith('idc.MakeRptCmt'):
addr, cmt = self.get_addr_data(line)
act = idac.add_comment(addr, cmt)
else:
app.logger.debug("Unknown IDA command %s" % (line))
continue
with app.app_context():
SampleController.add_idaaction(sid, act)
return funcs
def apply_result(self):
s_controller = SampleController()
sample = s_controller.get_by_id(self.sid)
app.logger.debug(self.tmessage + "APPLY_RESULT")
s_controller.add_multiple_strings(sample, self.resultstrings)
app.logger.debug(self.tmessage + "END - TIME %i" %
(int(time.time()) - self.tstart))
return True
def apply_result(self):
s_controller = SampleController()
sample = s_controller.get_by_id(self.sid)
app.logger.debug(self.tmessage + "APPLY_RESULT")
s_controller.add_multiple_strings(sample, self.resultstrings)
app.logger.debug(self.tmessage + "END - TIME %i" %
(int(time.time()) - self.tstart))
return True
def apply_result(self):
samplecontrol = SampleController()
sample = SampleController.get_by_id(self.sid)
if sample is None:
app.logger.error(self.tmessage + "Sample has disappeared...")
raise IOError
app.logger.debug(self.tmessage + "APPLY_RESULT")
# TXT report
app.logger.info("Creating new analyzeit report")
SampleController.create_analysis(sample, self.txt_report, "analyzeit",
True)
functions = self.parse_machoc_signatures()
# IDA COMMANDS report:
app.logger.info("Parsing idacommands")
functions = self.parse_ida_cmds(sample.id, functions)
# Functions: just push the list
app.logger.info("Storing functions")
samplecontrol.add_multiple_functions(sample, functions)
# global machoc match
app.logger.info("Calculating machoc80 matches")
samplecontrol.match_by_machoc80(sample)
return True
def execute(self):
"""
Extended yara execution. Stores hits on the yaramatched attribute.
"""
s_controller = SampleController()
sample = s_controller.get_by_id(self.sid)
self.tstart = int(time.time())
app.logger.debug(self.tmessage + "EXECUTE")
for yar in YaraRule.query.all():
if run_extended_yara(yar.raw_rule, sample) is True:
self.yaramatched.append(yar)
return True
def execute(self):
"""
Extended yara execution. Stores hits on the yaramatched attribute.
"""
s_controller = SampleController()
with app.app_context():
sample = s_controller.get_by_id(self.sid)
self.tstart = int(time.time())
app.logger.debug(self.tmessage + "EXECUTE")
for yar in YaraRule.query.all():
if run_extended_yara(yar.raw_rule, sample) is True:
self.yaramatched.append(yar)
return True
def apply_result(self):
"""
Commits to database.
"""
s_controller = SampleController()
sample = s_controller.get_by_id(self.sid)
app.logger.debug(self.tmessage + "APPLY_RESULT")
for match in self.yaramatched:
# use the static YaraController => the () will create a JobPool,
# causing exceptions (daemon => child).
YaraController.add_to_sample(sample, match)
app.logger.debug(self.tmessage + "END - TIME %i" %
(int(time.time()) - self.tstart))
return True
def apply_result(self):
"""
Commits to database.
"""
s_controller = SampleController()
sample = s_controller.get_by_id(self.sid)
app.logger.debug(self.tmessage + "APPLY_RESULT")
for match in self.yaramatched:
# use the static YaraController => the () will create a JobPool,
# causing exceptions (daemon => child).
YaraController.add_to_sample(sample, match)
app.logger.debug(self.tmessage + "END - TIME %i" %
(int(time.time()) - self.tstart))
return True
def apply_result(self):
with app.app_context():
samplecontrol = SampleController()
sample = SampleController.get_by_id(self.sid)
if sample is None:
app.logger.error(self.tmessage + "Sample has disappeared...")
raise IOError
app.logger.debug(self.tmessage + "APPLY_RESULT")
# TXT report
app.logger.info("Creating new analyzeit report")
SampleController.create_analysis(
sample, self.txt_report, "analyzeit", True)
functions = self.parse_machoc_signatures()
# IDA COMMANDS report:
app.logger.info("Parsing idacommands")
functions = self.parse_ida_cmds(sample.id, functions)
# Functions: just push the list
app.logger.info("Storing functions")
samplecontrol.add_multiple_functions(self.sid, functions)
# global machoc match
app.logger.info("Calculating machoc80 matches")
samplecontrol.match_by_machoc80(sample)
return True
def apply_result(self):
s_controller = SampleController()
sample = s_controller.get_by_id(self.sid)
app.logger.debug(self.tmessage + "APPLY_RESULT")
# Compilation timestamp (even when faked) IS a file date, so update it.
s_controller.add_multiple_metadata(sample, self.metadata_extracted)
s_controller.set_file_date(sample, self.compile_timestamp)
s_controller.set_import_hash(sample, self.import_hash)
app.logger.debug(self.tmessage + "END - TIME %i" %
(int(time.time()) - self.tstart))
return True
def apply_result(self):
s_controller = SampleController()
sample = s_controller.get_by_id(self.sid)
app.logger.debug(self.tmessage + "APPLY_RESULT")
# Compilation timestamp (even when faked) IS a file date, so update it.
s_controller.add_multiple_metadata(sample, self.metadata_extracted)
s_controller.set_file_date(sample, self.compile_timestamp)
s_controller.set_import_hash(sample, self.import_hash)
app.logger.debug(self.tmessage + "END - TIME %i" %
(int(time.time()) - self.tstart))
return True
class APIControl(object):
"""
Object used as a global API.
Data controllers are used for direct data manipulation.
Methods are used for complex (cross-data) manipulation.
TODO: create a brand new analysis scehduler working on database
samples status, and remove the analysis creation from this class.
"""
familycontrol = None
samplecontrol = None
usercontrol = None
analysiscontrol = None
familycontrol = FamilyController()
yaracontrol = YaraController()
samplecontrol = SampleController()
usercontrol = UserController()
analysiscontrol = AnalysisController(
app.config['ANALYSIS_PROCESS_POOL_SIZE'])
idacontrol = IDAActionsController()
def __init__(self):
"""
Initiate controllers.
"""
pass
def create_sample_and_run_analysis(
self,
file_data_stream,
originate_filename,
user=None,
tlp_level=TLPLevel.TLPWHITE,
family=None):
"""
Creates a new sample and a schedule an analysis. We also check the
file header for ZIP pattern: if a ZIP pattern is found, any file
inside the archive will be imported and scheduled for analysis.
TODO: move this to the SampleController, and start directly on new
file submission.
"""
file_data = file_data_stream.read()
if file_data.startswith("PK"):
with zipfile.ZipFile(file_data, "r") as zcl:
for name in zcl.namelist():
mfile = zcl.open(name, "r")
sample = self.samplecontrol.create_sample_from_file(
mfile, name, user, tlp_level)
if family is not None:
self.familycontrol.add_sample(sample, family)
if sample.analysis_status == AnalysisStatus.TOSTART:
self.analysiscontrol.schedule_sample_analysis(
sample.id)
zcl.close()
return None
sample = self.samplecontrol.create_sample_from_file(
file_data, originate_filename, user, tlp_level)
if sample.analysis_status == AnalysisStatus.TOSTART:
self.analysiscontrol.schedule_sample_analysis(sample.id)
if family is not None:
self.familycontrol.add_sample(sample, family)
return sample
def add_actions_fromfunc_infos(self, funcinfos, sample_dst, sample_src):
for fid_dst, fid_src in funcinfos:
fsrc = FunctionInfo.query.get(fid_src)
fdst = FunctionInfo.query.get(fid_dst)
if fsrc is None or fdst is None:
return False
if fsrc not in sample_src.functions:
return False
if fdst not in sample_dst.functions:
return False
if fsrc.name.startswith("sub_"):
continue
act = self.idacontrol.add_name(int(fdst.address), fsrc.name)
self.samplecontrol.add_idaaction(sample_dst.id, act)
db.session.commit()
return True
def apply_result(self):
sc = SampleController()
idac = IDAActionsController()
sample = SampleController.get_by_id(self.sid)
if sample is None:
app.logger.error(self.tmessage + "Sample has disappeared...")
raise IOError
app.logger.debug(self.tmessage + "APPLY_RESULT")
# TXT report
app.logger.info("Starting analysis creation")
SampleController.create_analysis(sample, self.txt_report, "analyzeit",
True)
# MACHOC report: we load the functions, hashes, etc.
app.logger.info("Starting functions")
fname = self.storage_file + '.sign'
functions = []
if os.path.exists(fname):
fdata = open(fname, 'rb').read()
items = fdata.split(";")
for i in items:
if ":" in i:
subitems = i.split(":")
machoc_h = subitems[0].strip()
address = subitems[1].strip()
functions.append([address, machoc_h, ""])
# IDA COMMANDS report:
# update functions list with idc.MakeName() information
# TODO: also store comments
app.logger.info("Starting idacommands")
fname = self.storage_file + '.idacmd'
if os.path.exists(fname):
fdata = open(fname, 'rb').read()
for line in fdata.split("\n"):
if line.startswith("idc.MakeName::"):
items = line.split("::")
if len(items) == 3:
addr = items[1]
name = items[2]
if addr.startswith("0x"):
addr = addr[2:]
for i in functions:
if i[0] == addr:
i[2] = name
name_action = idac.add_name(int(addr, 16), name)
SampleController.add_idaaction(sample.id, name_action)
elif line.startswith("idc.MakeRptCmt::"):
items = line.split("::")
if len(items) == 3:
addr = items[1]
value = items[2]
if addr.startswith("0x"):
addr = addr[2:]
try:
addr = int(addr, 16)
except Exception:
continue
act = idac.add_comment(addr, value)
SampleController.add_idaaction(sample.id, act)
# Functions: just push the list
app.logger.info("Storing actions")
if len(functions) > 0:
sc.add_multiple_functions(sample, functions)
# global machoc match
app.logger.info("Matching actions")
sc.match_by_machoc80(sample)
app.logger.debug(self.tmessage + "END - TIME %i" %
(int(time.time()) - self.tstart))
return True
class APIControl(object):
"""
Object used as a global API.
Data controllers are used for direct data manipulation.
Methods are used for complex (cross-data) manipulation.
TODO: create a brand new analysis scheduler working on database
samples status, and remove the analysis creation from this class.
"""
familycontrol = None
samplecontrol = None
usercontrol = None
analysiscontrol = None
familycontrol = FamilyController()
yaracontrol = YaraController()
samplecontrol = SampleController()
usercontrol = UserController()
analysiscontrol = AnalysisController(
app.config['ANALYSIS_PROCESS_POOL_SIZE'])
idacontrol = IDAActionsController()
def __init__(self):
"""
Initiate controllers.
"""
pass
def dispatch_sample_creation(self,
file_stream,
filename="",
user=None,
tlp=TLPLevel.TLPWHITE,
family=None):
"""
If the sample is a ZipFile, we unpack it and return
the last sample,otherwise we return a single sample.
"""
file_data = file_stream.read(4)
file_stream.seek(0)
if file_data.startswith("PK"):
samples = self.create_from_zip(file_stream, user, tlp, family)
else:
sample = self.create_sample_and_run_analysis(
file_stream, filename, user, tlp, family)
samples = [sample]
return samples
def create_from_zip(self, file_stream, user, tlp, family):
"""
Iterates over the samples in the zip
"""
output_samples = []
file_data = StringIO(file_stream.read())
with zipfile.ZipFile(file_data, "r") as zcl:
for name in zcl.namelist():
mfile = zcl.open(name, "r")
sample = self.create_sample_and_run_analysis(
mfile, name, user, tlp, family)
output_samples.append(sample)
zcl.close()
return output_samples
def create_sample_and_run_analysis(self,
file_data_stream,
originate_filename="",
user=None,
tlp_level=TLPLevel.TLPWHITE,
family=None):
"""
Creates a new sample and a schedule an analysis. We also check the
file header for ZIP pattern: if a ZIP pattern is found, any file
inside the archive will be imported and scheduled for analysis.
TODO: move this to the SampleController, and start directly on new
file submission.
"""
file_data = file_data_stream.read()
sample = self.samplecontrol.create_sample_from_file(
file_data, originate_filename, user, tlp_level)
if sample.analysis_status == AnalysisStatus.TOSTART:
self.analysiscontrol.schedule_sample_analysis(sample.id)
if family is not None:
self.familycontrol.add_sample(sample, family)
return sample
def add_actions_fromfunc_infos(self, funcinfos, sample_dst, sample_src):
"""
Create IDAActions from the samples's FuncInfos from AnalyzeIt
"""
for fid_dst, fid_src in funcinfos:
fsrc = FunctionInfo.query.get(fid_src)
fdst = FunctionInfo.query.get(fid_dst)
if fsrc is None or fdst is None:
return False
if fsrc not in sample_src.functions:
return False
if fdst not in sample_dst.functions:
return False
if fsrc.name.startswith("sub_"):
continue
act = self.idacontrol.add_name(int(fdst.address), fsrc.name)
self.samplecontrol.add_idaaction(sample_dst.id, act)
db.session.commit()
return True
def get_elem_by_type(self, element_type, element_id):
"""
Wrapper to get elements by ID in database.
@arg element_type string, "sample", "family",
"checklist", "family_file", "detection_item", "yara"
@arg element_id integer the id to search
@return if found the element,
abort 404 if not found,
and abort 500 if the type is incorrect
"""
elem_types = {
"sample": self.samplecontrol.get_by_id,
"family": self.familycontrol.get_by_id,
"checklist": self.samplecontrol.get_checklist_by_id,
"family_file": self.familycontrol.get_file_by_id,
"detection_item": self.familycontrol.get_detection_item_by_id,
"yara": self.yaracontrol.get_by_id
}
try:
elem = elem_types[element_type](element_id)
except KeyError:
app.logger.exception("Element type unknown")
abort(500)
if elem is None:
flash(element_type + " not found...", "error")
abort(404)
return elem
def remove_user_from_element(self, element_type, element_id, user):
"""
Remove a user from an element, be it a sample or a family
"""
elem_types = {
"family": self.familycontrol,
"sample": self.samplecontrol
}
elem = self.get_elem_by_type(element_type, element_id)
if user in elem.users:
elem_types[element_type].remove_user(user, elem)
else:
elem_types[element_type].add_user(user, elem)
return elem.id
def apply_result(self):
sc = SampleController()
idac = IDAActionsController()
sample = SampleController.get_by_id(self.sid)
if sample is None:
app.logger.error(self.tmessage + "Sample has disappeared...")
raise IOError
app.logger.debug(self.tmessage + "APPLY_RESULT")
# TXT report
app.logger.info("Starting analysis creation")
SampleController.create_analysis(
sample, self.txt_report, "analyzeit", True)
# MACHOC report: we load the functions, hashes, etc.
app.logger.info("Starting functions")
fname = self.storage_file + '.sign'
functions = []
if os.path.exists(fname):
fdata = open(fname, 'rb').read()
items = fdata.split(";")
for i in items:
if ":" in i:
subitems = i.split(":")
machoc_h = subitems[0].strip()
address = subitems[1].strip()
functions.append([address, machoc_h, ""])
# IDA COMMANDS report:
# update functions list with idc.MakeName() information
# TODO: also store comments
app.logger.info("Starting idacommands")
fname = self.storage_file + '.idacmd'
if os.path.exists(fname):
fdata = open(fname, 'rb').read()
for line in fdata.split("\n"):
if line.startswith("idc.MakeName::"):
items = line.split("::")
if len(items) == 3:
addr = items[1]
name = items[2]
if addr.startswith("0x"):
addr = addr[2:]
for i in functions:
if i[0] == addr:
i[2] = name
name_action = idac.add_name(int(addr, 16), name)
SampleController.add_idaaction(sample.id, name_action)
elif line.startswith("idc.MakeRptCmt::"):
items = line.split("::")
if len(items) == 3:
addr = items[1]
value = items[2]
if addr.startswith("0x"):
addr = addr[2:]
try:
addr = int(addr, 16)
except Exception:
continue
act = idac.add_comment(addr, value)
SampleController.add_idaaction(sample.id, act)
# Functions: just push the list
app.logger.info("Storing actions")
if len(functions) > 0:
sc.add_multiple_functions(sample, functions)
# global machoc match
app.logger.info("Matching actions")
sc.match_by_machoc80(sample)
app.logger.debug(self.tmessage + "END - TIME %i" %
(int(time.time()) - self.tstart))
return True
