def parse_ida_cmds(self, sid, functions): """ Parse and add IDA commands dumped by AnalyzeIt, and updates the functions names if needed """ idac = IDAActionsController() funcs = dict.copy(functions) fname = self.storage_file + '.idacmd' act = None if not os.path.exists(fname): return funcs with open(fname) as fdata: for line in fdata: if line.startswith('idc.MakeName'): addr, name = self.get_addr_data(line) try: # update functions list with idc.MakeName() information funcs[addr]['name'] = name except KeyError: app.logger.debug("No function found for %x" % (addr)) act = idac.add_name(addr, name) elif line.startswith('idc.MakeRptCmt'): addr, cmt = self.get_addr_data(line) act = idac.add_comment(addr, cmt) else: app.logger.debug("Unknown IDA command %s" % (line)) continue SampleController.add_idaaction(sid, act) return funcs
def parse_ida_cmds(self, sid, functions): """ Parse and add IDA commands dumped by AnalyzeIt, and updates the functions names if needed """ idac = IDAActionsController() funcs = dict.copy(functions) fname = self.storage_file + '.idacmd' act = None if not os.path.exists(fname): return funcs with open(fname) as fdata: for line in fdata: if line.startswith('idc.MakeName'): addr, name = self.get_addr_data(line) try: # update functions list with idc.MakeName() information funcs[addr]['name'] = name except KeyError: app.logger.debug("No function found for %x" % (addr)) act = idac.add_name(addr, name) elif line.startswith('idc.MakeRptCmt'): addr, cmt = self.get_addr_data(line) act = idac.add_comment(addr, cmt) else: app.logger.debug("Unknown IDA command %s" % (line)) continue with app.app_context(): SampleController.add_idaaction(sid, act) return funcs
def apply_result(self): s_controller = SampleController() sample = s_controller.get_by_id(self.sid) app.logger.debug(self.tmessage + "APPLY_RESULT") s_controller.add_multiple_strings(sample, self.resultstrings) app.logger.debug(self.tmessage + "END - TIME %i" % (int(time.time()) - self.tstart)) return True
def apply_result(self): samplecontrol = SampleController() sample = SampleController.get_by_id(self.sid) if sample is None: app.logger.error(self.tmessage + "Sample has disappeared...") raise IOError app.logger.debug(self.tmessage + "APPLY_RESULT") # TXT report app.logger.info("Creating new analyzeit report") SampleController.create_analysis(sample, self.txt_report, "analyzeit", True) functions = self.parse_machoc_signatures() # IDA COMMANDS report: app.logger.info("Parsing idacommands") functions = self.parse_ida_cmds(sample.id, functions) # Functions: just push the list app.logger.info("Storing functions") samplecontrol.add_multiple_functions(sample, functions) # global machoc match app.logger.info("Calculating machoc80 matches") samplecontrol.match_by_machoc80(sample) return True
def execute(self): """ Extended yara execution. Stores hits on the yaramatched attribute. """ s_controller = SampleController() sample = s_controller.get_by_id(self.sid) self.tstart = int(time.time()) app.logger.debug(self.tmessage + "EXECUTE") for yar in YaraRule.query.all(): if run_extended_yara(yar.raw_rule, sample) is True: self.yaramatched.append(yar) return True
def execute(self): """ Extended yara execution. Stores hits on the yaramatched attribute. """ s_controller = SampleController() with app.app_context(): sample = s_controller.get_by_id(self.sid) self.tstart = int(time.time()) app.logger.debug(self.tmessage + "EXECUTE") for yar in YaraRule.query.all(): if run_extended_yara(yar.raw_rule, sample) is True: self.yaramatched.append(yar) return True
def apply_result(self): """ Commits to database. """ s_controller = SampleController() sample = s_controller.get_by_id(self.sid) app.logger.debug(self.tmessage + "APPLY_RESULT") for match in self.yaramatched: # use the static YaraController => the () will create a JobPool, # causing exceptions (daemon => child). YaraController.add_to_sample(sample, match) app.logger.debug(self.tmessage + "END - TIME %i" % (int(time.time()) - self.tstart)) return True
def apply_result(self): with app.app_context(): samplecontrol = SampleController() sample = SampleController.get_by_id(self.sid) if sample is None: app.logger.error(self.tmessage + "Sample has disappeared...") raise IOError app.logger.debug(self.tmessage + "APPLY_RESULT") # TXT report app.logger.info("Creating new analyzeit report") SampleController.create_analysis( sample, self.txt_report, "analyzeit", True) functions = self.parse_machoc_signatures() # IDA COMMANDS report: app.logger.info("Parsing idacommands") functions = self.parse_ida_cmds(sample.id, functions) # Functions: just push the list app.logger.info("Storing functions") samplecontrol.add_multiple_functions(self.sid, functions) # global machoc match app.logger.info("Calculating machoc80 matches") samplecontrol.match_by_machoc80(sample) return True
def apply_result(self): s_controller = SampleController() sample = s_controller.get_by_id(self.sid) app.logger.debug(self.tmessage + "APPLY_RESULT") # Compilation timestamp (even when faked) IS a file date, so update it. s_controller.add_multiple_metadata(sample, self.metadata_extracted) s_controller.set_file_date(sample, self.compile_timestamp) s_controller.set_import_hash(sample, self.import_hash) app.logger.debug(self.tmessage + "END - TIME %i" % (int(time.time()) - self.tstart)) return True
class APIControl(object): """ Object used as a global API. Data controllers are used for direct data manipulation. Methods are used for complex (cross-data) manipulation. TODO: create a brand new analysis scehduler working on database samples status, and remove the analysis creation from this class. """ familycontrol = None samplecontrol = None usercontrol = None analysiscontrol = None familycontrol = FamilyController() yaracontrol = YaraController() samplecontrol = SampleController() usercontrol = UserController() analysiscontrol = AnalysisController( app.config['ANALYSIS_PROCESS_POOL_SIZE']) idacontrol = IDAActionsController() def __init__(self): """ Initiate controllers. """ pass def create_sample_and_run_analysis( self, file_data_stream, originate_filename, user=None, tlp_level=TLPLevel.TLPWHITE, family=None): """ Creates a new sample and a schedule an analysis. We also check the file header for ZIP pattern: if a ZIP pattern is found, any file inside the archive will be imported and scheduled for analysis. TODO: move this to the SampleController, and start directly on new file submission. """ file_data = file_data_stream.read() if file_data.startswith("PK"): with zipfile.ZipFile(file_data, "r") as zcl: for name in zcl.namelist(): mfile = zcl.open(name, "r") sample = self.samplecontrol.create_sample_from_file( mfile, name, user, tlp_level) if family is not None: self.familycontrol.add_sample(sample, family) if sample.analysis_status == AnalysisStatus.TOSTART: self.analysiscontrol.schedule_sample_analysis( sample.id) zcl.close() return None sample = self.samplecontrol.create_sample_from_file( file_data, originate_filename, user, tlp_level) if sample.analysis_status == AnalysisStatus.TOSTART: self.analysiscontrol.schedule_sample_analysis(sample.id) if family is not None: self.familycontrol.add_sample(sample, family) return sample def add_actions_fromfunc_infos(self, funcinfos, sample_dst, sample_src): for fid_dst, fid_src in funcinfos: fsrc = FunctionInfo.query.get(fid_src) fdst = FunctionInfo.query.get(fid_dst) if fsrc is None or fdst is None: return False if fsrc not in sample_src.functions: return False if fdst not in sample_dst.functions: return False if fsrc.name.startswith("sub_"): continue act = self.idacontrol.add_name(int(fdst.address), fsrc.name) self.samplecontrol.add_idaaction(sample_dst.id, act) db.session.commit() return True
def apply_result(self): sc = SampleController() idac = IDAActionsController() sample = SampleController.get_by_id(self.sid) if sample is None: app.logger.error(self.tmessage + "Sample has disappeared...") raise IOError app.logger.debug(self.tmessage + "APPLY_RESULT") # TXT report app.logger.info("Starting analysis creation") SampleController.create_analysis(sample, self.txt_report, "analyzeit", True) # MACHOC report: we load the functions, hashes, etc. app.logger.info("Starting functions") fname = self.storage_file + '.sign' functions = [] if os.path.exists(fname): fdata = open(fname, 'rb').read() items = fdata.split(";") for i in items: if ":" in i: subitems = i.split(":") machoc_h = subitems[0].strip() address = subitems[1].strip() functions.append([address, machoc_h, ""]) # IDA COMMANDS report: # update functions list with idc.MakeName() information # TODO: also store comments app.logger.info("Starting idacommands") fname = self.storage_file + '.idacmd' if os.path.exists(fname): fdata = open(fname, 'rb').read() for line in fdata.split("\n"): if line.startswith("idc.MakeName::"): items = line.split("::") if len(items) == 3: addr = items[1] name = items[2] if addr.startswith("0x"): addr = addr[2:] for i in functions: if i[0] == addr: i[2] = name name_action = idac.add_name(int(addr, 16), name) SampleController.add_idaaction(sample.id, name_action) elif line.startswith("idc.MakeRptCmt::"): items = line.split("::") if len(items) == 3: addr = items[1] value = items[2] if addr.startswith("0x"): addr = addr[2:] try: addr = int(addr, 16) except Exception: continue act = idac.add_comment(addr, value) SampleController.add_idaaction(sample.id, act) # Functions: just push the list app.logger.info("Storing actions") if len(functions) > 0: sc.add_multiple_functions(sample, functions) # global machoc match app.logger.info("Matching actions") sc.match_by_machoc80(sample) app.logger.debug(self.tmessage + "END - TIME %i" % (int(time.time()) - self.tstart)) return True
class APIControl(object): """ Object used as a global API. Data controllers are used for direct data manipulation. Methods are used for complex (cross-data) manipulation. TODO: create a brand new analysis scheduler working on database samples status, and remove the analysis creation from this class. """ familycontrol = None samplecontrol = None usercontrol = None analysiscontrol = None familycontrol = FamilyController() yaracontrol = YaraController() samplecontrol = SampleController() usercontrol = UserController() analysiscontrol = AnalysisController( app.config['ANALYSIS_PROCESS_POOL_SIZE']) idacontrol = IDAActionsController() def __init__(self): """ Initiate controllers. """ pass def dispatch_sample_creation(self, file_stream, filename="", user=None, tlp=TLPLevel.TLPWHITE, family=None): """ If the sample is a ZipFile, we unpack it and return the last sample,otherwise we return a single sample. """ file_data = file_stream.read(4) file_stream.seek(0) if file_data.startswith("PK"): samples = self.create_from_zip(file_stream, user, tlp, family) else: sample = self.create_sample_and_run_analysis( file_stream, filename, user, tlp, family) samples = [sample] return samples def create_from_zip(self, file_stream, user, tlp, family): """ Iterates over the samples in the zip """ output_samples = [] file_data = StringIO(file_stream.read()) with zipfile.ZipFile(file_data, "r") as zcl: for name in zcl.namelist(): mfile = zcl.open(name, "r") sample = self.create_sample_and_run_analysis( mfile, name, user, tlp, family) output_samples.append(sample) zcl.close() return output_samples def create_sample_and_run_analysis(self, file_data_stream, originate_filename="", user=None, tlp_level=TLPLevel.TLPWHITE, family=None): """ Creates a new sample and a schedule an analysis. We also check the file header for ZIP pattern: if a ZIP pattern is found, any file inside the archive will be imported and scheduled for analysis. TODO: move this to the SampleController, and start directly on new file submission. """ file_data = file_data_stream.read() sample = self.samplecontrol.create_sample_from_file( file_data, originate_filename, user, tlp_level) if sample.analysis_status == AnalysisStatus.TOSTART: self.analysiscontrol.schedule_sample_analysis(sample.id) if family is not None: self.familycontrol.add_sample(sample, family) return sample def add_actions_fromfunc_infos(self, funcinfos, sample_dst, sample_src): """ Create IDAActions from the samples's FuncInfos from AnalyzeIt """ for fid_dst, fid_src in funcinfos: fsrc = FunctionInfo.query.get(fid_src) fdst = FunctionInfo.query.get(fid_dst) if fsrc is None or fdst is None: return False if fsrc not in sample_src.functions: return False if fdst not in sample_dst.functions: return False if fsrc.name.startswith("sub_"): continue act = self.idacontrol.add_name(int(fdst.address), fsrc.name) self.samplecontrol.add_idaaction(sample_dst.id, act) db.session.commit() return True def get_elem_by_type(self, element_type, element_id): """ Wrapper to get elements by ID in database. @arg element_type string, "sample", "family", "checklist", "family_file", "detection_item", "yara" @arg element_id integer the id to search @return if found the element, abort 404 if not found, and abort 500 if the type is incorrect """ elem_types = { "sample": self.samplecontrol.get_by_id, "family": self.familycontrol.get_by_id, "checklist": self.samplecontrol.get_checklist_by_id, "family_file": self.familycontrol.get_file_by_id, "detection_item": self.familycontrol.get_detection_item_by_id, "yara": self.yaracontrol.get_by_id } try: elem = elem_types[element_type](element_id) except KeyError: app.logger.exception("Element type unknown") abort(500) if elem is None: flash(element_type + " not found...", "error") abort(404) return elem def remove_user_from_element(self, element_type, element_id, user): """ Remove a user from an element, be it a sample or a family """ elem_types = { "family": self.familycontrol, "sample": self.samplecontrol } elem = self.get_elem_by_type(element_type, element_id) if user in elem.users: elem_types[element_type].remove_user(user, elem) else: elem_types[element_type].add_user(user, elem) return elem.id
def apply_result(self): sc = SampleController() idac = IDAActionsController() sample = SampleController.get_by_id(self.sid) if sample is None: app.logger.error(self.tmessage + "Sample has disappeared...") raise IOError app.logger.debug(self.tmessage + "APPLY_RESULT") # TXT report app.logger.info("Starting analysis creation") SampleController.create_analysis( sample, self.txt_report, "analyzeit", True) # MACHOC report: we load the functions, hashes, etc. app.logger.info("Starting functions") fname = self.storage_file + '.sign' functions = [] if os.path.exists(fname): fdata = open(fname, 'rb').read() items = fdata.split(";") for i in items: if ":" in i: subitems = i.split(":") machoc_h = subitems[0].strip() address = subitems[1].strip() functions.append([address, machoc_h, ""]) # IDA COMMANDS report: # update functions list with idc.MakeName() information # TODO: also store comments app.logger.info("Starting idacommands") fname = self.storage_file + '.idacmd' if os.path.exists(fname): fdata = open(fname, 'rb').read() for line in fdata.split("\n"): if line.startswith("idc.MakeName::"): items = line.split("::") if len(items) == 3: addr = items[1] name = items[2] if addr.startswith("0x"): addr = addr[2:] for i in functions: if i[0] == addr: i[2] = name name_action = idac.add_name(int(addr, 16), name) SampleController.add_idaaction(sample.id, name_action) elif line.startswith("idc.MakeRptCmt::"): items = line.split("::") if len(items) == 3: addr = items[1] value = items[2] if addr.startswith("0x"): addr = addr[2:] try: addr = int(addr, 16) except Exception: continue act = idac.add_comment(addr, value) SampleController.add_idaaction(sample.id, act) # Functions: just push the list app.logger.info("Storing actions") if len(functions) > 0: sc.add_multiple_functions(sample, functions) # global machoc match app.logger.info("Matching actions") sc.match_by_machoc80(sample) app.logger.debug(self.tmessage + "END - TIME %i" % (int(time.time()) - self.tstart)) return True