def load(self, mailitem, mailbox_cfg): """ Load ExtractedRecord object with data from processed results. :type mailitem: ExtractedData :type mailbox_cfg: Mailbox :param mailitem: :param mailbox_cfg: :return: """ try: if mailbox_cfg.is_valid() and mailitem.has_data(): self.created_by = mailbox_cfg.created_by self.mailbox = mailbox_cfg self.email_subject = mailitem.subject self.email_sender = mailitem.sender self.email_receiver = mailitem.recipient self.email_body = mailitem.body self.extracted_ips = mailitem.get_ips() self.extracted_urls = mailitem.get_urls() self.extracted_emails = mailitem.get_emails() self.extracted_md5 = mailitem.get_md5s() self.extracted_sha1 = mailitem.get_sha1s() self.is_loaded = True self.save() else: self.is_loaded = False self.save() except Exception as e: crashlog.save(constants.ADAPTER_IDENTIFIER, str(e), traceback.format_exc())
def update_main(force_start=False, **kwargs): try: STIXBacklinks().run() except Exception as e: crashlog.save('mod_bl_job', e.message, traceback.format_exc()) raise return 0
def update_main(force_start=False, **kwargs): try: stix_dedup = STIXDedup(DedupConfiguration.get()) stix_dedup.run() except Exception as e: crashlog.save('dedup_job', e.message, traceback.format_exc()) raise return 0
def update_main(force_start=False, **kwargs): try: stix_fts = STIXFts(FtsConfiguration.get()) stix_fts.run() except Exception as e: crashlog.save('fts_job', e.message, traceback.format_exc()) raise return 0
def update_main(force_start=False, **kwargs): try: stix_purge = STIXPurge(RetentionConfiguration.get()) stix_purge.run() except Exception as e: crashlog.save('purge_job', e.message, traceback.format_exc()) raise return 0
def query_user_jobs(user): """ :param user: :return: """ jobs = {} try: jobs = JobInfo.objects(created_by=user.id).order_by('-job_start')[:MAX_JOB_HISTORY] except Exception as e: crashlog.save(constants.ADAPTER_IDENTIFIER, str(e), traceback.format_exc()) return jobs
def get_header_data(id): """ Given id, return the data for the given record. :param id: Id of result record :return: """ rawresdata = None try: rawresdata = ExtractedRecord.objects.get(id=id) except Exception as e: crashlog.save(constants.ADAPTER_IDENTIFIER, str(e), traceback.format_exc()) return rawresdata
def process_email(acct_host, acct_user, acct_pass, acct_port, ssl_enabled=False, mailbox="INBOX", do_ip_extract=True, do_url_extract=True, do_email_extract=True, max_to_process=constants.DEFAULT_MAX_EMAIL_TO_PROCESS): emaildata = [] # connect based on config, get email imapper = easyimap.connect(acct_host, acct_user, acct_pass, mailbox, ssl=ssl_enabled, port=acct_port) uids_to_process = imapper.listids(limit=max_to_process) for uid in uids_to_process: # fetch mail mail = imapper.mail(uid) subj = mail.title uid = str(mail.uid) recip = mail.to body = mail.body efrom = mail.from_addr edata = extracted_data.ExtractedData(uid=uid, subject=subj, sender=efrom, recipient=recip, body=body) try: # This mechanism for storing and saving results is quite convoluted and over-engineered # need to rewrite without trying to outsmart myself ips_from_body = extractors.extract_ipv4(body) md5s_from_body = extractors.extract_md5(body) sha1_from_body = extractors.extract_sha1(body) email_from_body = extractors.extract_email_addresses(body) urls_from_body = extractors.extract_urls(body) edata.add_ips(ips_from_body) edata.add_md5s(md5s_from_body) edata.add_sha1s(sha1_from_body) edata.add_emails(email_from_body) edata.add_urls(urls_from_body) edata.dedupe() except Exception as e: crashlog.save(constants.ADAPTER_IDENTIFIER, str(e), traceback.format_exc()) emaildata.append(edata) # if do_ip_extract: # #ips_from_body = extractors.extract_ips(body) # edata.add_ips(ips_from_body) # # if do_url_extract: # #urls_from_body = extractors.extract_urls(body) # edata.add_urls(urls_from_body) # emaildata.append(edata) return emaildata
def process_mailbox(mailbox_cfg): """ Celery task to extract data from specified inbox and return output for user import. :type mailbox_cfg: Mailbox :param mailbox_cfg: Configuration for mailbox to process, including which parsers to enable. :return: """ resultset = ExtractedRecord() if mailbox_cfg.is_valid(): rawresult = None try: rawresult = process.process_email( acct_host=mailbox_cfg.email_server_host, acct_user=mailbox_cfg.email_username, acct_pass=mailbox_cfg.email_password, acct_port=mailbox_cfg.email_server_port, ssl_enabled=mailbox_cfg.email_server_ssl_enabled, mailbox=mailbox_cfg.email_folder_name, do_ip_extract=mailbox_cfg.do_ip_extract, do_url_extract=mailbox_cfg.do_url_extract, do_email_extract=mailbox_cfg.do_email_address_extract, max_to_process=mailbox_cfg.max_email_to_process ) except Exception as e: crashlog.save(constants.ADAPTER_IDENTIFIER, str(e), traceback.format_exc()) # lets do it if rawresult: # build extractedRecord objects from result set processed_records = [] for index, mailitem in enumerate(rawresult): try: processed_record = ExtractedRecord() processed_record.load(mailitem, mailbox_cfg) processed_record.save() # out_file = "/tmp/STIX_" + str(index) + ".xml" # with open(out_file, 'w+') as fh: # fh.write(utils.pkg_builder(mailbox_cfg, processed_record)) except Exception as e: crashlog.save(constants.ADAPTER_IDENTIFIER, str(e), traceback.format_exc()) else: return False
def get_result_data(id): """ Given id, return the data for the given record. :param id: Id of result record :return: """ resdict = {} try: rawresdata = ExtractedRecord.objects.get(id=id) # don't try to be too fancy and just use a dict resdict[ADDR_TYPE] = rawresdata.extracted_ips resdict[MD5_TYPE] = rawresdata.extracted_md5 resdict[SHA1_TYPE] = rawresdata.extracted_sha1 resdict[EMAIL_ADDR_TYPE] = rawresdata.extracted_emails resdict[URL_TYPE] = rawresdata.extracted_urls except Exception as e: crashlog.save(constants.ADAPTER_IDENTIFIER, str(e), traceback.format_exc()) return resdict
def update_user_jobs(user): """ Update the status via celery for any jobs this user has :param user: :return: """ try: jobs = JobInfo.objects(created_by=user.id, job_active=True) if jobs: for job in jobs: # update job status for this user's active jobs result = AsyncResult(job.task_id) if result.state in CELERY_DONE_STATES: job.job_active = False job.job_state = result.state job.save() if job.job_state != result.state: job.job_state = result.state job.save() except Exception as e: crashlog.save('snort_adapter', str(e), traceback.format_exc())
def log_error(e, app_name, message=''): stack_trace = traceback.format_exc() crash_log.save(app_name, message + ': ' + str(e), stack_trace)