def load(self, mailitem, mailbox_cfg):
"""
Load ExtractedRecord object with data from processed results.
:type mailitem: ExtractedData
:type mailbox_cfg: Mailbox
:param mailitem:
:param mailbox_cfg:
:return:
"""
try:
if mailbox_cfg.is_valid() and mailitem.has_data():
self.created_by = mailbox_cfg.created_by
self.mailbox = mailbox_cfg
self.email_subject = mailitem.subject
self.email_sender = mailitem.sender
self.email_receiver = mailitem.recipient
self.email_body = mailitem.body
self.extracted_ips = mailitem.get_ips()
self.extracted_urls = mailitem.get_urls()
self.extracted_emails = mailitem.get_emails()
self.extracted_md5 = mailitem.get_md5s()
self.extracted_sha1 = mailitem.get_sha1s()
self.is_loaded = True
self.save()
else:
self.is_loaded = False
self.save()
except Exception as e:
crashlog.save(constants.ADAPTER_IDENTIFIER, str(e), traceback.format_exc())
def update_main(force_start=False, **kwargs):
try:
STIXBacklinks().run()
except Exception as e:
crashlog.save('mod_bl_job', e.message, traceback.format_exc())
raise
return 0
def update_main(force_start=False, **kwargs):
try:
stix_dedup = STIXDedup(DedupConfiguration.get())
stix_dedup.run()
except Exception as e:
crashlog.save('dedup_job', e.message, traceback.format_exc())
raise
return 0
def update_main(force_start=False, **kwargs):
try:
stix_fts = STIXFts(FtsConfiguration.get())
stix_fts.run()
except Exception as e:
crashlog.save('fts_job', e.message, traceback.format_exc())
raise
return 0
def update_main(force_start=False, **kwargs):
try:
stix_purge = STIXPurge(RetentionConfiguration.get())
stix_purge.run()
except Exception as e:
crashlog.save('purge_job', e.message, traceback.format_exc())
raise
return 0
def query_user_jobs(user):
"""
:param user:
:return:
"""
jobs = {}
try:
jobs = JobInfo.objects(created_by=user.id).order_by('-job_start')[:MAX_JOB_HISTORY]
except Exception as e:
crashlog.save(constants.ADAPTER_IDENTIFIER, str(e), traceback.format_exc())
return jobs
def get_header_data(id):
"""
Given id, return the data for the given record.
:param id: Id of result record
:return:
"""
rawresdata = None
try:
rawresdata = ExtractedRecord.objects.get(id=id)
except Exception as e:
crashlog.save(constants.ADAPTER_IDENTIFIER, str(e), traceback.format_exc())
return rawresdata
def process_email(acct_host, acct_user, acct_pass, acct_port, ssl_enabled=False, mailbox="INBOX", do_ip_extract=True,
do_url_extract=True, do_email_extract=True, max_to_process=constants.DEFAULT_MAX_EMAIL_TO_PROCESS):
emaildata = []
# connect based on config, get email
imapper = easyimap.connect(acct_host, acct_user, acct_pass, mailbox, ssl=ssl_enabled,
port=acct_port)
uids_to_process = imapper.listids(limit=max_to_process)
for uid in uids_to_process:
# fetch mail
mail = imapper.mail(uid)
subj = mail.title
uid = str(mail.uid)
recip = mail.to
body = mail.body
efrom = mail.from_addr
edata = extracted_data.ExtractedData(uid=uid, subject=subj, sender=efrom, recipient=recip, body=body)
try:
# This mechanism for storing and saving results is quite convoluted and over-engineered
# need to rewrite without trying to outsmart myself
ips_from_body = extractors.extract_ipv4(body)
md5s_from_body = extractors.extract_md5(body)
sha1_from_body = extractors.extract_sha1(body)
email_from_body = extractors.extract_email_addresses(body)
urls_from_body = extractors.extract_urls(body)
edata.add_ips(ips_from_body)
edata.add_md5s(md5s_from_body)
edata.add_sha1s(sha1_from_body)
edata.add_emails(email_from_body)
edata.add_urls(urls_from_body)
edata.dedupe()
except Exception as e:
crashlog.save(constants.ADAPTER_IDENTIFIER, str(e), traceback.format_exc())
emaildata.append(edata)
# if do_ip_extract:
# #ips_from_body = extractors.extract_ips(body)
# edata.add_ips(ips_from_body)
#
# if do_url_extract:
# #urls_from_body = extractors.extract_urls(body)
# edata.add_urls(urls_from_body)
# emaildata.append(edata)
return emaildata
def process_mailbox(mailbox_cfg):
"""
Celery task to extract data from specified inbox and return output for user import.
:type mailbox_cfg: Mailbox
:param mailbox_cfg: Configuration for mailbox to process, including which parsers to enable.
:return:
"""
resultset = ExtractedRecord()
if mailbox_cfg.is_valid():
rawresult = None
try:
rawresult = process.process_email(
acct_host=mailbox_cfg.email_server_host,
acct_user=mailbox_cfg.email_username,
acct_pass=mailbox_cfg.email_password,
acct_port=mailbox_cfg.email_server_port,
ssl_enabled=mailbox_cfg.email_server_ssl_enabled,
mailbox=mailbox_cfg.email_folder_name,
do_ip_extract=mailbox_cfg.do_ip_extract,
do_url_extract=mailbox_cfg.do_url_extract,
do_email_extract=mailbox_cfg.do_email_address_extract,
max_to_process=mailbox_cfg.max_email_to_process
)
except Exception as e:
crashlog.save(constants.ADAPTER_IDENTIFIER, str(e), traceback.format_exc())
# lets do it
if rawresult:
# build extractedRecord objects from result set
processed_records = []
for index, mailitem in enumerate(rawresult):
try:
processed_record = ExtractedRecord()
processed_record.load(mailitem, mailbox_cfg)
processed_record.save()
# out_file = "/tmp/STIX_" + str(index) + ".xml"
# with open(out_file, 'w+') as fh:
# fh.write(utils.pkg_builder(mailbox_cfg, processed_record))
except Exception as e:
crashlog.save(constants.ADAPTER_IDENTIFIER, str(e), traceback.format_exc())
else:
return False
def get_result_data(id):
"""
Given id, return the data for the given record.
:param id: Id of result record
:return:
"""
resdict = {}
try:
rawresdata = ExtractedRecord.objects.get(id=id)
# don't try to be too fancy and just use a dict
resdict[ADDR_TYPE] = rawresdata.extracted_ips
resdict[MD5_TYPE] = rawresdata.extracted_md5
resdict[SHA1_TYPE] = rawresdata.extracted_sha1
resdict[EMAIL_ADDR_TYPE] = rawresdata.extracted_emails
resdict[URL_TYPE] = rawresdata.extracted_urls
except Exception as e:
crashlog.save(constants.ADAPTER_IDENTIFIER, str(e), traceback.format_exc())
return resdict
def update_user_jobs(user):
"""
Update the status via celery for any jobs this user has
:param user:
:return:
"""
try:
jobs = JobInfo.objects(created_by=user.id, job_active=True)
if jobs:
for job in jobs:
# update job status for this user's active jobs
result = AsyncResult(job.task_id)
if result.state in CELERY_DONE_STATES:
job.job_active = False
job.job_state = result.state
job.save()
if job.job_state != result.state:
job.job_state = result.state
job.save()
except Exception as e:
crashlog.save('snort_adapter', str(e), traceback.format_exc())
def log_error(e, app_name, message=''):
stack_trace = traceback.format_exc()
crash_log.save(app_name, message + ': ' + str(e), stack_trace)
