def classify(eml):
try:
if eml:
content = eml.html_as_text
hosts = eml.hosts
# Preprocess, extract entities
words = juicer.extract_stanford(content,
named_only=False,
stemming=False)
category = katatasso.classifyv2(words, algo=ALGO)
return {
'label': category,
'class': categories[category],
'recipient': 'hidden',
'sender': eml.sender,
'subject': eml.subject,
'timedate': eml.date,
'hosts': hosts,
'file': eml.filepath
}
except Exception as e:
logger.error(
f'[PURA ] An error occurred while classifying the email.')
logger.error(e)
return None
def __add_comment(issue_key, body):
try:
logger.debug(f'[JIRA ] Adding comment to issue `{issue_key}`.')
return jc.add_comment(issue_key, body)
except JIRAError as jc_err:
logger.error(jc_err)
except Exception as err:
logger.error(err)
def __is_url(host):
try:
match = re.match(REGEX.URL, host, re.IGNORECASE)
except Exception as e:
logger.error(e)
if match:
return True
return False
def __search_assignable_users_for_projects():
try:
logger.debug(
f'[JIRA ] Searching for assignable users for project `{JIRA_PROJECT_KEY}`.'
)
assignable = jc.search_assignable_users_for_projects(
'', JIRA_PROJECT_KEY)
except JIRAError as jc_err:
logger.error(jc_err)
except Exception as err:
logger.error(err)
def __get_fqdn(host):
try:
o = urlparse(host)
if o.netloc:
return o.netloc
else:
logger.warning('[TH-INT] No netloc found in host.')
except ValueError as e:
logger.error(f'[TH-INT] An error occurred while parsing a host.')
logger.error(e)
return host
def add_comment_user_notified(issue_key,
notified_user,
message='',
via='email'):
try:
body = f'Response sent to user {notified_user} via {via}.'
if message:
body += f'\nMessage:\n {message}'
return __add_comment(issue_key, body)
except JIRAError as jc_err:
logger.error(jc_err)
except Exception as err:
logger.error(err)
def __get_fqdn_path(host):
try:
o = urlparse(host)
if o.netloc:
if not o.path:
logger.warning('[TH-INT] No path found in host.')
return f'{o.netloc}{o.path}'
else:
logger.warning('[TH-INT] No netloc found in host.')
except ValueError as e:
logger.error(f'[TH-INT] An error occurred while parsing a host.')
logger.error(e)
return host
def __parse_csv(response):
hosts = []
if response:
try:
if isinstance(response, list):
headers = response.pop(0).split(',')
try:
index = headers.index('url')
except ValueError:
try:
index = headers.index('ip')
except ValueError:
logger.error(f'[TH-INT] Unable to find either [url, ip] in headers of CSV. Returning empty list.')
return hosts
for line in response:
line = line.split(',')
try:
data = line[index]
if data:
if __is_ip(data) or __is_url(data):
hosts.append(data)
except IndexError:
pass
else:
logger.error('[TH-INT] Response is not of expected type `list`. Returning empty list.')
except Exception as e:
logger.error(e)
else:
logger.error('[TH-INT] Response is empty. No CSV to parse.')
return hosts
def main():
if len(sys.argv) > 1:
hosts = sys.argv[1]
hosts = hosts.split(',')
results = is_threat(hosts)
if results:
for result in results:
logger.debug(f'[TH-INT] From feed: {result["feed_url"]}')
print(f'Host: {result["host"]}')
print(f'Threat: {"true" if result["found"] else "false"}')
print(f'Confidence: {result["confidence"]}')
print()
else:
logger.error(f'[TH-INT] Missing argument for `host`')
print('Please specify a host to look for as the first argument.')
def __fetch_feed(feed):
with requests.Session() as session:
try:
res = session.get(feed)
res.raise_for_status()
except HTTPError as http_err:
logger.error(f'[TH-INT] HTTP error while fetching a feed\n{http_err}')
except Exception as err:
logger.error(f'[TH-INT] An error occurred while fetching a feed\n{err}')
if res and res.text:
try:
return res.text.split('\n')
except Exception:
return res.text
return None
def __set_priority(issue, priority_key):
try:
logger.debug(f'[JIRA ] Setting priority for issue `{issue.key}`.')
issue.update(fields={
'priority': {
'id': priority_key,
'name': PRIORITIES.get(priority_key)
}
})
logger.debug(
f'[JIRA ] Priority for issue `{issue.key}` set to `{PRIORITIES.get(priority_key)}` ({priority_key}).'
)
except JIRAError as jc_err:
logger.error(jc_err)
except Exception as err:
logger.error(err)
def fetch_emails(limit=10):
emls = []
i = 0
try:
client = FetchMail()
#client.set_mailbox('Junk')
res = client.search()
while i <= limit:
for _id in res:
eml_str = client.fetch(_id)
tempfile_path = client.save_tmp(_id, eml_str)
emls.append(emailyzer.from_eml(tempfile_path))
i += 1
return emls
except Exception as e:
logger.error(f'[PURA ] An error occurred while fetching the email.')
logger.error(e)
return emls
def __create_issue(summary, description, issue_type='Task'):
try:
logger.debug(
f'[JIRA ] Creating new issue for project `{JIRA_PROJECT_KEY}`.')
issue = {
'project': {
'key': JIRA_PROJECT_KEY
},
'summary': summary,
'description': description,
'issuetype': {
'name': issue_type
},
}
return jc.create_issue(fields=issue)
except JIRAError as jc_err:
logger.error(jc_err)
except Exception as err:
logger.error(err)
def __assign_user(issue_key, accountId=None):
assigned = False
try:
logger.debug(f'[JIRA ] Assigning user for issue `{issue_key}`..')
if not accountId:
logger.debug(
f'[JIRA ] No accountId provided. Selecting random user.')
users = __search_assignable_users_for_projects()
if users:
logger.debug(
f'[JIRA ] User search: {len(users)} users found.')
accountId = random.choice(users).accountId
else:
logger.error(
f'[JIRA ] No assignable users were found for this project.'
)
return assigned
assigned = jc.assign_issue(issue_key, accountId)
logger.debug(f'[JIRA ] User assigned: {assigned}')
except JIRAError as jc_err:
logger.error(jc_err)
except Exception as err:
logger.error(err)
return assigned
def is_threat(hosts):
"""Check whether a host is present in selected threat intelligence sources
Parameters
----------
hosts : list
A list of IP addresses, FQDNs and/or URLs to look for.
Returns
-------
results : list
A list of objects in the following format:
{ 'host': string, 'found': bool, 'confidence': float, 'feed_url': string }
host : string
The hostname (IP, FQDN or URL)
found : bool
Whether the host was found.
confidence : float
A confidence level from 0.0 to 1.0.
feed_url : string
The URL of the threat intel where the host was found.
"""
results = []
logger.info(f'[TH-INT] Checking host {len(hosts)} against threat intel feeds.')
for feed_url in FEEDS['plain']:
if len(results) == len(hosts):
return results
feed = __fetch_feed(feed_url)
if feed:
for host in hosts:
found = False
host = host.strip()
found, confidence = __is_in_feed(host, feed)
if found:
results.append({ 'host': host, 'found': found, 'confidence': confidence, 'feed_url': feed_url })
break
else:
logger.error(f'[TH-INT] Feed for {feed_url} is None or empty. Skipping.')
for feed_url in FEEDS['csv']:
if len(results) == len(hosts):
return results
feed = __fetch_feed(feed_url)
if feed:
feed = __parse_csv(feed)
if feed:
for host in hosts:
found = False
host = host.strip()
found, confidence = __is_in_feed(host, feed)
if found:
results.append({ 'host': host, 'found': found, 'confidence': confidence, 'feed_url': feed_url })
break
else:
logger.error(f'[TH-INT] No feed was returned from parsing the CSV.')
else:
logger.error(f'[TH-INT] Feed for {feed_url} is None or empty. Skipping.')
return results
def __add_attachment(issue_key, filepath, filename='email'):
try:
logger.debug(f'[JIRA ] Adding attachment to issue `{issue_key}`.')
return jc.add_attachment(issue_key, filepath, filename)
except JIRAError as jc_err:
logger.error(jc_err)
__add_comment(issue_key,
f'Uploading of email attachment `{filename}` failed.')
except FileNotFoundError:
logger.error(f'[JIRA ] File `{filepath}` does not exist.')
except Exception as err:
logger.error(
f'[JIRA ] An error occurred while uploading an attachment to issue `{issue_key}`.'
)
logger.error(err)
__add_comment(issue_key,
f'Uploading of email attachment `{filename}` failed.')
def create_issue(classification,
confidence_level,
recipient,
email_sender,
email_subject,
timedate,
attachment_filepath=None,
comment=''):
try:
summary, desc = __parse_template(classification, confidence_level,
recipient, email_sender,
email_subject, timedate)
issue = __create_issue(summary, desc)
if issue:
__determine_priority(issue, classification)
if int(float(confidence_level)) < MIN_CONFIDENCE_LEVEL:
logger.debug(
f'[JIRA ] Assigning user to handle manually due to low confidence level [level: {confidence_level}]'
)
assigned = __assign_user(issue.key)
logger.debug(
f'[JIRA ] Setting priority to `Highest` due to low confidence level [level: {confidence_level}]'
)
__set_priority(issue, '1')
if attachment_filepath:
__add_attachment(issue.key, attachment_filepath, 'email')
if comment:
__add_comment(issue.key, comment)
else:
logger.error(
f'[JIRA ] An error occurred while creating the issue in JIRA.'
)
except JIRAError as jc_err:
logger.error(jc_err)
except Exception as err:
logger.error(err)
import os
import sys
import random
from jira import JIRA, JIRAError
from pura.helpers.logger import rootLogger as logger
JIRA_SERVER = os.getenv('JIRA_SERVER', None)
JIRA_USER = os.getenv('JIRA_USER', None)
JIRA_TOKEN = os.getenv('JIRA_TOKEN', None)
if not JIRA_SERVER:
logger.error('[JIRA ] Missing environment variable `JIRA_SERVER`.')
sys.exit(2)
if not JIRA_USER and JIRA_TOKEN:
logger.error(
'[JIRA ] Missing environment variables for authentication (`JIRA_USER` and `JIRA_PASS`).'
)
sys.exit(2)
jc = JIRA(JIRA_SERVER, basic_auth=(JIRA_USER, JIRA_TOKEN))
JIRA_ASSIGNEES = os.getenv('JIRA_ASSIGNEES', '').split(',')
JIRA_PROJECT_KEY = os.getenv('JIRA_PROJECT_KEY', 'SEC')
MIN_CONFIDENCE_LEVEL = os.getenv('MIN_CONFIDENCE_LEVEL', 85)
templates = {
'summary':
'[%classification%] for user %recipient%',
'description':
'User %recipient% received a %classification% email on %timedate%.\nSender: %email_sender%\nSubject: %email_subject%\n\nConfidence level: %confidence_level%\nActions taken: See comments'