def classify(eml): try: if eml: content = eml.html_as_text hosts = eml.hosts # Preprocess, extract entities words = juicer.extract_stanford(content, named_only=False, stemming=False) category = katatasso.classifyv2(words, algo=ALGO) return { 'label': category, 'class': categories[category], 'recipient': 'hidden', 'sender': eml.sender, 'subject': eml.subject, 'timedate': eml.date, 'hosts': hosts, 'file': eml.filepath } except Exception as e: logger.error( f'[PURA ] An error occurred while classifying the email.') logger.error(e) return None
def __add_comment(issue_key, body): try: logger.debug(f'[JIRA ] Adding comment to issue `{issue_key}`.') return jc.add_comment(issue_key, body) except JIRAError as jc_err: logger.error(jc_err) except Exception as err: logger.error(err)
def __is_url(host): try: match = re.match(REGEX.URL, host, re.IGNORECASE) except Exception as e: logger.error(e) if match: return True return False
def __search_assignable_users_for_projects(): try: logger.debug( f'[JIRA ] Searching for assignable users for project `{JIRA_PROJECT_KEY}`.' ) assignable = jc.search_assignable_users_for_projects( '', JIRA_PROJECT_KEY) except JIRAError as jc_err: logger.error(jc_err) except Exception as err: logger.error(err)
def __get_fqdn(host): try: o = urlparse(host) if o.netloc: return o.netloc else: logger.warning('[TH-INT] No netloc found in host.') except ValueError as e: logger.error(f'[TH-INT] An error occurred while parsing a host.') logger.error(e) return host
def add_comment_user_notified(issue_key, notified_user, message='', via='email'): try: body = f'Response sent to user {notified_user} via {via}.' if message: body += f'\nMessage:\n {message}' return __add_comment(issue_key, body) except JIRAError as jc_err: logger.error(jc_err) except Exception as err: logger.error(err)
def __get_fqdn_path(host): try: o = urlparse(host) if o.netloc: if not o.path: logger.warning('[TH-INT] No path found in host.') return f'{o.netloc}{o.path}' else: logger.warning('[TH-INT] No netloc found in host.') except ValueError as e: logger.error(f'[TH-INT] An error occurred while parsing a host.') logger.error(e) return host
def __parse_csv(response): hosts = [] if response: try: if isinstance(response, list): headers = response.pop(0).split(',') try: index = headers.index('url') except ValueError: try: index = headers.index('ip') except ValueError: logger.error(f'[TH-INT] Unable to find either [url, ip] in headers of CSV. Returning empty list.') return hosts for line in response: line = line.split(',') try: data = line[index] if data: if __is_ip(data) or __is_url(data): hosts.append(data) except IndexError: pass else: logger.error('[TH-INT] Response is not of expected type `list`. Returning empty list.') except Exception as e: logger.error(e) else: logger.error('[TH-INT] Response is empty. No CSV to parse.') return hosts
def main(): if len(sys.argv) > 1: hosts = sys.argv[1] hosts = hosts.split(',') results = is_threat(hosts) if results: for result in results: logger.debug(f'[TH-INT] From feed: {result["feed_url"]}') print(f'Host: {result["host"]}') print(f'Threat: {"true" if result["found"] else "false"}') print(f'Confidence: {result["confidence"]}') print() else: logger.error(f'[TH-INT] Missing argument for `host`') print('Please specify a host to look for as the first argument.')
def __fetch_feed(feed): with requests.Session() as session: try: res = session.get(feed) res.raise_for_status() except HTTPError as http_err: logger.error(f'[TH-INT] HTTP error while fetching a feed\n{http_err}') except Exception as err: logger.error(f'[TH-INT] An error occurred while fetching a feed\n{err}') if res and res.text: try: return res.text.split('\n') except Exception: return res.text return None
def __set_priority(issue, priority_key): try: logger.debug(f'[JIRA ] Setting priority for issue `{issue.key}`.') issue.update(fields={ 'priority': { 'id': priority_key, 'name': PRIORITIES.get(priority_key) } }) logger.debug( f'[JIRA ] Priority for issue `{issue.key}` set to `{PRIORITIES.get(priority_key)}` ({priority_key}).' ) except JIRAError as jc_err: logger.error(jc_err) except Exception as err: logger.error(err)
def fetch_emails(limit=10): emls = [] i = 0 try: client = FetchMail() #client.set_mailbox('Junk') res = client.search() while i <= limit: for _id in res: eml_str = client.fetch(_id) tempfile_path = client.save_tmp(_id, eml_str) emls.append(emailyzer.from_eml(tempfile_path)) i += 1 return emls except Exception as e: logger.error(f'[PURA ] An error occurred while fetching the email.') logger.error(e) return emls
def __create_issue(summary, description, issue_type='Task'): try: logger.debug( f'[JIRA ] Creating new issue for project `{JIRA_PROJECT_KEY}`.') issue = { 'project': { 'key': JIRA_PROJECT_KEY }, 'summary': summary, 'description': description, 'issuetype': { 'name': issue_type }, } return jc.create_issue(fields=issue) except JIRAError as jc_err: logger.error(jc_err) except Exception as err: logger.error(err)
def __assign_user(issue_key, accountId=None): assigned = False try: logger.debug(f'[JIRA ] Assigning user for issue `{issue_key}`..') if not accountId: logger.debug( f'[JIRA ] No accountId provided. Selecting random user.') users = __search_assignable_users_for_projects() if users: logger.debug( f'[JIRA ] User search: {len(users)} users found.') accountId = random.choice(users).accountId else: logger.error( f'[JIRA ] No assignable users were found for this project.' ) return assigned assigned = jc.assign_issue(issue_key, accountId) logger.debug(f'[JIRA ] User assigned: {assigned}') except JIRAError as jc_err: logger.error(jc_err) except Exception as err: logger.error(err) return assigned
def is_threat(hosts): """Check whether a host is present in selected threat intelligence sources Parameters ---------- hosts : list A list of IP addresses, FQDNs and/or URLs to look for. Returns ------- results : list A list of objects in the following format: { 'host': string, 'found': bool, 'confidence': float, 'feed_url': string } host : string The hostname (IP, FQDN or URL) found : bool Whether the host was found. confidence : float A confidence level from 0.0 to 1.0. feed_url : string The URL of the threat intel where the host was found. """ results = [] logger.info(f'[TH-INT] Checking host {len(hosts)} against threat intel feeds.') for feed_url in FEEDS['plain']: if len(results) == len(hosts): return results feed = __fetch_feed(feed_url) if feed: for host in hosts: found = False host = host.strip() found, confidence = __is_in_feed(host, feed) if found: results.append({ 'host': host, 'found': found, 'confidence': confidence, 'feed_url': feed_url }) break else: logger.error(f'[TH-INT] Feed for {feed_url} is None or empty. Skipping.') for feed_url in FEEDS['csv']: if len(results) == len(hosts): return results feed = __fetch_feed(feed_url) if feed: feed = __parse_csv(feed) if feed: for host in hosts: found = False host = host.strip() found, confidence = __is_in_feed(host, feed) if found: results.append({ 'host': host, 'found': found, 'confidence': confidence, 'feed_url': feed_url }) break else: logger.error(f'[TH-INT] No feed was returned from parsing the CSV.') else: logger.error(f'[TH-INT] Feed for {feed_url} is None or empty. Skipping.') return results
def __add_attachment(issue_key, filepath, filename='email'): try: logger.debug(f'[JIRA ] Adding attachment to issue `{issue_key}`.') return jc.add_attachment(issue_key, filepath, filename) except JIRAError as jc_err: logger.error(jc_err) __add_comment(issue_key, f'Uploading of email attachment `{filename}` failed.') except FileNotFoundError: logger.error(f'[JIRA ] File `{filepath}` does not exist.') except Exception as err: logger.error( f'[JIRA ] An error occurred while uploading an attachment to issue `{issue_key}`.' ) logger.error(err) __add_comment(issue_key, f'Uploading of email attachment `{filename}` failed.')
def create_issue(classification, confidence_level, recipient, email_sender, email_subject, timedate, attachment_filepath=None, comment=''): try: summary, desc = __parse_template(classification, confidence_level, recipient, email_sender, email_subject, timedate) issue = __create_issue(summary, desc) if issue: __determine_priority(issue, classification) if int(float(confidence_level)) < MIN_CONFIDENCE_LEVEL: logger.debug( f'[JIRA ] Assigning user to handle manually due to low confidence level [level: {confidence_level}]' ) assigned = __assign_user(issue.key) logger.debug( f'[JIRA ] Setting priority to `Highest` due to low confidence level [level: {confidence_level}]' ) __set_priority(issue, '1') if attachment_filepath: __add_attachment(issue.key, attachment_filepath, 'email') if comment: __add_comment(issue.key, comment) else: logger.error( f'[JIRA ] An error occurred while creating the issue in JIRA.' ) except JIRAError as jc_err: logger.error(jc_err) except Exception as err: logger.error(err)
import os import sys import random from jira import JIRA, JIRAError from pura.helpers.logger import rootLogger as logger JIRA_SERVER = os.getenv('JIRA_SERVER', None) JIRA_USER = os.getenv('JIRA_USER', None) JIRA_TOKEN = os.getenv('JIRA_TOKEN', None) if not JIRA_SERVER: logger.error('[JIRA ] Missing environment variable `JIRA_SERVER`.') sys.exit(2) if not JIRA_USER and JIRA_TOKEN: logger.error( '[JIRA ] Missing environment variables for authentication (`JIRA_USER` and `JIRA_PASS`).' ) sys.exit(2) jc = JIRA(JIRA_SERVER, basic_auth=(JIRA_USER, JIRA_TOKEN)) JIRA_ASSIGNEES = os.getenv('JIRA_ASSIGNEES', '').split(',') JIRA_PROJECT_KEY = os.getenv('JIRA_PROJECT_KEY', 'SEC') MIN_CONFIDENCE_LEVEL = os.getenv('MIN_CONFIDENCE_LEVEL', 85) templates = { 'summary': '[%classification%] for user %recipient%', 'description': 'User %recipient% received a %classification% email on %timedate%.\nSender: %email_sender%\nSubject: %email_subject%\n\nConfidence level: %confidence_level%\nActions taken: See comments'