async def rest_api(self, request):
"""
Function to handle rest api calls
:param request: json data with rest request
:return: json response
"""
data = dict(await request.json())
index = data.pop('index')
options = dict(POST=dict(
false_positive=lambda d: self.rest_svc.false_positive(criteria=d),
true_positive=lambda d: self.rest_svc.true_positive(criteria=d),
false_negative=lambda d: self.rest_svc.false_negative(criteria=d),
set_status=lambda d: self.rest_svc.set_status(criteria=d),
insert_report=lambda d: self.rest_svc.insert_report(criteria=d),
insert_csv=lambda d: self.rest_svc.insert_csv(criteria=d),
remove_sentences=lambda d: self.rest_svc.remove_sentences(criteria=
d),
delete_report=lambda d: self.rest_svc.delete_report(criteria=d),
sentence_context=lambda d: self.rest_svc.sentence_context(criteria=
d),
confirmed_sentences=lambda d: self.rest_svc.confirmed_sentences(
criteria=d),
missing_technique=lambda d: self.rest_svc.missing_technique(
criteria=d),
update_report_status=lambda d: self.rest_svc.update_report_status(
uid=d)))
output = await options[request.method][index](data)
return web.json_response(output)
async def rest_api(self, request):
"""
Function to handle rest api calls
:param request: json data with rest request
:return: json response
"""
data = dict(await request.json())
try:
index = data.pop('index')
options = dict(
POST=dict(add_attack=lambda d: self.rest_svc.add_attack(
request=request, criteria=d),
reject_attack=lambda d: self.rest_svc.reject_attack(
request=request, criteria=d),
set_status=lambda d: self.rest_svc.set_status(
request=request, criteria=d),
insert_report=lambda d: self.rest_svc.insert_report(
request=request, criteria=d),
insert_csv=lambda d: self.rest_svc.insert_csv(
request=request, criteria=d),
remove_sentence=lambda d: self.rest_svc.
remove_sentence(request=request, criteria=d),
delete_report=lambda d: self.rest_svc.delete_report(
request=request, criteria=d),
rollback_report=lambda d: self.rest_svc.
rollback_report(request=request, criteria=d),
sentence_context=lambda d: self.rest_svc.
sentence_context(request=request, criteria=d),
confirmed_attacks=lambda d: self.rest_svc.
confirmed_attacks(request=request, criteria=d)))
method = options[request.method][index]
except KeyError:
return web.json_response(None, status=404)
output = await method(data)
status = 200
if output is not None and type(output) != dict:
pass
elif output is None or (output.get('success')
and not output.get('alert_user')):
status = 204
elif output.get('ignored'):
status = 202
elif output.get('error'):
status = 500
return web.json_response(output, status=status)
async def rest_api(self, request):
try:
data = dict(await request.json())
index = data.pop('index')
options = dict(POST=dict(
build_human=lambda d: self.human_svc.build_human(d),
load_human=lambda d: self.human_svc.load_humans(d),
))
return web.json_response(await options[request.method][index]
(data))
except Exception:
traceback.print_exc()
async def nav_export(self, request):
"""
Function to export confirmed sentences in layer json format
:param request: The title of the report information
:return: the layer json
"""
# Get the report from the database
report = await self.dao.get('reports',
dict(title=request.match_info.get('file')))
# Create the layer name and description
report_title = report[0]['title']
layer_name = f"{report_title}"
enterprise_layer_description = f"Enterprise techniques used by {report_title}, ATT&CK"
version = '1.0'
if (version): # add version number if it exists
enterprise_layer_description += f" v{version}"
# Enterprise navigator layer
enterprise_layer = {}
enterprise_layer['name'] = layer_name
enterprise_layer['description'] = enterprise_layer_description
enterprise_layer['domain'] = "mitre-enterprise"
enterprise_layer['version'] = "2.2"
enterprise_layer['techniques'] = []
enterprise_layer["gradient"] = { # white for nonused, blue for used
"colors": ["#ffffff", "#66b1ff"],
"minValue": 0,
"maxValue": 1
}
enterprise_layer['legendItems'] = [{
'label': f'used by {report_title}',
'color': "#66b1ff"
}]
# Get confirmed techniques for the report from the database
techniques = await self.data_svc.get_confirmed_techniques(
report[0]['uid'])
# Append techniques to enterprise layer
for technique in techniques:
enterprise_layer['techniques'].append(technique)
# Return the layer JSON in the response
layer = json.dumps(enterprise_layer)
return web.json_response(layer)
async def pdf_export(self, request):
"""
Function to export report in PDF format
:param request: The title of the report information
:return: response status of function
"""
report_needed = await self.dao.get(
'reports', dict(title=request.match_info.get('file')))
sentences_id = report_needed[0]['uid']
sentences = await self.data_svc.build_sentences(report_needed[0]['uid']
)
attack_uids = await self.dao.get('attack_uids')
# hits = await self.dao.get('report_sentence_hits', dict(report_uid=sentences_id))
dd = dict()
dd['content'] = []
dd['styles'] = dict()
# Document MetaData Info
# See https://pdfmake.github.io/docs/document-definition-object/document-medatadata/
dd['info'] = dict()
dd['info']['title'] = report_needed[0]['title']
dd['info']['creator'] = report_needed[0]['url']
table = {"body": []}
table["body"].append(["ID", "Name", "Identified Sentence"])
# Add the text to the document
for sentence in sentences:
dd['content'].append(sentence['text'])
if sentence['hits']:
for hit in sentence['hits']:
# 'hits' object doesn't provide all the information we need, so we
# do a makeshift join here to get that information from the attack_uid
# list. This is ineffecient, and a way to improve this would be to perform
# a join on the database side
matching_attacks = [
i for i in attack_uids if hit['attack_uid'] == i['uid']
]
for match in matching_attacks:
table["body"].append(
[match["tid"], match["name"], sentence['text']])
# Append table to the end
dd['content'].append({"table": table})
return web.json_response(dd)
async def nav_export(self, request):
"""
Function to export confirmed sentences in layer json format
:param request: The title of the report information
:return: the layer json
"""
# Get the report from the database
report_title = request.match_info.get(self.web_svc.REPORT_PARAM)
report = await self.dao.get('reports', dict(title=report_title))
try:
# Ensure a valid report title has been passed in the request
report_id, report_status = report[0]['uid'], report[0]['current_status']
except (KeyError, IndexError):
raise web.HTTPNotFound()
# Found a valid report, check if protected by token
await self.web_svc.action_allowed(request, 'nav-export', context=dict(report=report[0]))
# A queued report would pass the above checks but be blank; raise an error instead
if report_status not in [self.report_statuses.NEEDS_REVIEW.value, self.report_statuses.IN_REVIEW.value,
self.report_statuses.COMPLETED.value]:
raise web.HTTPNotFound()
# Create the layer name and description
enterprise_layer_description = f"Enterprise techniques used by '{report_title}', ATT&CK"
version = '1.0'
if version: # add version number if it exists
enterprise_layer_description += f" v{version}"
# Enterprise navigator layer
enterprise_layer = {
'filename': sanitise_filename(report_title), 'name': report_title, 'domain': 'mitre-enterprise',
'description': enterprise_layer_description, 'version': '2.2', 'techniques': [],
# white for non-used, blue for used
'gradient': {'colors': ['#ffffff', '#66b1ff'], 'minValue': 0, 'maxValue': 1},
'legendItems': [{'label': f'used by {report_title}', 'color': '#66b1ff'}]
}
# Get confirmed techniques for the report from the database
techniques = await self.data_svc.get_confirmed_techniques_for_nav_export(report_id)
# Append techniques to enterprise layer
for technique in techniques:
enterprise_layer['techniques'].append(technique)
# Return the layer JSON in the response
layer = json.dumps(enterprise_layer)
return web.json_response(layer)
async def rest_api(self, request):
data = dict(await request.json())
index = data.pop('index')
options = dict(
POST=dict(
false_positive=lambda d: self.rest_svc.false_positive(criteria=d),
true_positive=lambda d: self.rest_svc.true_positive(criteria=d),
false_negative=lambda d: self.rest_svc.false_negative(criteria=d),
set_status=lambda d: self.rest_svc.set_status(criteria=d),
insert_report=lambda d: self.rest_svc.insert_report(criteria=d),
remove_sentences=lambda d: self.rest_svc.remove_sentences(criteria=d),
delete_report=lambda d: self.rest_svc.delete_report(criteria=d),
sentence_context=lambda d: self.rest_svc.sentence_context(criteria=d),
confirmed_sentences=lambda d: self.rest_svc.confirmed_sentences(criteria=d),
missing_technique=lambda d: self.rest_svc.missing_technique(criteria=d)
))
output = await options[request.method][index](data)
return web.json_response(output)
async def pdf_export(self, request):
"""
Function to export report in PDF format
:param request: The title of the report information
:return: response status of function
"""
from helpers.bert_helpers import load_bert, preprocess_sents, make_predictions
from helpers.sql_helpers import insert_technique
from datetime import datetime
# Get the report
report = await self.dao.get('reports', dict(title=request.match_info.get('file')))
sentences = await self.data_svc.build_sentences(report[0]['uid'])
attack_uids = await self.dao.get('attack_uids')
dd = dict()
dd['content'] = []
dd['styles'] = dict()
# Document MetaData Info
# See https://pdfmake.github.io/docs/document-definition-object/document-medatadata/
dd['info'] = dict()
dd['info']['title'] = report[0]['title']
dd['info']['creator'] = report[0]['url']
table = {"body": []}
table["body"].append(["ID", "Name", "Identified Sentence"])
# * Load model once here
model, tokenizer = load_bert('torchBERT_allioc_model/')
# Add the text to the document
for sentence in sentences:
dd['content'].append(sentence['text'])
if sentence['hits']:
for hit in sentence['hits']:
# 'hits' object doesn't provide all the information we need, so we
# do a makeshift join here to get that information from the attack_uid
# list. This is ineffecient, and a way to improve this would be to perform
# a join on the database side
matching_attacks = [i for i in attack_uids if hit['attack_uid'] == i['uid']]
for match in matching_attacks:
table["body"].append([match["tid"], match["name"], sentence['text']])
# * ADDED
tech_id, t_name = match["tid"], match["name"]
source = report[0]['url']
sentences = sentence['text']
date_crawled = datetime.today().strftime('%Y-%m-%d')
# Preprocess sentences
sentences_list = re.split(' \n \n|\n\n | \n \n \n \n', sentences)
prep_sents = preprocess_sents(sentences_list)
# Predict malware name in sentences
malware_ids = make_predictions(model, tokenizer, prep_sents)
for i in range(len(prep_sents)):
sent = prep_sents[i]
m_id = malware_ids[i]
# Connect to database and insert data
try:
insert_technique(tech_id, t_name, sent, source, date_crawled, m_id)
except Exception as e:
print(f'FAILED INSERT with error {e}')
# Append table to the end
dd['content'].append({"table": table})
return web.json_response(dd)
async def pdf_export(self, request):
"""
Function to export report in PDF format
:param request: The title of the report information
:return: response status of function
"""
# Get the report and its sentences
title = request.match_info.get(self.web_svc.REPORT_PARAM)
report = await self.dao.get('reports', dict(title=title))
try:
# Ensure a valid report title has been passed in the request
report_id, report_status, report_url = report[0]['uid'], report[0]['current_status'], report[0]['url']
except (KeyError, IndexError):
raise web.HTTPNotFound()
# Found a valid report, check if protected by token
await self.web_svc.action_allowed(request, 'pdf-export', context=dict(report=report[0]))
# A queued report would pass the above checks but be blank; raise an error instead
if report_status not in [self.report_statuses.NEEDS_REVIEW.value, self.report_statuses.IN_REVIEW.value,
self.report_statuses.COMPLETED.value]:
raise web.HTTPNotFound()
# Continue with the method and retrieve the report's sentences
sentences = await self.data_svc.get_report_sentences_with_attacks(report_id=report_id)
dd = dict()
# Default background which will be replaced by logo via client-side
dd['background'] = 'Report by Arachne Digital'
dd['content'] = []
# The styles for this pdf - hyperlink styling needed to be added manually
dd['styles'] = dict(header=dict(fontSize=25, bold=True, alignment='center'), bold=dict(bold=True),
sub_header=dict(fontSize=15, bold=True), url=dict(color='blue', decoration='underline'))
# Document MetaData Info
# See https://pdfmake.github.io/docs/document-definition-object/document-medatadata/
dd['info'] = dict()
dd['info']['title'] = sanitise_filename(title)
dd['info']['creator'] = report_url
# Extra content if this report hasn't been completed: highlight it's a draft
if report_status != self.report_statuses.COMPLETED.value:
dd['content'].append(dict(text='DRAFT: Please note this report is still being analysed. '
'Techniques listed here may change later on.', style='sub_header'))
dd['content'].append(dict(text='\n')) # Blank line before report's title
dd['watermark'] = dict(text='DRAFT', opacity=0.3, bold=True, angle=70)
# Table for found attacks
table = {'body': []}
table['body'].append(['ID', 'Name', 'Identified Sentence'])
# Add the text to the document
dd['content'].append(dict(text=title, style='header')) # begin with title of document
dd['content'].append(dict(text='\n')) # Blank line after title
dd['content'].append(dict(text='URL:', style='bold')) # State report's source
dd['content'].append(dict(text=report_url, style='url'))
dd['content'].append(dict(text='\n')) # Blank line after URL
seen_sentences = set() # set to prevent duplicate sentences being exported
for sentence in sentences:
sen_id, sen_text = sentence['uid'], sentence['text']
if sen_id not in seen_sentences:
dd['content'].append(sen_text)
seen_sentences.add(sen_id)
if sentence['attack_tid'] and sentence['active_hit']:
# Append any attack for this sentence to the table; prefix parent-tech for any sub-technique
tech_name, parent_tech = sentence['attack_technique_name'], sentence.get('attack_parent_name')
tech_name = "%s: %s" % (parent_tech, tech_name) if parent_tech else tech_name
table['body'].append([sentence['attack_tid'], tech_name, sen_text])
# Append table to the end
dd['content'].append({'table': table})
return web.json_response(dd)
