def add_to_store(t, v, add_to_local=False):
body = {
"value" : escape(v),
"type" : escape(t)
}
doc_id = t + " " + v
get_es().index(ES_PREFIX + "threat_actor_simple", "data", body, doc_id)
if add_to_local:
global SIMPLE_CHOICES
SIMPLE_CHOICES[t].append(v)
print("Added {} {} to ES".format(v,t))
def fetch_related_elements(t):
if t == 'Actor':
index = "tp-threat_actors"
doc_type = "actor"
elif t == 'Report':
index = "tp-threat_reports"
doc_type = "report"
elif t == 'TTP':
index = "tp-threat_ttps"
doc_type = "ttp"
else:
return []
query = {
"query" : {
"match_all" : {}
},
"sort" : {
"name" : {
"order" : "asc"
}
},
"fields" : ["name"]
}
results = scan(get_es(),query=query,index=index,doc_type=doc_type,preserve_order=True)
choices = []
for r in results:
_id = r["_id"]
name = r["fields"]["name"][0]
choices.append((_id + ":::" + name, name))
return choices
def fetch_parent_data(_type, add_new=False, add_unknown=False):
d = []
#query
body = {
"query" : {
"term" : {
"type" : _type
}
},
"sort": {
"value": {
"order": "asc"
}
},
"size" : 1000
}
#get the data!
results = get_es().search(ES_PREFIX + 'threat_actor_pc', 'parent', body)
for r in results['hits']['hits']:
d.append((r['_source']['value'],r['_source']['value']))
if add_new:
d.append(("_NEW_","Other"))
if add_unknown:
d.insert(0, ("Unknown", "Unknown"))
#return the data
return d
def populate_simple_choices():
'''
Populates the SIMPLE_CHOICES dictionary with options
'''
global SIMPLE_CHOICES
SIMPLE_CHOICES = {}
try:
body = {
"query" : {
"match_all" : {}
},
"size" : 1000
}
results = get_es().search(ES_PREFIX + 'threat_actor_simple', 'data', body)
for r in results['hits']['hits']:
c_type = r['_source']['type']
c_value = r['_source']['value']
if c_type not in SIMPLE_CHOICES:
SIMPLE_CHOICES[c_type] = []
SIMPLE_CHOICES[c_type].append(c_value)
for k,v in SIMPLE_CHOICES.items():
SIMPLE_CHOICES[k] = sorted(v)
#print(SIMPLE_CHOICES)
except Exception as e:
error = "There was an error fetching choices. Details: {}".format(t, e)
flash(error,'danger')
log.exception(logging_prefix + error)
def delete(report_id, report_id_hash):
s = SALTS["report"] + report_id
hash_object = hashlib.sha256(s.encode("utf-8"))
hex_dig = hash_object.hexdigest()
if report_id_hash == hex_dig:
try:
get_es().delete(ES_PREFIX + "threat_reports", "report", report_id)
flash("TTP Deleted", "success")
except Exception as e:
flash("There was an error deleting the TTP. Error: {}".format(e), "danger")
else:
flash("There was an error deleting the TTP", "danger")
redirect_url = request.args.get("_r")
if not redirect_url:
redirec_url = "/"
# sleep a sec, make them think were busy, really its to give ES time to delete the doc
time.sleep(2)
return redirect(redirect_url, code=302)
def fetch_related_choices(t):
logging_prefix = logger_prefix + "fetch_related_choices({}) - ".format(t)
choices = [('_NONE_', 'n/a')]
if t == 'actor':
index = ES_PREFIX + "threat_actors"
doc_type = "actor"
elif t == 'report':
index = ES_PREFIX + "threat_reports"
doc_type = "report"
elif t == 'ttp':
index = ES_PREFIX + "threat_ttps"
doc_type = "ttp"
else:
raise Exception("Invalid type '{}'. Expected 'actor', 'ttp' or 'report'")
es_query = {
"query": {
"match_all": {}
},
"size": 1000,
"fields" : ["name"],
"sort": {
"name": {
"order": "asc"
}
}
}
try:
results = get_es().search(index, doc_type, es_query)
for r in results['hits']['hits']:
choices.append((r['_id'] + ":::" + r['fields']['name'][0],r['fields']['name'][0]))
except TransportError as te:
#if the index was not found, this is most likely becuase theres no data there
if te.status_code == 404:
log.warning("Index '{}' was not found".format(index))
else:
error = "There was an error fetching related {}s. Details: {}".format(t, te)
flash(error,'danger')
log.exception(logging_prefix + error)
except Exception as e:
error = "There was an error fetching related {}s. Details: {}".format(t, e)
flash(error,'danger')
log.exception(logging_prefix + error)
return choices
def delete(ttp_id, ttp_id_hash):
s = SALTS['ttp'] + ttp_id
hash_object = hashlib.sha256(s.encode('utf-8'))
hex_dig = hash_object.hexdigest()
if ttp_id_hash == hex_dig:
try:
get_es().delete(ES_PREFIX + "threat_ttps", "ttp", ttp_id)
flash("TTP Deleted", "success")
except Exception as e:
flash("There was an error deleting the TTP. Error: {}".format(e),
"danger")
else:
flash("There was an error deleting the TTP", "danger")
redirect_url = request.args.get("_r")
if not redirect_url:
redirec_url = "/"
#sleep a sec, make them think were busy, really its to give ES time to delete the doc
time.sleep(2)
return redirect(redirect_url, code=302)
def get_score(classification_family, classification_id):
#query
body = {
"query" : {
"query_string" : {
"query" : '+_parent:"tpx_classification {}" +value:"{}"'.format(escape(classification_family),escape(classification_id))
}
},
"sort": {
"value": {
"order": "asc"
}
},
"size" : 1
}
#get the data!
results = get_es().search(ES_PREFIX + 'threat_actor_pc', 'child', body)
if results['hits']['total'] != 1:
raise Exception("Unable to perform score look up for {} - {}".format(classification_family,classification_id))
return results['hits']['hits'][0]['_source']['score']
def index():
logging_prefix = logger_prefix + "index() - "
log.info(logging_prefix + "Loading home page")
form = forms.searchForm(request.form)
error = None
url = "/"
query_string_url = ""
try:
#this is the default query for actors in ES, i'd imagine this will be recently added/modified actors
es_query = {
"query": {
"match_all": {}
},
"size": 10,
"sort": {
"last_updated_s": {
"order": "desc"
}
}
}
#pull the query out of the url
query_string = request.args.get("q")
#someone is searching for something
if request.method == 'POST' and not query_string:
if form.validate():
#get the value
value = form.query.data
#redirect to this same page, but setting the query value in the url
return redirect("/?q={}".format(quote_plus(value)), code=307)
else:
#if there was an error print the error dictionary to the console
# temporary help, these should also appear under the form field
print(form.errors)
elif query_string:
#now that the query_string is provided as ?q=, perform the search
print("VALID SEARCH OPERATION DETECTED")
#do some searching...
es_query = {
"query": {
"query_string": {
"query": query_string
}
},
"size": 10
}
url += "?q=" + query_string
query_string_url = "?q=" + query_string
#set the form query value to what the user is searching for
form.query.data = query_string
'''
Fetch the data from ES
'''
actors = {}
actors['hits'] = {}
actors['hits']['hits'] = []
reports = dict(actors)
ttps = dict(actors)
try:
actors = get_es().search(ES_PREFIX + 'threat_actors', 'actor',
es_query)
except TransportError as te:
#if the index was not found, this is most likely becuase theres no data there
if te.status_code == 404:
log.warning("Index 'threat_actors' was not found")
else:
error = "There was an error fetching actors. Details: {}".format(
te)
flash(error, 'danger')
log.exception(logging_prefix + error)
except Exception as e:
error = "The was an error fetching Actors. Error: {}".format(e)
log.exception(error)
flash(error, "danger")
try:
reports = get_es().search(ES_PREFIX + 'threat_reports', 'report',
es_query)
except TransportError as te:
#if the index was not found, this is most likely becuase theres no data there
if te.status_code == 404:
log.warning("Index 'threat_reports' was not found")
else:
error = "There was an error fetching reports. Details: {}".format(
te)
flash(error, 'danger')
log.exception(logging_prefix + error)
except Exception as e:
error = "The was an error fetching Reports. Error: {}".format(e)
log.exception(error)
flash(error, "danger")
try:
ttps = get_es().search(ES_PREFIX + 'threat_ttps', 'ttp', es_query)
except TransportError as te:
#if the index was not found, this is most likely becuase theres no data there
if te.status_code == 404:
log.warning("Index 'threat_ttps' was not found")
else:
error = "There was an error ttps. Details: {}".format(te)
flash(error, 'danger')
log.exception(logging_prefix + error)
except Exception as e:
error = "The was an error fetching TTPs. Error: {}".format(e)
log.exception(error)
flash(error, "danger")
'''
Modify the data as needed
'''
for actor in actors['hits']['hits']:
s = SALTS['actor'] + actor['_id']
hash_object = hashlib.sha256(s.encode('utf-8'))
hex_dig = hash_object.hexdigest()
actor["_source"]['id_hash'] = hex_dig
for report in reports['hits']['hits']:
s = SALTS['report'] + report['_id']
hash_object = hashlib.sha256(s.encode('utf-8'))
hex_dig = hash_object.hexdigest()
report["_source"]['id_hash'] = hex_dig
for ttp in ttps['hits']['hits']:
s = SALTS['ttp'] + ttp['_id']
hash_object = hashlib.sha256(s.encode('utf-8'))
hex_dig = hash_object.hexdigest()
ttp["_source"]['id_hash'] = hex_dig
except Exception as e:
error = "There was an error completing your request. Details: {}".format(
e)
log.exception(error)
flash(error, "danger")
#render the template, passing the variables we need
# templates live in the templates folder
return render_template("index.html",
page_title="ActorTrackr",
form=form,
query_string_url=query_string_url,
actors=actors,
reports=reports,
ttps=ttps,
url=quote_plus(url))
def es_to_tpx(report_id):
'''
Build the TPX file from the data stored in Elasticsearch
'''
element_observables = {}
results = get_es().get(ES_PREFIX + "threat_reports",
doc_type="report",
id=report_id)
report_data = results['_source']
tpx = {}
tpx["schema_version_s"] = "2.2.0"
tpx["provider_s"] = "LookingGlass"
tpx["list_name_s"] = "Threat Intel Report"
tpx["created_t"] = report_data['created_milli']
tpx["created_s"] = report_data['created_s']
tpx["last_updated_t"] = report_data['last_updated_milli']
tpx["last_updated_s"] = report_data['last_updated_s']
tpx["score_i"] = 95
tpx["source_observable_s"] = "Cyveillance Threat Intel Report"
tpx["source_description_s"] = "This feed provides threat intelligence reports created by the LookingGlass Cyber Threat Intelligence Group"
tpx["observable_dictionary_c_array"] = []
observable_dict = {}
observable_dict["observable_id_s"] = report_data['name']
observable_dict["report_uuid_s"] = report_id
observable_dict["criticality_i"] = report_data['criticality']
observable_dict["classification_c_array"] = []
for i in report_data['classification']:
class_dict = {}
class_dict["score_i"] = i["score"]
#class_dict["score_i"] = get_score(i["family"], i["id"])
class_dict["classification_id_s"] = i["id"]
class_dict["classification_family_s"] = i["family"]
observable_dict["classification_c_array"].append(class_dict)
observable_dict["description_s"] = report_data['description']
observable_dict["report_s_map"] = {}
observable_dict["report_s_map"]["ctig_identifier_s"] = report_data[
"identifier"]
observable_dict["report_s_map"]["tlp_i"] = report_data["identifier"]
observable_dict["report_s_map"]["reliability_s"] = report_data[
"source_reliability"] + str(report_data["info_reliability"])
observable_dict["report_s_map"]["title_s"] = report_data['name']
observable_dict["report_s_map"]["sections_c_array"] = []
for i in report_data['section']:
section_dict = {}
section_dict["section_title_s"] = i["title"]
section_dict["section_tlp_i"] = i["tlp"]
section_dict["section_order_i"] = i["order"]
section_dict["section_content_s"] = i["content"]
observable_dict["report_s_map"]["sections_c_array"].append(
section_dict)
'''
Related elements
'''
relate_element_name_map = {
"FQDN": "subject_fqdn_s",
"IPv4": "subject_ipv4_s",
"TTP": "subject_ttp_s",
"CommAddr": "subject_address_s"
}
for i in report_data['related_element_choices']:
v = i['value']
print(v)
v_array = v.split(":::")
if len(v_array) == 2:
field_name = relate_element_name_map[v_array[0]]
field_type = "_DEFAULT_"
field_data = v_array[1]
elif len(v_array) == 3:
field_name = relate_element_name_map[v_array[1]]
field_type = v_array[0]
field_data = v_array[2]
else:
raise Exception("Unknown Related Element: '{}'".format(v))
#this field is used in the element observable c array, lets keep track of it
field_observable = report_data['name']
if field_name not in element_observables:
element_observables[field_name] = {}
if field_type not in element_observables[field_name]:
element_observables[field_name][field_type] = {}
if field_data not in element_observables[field_name][field_type]:
element_observables[field_name][field_type][field_data] = []
if field_observable not in element_observables[field_name][field_type][
field_data]:
element_observables[field_name][field_type][field_data].append(
field_observable)
observable_dict["report_s_map"]["related_ttps_c_array"] = []
for i in report_data['related_ttp']:
if i['name']:
observable_dict["report_s_map"]["related_ttps_c_array"].append({
"name_s":
i['name'],
"uuid_s":
i['id']
})
field_observable = i['name']
for j in i['elements']:
element_array = j.split(":::")
if len(element_array) == 3:
field_type = element_array[0]
field_name = relate_element_name_map[element_array[1]]
field_data = element_array[2]
elif len(element_array) == 2:
field_type = "_DEFAULT_"
field_name = relate_element_name_map[element_array[0]]
field_data = element_array[1]
else:
raise Exception("Invalid element '{}'".format(j))
if field_name not in element_observables:
element_observables[field_name] = {}
if field_type not in element_observables[field_name]:
element_observables[field_name][field_type] = {}
if field_data not in element_observables[field_name][
field_type]:
element_observables[field_name][field_type][
field_data] = []
if field_observable not in element_observables[field_name][
field_type][field_data]:
element_observables[field_name][field_type][
field_data].append(field_observable)
observable_dict["report_s_map"]["related_actors_c_array"] = []
for i in report_data['related_actor']:
if i['name']:
observable_dict["report_s_map"]["related_actors_c_array"].append({
"name_s":
i['name'],
"uuid_s":
i['id']
})
field_observable = i['name']
for j in i['elements']:
element_array = j.split(":::")
if len(element_array) == 3:
field_type = element_array[0]
field_name = relate_element_name_map[element_array[1]]
field_data = element_array[2]
elif len(element_array) == 2:
field_type = "_DEFAULT_"
field_name = relate_element_name_map[element_array[0]]
field_data = element_array[1]
else:
raise Exception("Invalid element '{}'".format(j))
if field_name not in element_observables:
element_observables[field_name] = {}
if field_type not in element_observables[field_name]:
element_observables[field_name][field_type] = {}
if field_data not in element_observables[field_name][
field_type]:
element_observables[field_name][field_type][
field_data] = []
if field_observable not in element_observables[field_name][
field_type][field_data]:
element_observables[field_name][field_type][
field_data].append(field_observable)
observable_dict["report_s_map"]["related_reports_c_array"] = []
for i in report_data['related_report']:
if i['name']:
observable_dict["report_s_map"]["related_reports_c_array"].append({
"name_s":
i['name'],
"uuid_s":
i['id']
})
field_observable = i['name']
for j in i['elements']:
element_array = j.split(":::")
if len(element_array) == 3:
field_type = element_array[0]
field_name = relate_element_name_map[element_array[1]]
field_data = element_array[2]
elif len(element_array) == 2:
field_type = "_DEFAULT_"
field_name = relate_element_name_map[element_array[0]]
field_data = element_array[1]
else:
raise Exception("Invalid element '{}'".format(j))
if field_name not in element_observables:
element_observables[field_name] = {}
if field_type not in element_observables[field_name]:
element_observables[field_name][field_type] = {}
if field_data not in element_observables[field_name][
field_type]:
element_observables[field_name][field_type][
field_data] = []
if field_observable not in element_observables[field_name][
field_type][field_data]:
element_observables[field_name][field_type][
field_data].append(field_observable)
tpx["observable_dictionary_c_array"].append(observable_dict)
tpx["element_observable_c_array"] = []
for field, val in element_observables.items():
for sub_type, val1 in val.items():
for data, val2 in val1.items():
e_dict = {}
e_dict[field] = data
if sub_type != "_DEFAULT_":
e_dict['type_s'] = sub_type
e_dict["threat_observable_c_map"] = {}
for observable_id in val2:
e_dict["threat_observable_c_map"][observable_id] = {}
e_dict["threat_observable_c_map"][observable_id][
"occurred_at_t"] = report_data['created_milli']
e_dict["threat_observable_c_map"][observable_id][
"occurred_at_s"] = report_data['created_s']
e_dict["threat_observable_c_map"][observable_id][
"last_seen_t"] = report_data['last_updated_milli']
e_dict["threat_observable_c_map"][observable_id][
"last_seen_s"] = report_data['last_updated_s']
tpx["element_observable_c_array"].append(e_dict)
return tpx
def view_all(t, page=1):
if t == 'favicon.ico':
return jsonify({}), 404
page = int(page)
logging_prefix = logger_prefix + "view_all() - "
log.info(logging_prefix +
"Loading view all page {} for {}".format(page, t))
form = forms.searchForm(request.form)
error = None
page_size = 50
offset = (page - 1) * page_size
url = "/{}/{}/".format(t, page)
search_url = ""
results_text = ""
try:
#this is the default query for actors in ES, i'd imagine this will be recently added/modified actors
es_query = {
"query": {
"match_all": {}
},
"size": page_size,
"from": offset,
"sort": {
"last_updated_s": {
"order": "desc"
}
}
}
#pull the query out of the url
query_string = request.args.get("q")
#someone is searching for something
if request.method == 'POST' and not query_string:
if form.validate():
print("VALID SEARCH OPERATION DETECTED, redirecting...")
#get the value
value = form.query.data
log.info(value)
#redirect to this same page, but setting the query value in the url
return redirect("/{}/1/?q={}".format(t, quote_plus(value)),
code=307)
else:
#if there was an error print the error dictionary to the console
# temporary help, these should also appear under the form field
print(form.errors)
elif query_string:
#now that the query_string is provided as ?q=, perform the search
print("VALID SEARCH OPERATION DETECTED")
#do some searching...
es_query = {
"query": {
"query_string": {
"query": query_string
}
},
"size": page_size,
"from": offset
}
search_url = "?q=" + query_string
#set the form query value to what the user is searching for
form.query.data = query_string
'''
Fetch the data from ES
'''
data = {}
data['hits'] = {}
data['hits']['hits'] = []
if t == 'actor':
index = ES_PREFIX + 'threat_actors'
doc_type = 'actor'
salt = SALTS['actor']
link_prefix = 'actor'
data_header = 'Actors'
field_header = 'Actor Name'
elif t == 'report':
index = ES_PREFIX + 'threat_reports'
doc_type = 'report'
salt = SALTS['report']
link_prefix = 'report'
data_header = 'Reports'
field_header = 'Report Title'
elif t == 'ttp':
index = ES_PREFIX + 'threat_ttps'
doc_type = 'ttp'
salt = SALTS['ttp']
link_prefix = 'ttp'
data_header = 'TTPs'
field_header = 'TTP Name'
else:
raise Exception("Unknown type {}".format(t))
try:
data = get_es().search(index, doc_type, es_query)
num_hits = len(data['hits']['hits'])
#set up previous link
if page == 1:
prev_url = None
else:
prev_url = "/{}/{}/{}".format(t, (page - 1), search_url)
if ((page - 1) * page_size) + num_hits < data['hits']['total']:
next_url = "/{}/{}/{}".format(t, (page + 1), search_url)
else:
next_url = None
url += search_url
for d in data['hits']['hits']:
s = salt + d['_id']
hash_object = hashlib.sha256(s.encode('utf-8'))
hex_dig = hash_object.hexdigest()
d["_source"]['id_hash'] = hex_dig
if num_hits == 0:
results_text = ""
else:
f = ((page - 1) * page_size) + 1
l = f + (num_hits - 1)
results_text = "Showing {} to {} of {} total results".format(
f, l, data['hits']['total'])
except TransportError as te:
#if the index was not found, this is most likely becuase theres no data there
if te.status_code == 404:
log.warning("Index '{}' was not found".format(index))
else:
error = "There was an error fetching {}. Details: {}".format(
t, te)
flash(error, 'danger')
log.exception(logging_prefix + error)
except Exception as e:
error = "The was an error fetching {}. Error: {}".format(t, e)
log.exception(error)
flash(error, "danger")
except Exception as e:
error = "There was an error completing your request. Details: {}".format(
e)
log.exception(error)
flash(error, "danger")
return redirect("/")
return render_template("view_all.html",
page_title="View All",
form=form,
data_header=data_header,
results_text=results_text,
field_header=field_header,
data=data,
link_prefix=link_prefix,
prev_url=prev_url,
next_url=next_url,
url=quote_plus(url))
def es_to_tpx(report_id):
"""
Build the TPX file from the data stored in Elasticsearch
"""
element_observables = {}
results = get_es().get(ES_PREFIX + "threat_reports", doc_type="report", id=report_id)
report_data = results["_source"]
tpx = {}
tpx["schema_version_s"] = "2.2.0"
tpx["provider_s"] = "LookingGlass"
tpx["list_name_s"] = "Threat Intel Report"
tpx["created_t"] = report_data["created_milli"]
tpx["created_s"] = report_data["created_s"]
tpx["last_updated_t"] = report_data["last_updated_milli"]
tpx["last_updated_s"] = report_data["last_updated_s"]
tpx["score_i"] = 95
tpx["source_observable_s"] = "Cyveillance Threat Intel Report"
tpx[
"source_description_s"
] = "This feed provides threat intelligence reports created by the LookingGlass Cyber Threat Intelligence Group"
tpx["observable_dictionary_c_array"] = []
observable_dict = {}
observable_dict["observable_id_s"] = report_data["name"]
observable_dict["report_uuid_s"] = report_id
observable_dict["criticality_i"] = report_data["criticality"]
observable_dict["classification_c_array"] = []
for i in report_data["classification"]:
class_dict = {}
class_dict["score_i"] = i["score"]
# class_dict["score_i"] = get_score(i["family"], i["id"])
class_dict["classification_id_s"] = i["id"]
class_dict["classification_family_s"] = i["family"]
observable_dict["classification_c_array"].append(class_dict)
observable_dict["description_s"] = report_data["description"]
observable_dict["report_s_map"] = {}
observable_dict["report_s_map"]["ctig_identifier_s"] = report_data["identifier"]
observable_dict["report_s_map"]["tlp_i"] = report_data["identifier"]
observable_dict["report_s_map"]["reliability_s"] = report_data["source_reliability"] + str(
report_data["info_reliability"]
)
observable_dict["report_s_map"]["title_s"] = report_data["name"]
observable_dict["report_s_map"]["sections_c_array"] = []
for i in report_data["section"]:
section_dict = {}
section_dict["section_title_s"] = i["title"]
section_dict["section_tlp_i"] = i["tlp"]
section_dict["section_order_i"] = i["order"]
section_dict["section_content_s"] = i["content"]
observable_dict["report_s_map"]["sections_c_array"].append(section_dict)
"""
Related elements
"""
relate_element_name_map = {
"FQDN": "subject_fqdn_s",
"IPv4": "subject_ipv4_s",
"TTP": "subject_ttp_s",
"CommAddr": "subject_address_s",
}
for i in report_data["related_element_choices"]:
v = i["value"]
print(v)
v_array = v.split(":::")
if len(v_array) == 2:
field_name = relate_element_name_map[v_array[0]]
field_type = "_DEFAULT_"
field_data = v_array[1]
elif len(v_array) == 3:
field_name = relate_element_name_map[v_array[1]]
field_type = v_array[0]
field_data = v_array[2]
else:
raise Exception("Unknown Related Element: '{}'".format(v))
# this field is used in the element observable c array, lets keep track of it
field_observable = report_data["name"]
if field_name not in element_observables:
element_observables[field_name] = {}
if field_type not in element_observables[field_name]:
element_observables[field_name][field_type] = {}
if field_data not in element_observables[field_name][field_type]:
element_observables[field_name][field_type][field_data] = []
if field_observable not in element_observables[field_name][field_type][field_data]:
element_observables[field_name][field_type][field_data].append(field_observable)
observable_dict["report_s_map"]["related_ttps_c_array"] = []
for i in report_data["related_ttp"]:
if i["name"]:
observable_dict["report_s_map"]["related_ttps_c_array"].append({"name_s": i["name"], "uuid_s": i["id"]})
field_observable = i["name"]
for j in i["elements"]:
element_array = j.split(":::")
if len(element_array) == 3:
field_type = element_array[0]
field_name = relate_element_name_map[element_array[1]]
field_data = element_array[2]
elif len(element_array) == 2:
field_type = "_DEFAULT_"
field_name = relate_element_name_map[element_array[0]]
field_data = element_array[1]
else:
raise Exception("Invalid element '{}'".format(j))
if field_name not in element_observables:
element_observables[field_name] = {}
if field_type not in element_observables[field_name]:
element_observables[field_name][field_type] = {}
if field_data not in element_observables[field_name][field_type]:
element_observables[field_name][field_type][field_data] = []
if field_observable not in element_observables[field_name][field_type][field_data]:
element_observables[field_name][field_type][field_data].append(field_observable)
observable_dict["report_s_map"]["related_actors_c_array"] = []
for i in report_data["related_actor"]:
if i["name"]:
observable_dict["report_s_map"]["related_actors_c_array"].append({"name_s": i["name"], "uuid_s": i["id"]})
field_observable = i["name"]
for j in i["elements"]:
element_array = j.split(":::")
if len(element_array) == 3:
field_type = element_array[0]
field_name = relate_element_name_map[element_array[1]]
field_data = element_array[2]
elif len(element_array) == 2:
field_type = "_DEFAULT_"
field_name = relate_element_name_map[element_array[0]]
field_data = element_array[1]
else:
raise Exception("Invalid element '{}'".format(j))
if field_name not in element_observables:
element_observables[field_name] = {}
if field_type not in element_observables[field_name]:
element_observables[field_name][field_type] = {}
if field_data not in element_observables[field_name][field_type]:
element_observables[field_name][field_type][field_data] = []
if field_observable not in element_observables[field_name][field_type][field_data]:
element_observables[field_name][field_type][field_data].append(field_observable)
observable_dict["report_s_map"]["related_reports_c_array"] = []
for i in report_data["related_report"]:
if i["name"]:
observable_dict["report_s_map"]["related_reports_c_array"].append({"name_s": i["name"], "uuid_s": i["id"]})
field_observable = i["name"]
for j in i["elements"]:
element_array = j.split(":::")
if len(element_array) == 3:
field_type = element_array[0]
field_name = relate_element_name_map[element_array[1]]
field_data = element_array[2]
elif len(element_array) == 2:
field_type = "_DEFAULT_"
field_name = relate_element_name_map[element_array[0]]
field_data = element_array[1]
else:
raise Exception("Invalid element '{}'".format(j))
if field_name not in element_observables:
element_observables[field_name] = {}
if field_type not in element_observables[field_name]:
element_observables[field_name][field_type] = {}
if field_data not in element_observables[field_name][field_type]:
element_observables[field_name][field_type][field_data] = []
if field_observable not in element_observables[field_name][field_type][field_data]:
element_observables[field_name][field_type][field_data].append(field_observable)
tpx["observable_dictionary_c_array"].append(observable_dict)
tpx["element_observable_c_array"] = []
for field, val in element_observables.items():
for sub_type, val1 in val.items():
for data, val2 in val1.items():
e_dict = {}
e_dict[field] = data
if sub_type != "_DEFAULT_":
e_dict["type_s"] = sub_type
e_dict["threat_observable_c_map"] = {}
for observable_id in val2:
e_dict["threat_observable_c_map"][observable_id] = {}
e_dict["threat_observable_c_map"][observable_id]["occurred_at_t"] = report_data["created_milli"]
e_dict["threat_observable_c_map"][observable_id]["occurred_at_s"] = report_data["created_s"]
e_dict["threat_observable_c_map"][observable_id]["last_seen_t"] = report_data["last_updated_milli"]
e_dict["threat_observable_c_map"][observable_id]["last_seen_s"] = report_data["last_updated_s"]
tpx["element_observable_c_array"].append(e_dict)
return tpx
def form_to_es(form, report_id):
logging_prefix = logger_prefix + "form_to_es() - "
log.info(logging_prefix + "Converting Form to ES for {}".format(report_id))
doc = {}
created_t = int(time.mktime(form.report_occurred_at.data.timetuple())) * 1000
created_s = form.report_occurred_at.data.strftime("%Y-%m-%dT%H:%M:%S")
now_t = int(time.time()) * 1000
now_s = datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
doc["created_milli"] = created_t
doc["created_s"] = created_s
doc["last_updated_milli"] = now_t
doc["last_updated_s"] = now_s
doc["name"] = escape(form.report_name.data)
doc["identifier"] = escape(form.report_id.data)
doc["description"] = escape(form.report_description.data)
doc["criticality"] = escape(int(form.report_criticality.data))
doc["classification"] = []
for sub_form in form.report_class.entries:
classification_dict = {}
classification_dict["score"] = int(get_score(sub_form.data["a_family"], sub_form.data["a_id"]))
classification_dict["id"] = escape(sub_form.data["a_id"])
classification_dict["family"] = escape(sub_form.data["a_family"])
if classification_dict not in doc["classification"]:
doc["classification"].append(classification_dict)
doc["tlp"] = int(escape(form.report_tlp.data))
doc["section"] = []
idx = 1
for sub_form in form.report_sections.entries:
section_dict = {}
section_dict["order"] = idx
section_dict["title"] = escape(sub_form.title.data)
section_dict["tlp"] = int(escape(sub_form.tlp.data))
section_dict["content"] = escape(sub_form.text.data)
idx += 1
doc["section"].append(section_dict)
doc["source_reliability"] = escape(form.report_source_reliability.data)
doc["info_reliability"] = int(escape(form.report_info_reliability.data))
doc["source"] = []
for sub_form in form.report_sources.entries:
source = sub_form.source.data
if source not in doc["source"]:
doc["source"].append(source)
"""
Related elements
"""
my_id = report_id
# Links to actors and reports
doc["related_actor"] = []
for sub_form in form.report_actors.entries:
r_dict = {}
data = escape(sub_form.data.data)
if data == "_NONE_":
continue
data_array = data.split(":::")
r_dict["id"] = escape(data_array[0])
# if the related element's id = my id, dont add to es, this should only apply to current type
if data_array[0] == my_id:
flash("A related actor matched this actor, this entry has been discarded", "warning")
continue
r_dict["name"] = escape(data_array[1])
# this is gonna be a nightmare to maintain,
# but it make sense to have this for searches
r_dict["elements"] = []
for sub_sub_form in sub_form.related_elements.entries:
if sub_sub_form.element.data:
if sub_sub_form.element_value.data not in r_dict["elements"]:
r_dict["elements"].append(sub_sub_form.element_value.data)
if r_dict not in doc["related_actor"]:
doc["related_actor"].append(r_dict)
doc["related_report"] = []
for sub_form in form.report_reports.entries:
r_dict = {}
data = escape(sub_form.data.data)
if data == "_NONE_":
continue
data_array = data.split(":::")
r_dict["id"] = escape(data_array[0])
# if the related element's id = my id, dont add to es, this should only apply to current type
if data_array[0] == my_id:
flash("A related report matched this report, this entry has been discarded", "warning")
continue
r_dict["name"] = escape(data_array[1])
# this is gonna be a nightmare to maintain,
# but it make sense to have this for searches
r_dict["elements"] = []
for sub_sub_form in sub_form.related_elements.entries:
if sub_sub_form.element.data:
if sub_sub_form.element_value.data not in r_dict["elements"]:
r_dict["elements"].append(sub_sub_form.element_value.data)
if r_dict not in doc["related_report"]:
doc["related_report"].append(r_dict)
doc["related_ttp"] = []
for sub_form in form.report_ttps.entries:
r_dict = {}
data = escape(sub_form.data.data)
if data == "_NONE_":
continue
data_array = data.split(":::")
r_dict["id"] = escape(data_array[0])
# if the related element's id = my id, dont add to es, this should only apply to current type
if data_array[0] == my_id:
flash("A related TTP matched this TTP, this entry has been discarded", "warning")
continue
r_dict["name"] = escape(data_array[1])
# this is gonna be a nightmare to maintain,
# but it make sense to have this for searches
r_dict["elements"] = []
for sub_sub_form in sub_form.related_elements.entries:
if sub_sub_form.element.data:
if sub_sub_form.element_value.data not in r_dict["elements"]:
r_dict["elements"].append(sub_sub_form.element_value.data)
if r_dict not in doc["related_ttp"]:
doc["related_ttp"].append(r_dict)
# store these to make life easier
doc["related_element_choices"] = []
rec = json.loads(form.related_element_choices.data)
for k, v in rec.items():
# dont include the n/a choice, as this will be first and handled manually
if v != "_NONE_":
# this will create a tuple ("display text", "value string")
doc["related_element_choices"].append({"display_text": k, "value": v})
"""
Edit Tracking
"""
doc["editor"] = get_editor_list(
es=get_es(),
index=ES_PREFIX + "threat_reports",
doc_type="report",
item_id=report_id,
user_id=session.get("id", None),
)
# index the doc
log.info(logging_prefix + "Start Indexing of {}".format(report_id))
response = get_es().index(ES_PREFIX + "threat_reports", "report", doc, report_id)
log.info(logging_prefix + "Done Indexing of {}".format(report_id))
return response, doc
def es_to_form(report_id):
form = forms.reportForm()
#get the values from ES
results = get_es().get(ES_PREFIX + "threat_reports",
doc_type="report",
id=report_id)
#store certain fields from ES, so this form can be used in an update
form.doc_index.data = results['_index']
form.doc_type.data = results['_type']
report_data = results['_source']
form.report_name.data = report_data['name']
form.report_id.data = report_data['identifier']
form.report_occurred_at.data = datetime.strptime(report_data['created_s'],
"%Y-%m-%dT%H:%M:%S")
form.report_description.data = report_data['description']
form.report_criticality.data = report_data['criticality']
idx = 0
for entry in range(len(form.report_class.entries)):
form.report_class.pop_entry()
for i in multikeysort(report_data['classification'], ['family', 'id']):
report_class_form = forms.TPXClassificationForm()
report_class_form.a_family = i['family']
report_class_form.a_id = i['id']
form.report_class.append_entry(report_class_form)
#set the options since this select is dynamic
form.report_class[idx].a_id.choices = fetch_child_data(
'tpx_classification', i['family'])
idx += 1
form.report_tlp.data = int(report_data['tlp'])
for entry in range(len(form.report_sections.entries)):
form.report_sections.pop_entry()
for i in multikeysort(report_data['section'], ['order']):
report_section_form = forms.ReportSectionsForm()
report_section_form.title = i['title']
report_section_form.tlp = i['tlp']
report_section_form.text = i['content']
form.report_sections.append_entry(report_section_form)
form.report_source_reliability.data = report_data["source_reliability"]
form.report_info_reliability.data = report_data["info_reliability"]
for entry in range(len(form.report_sources.entries)):
form.report_sources.pop_entry()
for i in sorted(report_data['source']):
report_source_form = forms.ReportSourcesForm()
report_source_form.source = i
form.report_sources.append_entry(report_source_form)
'''
Related element
'''
for entry in range(len(form.report_actors.entries)):
form.report_actors.pop_entry()
if report_data['related_actor']:
idx = 0
for i in multikeysort(report_data['related_actor'], ['name', 'id']):
sub_form = forms.RelatedActorsForm()
sub_form.data = i['id'] + ":::" + i['name']
has_related_elements = False
idx2 = 0
for entry in range(len(sub_form.related_elements.entries)):
sub_form.related_elements.pop_entry()
for j in multikeysort(report_data['related_element_choices'],
['display_text']):
sub_sub_form = forms.ElementObservablesFrom()
sub_form.related_elements.append_entry(sub_sub_form)
is_related = (j['value'] in i['elements'])
sub_form.related_elements[idx2].element = is_related
sub_form.related_elements[idx2].element_value = j['value']
sub_form.related_elements[idx2].element_text = j[
'display_text']
if is_related:
has_related_elements = True
idx2 += 1
form.report_actors.append_entry(sub_form)
form.report_actors[
idx].has_related_elements.data = has_related_elements
idx += 1
else:
sub_form = forms.RelatedActorsForm()
sub_form.data = "_NONE_"
form.report_actors.append_entry(sub_form)
for entry in range(len(form.report_reports.entries)):
form.report_reports.pop_entry()
if report_data['related_report']:
idx = 0
for i in multikeysort(report_data['related_report'], ['name', 'id']):
sub_form = forms.RelatedReportsForm()
sub_form.data = i['id'] + ":::" + i['name']
has_related_elements = False
idx2 = 0
for entry in range(len(sub_form.related_elements.entries)):
sub_form.related_elements.pop_entry()
for j in multikeysort(report_data['related_element_choices'],
['display_text']):
sub_sub_form = forms.ElementObservablesFrom()
sub_form.related_elements.append_entry(sub_sub_form)
is_related = (j['value'] in i['elements'])
sub_form.related_elements[idx2].element = is_related
sub_form.related_elements[idx2].element_value = j['value']
sub_form.related_elements[idx2].element_text = j[
'display_text']
if is_related:
has_related_elements = True
idx2 += 1
form.report_reports.append_entry(sub_form)
form.report_reports[
idx].has_related_elements.data = has_related_elements
idx += 1
else:
sub_form = forms.RelatedReportsForm()
sub_form.data = "_NONE_"
form.report_reports.append_entry(sub_form)
for entry in range(len(form.report_ttps.entries)):
form.report_ttps.pop_entry()
if report_data['related_ttp']:
idx = 0
for i in multikeysort(report_data['related_ttp'], ['name', 'id']):
sub_form = forms.RelatedTTPsForm()
sub_form.data = i['id'] + ":::" + i['name']
has_related_elements = False
idx2 = 0
for entry in range(len(sub_form.related_elements.entries)):
sub_form.related_elements.pop_entry()
for j in multikeysort(report_data['related_element_choices'],
['display_text']):
sub_sub_form = forms.ElementObservablesFrom()
sub_form.related_elements.append_entry(sub_sub_form)
is_related = (j['value'] in i['elements'])
sub_form.related_elements[idx2].element = is_related
sub_form.related_elements[idx2].element_value = j['value']
sub_form.related_elements[idx2].element_text = j[
'display_text']
if is_related:
has_related_elements = True
idx2 += 1
form.report_ttps.append_entry(sub_form)
form.report_ttps[
idx].has_related_elements.data = has_related_elements
idx += 1
else:
sub_form = forms.RelatedTTPsForm()
sub_form.data = "_NONE_"
form.report_ttps.append_entry(sub_form)
#convert editor dictionary of ids and times to names and times
editors = get_editor_names(get_mysql(), report_data['editor'])
return form, editors
def es_to_form(ttp_id):
form = forms.ttpForm()
#get the values from ES
results = get_es().get(ES_PREFIX + "threat_ttps",
doc_type="ttp",
id=ttp_id)
#store certain fields from ES, so this form can be used in an update
form.doc_index.data = results['_index']
form.doc_type.data = results['_type']
ttp_data = results['_source']
form.ttp_name.data = ttp_data['name']
form.ttp_first_observed.data = datetime.strptime(ttp_data['created_s'],
"%Y-%m-%dT%H:%M:%S")
form.ttp_description.data = ttp_data['description']
form.ttp_criticality.data = int(ttp_data['criticality'])
idx = 0
for entry in range(len(form.ttp_class.entries)):
form.ttp_class.pop_entry()
for i in multikeysort(ttp_data['classification'], ['family', 'id']):
ttp_class_form = forms.TPXClassificationForm()
ttp_class_form.a_family = i['family']
ttp_class_form.a_id = i['id']
form.ttp_class.append_entry(ttp_class_form)
#set the options since this select is dynamic
form.ttp_class[idx].a_id.choices = fetch_child_data(
'tpx_classification', i['family'])
idx += 1
if ttp_data['related_actor']:
for entry in range(len(form.ttp_actors.entries)):
form.ttp_actors.pop_entry()
for i in multikeysort(ttp_data['related_actor'], ['name', 'id']):
sub_form = forms.RelatedActorsForm()
sub_form.data = i['id'] + ":::" + i['name']
form.ttp_actors.append_entry(sub_form)
if ttp_data['related_report']:
for entry in range(len(form.ttp_reports.entries)):
form.ttp_reports.pop_entry()
for i in multikeysort(ttp_data['related_report'], ['name', 'id']):
sub_form = forms.RelatedReportsForm()
sub_form.data = i['id'] + ":::" + i['name']
form.ttp_reports.append_entry(sub_form)
if ttp_data['related_ttp']:
for entry in range(len(form.ttp_ttps.entries)):
form.ttp_ttps.pop_entry()
for i in multikeysort(ttp_data['related_ttp'], ['name', 'id']):
sub_form = forms.RelatedTTPsForm()
sub_form.data = i['id'] + ":::" + i['name']
form.ttp_ttps.append_entry(sub_form)
#convert editor dictionary of ids and times to names and times
editors = get_editor_names(get_mysql(), ttp_data['editor'])
return form, editors
def form_to_es(form, ttp_id):
logging_prefix = logger_prefix + "form_to_es() - "
log.info(logging_prefix + "Converting Form to ES for {}".format(ttp_id))
doc = {}
created_t = int(time.mktime(
form.ttp_first_observed.data.timetuple())) * 1000
created_s = form.ttp_first_observed.data.strftime("%Y-%m-%dT%H:%M:%S")
now_t = int(time.time()) * 1000
now_s = datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
doc["created_milli"] = created_t
doc["created_s"] = created_s
doc["last_updated_milli"] = now_t
doc["last_updated_s"] = now_s
doc['name'] = escape(form.ttp_name.data)
doc['description'] = escape(form.ttp_description.data)
doc['criticality'] = int(escape(form.ttp_criticality.data))
doc['classification'] = []
for sub_form in form.ttp_class.entries:
classification_dict = {}
classification_dict['score'] = int(
get_score(sub_form.data['a_family'], sub_form.data['a_id']))
classification_dict['id'] = escape(sub_form.data['a_id'])
classification_dict['family'] = escape(sub_form.data['a_family'])
if classification_dict not in doc['classification']:
doc['classification'].append(classification_dict)
#Links to actors and reports
doc['related_actor'] = []
for sub_form in form.ttp_actors.entries:
r_dict = {}
data = escape(sub_form.data.data)
if data == "_NONE_":
continue
data_array = data.split(":::")
r_dict['id'] = escape(data_array[0])
r_dict['name'] = escape(data_array[1])
#this is gonna be a nightmare to maintain,
# but it make sense to have this for searches
if r_dict not in doc['related_actor']:
doc['related_actor'].append(r_dict)
doc['related_report'] = []
for sub_form in form.ttp_reports.entries:
r_dict = {}
data = escape(sub_form.data.data)
if data == "_NONE_":
continue
data_array = data.split(":::")
r_dict['id'] = escape(data_array[0])
r_dict['name'] = escape(data_array[1])
#this is gonna be a nightmare to maintain,
# but it make sense to have this for searches
if r_dict not in doc['related_report']:
doc['related_report'].append(r_dict)
doc['related_ttp'] = []
for sub_form in form.ttp_ttps.entries:
r_dict = {}
data = escape(sub_form.data.data)
if data == "_NONE_":
continue
data_array = data.split(":::")
r_dict['id'] = escape(data_array[0])
r_dict['name'] = escape(data_array[1])
#this is gonna be a nightmare to maintain,
# but it make sense to have this for searches
if r_dict not in doc['related_ttp']:
doc['related_ttp'].append(r_dict)
'''
Edit Tracking
'''
doc['editor'] = get_editor_list(es=get_es(),
index=ES_PREFIX + "threat_ttps",
doc_type="ttp",
item_id=ttp_id,
user_id=session.get('id', None))
#print_tpx(doc)
#index the doc
log.info(logging_prefix + "Start Indexing of {}".format(ttp_id))
response = get_es().index(ES_PREFIX + "threat_ttps", "ttp", doc, ttp_id)
log.info(logging_prefix + "Done Indexing of {}".format(ttp_id))
return response, doc
def es_to_tpx(ttp_id):
'''
Build the TPX file from the data stored in Elasticsearch
'''
element_observables = {}
results = get_es().get(ES_PREFIX + "threat_ttps",
doc_type="ttp",
id=ttp_id)
ttp_data = results['_source']
tpx = {}
tpx["schema_version_s"] = "2.2.0"
tpx["provider_s"] = "LookingGlass"
tpx["list_name_s"] = "Threat Actor"
tpx["created_t"] = ttp_data['created_milli']
tpx["created_s"] = ttp_data['created_s']
tpx["last_updated_t"] = ttp_data['last_updated_milli']
tpx["last_updated_s"] = ttp_data['last_updated_s']
tpx["score_i"] = 95
tpx["source_observable_s"] = "Cyveillance Threat Actor"
tpx["source_description_s"] = "This feed provides threat actor or threat actor group profiles and characterizations created by the LookingGlass Cyber Threat Intelligence Group"
tpx["observable_dictionary_c_array"] = []
observable_dict = {}
observable_dict["ttp_uuid_s"] = ttp_id
observable_dict["observable_id_s"] = ttp_data['name']
observable_dict["description_s"] = ttp_data['description']
observable_dict["criticality_i"] = ttp_data['criticality']
observable_dict["classification_c_array"] = []
class_dict = {}
class_dict["score_i"] = 70
class_dict["classification_id_s"] = "Intel"
class_dict["classification_family_s"] = "TTP"
observable_dict["classification_c_array"].append(class_dict)
for i in ttp_data['classification']:
class_dict = {}
class_dict["score_i"] = i["score"]
class_dict["classification_id_s"] = i["id"]
class_dict["classification_family_s"] = i["family"]
if class_dict not in observable_dict["classification_c_array"]:
observable_dict["classification_c_array"].append(class_dict)
observable_dict["related_ttps_c_array"] = []
for i in ttp_data['related_ttp']:
if i['name']:
observable_dict["related_ttps_c_array"].append({
"name_s": i['name'],
"uuid_s": i['id']
})
observable_dict["related_actors_c_array"] = []
for i in ttp_data['related_actor']:
if i['name']:
observable_dict["related_actors_c_array"].append({
"name_s":
i['name'],
"uuid_s":
i['id']
})
observable_dict["related_reports_c_array"] = []
for i in ttp_data['related_report']:
if i['name']:
observable_dict["related_reports_c_array"].append({
"name_s":
i['name'],
"uuid_s":
i['id']
})
'''
Related elements
'''
relate_element_name_map = {
"FQDN": "subject_fqdn_s",
"IPv4": "subject_ipv4_s",
"TTP": "subject_ttp_s",
"CommAddr": "subject_address_s"
}
tpx["observable_dictionary_c_array"].append(observable_dict)
return tpx
def form_to_es(form, report_id):
logging_prefix = logger_prefix + "form_to_es() - "
log.info(logging_prefix + "Converting Form to ES for {}".format(report_id))
doc = {}
created_t = int(time.mktime(
form.report_occurred_at.data.timetuple())) * 1000
created_s = form.report_occurred_at.data.strftime("%Y-%m-%dT%H:%M:%S")
now_t = int(time.time()) * 1000
now_s = datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
doc["created_milli"] = created_t
doc["created_s"] = created_s
doc["last_updated_milli"] = now_t
doc["last_updated_s"] = now_s
doc["name"] = escape(form.report_name.data)
doc["identifier"] = escape(form.report_id.data)
doc["description"] = escape(form.report_description.data)
doc["criticality"] = escape(int(form.report_criticality.data))
doc['classification'] = []
for sub_form in form.report_class.entries:
classification_dict = {}
classification_dict['score'] = int(
get_score(sub_form.data['a_family'], sub_form.data['a_id']))
classification_dict['id'] = escape(sub_form.data['a_id'])
classification_dict['family'] = escape(sub_form.data['a_family'])
if classification_dict not in doc['classification']:
doc['classification'].append(classification_dict)
doc["tlp"] = int(escape(form.report_tlp.data))
doc['section'] = []
idx = 1
for sub_form in form.report_sections.entries:
section_dict = {}
section_dict['order'] = idx
section_dict['title'] = escape(sub_form.title.data)
section_dict['tlp'] = int(escape(sub_form.tlp.data))
section_dict['content'] = escape(sub_form.text.data)
idx += 1
doc['section'].append(section_dict)
doc["source_reliability"] = escape(form.report_source_reliability.data)
doc["info_reliability"] = int(escape(form.report_info_reliability.data))
doc['source'] = []
for sub_form in form.report_sources.entries:
source = sub_form.source.data
if source not in doc['source']:
doc['source'].append(source)
'''
Related elements
'''
my_id = report_id
#Links to actors and reports
doc['related_actor'] = []
for sub_form in form.report_actors.entries:
r_dict = {}
data = escape(sub_form.data.data)
if data == "_NONE_":
continue
data_array = data.split(":::")
r_dict['id'] = escape(data_array[0])
#if the related element's id = my id, dont add to es, this should only apply to current type
if data_array[0] == my_id:
flash(
"A related actor matched this actor, this entry has been discarded",
"warning")
continue
r_dict['name'] = escape(data_array[1])
#this is gonna be a nightmare to maintain,
# but it make sense to have this for searches
r_dict['elements'] = []
for sub_sub_form in sub_form.related_elements.entries:
if sub_sub_form.element.data:
if sub_sub_form.element_value.data not in r_dict['elements']:
r_dict['elements'].append(sub_sub_form.element_value.data)
if r_dict not in doc['related_actor']:
doc['related_actor'].append(r_dict)
doc['related_report'] = []
for sub_form in form.report_reports.entries:
r_dict = {}
data = escape(sub_form.data.data)
if data == "_NONE_":
continue
data_array = data.split(":::")
r_dict['id'] = escape(data_array[0])
#if the related element's id = my id, dont add to es, this should only apply to current type
if data_array[0] == my_id:
flash(
"A related report matched this report, this entry has been discarded",
"warning")
continue
r_dict['name'] = escape(data_array[1])
#this is gonna be a nightmare to maintain,
# but it make sense to have this for searches
r_dict['elements'] = []
for sub_sub_form in sub_form.related_elements.entries:
if sub_sub_form.element.data:
if sub_sub_form.element_value.data not in r_dict['elements']:
r_dict['elements'].append(sub_sub_form.element_value.data)
if r_dict not in doc['related_report']:
doc['related_report'].append(r_dict)
doc['related_ttp'] = []
for sub_form in form.report_ttps.entries:
r_dict = {}
data = escape(sub_form.data.data)
if data == "_NONE_":
continue
data_array = data.split(":::")
r_dict['id'] = escape(data_array[0])
#if the related element's id = my id, dont add to es, this should only apply to current type
if data_array[0] == my_id:
flash(
"A related TTP matched this TTP, this entry has been discarded",
"warning")
continue
r_dict['name'] = escape(data_array[1])
#this is gonna be a nightmare to maintain,
# but it make sense to have this for searches
r_dict['elements'] = []
for sub_sub_form in sub_form.related_elements.entries:
if sub_sub_form.element.data:
if sub_sub_form.element_value.data not in r_dict['elements']:
r_dict['elements'].append(sub_sub_form.element_value.data)
if r_dict not in doc['related_ttp']:
doc['related_ttp'].append(r_dict)
#store these to make life easier
doc['related_element_choices'] = []
rec = json.loads(form.related_element_choices.data)
for k, v in rec.items():
#dont include the n/a choice, as this will be first and handled manually
if v != "_NONE_":
#this will create a tuple ("display text", "value string")
doc['related_element_choices'].append({
"display_text": k,
"value": v
})
'''
Edit Tracking
'''
doc['editor'] = get_editor_list(es=get_es(),
index=ES_PREFIX + "threat_reports",
doc_type="report",
item_id=report_id,
user_id=session.get('id', None))
#index the doc
log.info(logging_prefix + "Start Indexing of {}".format(report_id))
response = get_es().index(ES_PREFIX + "threat_reports", "report", doc,
report_id)
log.info(logging_prefix + "Done Indexing of {}".format(report_id))
return response, doc
def choices():
logging_prefix = logger_prefix + "choices() - "
log.info(logging_prefix + "Starting")
simple_choices = None
try:
body = {
"query" : {
"match_all" : {}
},
"size" : 1000
}
results = get_es().search(ES_PREFIX + 'threat_actor_simple', 'data', body)
parsed_results = []
for r in results['hits']['hits']:
d = {}
d['type'] = r['_source']['type']
d['value'] = r['_source']['value']
d['id'] = r['_id']
#determine how many actor profiles use this value
query_string = None
if d['type'] == "classification":
query_string = "type:\"" + escape(d['value']) + "\""
elif d['type'] == "communication":
query_string = "communication_address.type:\"" + escape(d['value']) + "\""
elif d['type'] == "country":
query_string = "country_affiliation:\"" + escape(d['value']) + "\" origin:\"" + escape(d['value']) + "\""
if query_string:
body = {
"query" : {
"query_string" : {
"query" : query_string
}
},
"size" : 0
}
count = get_es().search(ES_PREFIX + 'threat_actors', 'actor', body)
d['count'] = count['hits']['total']
d['q'] = quote_plus(query_string)
else:
d['count'] = "-"
d['q'] = ""
parsed_results.append(d)
simple_choices = multikeysort(parsed_results, ['type', 'value'])
except Exception as e:
error = "There was an error completing your request. Details: {}".format(e)
flash(error,'danger')
log.exception(logging_prefix + error)
return render_template("admin_choices.html",
page_title="Admin",
simple_choices = simple_choices
)
def es_to_tpx(actor_id):
'''
Build the TPX file from the data stored in Elasticsearch
'''
element_observables = {}
results = get_es().get(ES_PREFIX + "threat_actors", doc_type="actor", id=actor_id)
actor_data = results['_source']
tpx = {}
tpx["schema_version_s"] = "2.2.0"
tpx["provider_s"] = "LookingGlass"
tpx["list_name_s"] = "Threat Actor"
tpx["created_t"] = actor_data['created_milli']
tpx["created_s"] = actor_data['created_s']
tpx["last_updated_t"] = actor_data['last_updated_milli']
tpx["last_updated_s"] = actor_data['last_updated_s']
tpx["score_i"] = 95
tpx["source_observable_s"] = "Cyveillance Threat Actor"
tpx["source_description_s"] = "This feed provides threat actor or threat actor group profiles and characterizations created by the LookingGlass Cyber Threat Intelligence Group"
tpx["observable_dictionary_c_array"] = []
observable_dict = {}
observable_dict["observable_id_s"] = actor_data['name']
observable_dict["actor_uuid_s"] = actor_id
observable_dict["criticality_i"] = actor_data['criticality']
observable_dict["classification_c_array"] = []
for i in actor_data['classification']:
class_dict = {}
class_dict["score_i"] = i["score"]
#class_dict["score_i"] = get_score(i["family"], i["id"])
class_dict["classification_id_s"] = i["id"]
class_dict["classification_family_s"] = i["family"]
observable_dict["classification_c_array"].append(class_dict)
observable_dict["description_s"] = actor_data['description']
observable_dict["attribute_c_map"] = {}
observable_dict["attribute_c_map"]["last_seen_t"] = actor_data['last_updated_milli']
observable_dict["attribute_c_map"]["last_seen_s"] = actor_data['last_updated_s']
observable_dict["attribute_c_map"]["tlp_i"] = actor_data['tlp']
observable_dict["attribute_c_map"]["motivation_s_array"] = []
for i in actor_data["motivation"]:
observable_dict["attribute_c_map"]["motivation_s_array"].append(i)
observable_dict["attribute_c_map"]["aliases_s_array"] = []
for i in actor_data["alias"]:
if i: #make sure its not blank
observable_dict["attribute_c_map"]["aliases_s_array"].append(i)
observable_dict["attribute_c_map"]["communications_c_map"] = {}
if "communication_address" in actor_data:
for i in actor_data["communication_address"]:
#build the key, this is a bit tricky if the user enters special characters,
#or has multiple spaces (possible trailing space, should address this when the
# data is submitted) the key could get messed up.
# Example: Jabber -> jabber_s_array
key = i['type'].lower().replace(" ","_") +"_s_array"
if key not in observable_dict["attribute_c_map"]["communications_c_map"]:
observable_dict["attribute_c_map"]["communications_c_map"][key] = []
if i['value']:
observable_dict["attribute_c_map"]["communications_c_map"][key].append(i['value'])
observable_dict["attribute_c_map"]["financial_accounts_c_map"] = {}
if "financial_account" in actor_data:
for i in actor_data["financial_account"]:
#build the key, this is a bit tricky if the user enters special characters,
#or has multiple spaces (possible trailing space, should address this when the
# data is submitted) the key could get messed up.
# Example: Jabber -> jabber_s_array
key = i['type'].lower().replace(" ","_") +"_s_array"
if key not in observable_dict["attribute_c_map"]["financial_accounts_c_map"]:
observable_dict["attribute_c_map"]["financial_accounts_c_map"][key] = []
if i['value']:
observable_dict["attribute_c_map"]["financial_accounts_c_map"][key].append(i['value'])
observable_dict["attribute_c_map"]["frequent_locations_s_array"] = []
for i in actor_data["frequented_location"]:
if i:
observable_dict["attribute_c_map"]["frequent_locations_s_array"].append(i)
observable_dict["attribute_c_map"]["affiliations_s_array"] = []
for i in actor_data["country_affiliation"]:
observable_dict["attribute_c_map"]["affiliations_s_array"].append(i)
observable_dict["attribute_c_map"]["known_targets_s_array"] = []
for i in actor_data["known_target"]:
observable_dict["attribute_c_map"]["known_targets_s_array"].append(i)
observable_dict["attribute_c_map"]["suspected_point_of_origin_s"] = actor_data['origin']
observable_dict["attribute_c_map"]["infrastructure_ipv4_s_array"] = []
for i in actor_data["infrastructure"]["ipv4"]:
if i:
observable_dict["attribute_c_map"]["infrastructure_ipv4_s_array"].append(i)
observable_dict["attribute_c_map"]["infrastructure_fqdn_s_array"] = []
for i in actor_data["infrastructure"]["fqdn"]:
if i:
observable_dict["attribute_c_map"]["infrastructure_fqdn_s_array"].append(i)
observable_dict["attribute_c_map"]["infrastructure_action_s"] = actor_data["infrastructure"]["action"]
observable_dict["attribute_c_map"]["infrastructure_operation_s"] = actor_data["infrastructure"]["operation"]
observable_dict["attribute_c_map"]["infrastructure_status_s"] = actor_data["infrastructure"]["status"]
observable_dict["attribute_c_map"]["infrastructure_type_s_array"] = []
for i in actor_data["infrastructure"]["type"]:
observable_dict["attribute_c_map"]["infrastructure_type_s_array"].append(i)
observable_dict["attribute_c_map"]["detection_c_map"] = {}
for i in actor_data["detection_rule"]:
#build the key, this is a bit tricky if the user enters special characters,
#or has multiple spaces (possible trailing space, should address this when the
# data is submitted) the key could get messed up.
# Example: Snort -> snort_rule_s_map
key = i['type'].lower().replace(" ","_") +"_s_map"
if not i['value']:
continue
if key not in observable_dict["attribute_c_map"]["detection_c_map"]:
observable_dict["attribute_c_map"]["detection_c_map"][key] = {}
observable_dict["attribute_c_map"]["detection_c_map"][key][i['id']] = i['value']
'''
Related elements
'''
relate_element_name_map = {
"FQDN" : "subject_fqdn_s",
"IPv4" : "subject_ipv4_s",
"TTP" : "subject_ttp_s",
"CommAddr" : "subject_address_s"
}
for i in actor_data['related_element_choices']:
v = i['value']
print(v)
v_array = v.split(":::")
if len(v_array) == 2:
field_name = relate_element_name_map[v_array[0]]
field_type = "_DEFAULT_"
field_data = v_array[1]
elif len(v_array) == 3:
field_name = relate_element_name_map[v_array[1]]
field_type = v_array[0]
field_data = v_array[2]
else:
raise Exception("Unknown Related Element: '{}'".format(v))
field_observable = actor_data['name']
if field_name not in element_observables:
element_observables[field_name] = {}
if field_type not in element_observables[field_name]:
element_observables[field_name][field_type] = {}
if field_data not in element_observables[field_name][field_type]:
element_observables[field_name][field_type][field_data] = []
if field_observable not in element_observables[field_name][field_type][field_data]:
element_observables[field_name][field_type][field_data].append(field_observable)
observable_dict["attribute_c_map"]["related_ttps_c_array"] = []
for i in actor_data['related_ttp']:
if i['name']:
observable_dict["attribute_c_map"]["related_ttps_c_array"].append({ "name_s" : i['name'], "uuid_s" : i['id'] })
field_observable = i['name']
for j in i['elements']:
element_array = j.split(":::")
if len(element_array) == 3:
field_type = element_array[0]
field_name = relate_element_name_map[element_array[1]]
field_data = element_array[2]
elif len(element_array) == 2:
field_type = "_DEFAULT_"
field_name = relate_element_name_map[element_array[0]]
field_data = element_array[1]
else:
raise Exception("Invalid element '{}'".format(j))
if field_name not in element_observables:
element_observables[field_name] = {}
if field_type not in element_observables[field_name]:
element_observables[field_name][field_type] = {}
if field_data not in element_observables[field_name][field_type]:
element_observables[field_name][field_type][field_data] = []
if field_observable not in element_observables[field_name][field_type][field_data]:
element_observables[field_name][field_type][field_data].append(field_observable)
observable_dict["attribute_c_map"]["related_actors_c_array"] = []
for i in actor_data['related_actor']:
if i['name']:
observable_dict["attribute_c_map"]["related_actors_c_array"].append({ "name_s" : i['name'], "uuid_s" : i['id'] })
field_observable = i['name']
for j in i['elements']:
element_array = j.split(":::")
if len(element_array) == 3:
field_type = element_array[0]
field_name = relate_element_name_map[element_array[1]]
field_data = element_array[2]
elif len(element_array) == 2:
field_type = "_DEFAULT_"
field_name = relate_element_name_map[element_array[0]]
field_data = element_array[1]
else:
raise Exception("Invalid element '{}'".format(j))
if field_name not in element_observables:
element_observables[field_name] = {}
if field_type not in element_observables[field_name]:
element_observables[field_name][field_type] = {}
if field_data not in element_observables[field_name][field_type]:
element_observables[field_name][field_type][field_data] = []
if field_observable not in element_observables[field_name][field_type][field_data]:
element_observables[field_name][field_type][field_data].append(field_observable)
observable_dict["attribute_c_map"]["related_reports_c_array"] = []
for i in actor_data['related_report']:
if i['name']:
observable_dict["attribute_c_map"]["related_reports_c_array"].append({ "name_s" : i['name'], "uuid_s" : i['id'] })
field_observable = i['name']
for j in i['elements']:
element_array = j.split(":::")
if len(element_array) == 3:
field_type = element_array[0]
field_name = relate_element_name_map[element_array[1]]
field_data = element_array[2]
elif len(element_array) == 2:
field_type = "_DEFAULT_"
field_name = relate_element_name_map[element_array[0]]
field_data = element_array[1]
else:
raise Exception("Invalid element '{}'".format(j))
if field_name not in element_observables:
element_observables[field_name] = {}
if field_type not in element_observables[field_name]:
element_observables[field_name][field_type] = {}
if field_data not in element_observables[field_name][field_type]:
element_observables[field_name][field_type][field_data] = []
if field_observable not in element_observables[field_name][field_type][field_data]:
element_observables[field_name][field_type][field_data].append(field_observable)
tpx["observable_dictionary_c_array"].append(observable_dict)
'''
Element Observables - the reverse linkage
'''
tpx["element_observable_c_array"] = []
for field,val in element_observables.items():
for sub_type, val1 in val.items():
for data, val2 in val1.items():
e_dict = {}
e_dict[field] = data
if sub_type != "_DEFAULT_":
e_dict['type_s'] = sub_type
e_dict["threat_observable_c_map"] = {}
for observable_id in val2:
e_dict["threat_observable_c_map"][observable_id] = {}
e_dict["threat_observable_c_map"][observable_id]["occurred_at_t"] = actor_data['created_milli']
e_dict["threat_observable_c_map"][observable_id]["occurred_at_s"] = actor_data['created_s']
e_dict["threat_observable_c_map"][observable_id]["last_seen_t"] = actor_data['last_updated_milli']
e_dict["threat_observable_c_map"][observable_id]["last_seen_s"] = actor_data['last_updated_s']
tpx["element_observable_c_array"].append(e_dict)
return tpx
def es_to_form(report_id):
form = forms.reportForm()
# get the values from ES
results = get_es().get(ES_PREFIX + "threat_reports", doc_type="report", id=report_id)
# store certain fields from ES, so this form can be used in an update
form.doc_index.data = results["_index"]
form.doc_type.data = results["_type"]
report_data = results["_source"]
form.report_name.data = report_data["name"]
form.report_id.data = report_data["identifier"]
form.report_occurred_at.data = datetime.strptime(report_data["created_s"], "%Y-%m-%dT%H:%M:%S")
form.report_description.data = report_data["description"]
form.report_criticality.data = report_data["criticality"]
idx = 0
for entry in range(len(form.report_class.entries)):
form.report_class.pop_entry()
for i in multikeysort(report_data["classification"], ["family", "id"]):
report_class_form = forms.TPXClassificationForm()
report_class_form.a_family = i["family"]
report_class_form.a_id = i["id"]
form.report_class.append_entry(report_class_form)
# set the options since this select is dynamic
form.report_class[idx].a_id.choices = fetch_child_data("tpx_classification", i["family"])
idx += 1
form.report_tlp.data = int(report_data["tlp"])
for entry in range(len(form.report_sections.entries)):
form.report_sections.pop_entry()
for i in multikeysort(report_data["section"], ["order"]):
report_section_form = forms.ReportSectionsForm()
report_section_form.title = i["title"]
report_section_form.tlp = i["tlp"]
report_section_form.text = i["content"]
form.report_sections.append_entry(report_section_form)
form.report_source_reliability.data = report_data["source_reliability"]
form.report_info_reliability.data = report_data["info_reliability"]
for entry in range(len(form.report_sources.entries)):
form.report_sources.pop_entry()
for i in sorted(report_data["source"]):
report_source_form = forms.ReportSourcesForm()
report_source_form.source = i
form.report_sources.append_entry(report_source_form)
"""
Related element
"""
for entry in range(len(form.report_actors.entries)):
form.report_actors.pop_entry()
if report_data["related_actor"]:
idx = 0
for i in multikeysort(report_data["related_actor"], ["name", "id"]):
sub_form = forms.RelatedActorsForm()
sub_form.data = i["id"] + ":::" + i["name"]
has_related_elements = False
idx2 = 0
for entry in range(len(sub_form.related_elements.entries)):
sub_form.related_elements.pop_entry()
for j in multikeysort(report_data["related_element_choices"], ["display_text"]):
sub_sub_form = forms.ElementObservablesFrom()
sub_form.related_elements.append_entry(sub_sub_form)
is_related = j["value"] in i["elements"]
sub_form.related_elements[idx2].element = is_related
sub_form.related_elements[idx2].element_value = j["value"]
sub_form.related_elements[idx2].element_text = j["display_text"]
if is_related:
has_related_elements = True
idx2 += 1
form.report_actors.append_entry(sub_form)
form.report_actors[idx].has_related_elements.data = has_related_elements
idx += 1
else:
sub_form = forms.RelatedActorsForm()
sub_form.data = "_NONE_"
form.report_actors.append_entry(sub_form)
for entry in range(len(form.report_reports.entries)):
form.report_reports.pop_entry()
if report_data["related_report"]:
idx = 0
for i in multikeysort(report_data["related_report"], ["name", "id"]):
sub_form = forms.RelatedReportsForm()
sub_form.data = i["id"] + ":::" + i["name"]
has_related_elements = False
idx2 = 0
for entry in range(len(sub_form.related_elements.entries)):
sub_form.related_elements.pop_entry()
for j in multikeysort(report_data["related_element_choices"], ["display_text"]):
sub_sub_form = forms.ElementObservablesFrom()
sub_form.related_elements.append_entry(sub_sub_form)
is_related = j["value"] in i["elements"]
sub_form.related_elements[idx2].element = is_related
sub_form.related_elements[idx2].element_value = j["value"]
sub_form.related_elements[idx2].element_text = j["display_text"]
if is_related:
has_related_elements = True
idx2 += 1
form.report_reports.append_entry(sub_form)
form.report_reports[idx].has_related_elements.data = has_related_elements
idx += 1
else:
sub_form = forms.RelatedReportsForm()
sub_form.data = "_NONE_"
form.report_reports.append_entry(sub_form)
for entry in range(len(form.report_ttps.entries)):
form.report_ttps.pop_entry()
if report_data["related_ttp"]:
idx = 0
for i in multikeysort(report_data["related_ttp"], ["name", "id"]):
sub_form = forms.RelatedTTPsForm()
sub_form.data = i["id"] + ":::" + i["name"]
has_related_elements = False
idx2 = 0
for entry in range(len(sub_form.related_elements.entries)):
sub_form.related_elements.pop_entry()
for j in multikeysort(report_data["related_element_choices"], ["display_text"]):
sub_sub_form = forms.ElementObservablesFrom()
sub_form.related_elements.append_entry(sub_sub_form)
is_related = j["value"] in i["elements"]
sub_form.related_elements[idx2].element = is_related
sub_form.related_elements[idx2].element_value = j["value"]
sub_form.related_elements[idx2].element_text = j["display_text"]
if is_related:
has_related_elements = True
idx2 += 1
form.report_ttps.append_entry(sub_form)
form.report_ttps[idx].has_related_elements.data = has_related_elements
idx += 1
else:
sub_form = forms.RelatedTTPsForm()
sub_form.data = "_NONE_"
form.report_ttps.append_entry(sub_form)
# convert editor dictionary of ids and times to names and times
editors = get_editor_names(get_mysql(), report_data["editor"])
return form, editors
def es_to_form(actor_id):
form = forms.actorForm()
editors = []
#get the values from ES
results = get_es().get(ES_PREFIX + "threat_actors", doc_type="actor", id=actor_id)
#store certain fields from ES, so this form can be used in an update
form.doc_index.data = results['_index']
form.doc_type.data = results['_type']
actor_data = results['_source']
form.actor_name.data = actor_data['name']
form.actor_occurred_at.data = datetime.strptime(actor_data['created_s'],"%Y-%m-%dT%H:%M:%S")
form.actor_description.data = actor_data['description']
form.actor_criticality.data = actor_data['criticality']
#blank this field, and then add data
idx = 0
for entry in range(len(form.actor_class.entries)): form.actor_class.pop_entry()
for i in multikeysort(actor_data['classification'], ['family', 'id']):
actor_class_form = forms.TPXClassificationForm()
actor_class_form.a_family = i['family']
actor_class_form.a_id = i['id']
form.actor_class.append_entry(actor_class_form)
#set the options since this select is dynamic
form.actor_class[idx].a_id.choices = fetch_child_data('tpx_classification',i['family'])
idx += 1
form.actor_tlp.data = str(actor_data['tlp'])
for entry in range(len(form.actor_type.entries)): form.actor_type.pop_entry()
for i in sorted(actor_data['type']):
actor_type_form = forms.TypeForm()
actor_type_form.a_type = i
form.actor_type.append_entry(actor_type_form)
for entry in range(len(form.actor_motivations.entries)): form.actor_motivations.pop_entry()
for i in sorted(actor_data['motivation']):
actor_motivation_form = forms.MotivationForm()
actor_motivation_form.motivation = i
actor_motivation_form.motivation_other = ""
form.actor_motivations.append_entry(actor_motivation_form)
for entry in range(len(form.actor_aliases.entries)): form.actor_aliases.pop_entry()
for i in sorted(actor_data['alias']):
actor_alias_form = forms.AliasForm()
actor_alias_form.alias = i
form.actor_aliases.append_entry(actor_alias_form)
for entry in range(len(form.actor_comms.entries)): form.actor_comms.pop_entry()
for i in multikeysort(actor_data['communication_address'], ['type', 'value']):
actor_comms_form = forms.CommunicationsForm()
actor_comms_form.a_type = i['type']
actor_comms_form.a_type_other = ""
actor_comms_form.address = i['value']
form.actor_comms.append_entry(actor_comms_form)
for entry in range(len(form.actor_financials.entries)): form.actor_financials.pop_entry()
for i in multikeysort(actor_data['financial_account'], ['type', 'value']):
actor_fin_form = forms.FinancialsForm()
actor_fin_form.f_type = i['type']
actor_fin_form.f_type_other = ""
actor_fin_form.account = i['value']
form.actor_financials.append_entry(actor_fin_form)
for entry in range(len(form.actor_locations.entries)): form.actor_locations.pop_entry()
for i in sorted(actor_data['frequented_location']):
actor_locations_form = forms.LocationsForm()
actor_locations_form.location = i
form.actor_locations.append_entry(actor_locations_form)
for entry in range(len(form.actor_affliations.entries)): form.actor_affliations.pop_entry()
for i in sorted(actor_data['country_affiliation']):
actor_affil_form = forms.AffiliationsForm()
actor_affil_form.affiliation = i
actor_affil_form.affiliation_other = ""
form.actor_affliations.append_entry(actor_affil_form)
for entry in range(len(form.actor_known_targets.entries)): form.actor_known_targets.pop_entry()
for i in sorted(actor_data['known_target']):
actor_known_target_form = forms.KnownTargetsForm()
actor_known_target_form.target = i
actor_known_target_form.target_other = ""
form.actor_known_targets.append_entry(actor_known_target_form)
for entry in range(len(form.actor_origin.entries)): form.actor_origin.pop_entry()
#theres only one of these, at least there should be
actor_affil_form = forms.AffiliationsForm()
actor_affil_form.affiliation = actor_data['origin']
actor_affil_form.affiliation_other = ""
form.actor_origin.append_entry(actor_affil_form)
for entry in range(len(form.actor_infra_ipv4.entries)): form.actor_infra_ipv4.pop_entry()
for i in sorted(actor_data['infrastructure']['ipv4']):
actor_infra_ipv4_form = forms.InfrastructureIPv4Form()
actor_infra_ipv4_form.ipv4 = i
form.actor_infra_ipv4.append_entry(actor_infra_ipv4_form)
for entry in range(len(form.actor_infra_fqdn.entries)): form.actor_infra_fqdn.pop_entry()
for i in sorted(actor_data['infrastructure']['fqdn']):
actor_infra_fqdn_form = forms.InfrastructureIPv4Form()
actor_infra_fqdn_form.fqdn = i
form.actor_infra_fqdn.append_entry(actor_infra_fqdn_form)
form.actor_infra_action.data = actor_data['infrastructure']['action']
form.actor_infra_operation.data = actor_data['infrastructure']['operation']
form.actor_infra_status.data = actor_data['infrastructure']['status']
for entry in range(len(form.actor_infra_types.entries)): form.actor_infra_types.pop_entry()
for i in sorted(actor_data['infrastructure']['type']):
actor_infra_type_form = forms.InfrastructureTypesForm()
actor_infra_type_form.infra_type = i
form.actor_infra_types.append_entry(actor_infra_type_form)
for entry in range(len(form.actor_detections.entries)): form.actor_detections.pop_entry()
for i in multikeysort(actor_data['detection_rule'], ["type", "id"]):
actor_detection_form = forms.DetectionsForm()
actor_detection_form.d_type = i['type']
actor_detection_form.rule_id = i['id']
actor_detection_form.rule = i['value']
form.actor_detections.append_entry(actor_detection_form)
'''
Related element
'''
for entry in range(len(form.actor_actors.entries)): form.actor_actors.pop_entry()
if actor_data['related_actor']:
idx = 0
for i in multikeysort(actor_data['related_actor'], ['name', 'id']):
sub_form = forms.RelatedActorsForm()
sub_form.data = i['id'] + ":::" + i['name']
has_related_elements = False
idx2=0
for entry in range(len(sub_form.related_elements.entries)): sub_form.related_elements.pop_entry()
for j in multikeysort(actor_data['related_element_choices'],['display_text']):
sub_sub_form = forms.ElementObservablesFrom()
sub_form.related_elements.append_entry(sub_sub_form)
is_related = (j['value'] in i['elements'])
sub_form.related_elements[idx2].element = is_related
sub_form.related_elements[idx2].element_value = j['value']
sub_form.related_elements[idx2].element_text = j['display_text']
if is_related:
has_related_elements = True
idx2 +=1
form.actor_actors.append_entry(sub_form)
form.actor_actors[idx].has_related_elements.data = has_related_elements
idx+=1
else:
sub_form = forms.RelatedActorsForm()
sub_form.data = "_NONE_"
form.actor_actors.append_entry(sub_form)
for entry in range(len(form.actor_reports.entries)): form.actor_reports.pop_entry()
if actor_data['related_report']:
idx = 0
for i in multikeysort(actor_data['related_report'], ['name', 'id']):
sub_form = forms.RelatedReportsForm()
sub_form.data = i['id'] + ":::" + i['name']
has_related_elements = False
idx2=0
for entry in range(len(sub_form.related_elements.entries)): sub_form.related_elements.pop_entry()
for j in multikeysort(actor_data['related_element_choices'],['display_text']):
sub_sub_form = forms.ElementObservablesFrom()
sub_form.related_elements.append_entry(sub_sub_form)
is_related = (j['value'] in i['elements'])
sub_form.related_elements[idx2].element = is_related
sub_form.related_elements[idx2].element_value = j['value']
sub_form.related_elements[idx2].element_text = j['display_text']
if is_related:
has_related_elements = True
idx2 +=1
form.actor_reports.append_entry(sub_form)
form.actor_reports[idx].has_related_elements.data = has_related_elements
idx+=1
else:
sub_form = forms.RelatedReportsForm()
sub_form.data = "_NONE_"
form.actor_reports.append_entry(sub_form)
for entry in range(len(form.actor_ttps.entries)): form.actor_ttps.pop_entry()
if actor_data['related_ttp']:
idx = 0
for i in multikeysort(actor_data['related_ttp'], ['name', 'id']):
sub_form = forms.RelatedTTPsForm()
sub_form.data = i['id'] + ":::" + i['name']
has_related_elements = False
idx2=0
for entry in range(len(sub_form.related_elements.entries)): sub_form.related_elements.pop_entry()
for j in multikeysort(actor_data['related_element_choices'],['display_text']):
sub_sub_form = forms.ElementObservablesFrom()
sub_form.related_elements.append_entry(sub_sub_form)
is_related = (j['value'] in i['elements'])
sub_form.related_elements[idx2].element = is_related
sub_form.related_elements[idx2].element_value = j['value']
sub_form.related_elements[idx2].element_text = j['display_text']
if is_related:
has_related_elements = True
idx2 +=1
form.actor_ttps.append_entry(sub_form)
form.actor_ttps[idx].has_related_elements.data = has_related_elements
idx+=1
else:
sub_form = forms.RelatedTTPsForm()
sub_form.data = "_NONE_"
form.actor_ttps.append_entry(sub_form)
#convert editor dictionary of ids and times to names and times
editors = get_editor_names(get_mysql(), actor_data['editor'])
return form, editors
def form_to_es(form, actor_id, update=False):
logging_prefix = logger_prefix + "form_to_es() - "
log.info(logging_prefix + "Converting Form to ES for {}".format(actor_id))
doc = {}
element_observables = {}
doc["approval_state"] = 0 # 0 - newly added
# 1 - scores have been approved and this actor is ready for SV
created_t = int(time.mktime(form.actor_occurred_at.data.timetuple())) * 1000
created_s = form.actor_occurred_at.data.strftime("%Y-%m-%dT%H:%M:%S")
now_t = int(time.time()) * 1000
now_s = datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
doc["created_milli"] = created_t
doc["created_s"] = created_s
doc["last_updated_milli"] = now_t
doc["last_updated_s"] = now_s
observable_dictionary = {}
actor_name = form.actor_name.data
doc['name'] = escape(actor_name)
doc['criticality'] = int(escape(form.actor_criticality.data))
doc['classification'] = []
for sub_form in form.actor_class.entries:
classification_dict = {}
classification_dict['score'] = int(get_score(sub_form.data['a_family'], sub_form.data['a_id']))
classification_dict['id'] = escape(sub_form.data['a_id'])
classification_dict['family'] = escape(sub_form.data['a_family'])
if classification_dict not in doc['classification']:
doc['classification'].append(classification_dict)
doc["description"] = escape(form.actor_description.data)
doc["tlp"] = int(form.actor_tlp.data)
'''
Types
'''
doc["type"] = [] # ??? should we change this to actor_type_s_array
for sub_form in form.actor_type.entries: # since the field is labeled Actor Types
actor_type = sub_form.data['a_type']
if actor_type not in doc["type"]:
doc["type"].append(escape(actor_type))
'''
Motivations
'''
doc['motivation'] = []
for sub_form in form.actor_motivations.entries:
motivation = sub_form.data['motivation']
if motivation == '_NEW_':
motivation = sub_form.data['motivation_other']
add_to_store('motivation', motivation, True)
if motivation not in doc['motivation']:
doc['motivation'].append(escape(motivation))
'''
Aliases
'''
doc['alias'] = []
for sub_form in form.actor_aliases.entries:
alias = sub_form.data['alias']
if alias not in doc['alias']:
doc['alias'].append(escape(alias))
'''
Communications
'''
doc['communication_address'] = []
for sub_form in form.actor_comms.entries:
address = sub_form.data['address']
comm_type = sub_form.data['a_type']
if comm_type == "_NEW_":
comm_type = sub_form.data['a_type_other']
add_to_store('communication', comm_type, True)
#since comm_type is used as a key in the tpx file, sanitize it first
#comm_type = comm_type.lower().replace(" "," ").replace(" ","_") + "_s_array"
data = {}
data['type'] = escape(comm_type)
data['value'] = escape(address)
if data not in doc['communication_address']:
doc['communication_address'].append(data)
'''
Financials
'''
doc["financial_account"] = []
for sub_form in form.actor_financials.entries:
account = sub_form.data['account']
fin_type = sub_form.data['f_type']
if fin_type == "_NEW_":
fin_type = sub_form.data['f_type_other']
add_to_store('financial', fin_type, True)
#since fin_type is used as a key in the tpx file, sanitize it first
#fin_type = fin_type.lower().replace(" "," ").replace(" ","_") + "_s_array"
data = {}
data['type'] = escape(fin_type)
data['value'] = escape(account)
if data not in doc["financial_account"]:
doc["financial_account"].append(data)
'''
Frequented Locations
'''
doc["frequented_location"] = []
for sub_form in form.actor_locations.entries:
location = sub_form.data['location']
if location not in doc["frequented_location"]:
doc["frequented_location"].append(escape(location))
'''
Country Affliations
'''
doc["country_affiliation"] = []
for sub_form in form.actor_affliations.entries:
affiliation = sub_form.data['affiliation']
if affiliation == '_NEW_':
affiliation = sub_form.data['affiliation_other']
add_to_store('country', affiliation, True)
if affiliation not in doc["country_affiliation"]:
doc["country_affiliation"].append(escape(affiliation))
'''
Known Targets
'''
doc["known_target"] = []
for sub_form in form.actor_known_targets.entries:
target = sub_form.data['target']
if target == '_NEW_':
target = sub_form.data['target_other']
add_to_store('known_target', target, True)
if target not in doc["known_target"]:
doc["known_target"].append(escape(target))
'''
Origin
'''
#this one is different, because i borrowed from the country_affliation sub form
affiliation = form.actor_origin.data[0]['affiliation']
if affiliation == '_NEW_':
affiliation = form.actor_origin.data[0]['affiliation_other']
add_to_store('country', affiliation, True)
doc["origin"] = escape(affiliation)
'''
Infrastructure IPv4s
'''
doc['infrastructure'] = {}
doc['infrastructure']['ipv4'] = []
for sub_form in form.actor_infra_ipv4.entries:
ip = sub_form.data['ipv4']
if ip not in doc['infrastructure']['ipv4']:
doc['infrastructure']['ipv4'].append(escape(ip))
'''
Infrastructure FQDNs
'''
doc['infrastructure']['fqdn'] = []
for sub_form in form.actor_infra_fqdn.entries:
fqdn = sub_form.data['fqdn']
if fqdn not in doc['infrastructure']['fqdn']:
doc['infrastructure']['fqdn'].append(escape(fqdn))
'''
Infrastructure Action, Operation, Status
'''
doc['infrastructure']['action'] = escape(form.actor_infra_action.data)
doc['infrastructure']['operation'] = escape(form.actor_infra_operation.data)
doc['infrastructure']['status'] = escape(form.actor_infra_status.data)
'''
Infrastructure Types
'''
doc['infrastructure']['type'] = []
for sub_form in form.actor_infra_types.entries:
_type = sub_form.data['infra_type']
if location not in doc['infrastructure']['type']:
doc['infrastructure']['type'].append(escape(_type))
'''
Detection Rules
'''
doc['detection_rule'] = []
for sub_form in form.actor_detections.entries:
rule = sub_form.data['rule']
rule_id = sub_form.data['rule_id']
rule_type = sub_form.data['d_type']
#since comm_type is used as a key in the tpx file, sanitize it first
#rule_type = rule_type.lower().replace(" "," ").replace(" ","_") + "_rule_s_array"
data = {}
data['type'] = escape(rule_type)
data['value'] = escape(rule)
data['id'] = escape(rule_id)
if data not in doc['detection_rule']:
doc['detection_rule'].append(data)
'''
Related elements
'''
my_id = actor_id
#Links to actors and reports
doc['related_actor'] = []
for sub_form in form.actor_actors.entries:
r_dict = {}
data = escape(sub_form.data.data)
if data == "_NONE_":
continue
data_array = data.split(":::")
r_dict['id'] = escape(data_array[0])
#if the related element's id = my id, dont add to es, this should only apply to current type
if data_array[0] == my_id:
flash("A related actor matched this actor, this entry has been discarded", "warning")
continue
r_dict['name'] = escape(data_array[1])
#this is gonna be a nightmare to maintain,
# but it make sense to have this for searches
r_dict['elements'] = []
for sub_sub_form in sub_form.related_elements.entries:
if sub_sub_form.element.data:
if sub_sub_form.element_value.data not in r_dict['elements']:
r_dict['elements'].append(sub_sub_form.element_value.data)
if r_dict not in doc['related_actor']:
doc['related_actor'].append(r_dict)
doc['related_report'] = []
for sub_form in form.actor_reports.entries:
r_dict = {}
data = escape(sub_form.data.data)
if data == "_NONE_":
continue
data_array = data.split(":::")
r_dict['id'] = escape(data_array[0])
#if the related element's id = my id, dont add to es, this should only apply to current type
if data_array[0] == my_id:
flash("A related report matched this report, this entry has been discarded", "warning")
continue
r_dict['name'] = escape(data_array[1])
#this is gonna be a nightmare to maintain,
# but it make sense to have this for searches
r_dict['elements'] = []
for sub_sub_form in sub_form.related_elements.entries:
if sub_sub_form.element.data:
if sub_sub_form.element_value.data not in r_dict['elements']:
r_dict['elements'].append(sub_sub_form.element_value.data)
if r_dict not in doc['related_report']:
doc['related_report'].append(r_dict)
doc['related_ttp'] = []
for sub_form in form.actor_ttps.entries:
r_dict = {}
data = escape(sub_form.data.data)
if data == "_NONE_":
continue
data_array = data.split(":::")
r_dict['id'] = escape(data_array[0])
#if the related element's id = my id, dont add to es, this should only apply to current type
if data_array[0] == my_id:
flash("A related TTP matched this TTP, this entry has been discarded", "warning")
continue
r_dict['name'] = escape(data_array[1])
#this is gonna be a nightmare to maintain,
# but it make sense to have this for searches
r_dict['elements'] = []
for sub_sub_form in sub_form.related_elements.entries:
if sub_sub_form.element.data:
if sub_sub_form.element_value.data not in r_dict['elements']:
r_dict['elements'].append(sub_sub_form.element_value.data)
if r_dict not in doc['related_ttp']:
doc['related_ttp'].append(r_dict)
#store these to make life easier
doc['related_element_choices'] = []
rec = json.loads(form.related_element_choices.data)
for k,v in rec.items():
#dont include the n/a choice, as this will be first and handled manually
if v != "_NONE_":
#this will create a tuple ("display text", "value string")
doc['related_element_choices'].append({"display_text" : k, "value" : v})
'''
Edit Tracking
'''
doc['editor'] = get_editor_list(
es=get_es(),
index=ES_PREFIX + "threat_actors",
doc_type="actor",
item_id=actor_id,
user_id=session.get('id',None)
)
#index the doc
#print_tpx(doc['related_ttp'])
#print_tpx(doc['related_element_choices'])
#response = None
log.info(logging_prefix + "Start Indexing of {}".format(actor_id))
response = get_es().index(ES_PREFIX + "threat_actors", "actor", doc, actor_id)
log.info(logging_prefix + "Done Indexing of {}".format(actor_id))
return response, doc
def export_all_the_data():
logging_prefix = logger_prefix + "export_all_the_data() - "
log.info(logging_prefix + "Exporting")
try:
dump = {}
dump['actors'] = []
dump['reports'] = []
dump['ttps'] = []
dump['choices'] = {}
query = {"query": {"match_all": {}}}
results = scan(get_es(),
query=query,
index=ES_PREFIX + "threat_actors",
doc_type="actor")
for i in results:
dump['actors'].append(i)
results = scan(get_es(),
query=query,
index=ES_PREFIX + "threat_reports",
doc_type="report")
for i in results:
dump['reports'].append(i)
results = scan(get_es(),
query=query,
index=ES_PREFIX + "threat_ttps",
doc_type="ttp")
for i in results:
dump['ttps'].append(i)
results = scan(get_es(),
query=query,
index=ES_PREFIX + "threat_actor_pc",
doc_type="parent")
dump['choices']['parents'] = []
for i in results:
dump['choices']['parents'].append(i)
results = scan(get_es(),
query=query,
index=ES_PREFIX + "threat_actor_pc",
doc_type="child")
dump['choices']['children'] = []
for i in results:
dump['choices']['children'].append(i)
results = scan(get_es(),
query=query,
index=ES_PREFIX + "threat_actor_simple",
doc_type="data")
dump['choices']['simple'] = []
for i in results:
dump['choices']['simple'].append(i)
# We need to modify the response, so the first thing we
# need to do is create a response out of the Dictionary
response = make_response(json.dumps(dump))
# This is the key: Set the right header for the response
# to be downloaded, instead of just printed on the browser
response.headers[
"Content-Disposition"] = "attachment; filename=export.json"
response.headers["Content-Type"] = "text/json; charset=utf-8"
return response
except Exception as e:
error = "There was an error completing your request. Details: {}".format(
e)
log.exception(error)
flash(error, "danger")
return redirect("/")
