def add_obj_to_create_by_lvl(all_obj_to_export, set_relationship, dict_obj, lvl):
# # TODO: filter by export mode or filter on all global ?
if lvl >= 0:
add_obj_to_create(all_obj_to_export, add_obj_to_create, dict_obj)
if lvl > 0:
lvl = lvl - 1
# # TODO: filter by correlation types
obj_correlations = Correlate_object.get_object_correlation(dict_obj['type'], dict_obj['id'], requested_correl_type=dict_obj.get('subtype', None))
for obj_type in obj_correlations:
dict_new_obj = {'type': obj_type}
if obj_type=='pgp' or obj_type=='cryptocurrency':
for subtype in obj_correlations[obj_type]:
dict_new_obj['subtype'] = subtype
for obj_id in obj_correlations[obj_type][subtype]:
dict_new_obj['id'] = obj_id
add_obj_to_create_by_lvl(all_obj_to_export, set_relationship, dict_new_obj, lvl)
add_relation_ship_to_create(set_relationship, dict_obj, dict_new_obj)
else:
for obj_id in obj_correlations[obj_type]:
dict_new_obj['id'] = obj_id
add_obj_to_create_by_lvl(all_obj_to_export, set_relationship, dict_new_obj, lvl)
add_relation_ship_to_create(set_relationship, dict_obj, dict_new_obj)
add_obj_to_create_by_lvl(all_obj_to_export, set_relationship, dict_obj, lvl)
def get_domain_all_correlation(domain, correlation_names=[], get_nb=False):
'''
Retun all correlation of a given domain.
:param domain: crawled domain
:type domain: str
:return: a dict of all correlation for a given domain
:rtype: dict
'''
if not correlation_names:
correlation_names = Correlate_object.get_all_correlation_names()
domain_correl = {}
for correlation_name in correlation_names:
if correlation_name == 'cryptocurrency':
res = get_domain_cryptocurrency(domain, get_nb=get_nb)
elif correlation_name == 'pgp':
res = get_domain_pgp(domain, get_nb=get_nb)
elif correlation_name == 'username':
res = get_domain_username(domain, get_nb=get_nb)
elif correlation_name == 'decoded':
res = get_domain_decoded(domain)
elif correlation_name == 'screenshot':
res = get_domain_screenshot(domain)
else:
res = None
# add correllation to dict
if res:
domain_correl[correlation_name] = res
return domain_correl
def get_item_all_correlation(item_id, correlation_names=[], get_nb=False):
'''
Retun all correlation of a given item id.
:param item_id: item id
:type domain: str
:return: a dict of all correlation for a item id
:rtype: dict
'''
if not correlation_names:
correlation_names = Correlate_object.get_all_correlation_names()
item_correl = {}
for correlation_name in correlation_names:
if correlation_name == 'cryptocurrency':
res = get_item_cryptocurrency(item_id, get_nb=get_nb)
elif correlation_name == 'pgp':
res = get_item_pgp(item_id, get_nb=get_nb)
elif correlation_name == 'username':
res = get_item_username(item_id, get_nb=get_nb)
elif correlation_name == 'decoded':
res = get_item_decoded(item_id)
elif correlation_name == 'screenshot':
res = get_item_all_screenshot(item_id)
else:
res = None
# add correllation to dict
if res:
item_correl[correlation_name] = res
return item_correl
def graph_node_json():
correlation_id = request.args.get('correlation_id')
type_id = request.args.get('type_id')
object_type = request.args.get('object_type')
max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes'))
correlation_names = sanitise_correlation_names(
request.args.get('correlation_names'))
correlation_objects = sanitise_correlation_objects(
request.args.get('correlation_objects'))
# # TODO: remove me, rename screenshot to image
if object_type == 'image':
object_type == 'screenshot'
mode = sanitise_graph_mode(request.args.get('mode'))
res = Correlate_object.get_graph_node_object_correlation(
object_type,
correlation_id,
mode,
correlation_names,
correlation_objects,
requested_correl_type=type_id,
max_nodes=max_nodes)
return jsonify(res)
def delete_user_object_to_export(user_id, obj_type, obj_id, obj_subtype=None):
## TODO: check if user exist
global_id = Correlate_object.get_obj_global_id(obj_type,
obj_id,
obj_sub_type=obj_subtype)
r_serv_objects.hdel('user:all_objs_to_export:{}'.format(user_id),
global_id)
def get_user_list_of_obj_to_export(user_id, add_uuid=False):
set_objs_to_export = []
res = r_serv_objects.hgetall('user:all_objs_to_export:{}'.format(user_id))
for global_id in res:
dict_obj = Correlate_object.get_global_id_from_id(global_id)
dict_obj['lvl'] = int(res[global_id])
if add_uuid:
obj_dict['uuid'] = str(uuid.uuid4())
set_objs_to_export.append(dict_obj)
return set_objs_to_export
def delete_item(obj_id):
# check if item exists
if not exist_item(obj_id):
return False
else:
Tag.delete_obj_tags(obj_id, 'item', Tag.get_obj_tag(obj_id))
delete_item_duplicate(obj_id)
# delete MISP event
r_serv_metadata.delete('misp_events:{}'.format(obj_id))
r_serv_metadata.delete('hive_cases:{}'.format(obj_id))
os.remove(get_item_filename(obj_id))
# get all correlation
obj_correlations = get_item_all_correlation(obj_id)
for correlation in obj_correlations:
if correlation == 'cryptocurrency' or correlation == 'pgp':
for obj2_subtype in obj_correlations[correlation]:
for obj2_id in obj_correlations[correlation][obj2_subtype]:
Correlate_object.delete_obj_relationship(
correlation,
obj2_id,
'item',
obj_id,
obj1_subtype=obj2_subtype)
else:
for obj2_id in obj_correlations[correlation]:
Correlate_object.delete_obj_relationship(
correlation, obj2_id, 'item', obj_id)
# delete father/child
delete_node(obj_id)
# delete item metadata
r_serv_metadata.delete('paste_metadata:{}'.format(obj_id))
return True
### TODO in inport V2
# delete from tracked items
# delete from queue
###
return False
def delete_tag():
object_type = request.args.get('object_type')
object_id = request.args.get('object_id')
tag = request.args.get('tag')
res = Tag.api_delete_obj_tags(tags=[tag],
object_id=object_id,
object_type=object_type)
if res[1] != 200:
return str(res[0])
return redirect(Correlate_object.get_item_url(object_type, object_id))
def create_obj_relationships(map_uuid_global_id, misp_obj):
if misp_obj.uuid in map_uuid_global_id:
for relationship in misp_obj.ObjectReference:
if relationship.referenced_uuid in map_uuid_global_id:
obj_meta_src = get_global_id_from_id(
map_uuid_global_id[relationship.object_uuid])
obj_meta_target = get_global_id_from_id(
map_uuid_global_id[relationship.referenced_uuid])
if obj_meta_src == 'decoded' or obj_meta_src == 'item':
print('000000')
print(obj_meta_src)
print(obj_meta_target)
print('111111')
Correlate_object.create_obj_relationship(
obj_meta_src['type'],
obj_meta_src['id'],
obj_meta_target['type'],
obj_meta_target['id'],
obj1_subtype=obj_meta_src['subtype'],
obj2_subtype=obj_meta_target['subtype'])
def add_user_object_to_export(user_id,
obj_type,
obj_id,
lvl,
obj_subtype=None):
## TODO: check if user exist
# # TODO: check if valid object
# # TODO: check lvl
global_id = Correlate_object.get_obj_global_id(obj_type,
obj_id,
obj_sub_type=obj_subtype)
return r_serv_objects.hset('user:all_objs_to_export:{}'.format(user_id),
global_id, lvl)
def get_card_metadata(object_type,
correlation_id,
type_id=None,
expand_card=False):
card_dict = {}
if object_type == 'cryptocurrency':
card_dict[
"sparkline"] = Cryptocurrency.cryptocurrency.get_list_nb_previous_correlation_object(
type_id, correlation_id, 6)
card_dict["icon"] = Correlate_object.get_correlation_node_icon(
object_type, type_id)
if type_id == 'bitcoin' and expand_card:
card_dict["related_btc"] = btc_ail.get_bitcoin_info(correlation_id)
elif object_type == 'pgp':
card_dict[
"sparkline"] = Pgp.pgp.get_list_nb_previous_correlation_object(
type_id, correlation_id, 6)
card_dict["icon"] = Correlate_object.get_correlation_node_icon(
object_type, type_id)
elif object_type == 'decoded':
card_dict["sparkline"] = Decoded.get_list_nb_previous_hash(
correlation_id, 6)
card_dict["icon"] = Correlate_object.get_correlation_node_icon(
object_type, value=correlation_id)
card_dict["vt"] = Decoded.get_decoded_vt_report(correlation_id)
card_dict["vt"]["status"] = vt_enabled
card_dict["add_tags_modal"] = Tag.get_modal_add_tags(
correlation_id, object_type='decoded')
elif object_type == 'domain':
card_dict["icon"] = Correlate_object.get_correlation_node_icon(
object_type, value=correlation_id)
card_dict["tags"] = Domain.get_domain_tags(correlation_id)
elif object_type == 'screenshot':
card_dict["add_tags_modal"] = Tag.get_modal_add_tags(
correlation_id, object_type='image')
elif object_type == 'paste':
card_dict["icon"] = Correlate_object.get_correlation_node_icon(
object_type, value=correlation_id)
return card_dict
def import_object_file():
error = None
is_file = False
if 'file' in request.files:
file = request.files['file']
if file:
if file.filename:
is_file = True
all_imported_obj = []
if is_file:
filename = MispImport.sanitize_import_file_path(file.filename)
file.save(filename)
map_uuid_global_id = MispImport.import_objs_from_file(filename)
os.remove(filename)
for obj_uuid in map_uuid_global_id:
dict_obj = Correlate_object.get_global_id_from_id(
map_uuid_global_id[obj_uuid])
dict_obj['uuid'] = obj_uuid
dict_obj['url'] = Correlate_object.get_item_url(
dict_obj['type'],
dict_obj['id'],
correlation_type=dict_obj['subtype'])
dict_obj['node'] = Correlate_object.get_correlation_node_icon(
dict_obj['type'],
correlation_type=dict_obj['subtype'],
value=dict_obj['id'])
all_imported_obj.append(dict_obj)
if not all_imported_obj:
error = "error: Empty or invalid JSON file"
return render_template("import_object.html",
all_imported_obj=all_imported_obj,
error=error)
def sanitise_correlation_names(correlation_names):
'''
correlation_names ex = 'pgp,crypto'
'''
all_correlation_names = Correlate_object.get_all_correlation_names()
if correlation_names is None:
return all_correlation_names
else:
l_correlation_names = []
for correl in correlation_names.split(','):
if correl in all_correlation_names:
l_correlation_names.append(correl)
if l_correlation_names:
return l_correlation_names
else:
return all_correlation_names
def sanitise_correlation_objects(correlation_objects):
'''
correlation_objects ex = 'domain,decoded'
'''
all_correlation_objects = Correlate_object.get_all_correlation_objects()
if correlation_objects is None:
return all_correlation_objects
else:
l_correlation_objects = []
for correl in correlation_objects.split(','):
if correl in all_correlation_objects:
l_correlation_objects.append(correl)
if l_correlation_objects:
return l_correlation_objects
else:
return all_correlation_objects
def update_tag_global_by_obj_type(object_type, tag):
tag_deleted = False
if object_type=='item':
if not r_serv_tags.exists('tag_metadata:{}'.format(tag)):
tag_deleted = True
else:
if not r_serv_tags.exists('{}:{}'.format(object_type, tag)):
tag_deleted = True
if tag_deleted:
# update object global tags
r_serv_tags.srem('list_tags:{}'.format(object_type), tag)
# update global tags
for obj_type in Correlate_object.get_all_objects():
if r_serv_tags.exists('{}:{}'.format(obj_type, tag)):
tag_deleted = False
if tag_deleted:
r_serv_tags.srem('list_tags', tag)
def add_tags():
tags = request.args.get('tags')
tagsgalaxies = request.args.get('tagsgalaxies')
object_id = request.args.get('object_id')
object_type = request.args.get('object_type')
list_tag = tags.split(',')
list_tag_galaxies = tagsgalaxies.split(',')
res = Tag.api_add_obj_tags(tags=list_tag,
galaxy_tags=list_tag_galaxies,
object_id=object_id,
object_type=object_type)
# error
if res[1] != 200:
return str(res[0])
return redirect(Correlate_object.get_item_url(object_type, object_id))
def graph_node_json(): # # TODO: use post
correlation_id = request.args.get('correlation_id')
type_id = request.args.get('type_id')
object_type = request.args.get('object_type')
max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes'))
correlation_names = sanitise_correlation_names(
request.args.get('correlation_names'))
correlation_objects = sanitise_correlation_objects(
request.args.get('correlation_objects'))
mode = sanitise_graph_mode(request.args.get('mode'))
res = Correlate_object.get_graph_node_object_correlation(
object_type,
correlation_id,
mode,
correlation_names,
correlation_objects,
requested_correl_type=type_id,
max_nodes=max_nodes)
return jsonify(res)
def get_screenshot_correlated_object(sha256_string, correlation_objects=[]):
'''
Retun all correlation of a given sha256.
:param sha1_string: sha256
:type sha1_string: str
:return: a dict of all correlation for a given sha256
:rtype: dict
'''
if not correlation_objects:
correlation_objects = Correlate_object.get_all_correlation_objects()
decoded_correlation = {}
for correlation_object in correlation_objects:
if correlation_object == 'paste':
res = get_screenshot_items_list(sha256_string)
elif correlation_object == 'domain':
res = get_screenshot_domain(sha256_string)
else:
res = None
if res:
decoded_correlation[correlation_object] = res
return decoded_correlation
def export_object_file():
user_id = current_user.get_id()
l_obj_to_export = []
l_obj_invalid = []
export_to_misp = False
dict_misp_event_export = {}
# Get new added Object
for obj_tuple in list(request.form):
l_input = request.form.getlist(obj_tuple)
if len(l_input) == 3:
obj_type = l_input[0]
obj_id = l_input[1]
lvl = l_input[2]
lvl = MispExport.sanitize_obj_export_lvl(lvl)
obj_subtype = obj_type.split(';')
if len(obj_subtype) == 2:
obj_type = obj_subtype[0]
obj_subtype = obj_subtype[1]
else:
obj_subtype = None
obj_dict = {'id': obj_id, 'type': obj_type, 'lvl': lvl}
if obj_subtype:
obj_dict['subtype'] = obj_subtype
if MispExport.is_valid_obj_to_export(obj_type, obj_subtype,
obj_id):
l_obj_to_export.append(obj_dict)
AILObjects.add_user_object_to_export(user_id,
obj_dict['type'],
obj_dict['id'],
obj_dict['lvl'],
obj_subtype=obj_dict.get(
'subtype', None))
else:
if obj_id:
l_obj_invalid.append(obj_dict)
else:
dict_misp_event_export[str(obj_tuple)] = request.form.get(
obj_tuple)
if dict_misp_event_export.get('export_to_misp', None):
export_to_misp = True
else:
dict_misp_event_export = None
if l_obj_invalid:
for obj_dict in l_obj_to_export:
obj_dict['uuid'] = str(uuid.uuid4())
obj_dict['type'] = Correlate_object.get_obj_str_type_subtype(
obj_dict['type'], obj_dict.get('subtype', None))
# get user saved obj to export # # TODO: # performance
l_obj_to_export = AILObjects.get_user_list_of_obj_to_export(user_id)
for obj_dict in l_obj_invalid:
obj_dict['uuid'] = str(uuid.uuid4())
obj_dict['type'] = Correlate_object.get_obj_str_type_subtype(
obj_dict['type'], obj_dict.get('subtype', None))
return render_template("export_object.html",
l_obj_to_export=l_obj_to_export,
l_obj_invalid=l_obj_invalid,
dict_misp_event_export=dict_misp_event_export)
else:
if export_to_misp and MispExport.ping_misp():
l_obj_to_export = AILObjects.get_user_list_of_obj_to_export(
user_id)
event = MispExport.create_list_of_objs_to_export(l_obj_to_export,
r_type='event')
event_metadata = MispExport.create_misp_event(
event,
distribution=dict_misp_event_export.get(
'export_to_misp', None),
threat_level_id=dict_misp_event_export.get(
'misp_threat_level_id', None),
publish=dict_misp_event_export.get('misp_publish', None),
analysis=dict_misp_event_export.get('misp_event_analysis',
None),
event_info=dict_misp_event_export.get('misp_event_info', None))
AILObjects.delete_all_user_object_to_export(user_id)
return render_template("export_object.html",
l_obj_to_export=l_obj_to_export,
event_metadata=event_metadata,
l_obj_invalid=[],
dict_misp_event_export=[])
else:
# get user saved obj to export # # TODO: # performance
l_obj_to_export = AILObjects.get_user_list_of_obj_to_export(
user_id)
json_export = MispExport.create_list_of_objs_to_export(
l_obj_to_export)
export_filename = MispExport.get_export_filename(json_export)
json_export = MispExport.create_in_memory_file(
json_export.to_json())
AILObjects.delete_all_user_object_to_export(user_id)
return send_file(json_export,
as_attachment=True,
attachment_filename=export_filename)
def show_correlation():
if request.method == 'POST':
object_type = request.form.get('object_type')
type_id = request.form.get('type_id')
correlation_id = request.form.get('correlation_id')
max_nodes = request.form.get('max_nb_nodes_in')
mode = request.form.get('mode')
if mode:
mode = 'inter'
else:
mode = 'union'
## get all selected correlations
correlation_names = []
correlation_objects = []
#correlation_names
correl_option = request.form.get('CryptocurrencyCheck')
if correl_option:
correlation_names.append('cryptocurrency')
correl_option = request.form.get('PgpCheck')
if correl_option:
correlation_names.append('pgp')
correl_option = request.form.get('UsernameCheck')
if correl_option:
correlation_names.append('username')
correl_option = request.form.get('DecodedCheck')
if correl_option:
correlation_names.append('decoded')
correl_option = request.form.get('ScreenshotCheck')
if correl_option:
correlation_names.append('screenshot')
# correlation_objects
correl_option = request.form.get('DomainCheck')
if correl_option:
correlation_objects.append('domain')
correl_option = request.form.get('PasteCheck')
if correl_option:
correlation_objects.append('paste')
# list as params
correlation_names = ",".join(correlation_names)
correlation_objects = ",".join(correlation_objects)
# redirect to keep history and bookmark
return redirect(
url_for('correlation.show_correlation',
object_type=object_type,
type_id=type_id,
correlation_id=correlation_id,
mode=mode,
max_nodes=max_nodes,
correlation_names=correlation_names,
correlation_objects=correlation_objects))
# request.method == 'GET'
else:
object_type = request.args.get('object_type')
type_id = request.args.get('type_id')
correlation_id = request.args.get('correlation_id')
max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes'))
mode = sanitise_graph_mode(request.args.get('mode'))
expand_card = request.args.get('expand_card')
correlation_names = sanitise_correlation_names(
request.args.get('correlation_names'))
correlation_objects = sanitise_correlation_objects(
request.args.get('correlation_objects'))
# # TODO: remove me, rename screenshot to image
if object_type == 'image':
object_type == 'screenshot'
# check if correlation_id exist
if not Correlate_object.exist_object(
object_type, correlation_id, type_id=type_id):
abort(404) # return 404
# oject exist
else:
dict_object = {
"object_type": object_type,
"correlation_id": correlation_id
}
dict_object["max_nodes"] = max_nodes
dict_object["mode"] = mode
dict_object["correlation_names"] = correlation_names
dict_object["correlation_names_str"] = ",".join(correlation_names)
dict_object["correlation_objects"] = correlation_objects
dict_object["correlation_objects_str"] = ",".join(
correlation_objects)
dict_object["metadata"] = Correlate_object.get_object_metadata(
object_type, correlation_id, type_id=type_id)
if type_id:
dict_object["metadata"]['type_id'] = type_id
dict_object["metadata_card"] = get_card_metadata(
object_type,
correlation_id,
type_id=type_id,
expand_card=expand_card)
return render_template("show_correlation.html",
dict_object=dict_object,
bootstrap_label=bootstrap_label)
def add_obj_to_create(all_obj_to_export, set_relationship, dict_obj):
all_obj_to_export.add(
Correlate_object.get_obj_global_id(dict_obj['type'], dict_obj['id'],
dict_obj.get('subtype', None)))
def filter_obj_linked(l_obj):
for obj in l_obj:
res = Correlate_object.get_object_correlation(obj['type'], obj['id'],
obj.get('subtype', None))
print(res)
def get_all_obj_tags():
object_type = request.args.get('object_type')
res = Correlate_object.sanitize_object_type(object_type)
if res:
return jsonify(res)
return jsonify(Tag.get_all_obj_tags(object_type))
def get_obj_by_tags():
# # TODO: sanityze all
object_type = request.args.get('object_type')
ltags = request.args.get('ltags')
page = request.args.get('page')
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
# remove date separator
if date_from:
date_from = date_from.replace('-', '')
if date_to:
date_to = date_to.replace('-', '')
# unpack tags
list_tags = ltags.split(',')
list_tag = []
for tag in list_tags:
list_tag.append(tag.replace('"', '\"'))
# object_type
res = Correlate_object.sanitize_object_type(object_type)
if res:
return jsonify(res)
# page
try:
page = int(page)
except:
page = 1
dict_obj = Tag.get_obj_by_tags(object_type,
list_tag,
date_from=date_from,
date_to=date_to,
page=page)
if dict_obj['tagged_obj']:
dict_tagged = {
"object_type": object_type,
"object_name": object_type.title() + "s",
"tagged_obj": [],
"page": dict_obj['page'],
"nb_pages": dict_obj['nb_pages'],
"nb_first_elem": dict_obj['nb_first_elem'],
"nb_last_elem": dict_obj['nb_last_elem'],
"nb_all_elem": dict_obj['nb_all_elem']
}
for obj_id in dict_obj['tagged_obj']:
obj_metadata = Correlate_object.get_object_metadata(
object_type, obj_id)
obj_metadata['id'] = obj_id
dict_tagged["tagged_obj"].append(obj_metadata)
dict_tagged['tab_keys'] = Correlate_object.get_obj_tag_table_keys(
object_type)
if len(list_tag) == 1:
dict_tagged['current_tags'] = [ltags.replace('"', '\"')]
else:
dict_tagged['current_tags'] = list_tag
dict_tagged['current_tags_str'] = ltags
#return jsonify(dict_tagged)
else:
dict_tagged = {
"object_type": object_type,
"object_name": object_type.title() + "s"
}
if 'date' in dict_obj:
dict_tagged['date'] = dict_obj['date']
return render_template("tags/search_obj_by_tags.html",
bootstrap_label=bootstrap_label,
dict_tagged=dict_tagged)