def add_obj_to_create_by_lvl(all_obj_to_export, set_relationship, dict_obj, lvl): # # TODO: filter by export mode or filter on all global ? if lvl >= 0: add_obj_to_create(all_obj_to_export, add_obj_to_create, dict_obj) if lvl > 0: lvl = lvl - 1 # # TODO: filter by correlation types obj_correlations = Correlate_object.get_object_correlation(dict_obj['type'], dict_obj['id'], requested_correl_type=dict_obj.get('subtype', None)) for obj_type in obj_correlations: dict_new_obj = {'type': obj_type} if obj_type=='pgp' or obj_type=='cryptocurrency': for subtype in obj_correlations[obj_type]: dict_new_obj['subtype'] = subtype for obj_id in obj_correlations[obj_type][subtype]: dict_new_obj['id'] = obj_id add_obj_to_create_by_lvl(all_obj_to_export, set_relationship, dict_new_obj, lvl) add_relation_ship_to_create(set_relationship, dict_obj, dict_new_obj) else: for obj_id in obj_correlations[obj_type]: dict_new_obj['id'] = obj_id add_obj_to_create_by_lvl(all_obj_to_export, set_relationship, dict_new_obj, lvl) add_relation_ship_to_create(set_relationship, dict_obj, dict_new_obj) add_obj_to_create_by_lvl(all_obj_to_export, set_relationship, dict_obj, lvl)
def get_domain_all_correlation(domain, correlation_names=[], get_nb=False): ''' Retun all correlation of a given domain. :param domain: crawled domain :type domain: str :return: a dict of all correlation for a given domain :rtype: dict ''' if not correlation_names: correlation_names = Correlate_object.get_all_correlation_names() domain_correl = {} for correlation_name in correlation_names: if correlation_name == 'cryptocurrency': res = get_domain_cryptocurrency(domain, get_nb=get_nb) elif correlation_name == 'pgp': res = get_domain_pgp(domain, get_nb=get_nb) elif correlation_name == 'username': res = get_domain_username(domain, get_nb=get_nb) elif correlation_name == 'decoded': res = get_domain_decoded(domain) elif correlation_name == 'screenshot': res = get_domain_screenshot(domain) else: res = None # add correllation to dict if res: domain_correl[correlation_name] = res return domain_correl
def get_item_all_correlation(item_id, correlation_names=[], get_nb=False): ''' Retun all correlation of a given item id. :param item_id: item id :type domain: str :return: a dict of all correlation for a item id :rtype: dict ''' if not correlation_names: correlation_names = Correlate_object.get_all_correlation_names() item_correl = {} for correlation_name in correlation_names: if correlation_name == 'cryptocurrency': res = get_item_cryptocurrency(item_id, get_nb=get_nb) elif correlation_name == 'pgp': res = get_item_pgp(item_id, get_nb=get_nb) elif correlation_name == 'username': res = get_item_username(item_id, get_nb=get_nb) elif correlation_name == 'decoded': res = get_item_decoded(item_id) elif correlation_name == 'screenshot': res = get_item_all_screenshot(item_id) else: res = None # add correllation to dict if res: item_correl[correlation_name] = res return item_correl
def graph_node_json(): correlation_id = request.args.get('correlation_id') type_id = request.args.get('type_id') object_type = request.args.get('object_type') max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes')) correlation_names = sanitise_correlation_names( request.args.get('correlation_names')) correlation_objects = sanitise_correlation_objects( request.args.get('correlation_objects')) # # TODO: remove me, rename screenshot to image if object_type == 'image': object_type == 'screenshot' mode = sanitise_graph_mode(request.args.get('mode')) res = Correlate_object.get_graph_node_object_correlation( object_type, correlation_id, mode, correlation_names, correlation_objects, requested_correl_type=type_id, max_nodes=max_nodes) return jsonify(res)
def delete_user_object_to_export(user_id, obj_type, obj_id, obj_subtype=None): ## TODO: check if user exist global_id = Correlate_object.get_obj_global_id(obj_type, obj_id, obj_sub_type=obj_subtype) r_serv_objects.hdel('user:all_objs_to_export:{}'.format(user_id), global_id)
def get_user_list_of_obj_to_export(user_id, add_uuid=False): set_objs_to_export = [] res = r_serv_objects.hgetall('user:all_objs_to_export:{}'.format(user_id)) for global_id in res: dict_obj = Correlate_object.get_global_id_from_id(global_id) dict_obj['lvl'] = int(res[global_id]) if add_uuid: obj_dict['uuid'] = str(uuid.uuid4()) set_objs_to_export.append(dict_obj) return set_objs_to_export
def delete_item(obj_id): # check if item exists if not exist_item(obj_id): return False else: Tag.delete_obj_tags(obj_id, 'item', Tag.get_obj_tag(obj_id)) delete_item_duplicate(obj_id) # delete MISP event r_serv_metadata.delete('misp_events:{}'.format(obj_id)) r_serv_metadata.delete('hive_cases:{}'.format(obj_id)) os.remove(get_item_filename(obj_id)) # get all correlation obj_correlations = get_item_all_correlation(obj_id) for correlation in obj_correlations: if correlation == 'cryptocurrency' or correlation == 'pgp': for obj2_subtype in obj_correlations[correlation]: for obj2_id in obj_correlations[correlation][obj2_subtype]: Correlate_object.delete_obj_relationship( correlation, obj2_id, 'item', obj_id, obj1_subtype=obj2_subtype) else: for obj2_id in obj_correlations[correlation]: Correlate_object.delete_obj_relationship( correlation, obj2_id, 'item', obj_id) # delete father/child delete_node(obj_id) # delete item metadata r_serv_metadata.delete('paste_metadata:{}'.format(obj_id)) return True ### TODO in inport V2 # delete from tracked items # delete from queue ### return False
def delete_tag(): object_type = request.args.get('object_type') object_id = request.args.get('object_id') tag = request.args.get('tag') res = Tag.api_delete_obj_tags(tags=[tag], object_id=object_id, object_type=object_type) if res[1] != 200: return str(res[0]) return redirect(Correlate_object.get_item_url(object_type, object_id))
def create_obj_relationships(map_uuid_global_id, misp_obj): if misp_obj.uuid in map_uuid_global_id: for relationship in misp_obj.ObjectReference: if relationship.referenced_uuid in map_uuid_global_id: obj_meta_src = get_global_id_from_id( map_uuid_global_id[relationship.object_uuid]) obj_meta_target = get_global_id_from_id( map_uuid_global_id[relationship.referenced_uuid]) if obj_meta_src == 'decoded' or obj_meta_src == 'item': print('000000') print(obj_meta_src) print(obj_meta_target) print('111111') Correlate_object.create_obj_relationship( obj_meta_src['type'], obj_meta_src['id'], obj_meta_target['type'], obj_meta_target['id'], obj1_subtype=obj_meta_src['subtype'], obj2_subtype=obj_meta_target['subtype'])
def add_user_object_to_export(user_id, obj_type, obj_id, lvl, obj_subtype=None): ## TODO: check if user exist # # TODO: check if valid object # # TODO: check lvl global_id = Correlate_object.get_obj_global_id(obj_type, obj_id, obj_sub_type=obj_subtype) return r_serv_objects.hset('user:all_objs_to_export:{}'.format(user_id), global_id, lvl)
def get_card_metadata(object_type, correlation_id, type_id=None, expand_card=False): card_dict = {} if object_type == 'cryptocurrency': card_dict[ "sparkline"] = Cryptocurrency.cryptocurrency.get_list_nb_previous_correlation_object( type_id, correlation_id, 6) card_dict["icon"] = Correlate_object.get_correlation_node_icon( object_type, type_id) if type_id == 'bitcoin' and expand_card: card_dict["related_btc"] = btc_ail.get_bitcoin_info(correlation_id) elif object_type == 'pgp': card_dict[ "sparkline"] = Pgp.pgp.get_list_nb_previous_correlation_object( type_id, correlation_id, 6) card_dict["icon"] = Correlate_object.get_correlation_node_icon( object_type, type_id) elif object_type == 'decoded': card_dict["sparkline"] = Decoded.get_list_nb_previous_hash( correlation_id, 6) card_dict["icon"] = Correlate_object.get_correlation_node_icon( object_type, value=correlation_id) card_dict["vt"] = Decoded.get_decoded_vt_report(correlation_id) card_dict["vt"]["status"] = vt_enabled card_dict["add_tags_modal"] = Tag.get_modal_add_tags( correlation_id, object_type='decoded') elif object_type == 'domain': card_dict["icon"] = Correlate_object.get_correlation_node_icon( object_type, value=correlation_id) card_dict["tags"] = Domain.get_domain_tags(correlation_id) elif object_type == 'screenshot': card_dict["add_tags_modal"] = Tag.get_modal_add_tags( correlation_id, object_type='image') elif object_type == 'paste': card_dict["icon"] = Correlate_object.get_correlation_node_icon( object_type, value=correlation_id) return card_dict
def import_object_file(): error = None is_file = False if 'file' in request.files: file = request.files['file'] if file: if file.filename: is_file = True all_imported_obj = [] if is_file: filename = MispImport.sanitize_import_file_path(file.filename) file.save(filename) map_uuid_global_id = MispImport.import_objs_from_file(filename) os.remove(filename) for obj_uuid in map_uuid_global_id: dict_obj = Correlate_object.get_global_id_from_id( map_uuid_global_id[obj_uuid]) dict_obj['uuid'] = obj_uuid dict_obj['url'] = Correlate_object.get_item_url( dict_obj['type'], dict_obj['id'], correlation_type=dict_obj['subtype']) dict_obj['node'] = Correlate_object.get_correlation_node_icon( dict_obj['type'], correlation_type=dict_obj['subtype'], value=dict_obj['id']) all_imported_obj.append(dict_obj) if not all_imported_obj: error = "error: Empty or invalid JSON file" return render_template("import_object.html", all_imported_obj=all_imported_obj, error=error)
def sanitise_correlation_names(correlation_names): ''' correlation_names ex = 'pgp,crypto' ''' all_correlation_names = Correlate_object.get_all_correlation_names() if correlation_names is None: return all_correlation_names else: l_correlation_names = [] for correl in correlation_names.split(','): if correl in all_correlation_names: l_correlation_names.append(correl) if l_correlation_names: return l_correlation_names else: return all_correlation_names
def sanitise_correlation_objects(correlation_objects): ''' correlation_objects ex = 'domain,decoded' ''' all_correlation_objects = Correlate_object.get_all_correlation_objects() if correlation_objects is None: return all_correlation_objects else: l_correlation_objects = [] for correl in correlation_objects.split(','): if correl in all_correlation_objects: l_correlation_objects.append(correl) if l_correlation_objects: return l_correlation_objects else: return all_correlation_objects
def update_tag_global_by_obj_type(object_type, tag): tag_deleted = False if object_type=='item': if not r_serv_tags.exists('tag_metadata:{}'.format(tag)): tag_deleted = True else: if not r_serv_tags.exists('{}:{}'.format(object_type, tag)): tag_deleted = True if tag_deleted: # update object global tags r_serv_tags.srem('list_tags:{}'.format(object_type), tag) # update global tags for obj_type in Correlate_object.get_all_objects(): if r_serv_tags.exists('{}:{}'.format(obj_type, tag)): tag_deleted = False if tag_deleted: r_serv_tags.srem('list_tags', tag)
def add_tags(): tags = request.args.get('tags') tagsgalaxies = request.args.get('tagsgalaxies') object_id = request.args.get('object_id') object_type = request.args.get('object_type') list_tag = tags.split(',') list_tag_galaxies = tagsgalaxies.split(',') res = Tag.api_add_obj_tags(tags=list_tag, galaxy_tags=list_tag_galaxies, object_id=object_id, object_type=object_type) # error if res[1] != 200: return str(res[0]) return redirect(Correlate_object.get_item_url(object_type, object_id))
def graph_node_json(): # # TODO: use post correlation_id = request.args.get('correlation_id') type_id = request.args.get('type_id') object_type = request.args.get('object_type') max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes')) correlation_names = sanitise_correlation_names( request.args.get('correlation_names')) correlation_objects = sanitise_correlation_objects( request.args.get('correlation_objects')) mode = sanitise_graph_mode(request.args.get('mode')) res = Correlate_object.get_graph_node_object_correlation( object_type, correlation_id, mode, correlation_names, correlation_objects, requested_correl_type=type_id, max_nodes=max_nodes) return jsonify(res)
def get_screenshot_correlated_object(sha256_string, correlation_objects=[]): ''' Retun all correlation of a given sha256. :param sha1_string: sha256 :type sha1_string: str :return: a dict of all correlation for a given sha256 :rtype: dict ''' if not correlation_objects: correlation_objects = Correlate_object.get_all_correlation_objects() decoded_correlation = {} for correlation_object in correlation_objects: if correlation_object == 'paste': res = get_screenshot_items_list(sha256_string) elif correlation_object == 'domain': res = get_screenshot_domain(sha256_string) else: res = None if res: decoded_correlation[correlation_object] = res return decoded_correlation
def export_object_file(): user_id = current_user.get_id() l_obj_to_export = [] l_obj_invalid = [] export_to_misp = False dict_misp_event_export = {} # Get new added Object for obj_tuple in list(request.form): l_input = request.form.getlist(obj_tuple) if len(l_input) == 3: obj_type = l_input[0] obj_id = l_input[1] lvl = l_input[2] lvl = MispExport.sanitize_obj_export_lvl(lvl) obj_subtype = obj_type.split(';') if len(obj_subtype) == 2: obj_type = obj_subtype[0] obj_subtype = obj_subtype[1] else: obj_subtype = None obj_dict = {'id': obj_id, 'type': obj_type, 'lvl': lvl} if obj_subtype: obj_dict['subtype'] = obj_subtype if MispExport.is_valid_obj_to_export(obj_type, obj_subtype, obj_id): l_obj_to_export.append(obj_dict) AILObjects.add_user_object_to_export(user_id, obj_dict['type'], obj_dict['id'], obj_dict['lvl'], obj_subtype=obj_dict.get( 'subtype', None)) else: if obj_id: l_obj_invalid.append(obj_dict) else: dict_misp_event_export[str(obj_tuple)] = request.form.get( obj_tuple) if dict_misp_event_export.get('export_to_misp', None): export_to_misp = True else: dict_misp_event_export = None if l_obj_invalid: for obj_dict in l_obj_to_export: obj_dict['uuid'] = str(uuid.uuid4()) obj_dict['type'] = Correlate_object.get_obj_str_type_subtype( obj_dict['type'], obj_dict.get('subtype', None)) # get user saved obj to export # # TODO: # performance l_obj_to_export = AILObjects.get_user_list_of_obj_to_export(user_id) for obj_dict in l_obj_invalid: obj_dict['uuid'] = str(uuid.uuid4()) obj_dict['type'] = Correlate_object.get_obj_str_type_subtype( obj_dict['type'], obj_dict.get('subtype', None)) return render_template("export_object.html", l_obj_to_export=l_obj_to_export, l_obj_invalid=l_obj_invalid, dict_misp_event_export=dict_misp_event_export) else: if export_to_misp and MispExport.ping_misp(): l_obj_to_export = AILObjects.get_user_list_of_obj_to_export( user_id) event = MispExport.create_list_of_objs_to_export(l_obj_to_export, r_type='event') event_metadata = MispExport.create_misp_event( event, distribution=dict_misp_event_export.get( 'export_to_misp', None), threat_level_id=dict_misp_event_export.get( 'misp_threat_level_id', None), publish=dict_misp_event_export.get('misp_publish', None), analysis=dict_misp_event_export.get('misp_event_analysis', None), event_info=dict_misp_event_export.get('misp_event_info', None)) AILObjects.delete_all_user_object_to_export(user_id) return render_template("export_object.html", l_obj_to_export=l_obj_to_export, event_metadata=event_metadata, l_obj_invalid=[], dict_misp_event_export=[]) else: # get user saved obj to export # # TODO: # performance l_obj_to_export = AILObjects.get_user_list_of_obj_to_export( user_id) json_export = MispExport.create_list_of_objs_to_export( l_obj_to_export) export_filename = MispExport.get_export_filename(json_export) json_export = MispExport.create_in_memory_file( json_export.to_json()) AILObjects.delete_all_user_object_to_export(user_id) return send_file(json_export, as_attachment=True, attachment_filename=export_filename)
def show_correlation(): if request.method == 'POST': object_type = request.form.get('object_type') type_id = request.form.get('type_id') correlation_id = request.form.get('correlation_id') max_nodes = request.form.get('max_nb_nodes_in') mode = request.form.get('mode') if mode: mode = 'inter' else: mode = 'union' ## get all selected correlations correlation_names = [] correlation_objects = [] #correlation_names correl_option = request.form.get('CryptocurrencyCheck') if correl_option: correlation_names.append('cryptocurrency') correl_option = request.form.get('PgpCheck') if correl_option: correlation_names.append('pgp') correl_option = request.form.get('UsernameCheck') if correl_option: correlation_names.append('username') correl_option = request.form.get('DecodedCheck') if correl_option: correlation_names.append('decoded') correl_option = request.form.get('ScreenshotCheck') if correl_option: correlation_names.append('screenshot') # correlation_objects correl_option = request.form.get('DomainCheck') if correl_option: correlation_objects.append('domain') correl_option = request.form.get('PasteCheck') if correl_option: correlation_objects.append('paste') # list as params correlation_names = ",".join(correlation_names) correlation_objects = ",".join(correlation_objects) # redirect to keep history and bookmark return redirect( url_for('correlation.show_correlation', object_type=object_type, type_id=type_id, correlation_id=correlation_id, mode=mode, max_nodes=max_nodes, correlation_names=correlation_names, correlation_objects=correlation_objects)) # request.method == 'GET' else: object_type = request.args.get('object_type') type_id = request.args.get('type_id') correlation_id = request.args.get('correlation_id') max_nodes = sanitise_nb_max_nodes(request.args.get('max_nodes')) mode = sanitise_graph_mode(request.args.get('mode')) expand_card = request.args.get('expand_card') correlation_names = sanitise_correlation_names( request.args.get('correlation_names')) correlation_objects = sanitise_correlation_objects( request.args.get('correlation_objects')) # # TODO: remove me, rename screenshot to image if object_type == 'image': object_type == 'screenshot' # check if correlation_id exist if not Correlate_object.exist_object( object_type, correlation_id, type_id=type_id): abort(404) # return 404 # oject exist else: dict_object = { "object_type": object_type, "correlation_id": correlation_id } dict_object["max_nodes"] = max_nodes dict_object["mode"] = mode dict_object["correlation_names"] = correlation_names dict_object["correlation_names_str"] = ",".join(correlation_names) dict_object["correlation_objects"] = correlation_objects dict_object["correlation_objects_str"] = ",".join( correlation_objects) dict_object["metadata"] = Correlate_object.get_object_metadata( object_type, correlation_id, type_id=type_id) if type_id: dict_object["metadata"]['type_id'] = type_id dict_object["metadata_card"] = get_card_metadata( object_type, correlation_id, type_id=type_id, expand_card=expand_card) return render_template("show_correlation.html", dict_object=dict_object, bootstrap_label=bootstrap_label)
def add_obj_to_create(all_obj_to_export, set_relationship, dict_obj): all_obj_to_export.add( Correlate_object.get_obj_global_id(dict_obj['type'], dict_obj['id'], dict_obj.get('subtype', None)))
def filter_obj_linked(l_obj): for obj in l_obj: res = Correlate_object.get_object_correlation(obj['type'], obj['id'], obj.get('subtype', None)) print(res)
def get_all_obj_tags(): object_type = request.args.get('object_type') res = Correlate_object.sanitize_object_type(object_type) if res: return jsonify(res) return jsonify(Tag.get_all_obj_tags(object_type))
def get_obj_by_tags(): # # TODO: sanityze all object_type = request.args.get('object_type') ltags = request.args.get('ltags') page = request.args.get('page') date_from = request.args.get('date_from') date_to = request.args.get('date_to') # remove date separator if date_from: date_from = date_from.replace('-', '') if date_to: date_to = date_to.replace('-', '') # unpack tags list_tags = ltags.split(',') list_tag = [] for tag in list_tags: list_tag.append(tag.replace('"', '\"')) # object_type res = Correlate_object.sanitize_object_type(object_type) if res: return jsonify(res) # page try: page = int(page) except: page = 1 dict_obj = Tag.get_obj_by_tags(object_type, list_tag, date_from=date_from, date_to=date_to, page=page) if dict_obj['tagged_obj']: dict_tagged = { "object_type": object_type, "object_name": object_type.title() + "s", "tagged_obj": [], "page": dict_obj['page'], "nb_pages": dict_obj['nb_pages'], "nb_first_elem": dict_obj['nb_first_elem'], "nb_last_elem": dict_obj['nb_last_elem'], "nb_all_elem": dict_obj['nb_all_elem'] } for obj_id in dict_obj['tagged_obj']: obj_metadata = Correlate_object.get_object_metadata( object_type, obj_id) obj_metadata['id'] = obj_id dict_tagged["tagged_obj"].append(obj_metadata) dict_tagged['tab_keys'] = Correlate_object.get_obj_tag_table_keys( object_type) if len(list_tag) == 1: dict_tagged['current_tags'] = [ltags.replace('"', '\"')] else: dict_tagged['current_tags'] = list_tag dict_tagged['current_tags_str'] = ltags #return jsonify(dict_tagged) else: dict_tagged = { "object_type": object_type, "object_name": object_type.title() + "s" } if 'date' in dict_obj: dict_tagged['date'] = dict_obj['date'] return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged)