Ejemplo n.º 1
0
def add_to_store(t, v, add_to_local=False):
    body = {
        "value" : escape(v),
        "type" : escape(t)
    }
    doc_id = t + " " + v
    get_es().index(ES_PREFIX + "threat_actor_simple", "data", body, doc_id)

    if add_to_local:
        global SIMPLE_CHOICES
        SIMPLE_CHOICES[t].append(v)

    print("Added {} {} to ES".format(v,t))
Ejemplo n.º 2
0
def fetch_related_elements(t):
    if t == 'Actor':
        index = "tp-threat_actors"
        doc_type = "actor"
    elif t == 'Report':
        index = "tp-threat_reports"
        doc_type = "report"
    elif t == 'TTP':
        index = "tp-threat_ttps"
        doc_type = "ttp"
    else:
        return []

    query = {
        "query" : {
            "match_all" : {}
        },
        "sort" : {
            "name" : {
                "order" : "asc"
            }
        },
        "fields" : ["name"]
    }

    results = scan(get_es(),query=query,index=index,doc_type=doc_type,preserve_order=True)

    choices = []
    for r in results:
        _id = r["_id"]
        name = r["fields"]["name"][0]
        choices.append((_id + ":::" + name, name))

    return choices
Ejemplo n.º 3
0
def fetch_parent_data(_type, add_new=False, add_unknown=False):
    d = []

    #query
    body = {
        "query" : {
            "term" : {
                "type" : _type
            }
        },
        "sort": {
            "value": {
                "order": "asc"
            }
        },
        "size" : 1000
    }

    #get the data!
    results = get_es().search(ES_PREFIX + 'threat_actor_pc', 'parent', body)
    for r in results['hits']['hits']:
        d.append((r['_source']['value'],r['_source']['value']))

    if add_new:
        d.append(("_NEW_","Other"))

    if add_unknown:
        d.insert(0, ("Unknown", "Unknown"))
    
    #return the data
    return d
Ejemplo n.º 4
0
def populate_simple_choices():
    '''
    Populates the SIMPLE_CHOICES dictionary with options
    '''

    global SIMPLE_CHOICES

    SIMPLE_CHOICES = {}
    try:
        body = {
            "query" : {
                "match_all" : {}
            },
            "size" : 1000
        }
        results = get_es().search(ES_PREFIX + 'threat_actor_simple', 'data', body)

        for r in results['hits']['hits']:
            c_type = r['_source']['type']
            c_value = r['_source']['value']

            if c_type not in SIMPLE_CHOICES:
                SIMPLE_CHOICES[c_type] = []

            SIMPLE_CHOICES[c_type].append(c_value)

        for k,v in SIMPLE_CHOICES.items():
            SIMPLE_CHOICES[k] = sorted(v)

        #print(SIMPLE_CHOICES)

    except Exception as e:
        error = "There was an error fetching choices. Details: {}".format(t, e)
        flash(error,'danger')
        log.exception(logging_prefix + error)
Ejemplo n.º 5
0
def delete(report_id, report_id_hash):
    s = SALTS["report"] + report_id
    hash_object = hashlib.sha256(s.encode("utf-8"))
    hex_dig = hash_object.hexdigest()

    if report_id_hash == hex_dig:
        try:
            get_es().delete(ES_PREFIX + "threat_reports", "report", report_id)
            flash("TTP Deleted", "success")
        except Exception as e:
            flash("There was an error deleting the TTP. Error: {}".format(e), "danger")
    else:
        flash("There was an error deleting the TTP", "danger")

    redirect_url = request.args.get("_r")
    if not redirect_url:
        redirec_url = "/"

    # sleep a sec, make them think were busy, really its to give ES time to delete the doc
    time.sleep(2)

    return redirect(redirect_url, code=302)
Ejemplo n.º 6
0
def fetch_related_choices(t):
    logging_prefix = logger_prefix + "fetch_related_choices({}) - ".format(t)

    choices = [('_NONE_', 'n/a')]

    if t == 'actor':
        index = ES_PREFIX + "threat_actors"
        doc_type = "actor"
    elif t == 'report':
        index = ES_PREFIX + "threat_reports"
        doc_type = "report"
    elif t == 'ttp':
        index = ES_PREFIX + "threat_ttps"
        doc_type = "ttp"
    else:
        raise Exception("Invalid type '{}'. Expected 'actor', 'ttp' or 'report'")

    es_query = {
            "query": {
                "match_all": {}
            },
            "size": 1000,
            "fields" : ["name"],
            "sort": {
                "name": {
                    "order": "asc"
                }
            }
        }

    try:
        results = get_es().search(index, doc_type, es_query)
        for r in results['hits']['hits']:
            choices.append((r['_id'] + ":::" + r['fields']['name'][0],r['fields']['name'][0]))

    except TransportError as te:

        #if the index was not found, this is most likely becuase theres no data there
        if te.status_code == 404:
            log.warning("Index '{}' was not found".format(index))
        else:
            error = "There was an error fetching related {}s. Details: {}".format(t, te)
            flash(error,'danger')
            log.exception(logging_prefix + error)

    except Exception as e:
        error = "There was an error fetching related {}s. Details: {}".format(t, e)
        flash(error,'danger')
        log.exception(logging_prefix + error)

    return choices
Ejemplo n.º 7
0
def delete(ttp_id, ttp_id_hash):
    s = SALTS['ttp'] + ttp_id
    hash_object = hashlib.sha256(s.encode('utf-8'))
    hex_dig = hash_object.hexdigest()

    if ttp_id_hash == hex_dig:
        try:
            get_es().delete(ES_PREFIX + "threat_ttps", "ttp", ttp_id)
            flash("TTP Deleted", "success")
        except Exception as e:
            flash("There was an error deleting the TTP. Error: {}".format(e),
                  "danger")
    else:
        flash("There was an error deleting the TTP", "danger")

    redirect_url = request.args.get("_r")
    if not redirect_url:
        redirec_url = "/"

    #sleep a sec, make them think were busy, really its to give ES time to delete the doc
    time.sleep(2)

    return redirect(redirect_url, code=302)
Ejemplo n.º 8
0
def get_score(classification_family, classification_id):
    #query
    body = {
        "query" : {
            "query_string" : {
                "query" : '+_parent:"tpx_classification {}" +value:"{}"'.format(escape(classification_family),escape(classification_id))
            }
        },
        "sort": {
            "value": {
                "order": "asc"
            }
        },
        "size" : 1
    }

    #get the data!
    results = get_es().search(ES_PREFIX + 'threat_actor_pc', 'child', body)
    if results['hits']['total'] != 1:
        raise Exception("Unable to perform score look up for {} - {}".format(classification_family,classification_id))

    return results['hits']['hits'][0]['_source']['score']
Ejemplo n.º 9
0
def index():
    logging_prefix = logger_prefix + "index() - "
    log.info(logging_prefix + "Loading home page")

    form = forms.searchForm(request.form)
    error = None
    url = "/"
    query_string_url = ""
    try:
        #this is the default query for actors in ES, i'd imagine this will be recently added/modified actors
        es_query = {
            "query": {
                "match_all": {}
            },
            "size": 10,
            "sort": {
                "last_updated_s": {
                    "order": "desc"
                }
            }
        }

        #pull the query out of the url
        query_string = request.args.get("q")

        #someone is searching for something
        if request.method == 'POST' and not query_string:
            if form.validate():

                #get the value
                value = form.query.data

                #redirect to this same page, but setting the query value in the url
                return redirect("/?q={}".format(quote_plus(value)), code=307)

            else:
                #if there was an error print the error dictionary to the console
                #   temporary help, these should also appear under the form field
                print(form.errors)

        elif query_string:
            #now that the query_string is provided as ?q=, perform the search
            print("VALID SEARCH OPERATION DETECTED")

            #do some searching...
            es_query = {
                "query": {
                    "query_string": {
                        "query": query_string
                    }
                },
                "size": 10
            }

            url += "?q=" + query_string
            query_string_url = "?q=" + query_string

            #set the form query value to what the user is searching for
            form.query.data = query_string
        '''
        Fetch the data from ES
        '''

        actors = {}
        actors['hits'] = {}
        actors['hits']['hits'] = []

        reports = dict(actors)

        ttps = dict(actors)

        try:
            actors = get_es().search(ES_PREFIX + 'threat_actors', 'actor',
                                     es_query)
        except TransportError as te:
            #if the index was not found, this is most likely becuase theres no data there
            if te.status_code == 404:
                log.warning("Index 'threat_actors' was not found")
            else:
                error = "There was an error fetching actors. Details: {}".format(
                    te)
                flash(error, 'danger')
                log.exception(logging_prefix + error)
        except Exception as e:
            error = "The was an error fetching Actors. Error: {}".format(e)
            log.exception(error)
            flash(error, "danger")

        try:
            reports = get_es().search(ES_PREFIX + 'threat_reports', 'report',
                                      es_query)
        except TransportError as te:
            #if the index was not found, this is most likely becuase theres no data there
            if te.status_code == 404:
                log.warning("Index 'threat_reports' was not found")
            else:
                error = "There was an error fetching reports. Details: {}".format(
                    te)
                flash(error, 'danger')
                log.exception(logging_prefix + error)
        except Exception as e:
            error = "The was an error fetching Reports. Error: {}".format(e)
            log.exception(error)
            flash(error, "danger")

        try:
            ttps = get_es().search(ES_PREFIX + 'threat_ttps', 'ttp', es_query)
        except TransportError as te:
            #if the index was not found, this is most likely becuase theres no data there
            if te.status_code == 404:
                log.warning("Index 'threat_ttps' was not found")
            else:
                error = "There was an error ttps. Details: {}".format(te)
                flash(error, 'danger')
                log.exception(logging_prefix + error)
        except Exception as e:
            error = "The was an error fetching TTPs. Error: {}".format(e)
            log.exception(error)
            flash(error, "danger")
        '''
        Modify the data as needed
        '''

        for actor in actors['hits']['hits']:
            s = SALTS['actor'] + actor['_id']
            hash_object = hashlib.sha256(s.encode('utf-8'))
            hex_dig = hash_object.hexdigest()
            actor["_source"]['id_hash'] = hex_dig

        for report in reports['hits']['hits']:
            s = SALTS['report'] + report['_id']
            hash_object = hashlib.sha256(s.encode('utf-8'))
            hex_dig = hash_object.hexdigest()
            report["_source"]['id_hash'] = hex_dig

        for ttp in ttps['hits']['hits']:
            s = SALTS['ttp'] + ttp['_id']
            hash_object = hashlib.sha256(s.encode('utf-8'))
            hex_dig = hash_object.hexdigest()
            ttp["_source"]['id_hash'] = hex_dig

    except Exception as e:
        error = "There was an error completing your request. Details: {}".format(
            e)
        log.exception(error)
        flash(error, "danger")

    #render the template, passing the variables we need
    #   templates live in the templates folder
    return render_template("index.html",
                           page_title="ActorTrackr",
                           form=form,
                           query_string_url=query_string_url,
                           actors=actors,
                           reports=reports,
                           ttps=ttps,
                           url=quote_plus(url))
Ejemplo n.º 10
0
def es_to_tpx(report_id):
    '''
    Build the TPX file from the data stored in Elasticsearch
    '''
    element_observables = {}

    results = get_es().get(ES_PREFIX + "threat_reports",
                           doc_type="report",
                           id=report_id)
    report_data = results['_source']

    tpx = {}
    tpx["schema_version_s"] = "2.2.0"
    tpx["provider_s"] = "LookingGlass"
    tpx["list_name_s"] = "Threat Intel Report"
    tpx["created_t"] = report_data['created_milli']
    tpx["created_s"] = report_data['created_s']
    tpx["last_updated_t"] = report_data['last_updated_milli']
    tpx["last_updated_s"] = report_data['last_updated_s']
    tpx["score_i"] = 95
    tpx["source_observable_s"] = "Cyveillance Threat Intel Report"
    tpx["source_description_s"] = "This feed provides threat intelligence reports created by the LookingGlass Cyber Threat Intelligence Group"

    tpx["observable_dictionary_c_array"] = []

    observable_dict = {}
    observable_dict["observable_id_s"] = report_data['name']
    observable_dict["report_uuid_s"] = report_id
    observable_dict["criticality_i"] = report_data['criticality']

    observable_dict["classification_c_array"] = []
    for i in report_data['classification']:
        class_dict = {}
        class_dict["score_i"] = i["score"]
        #class_dict["score_i"] = get_score(i["family"], i["id"])
        class_dict["classification_id_s"] = i["id"]
        class_dict["classification_family_s"] = i["family"]

        observable_dict["classification_c_array"].append(class_dict)

    observable_dict["description_s"] = report_data['description']

    observable_dict["report_s_map"] = {}
    observable_dict["report_s_map"]["ctig_identifier_s"] = report_data[
        "identifier"]
    observable_dict["report_s_map"]["tlp_i"] = report_data["identifier"]
    observable_dict["report_s_map"]["reliability_s"] = report_data[
        "source_reliability"] + str(report_data["info_reliability"])
    observable_dict["report_s_map"]["title_s"] = report_data['name']

    observable_dict["report_s_map"]["sections_c_array"] = []
    for i in report_data['section']:

        section_dict = {}
        section_dict["section_title_s"] = i["title"]
        section_dict["section_tlp_i"] = i["tlp"]
        section_dict["section_order_i"] = i["order"]
        section_dict["section_content_s"] = i["content"]

        observable_dict["report_s_map"]["sections_c_array"].append(
            section_dict)
    '''
    Related elements
    '''

    relate_element_name_map = {
        "FQDN": "subject_fqdn_s",
        "IPv4": "subject_ipv4_s",
        "TTP": "subject_ttp_s",
        "CommAddr": "subject_address_s"
    }

    for i in report_data['related_element_choices']:
        v = i['value']

        print(v)

        v_array = v.split(":::")

        if len(v_array) == 2:
            field_name = relate_element_name_map[v_array[0]]
            field_type = "_DEFAULT_"
            field_data = v_array[1]
        elif len(v_array) == 3:
            field_name = relate_element_name_map[v_array[1]]
            field_type = v_array[0]
            field_data = v_array[2]
        else:
            raise Exception("Unknown Related Element: '{}'".format(v))

        #this field is used in the element observable c array, lets keep track of it

        field_observable = report_data['name']

        if field_name not in element_observables:
            element_observables[field_name] = {}

        if field_type not in element_observables[field_name]:
            element_observables[field_name][field_type] = {}

        if field_data not in element_observables[field_name][field_type]:
            element_observables[field_name][field_type][field_data] = []

        if field_observable not in element_observables[field_name][field_type][
                field_data]:
            element_observables[field_name][field_type][field_data].append(
                field_observable)

    observable_dict["report_s_map"]["related_ttps_c_array"] = []
    for i in report_data['related_ttp']:
        if i['name']:
            observable_dict["report_s_map"]["related_ttps_c_array"].append({
                "name_s":
                i['name'],
                "uuid_s":
                i['id']
            })
            field_observable = i['name']

            for j in i['elements']:
                element_array = j.split(":::")

                if len(element_array) == 3:
                    field_type = element_array[0]
                    field_name = relate_element_name_map[element_array[1]]
                    field_data = element_array[2]
                elif len(element_array) == 2:
                    field_type = "_DEFAULT_"
                    field_name = relate_element_name_map[element_array[0]]
                    field_data = element_array[1]
                else:
                    raise Exception("Invalid element '{}'".format(j))

                if field_name not in element_observables:
                    element_observables[field_name] = {}

                if field_type not in element_observables[field_name]:
                    element_observables[field_name][field_type] = {}

                if field_data not in element_observables[field_name][
                        field_type]:
                    element_observables[field_name][field_type][
                        field_data] = []

                if field_observable not in element_observables[field_name][
                        field_type][field_data]:
                    element_observables[field_name][field_type][
                        field_data].append(field_observable)

    observable_dict["report_s_map"]["related_actors_c_array"] = []
    for i in report_data['related_actor']:
        if i['name']:
            observable_dict["report_s_map"]["related_actors_c_array"].append({
                "name_s":
                i['name'],
                "uuid_s":
                i['id']
            })
            field_observable = i['name']

            for j in i['elements']:
                element_array = j.split(":::")

                if len(element_array) == 3:
                    field_type = element_array[0]
                    field_name = relate_element_name_map[element_array[1]]
                    field_data = element_array[2]
                elif len(element_array) == 2:
                    field_type = "_DEFAULT_"
                    field_name = relate_element_name_map[element_array[0]]
                    field_data = element_array[1]
                else:
                    raise Exception("Invalid element '{}'".format(j))

                if field_name not in element_observables:
                    element_observables[field_name] = {}

                if field_type not in element_observables[field_name]:
                    element_observables[field_name][field_type] = {}

                if field_data not in element_observables[field_name][
                        field_type]:
                    element_observables[field_name][field_type][
                        field_data] = []

                if field_observable not in element_observables[field_name][
                        field_type][field_data]:
                    element_observables[field_name][field_type][
                        field_data].append(field_observable)

    observable_dict["report_s_map"]["related_reports_c_array"] = []
    for i in report_data['related_report']:
        if i['name']:
            observable_dict["report_s_map"]["related_reports_c_array"].append({
                "name_s":
                i['name'],
                "uuid_s":
                i['id']
            })
            field_observable = i['name']

            for j in i['elements']:
                element_array = j.split(":::")

                if len(element_array) == 3:
                    field_type = element_array[0]
                    field_name = relate_element_name_map[element_array[1]]
                    field_data = element_array[2]
                elif len(element_array) == 2:
                    field_type = "_DEFAULT_"
                    field_name = relate_element_name_map[element_array[0]]
                    field_data = element_array[1]
                else:
                    raise Exception("Invalid element '{}'".format(j))

                if field_name not in element_observables:
                    element_observables[field_name] = {}

                if field_type not in element_observables[field_name]:
                    element_observables[field_name][field_type] = {}

                if field_data not in element_observables[field_name][
                        field_type]:
                    element_observables[field_name][field_type][
                        field_data] = []

                if field_observable not in element_observables[field_name][
                        field_type][field_data]:
                    element_observables[field_name][field_type][
                        field_data].append(field_observable)

    tpx["observable_dictionary_c_array"].append(observable_dict)

    tpx["element_observable_c_array"] = []
    for field, val in element_observables.items():
        for sub_type, val1 in val.items():
            for data, val2 in val1.items():
                e_dict = {}

                e_dict[field] = data

                if sub_type != "_DEFAULT_":
                    e_dict['type_s'] = sub_type

                e_dict["threat_observable_c_map"] = {}
                for observable_id in val2:
                    e_dict["threat_observable_c_map"][observable_id] = {}

                    e_dict["threat_observable_c_map"][observable_id][
                        "occurred_at_t"] = report_data['created_milli']
                    e_dict["threat_observable_c_map"][observable_id][
                        "occurred_at_s"] = report_data['created_s']
                    e_dict["threat_observable_c_map"][observable_id][
                        "last_seen_t"] = report_data['last_updated_milli']
                    e_dict["threat_observable_c_map"][observable_id][
                        "last_seen_s"] = report_data['last_updated_s']

                tpx["element_observable_c_array"].append(e_dict)
    return tpx
Ejemplo n.º 11
0
def view_all(t, page=1):
    if t == 'favicon.ico':
        return jsonify({}), 404

    page = int(page)

    logging_prefix = logger_prefix + "view_all() - "
    log.info(logging_prefix +
             "Loading view all page {} for {}".format(page, t))

    form = forms.searchForm(request.form)
    error = None
    page_size = 50
    offset = (page - 1) * page_size
    url = "/{}/{}/".format(t, page)
    search_url = ""
    results_text = ""
    try:

        #this is the default query for actors in ES, i'd imagine this will be recently added/modified actors
        es_query = {
            "query": {
                "match_all": {}
            },
            "size": page_size,
            "from": offset,
            "sort": {
                "last_updated_s": {
                    "order": "desc"
                }
            }
        }

        #pull the query out of the url
        query_string = request.args.get("q")

        #someone is searching for something
        if request.method == 'POST' and not query_string:
            if form.validate():
                print("VALID SEARCH OPERATION DETECTED, redirecting...")

                #get the value
                value = form.query.data

                log.info(value)

                #redirect to this same page, but setting the query value in the url
                return redirect("/{}/1/?q={}".format(t, quote_plus(value)),
                                code=307)

            else:
                #if there was an error print the error dictionary to the console
                #   temporary help, these should also appear under the form field
                print(form.errors)

        elif query_string:
            #now that the query_string is provided as ?q=, perform the search
            print("VALID SEARCH OPERATION DETECTED")

            #do some searching...
            es_query = {
                "query": {
                    "query_string": {
                        "query": query_string
                    }
                },
                "size": page_size,
                "from": offset
            }

            search_url = "?q=" + query_string
            #set the form query value to what the user is searching for
            form.query.data = query_string
        '''
        Fetch the data from ES
        '''

        data = {}
        data['hits'] = {}
        data['hits']['hits'] = []

        if t == 'actor':
            index = ES_PREFIX + 'threat_actors'
            doc_type = 'actor'
            salt = SALTS['actor']
            link_prefix = 'actor'
            data_header = 'Actors'
            field_header = 'Actor Name'
        elif t == 'report':
            index = ES_PREFIX + 'threat_reports'
            doc_type = 'report'
            salt = SALTS['report']
            link_prefix = 'report'
            data_header = 'Reports'
            field_header = 'Report Title'
        elif t == 'ttp':
            index = ES_PREFIX + 'threat_ttps'
            doc_type = 'ttp'
            salt = SALTS['ttp']
            link_prefix = 'ttp'
            data_header = 'TTPs'
            field_header = 'TTP Name'
        else:
            raise Exception("Unknown type {}".format(t))

        try:
            data = get_es().search(index, doc_type, es_query)
            num_hits = len(data['hits']['hits'])

            #set up previous link
            if page == 1:
                prev_url = None
            else:
                prev_url = "/{}/{}/{}".format(t, (page - 1), search_url)

            if ((page - 1) * page_size) + num_hits < data['hits']['total']:
                next_url = "/{}/{}/{}".format(t, (page + 1), search_url)
            else:
                next_url = None

            url += search_url

            for d in data['hits']['hits']:
                s = salt + d['_id']
                hash_object = hashlib.sha256(s.encode('utf-8'))
                hex_dig = hash_object.hexdigest()
                d["_source"]['id_hash'] = hex_dig

            if num_hits == 0:
                results_text = ""
            else:
                f = ((page - 1) * page_size) + 1
                l = f + (num_hits - 1)
                results_text = "Showing {} to {} of {} total results".format(
                    f, l, data['hits']['total'])

        except TransportError as te:

            #if the index was not found, this is most likely becuase theres no data there
            if te.status_code == 404:
                log.warning("Index '{}' was not found".format(index))
            else:
                error = "There was an error fetching {}. Details: {}".format(
                    t, te)
                flash(error, 'danger')
                log.exception(logging_prefix + error)

        except Exception as e:
            error = "The was an error fetching {}. Error: {}".format(t, e)
            log.exception(error)
            flash(error, "danger")

    except Exception as e:
        error = "There was an error completing your request. Details: {}".format(
            e)
        log.exception(error)
        flash(error, "danger")
        return redirect("/")

    return render_template("view_all.html",
                           page_title="View All",
                           form=form,
                           data_header=data_header,
                           results_text=results_text,
                           field_header=field_header,
                           data=data,
                           link_prefix=link_prefix,
                           prev_url=prev_url,
                           next_url=next_url,
                           url=quote_plus(url))
Ejemplo n.º 12
0
def es_to_tpx(report_id):
    """
    Build the TPX file from the data stored in Elasticsearch
    """
    element_observables = {}

    results = get_es().get(ES_PREFIX + "threat_reports", doc_type="report", id=report_id)
    report_data = results["_source"]

    tpx = {}
    tpx["schema_version_s"] = "2.2.0"
    tpx["provider_s"] = "LookingGlass"
    tpx["list_name_s"] = "Threat Intel Report"
    tpx["created_t"] = report_data["created_milli"]
    tpx["created_s"] = report_data["created_s"]
    tpx["last_updated_t"] = report_data["last_updated_milli"]
    tpx["last_updated_s"] = report_data["last_updated_s"]
    tpx["score_i"] = 95
    tpx["source_observable_s"] = "Cyveillance Threat Intel Report"
    tpx[
        "source_description_s"
    ] = "This feed provides threat intelligence reports created by the LookingGlass Cyber Threat Intelligence Group"

    tpx["observable_dictionary_c_array"] = []

    observable_dict = {}
    observable_dict["observable_id_s"] = report_data["name"]
    observable_dict["report_uuid_s"] = report_id
    observable_dict["criticality_i"] = report_data["criticality"]

    observable_dict["classification_c_array"] = []
    for i in report_data["classification"]:
        class_dict = {}
        class_dict["score_i"] = i["score"]
        # class_dict["score_i"] = get_score(i["family"], i["id"])
        class_dict["classification_id_s"] = i["id"]
        class_dict["classification_family_s"] = i["family"]

        observable_dict["classification_c_array"].append(class_dict)

    observable_dict["description_s"] = report_data["description"]

    observable_dict["report_s_map"] = {}
    observable_dict["report_s_map"]["ctig_identifier_s"] = report_data["identifier"]
    observable_dict["report_s_map"]["tlp_i"] = report_data["identifier"]
    observable_dict["report_s_map"]["reliability_s"] = report_data["source_reliability"] + str(
        report_data["info_reliability"]
    )
    observable_dict["report_s_map"]["title_s"] = report_data["name"]

    observable_dict["report_s_map"]["sections_c_array"] = []
    for i in report_data["section"]:

        section_dict = {}
        section_dict["section_title_s"] = i["title"]
        section_dict["section_tlp_i"] = i["tlp"]
        section_dict["section_order_i"] = i["order"]
        section_dict["section_content_s"] = i["content"]

        observable_dict["report_s_map"]["sections_c_array"].append(section_dict)

    """
    Related elements
    """

    relate_element_name_map = {
        "FQDN": "subject_fqdn_s",
        "IPv4": "subject_ipv4_s",
        "TTP": "subject_ttp_s",
        "CommAddr": "subject_address_s",
    }

    for i in report_data["related_element_choices"]:
        v = i["value"]

        print(v)

        v_array = v.split(":::")

        if len(v_array) == 2:
            field_name = relate_element_name_map[v_array[0]]
            field_type = "_DEFAULT_"
            field_data = v_array[1]
        elif len(v_array) == 3:
            field_name = relate_element_name_map[v_array[1]]
            field_type = v_array[0]
            field_data = v_array[2]
        else:
            raise Exception("Unknown Related Element: '{}'".format(v))

        # this field is used in the element observable c array, lets keep track of it

        field_observable = report_data["name"]

        if field_name not in element_observables:
            element_observables[field_name] = {}

        if field_type not in element_observables[field_name]:
            element_observables[field_name][field_type] = {}

        if field_data not in element_observables[field_name][field_type]:
            element_observables[field_name][field_type][field_data] = []

        if field_observable not in element_observables[field_name][field_type][field_data]:
            element_observables[field_name][field_type][field_data].append(field_observable)

    observable_dict["report_s_map"]["related_ttps_c_array"] = []
    for i in report_data["related_ttp"]:
        if i["name"]:
            observable_dict["report_s_map"]["related_ttps_c_array"].append({"name_s": i["name"], "uuid_s": i["id"]})
            field_observable = i["name"]

            for j in i["elements"]:
                element_array = j.split(":::")

                if len(element_array) == 3:
                    field_type = element_array[0]
                    field_name = relate_element_name_map[element_array[1]]
                    field_data = element_array[2]
                elif len(element_array) == 2:
                    field_type = "_DEFAULT_"
                    field_name = relate_element_name_map[element_array[0]]
                    field_data = element_array[1]
                else:
                    raise Exception("Invalid element '{}'".format(j))

                if field_name not in element_observables:
                    element_observables[field_name] = {}

                if field_type not in element_observables[field_name]:
                    element_observables[field_name][field_type] = {}

                if field_data not in element_observables[field_name][field_type]:
                    element_observables[field_name][field_type][field_data] = []

                if field_observable not in element_observables[field_name][field_type][field_data]:
                    element_observables[field_name][field_type][field_data].append(field_observable)

    observable_dict["report_s_map"]["related_actors_c_array"] = []
    for i in report_data["related_actor"]:
        if i["name"]:
            observable_dict["report_s_map"]["related_actors_c_array"].append({"name_s": i["name"], "uuid_s": i["id"]})
            field_observable = i["name"]

            for j in i["elements"]:
                element_array = j.split(":::")

                if len(element_array) == 3:
                    field_type = element_array[0]
                    field_name = relate_element_name_map[element_array[1]]
                    field_data = element_array[2]
                elif len(element_array) == 2:
                    field_type = "_DEFAULT_"
                    field_name = relate_element_name_map[element_array[0]]
                    field_data = element_array[1]
                else:
                    raise Exception("Invalid element '{}'".format(j))

                if field_name not in element_observables:
                    element_observables[field_name] = {}

                if field_type not in element_observables[field_name]:
                    element_observables[field_name][field_type] = {}

                if field_data not in element_observables[field_name][field_type]:
                    element_observables[field_name][field_type][field_data] = []

                if field_observable not in element_observables[field_name][field_type][field_data]:
                    element_observables[field_name][field_type][field_data].append(field_observable)

    observable_dict["report_s_map"]["related_reports_c_array"] = []
    for i in report_data["related_report"]:
        if i["name"]:
            observable_dict["report_s_map"]["related_reports_c_array"].append({"name_s": i["name"], "uuid_s": i["id"]})
            field_observable = i["name"]

            for j in i["elements"]:
                element_array = j.split(":::")

                if len(element_array) == 3:
                    field_type = element_array[0]
                    field_name = relate_element_name_map[element_array[1]]
                    field_data = element_array[2]
                elif len(element_array) == 2:
                    field_type = "_DEFAULT_"
                    field_name = relate_element_name_map[element_array[0]]
                    field_data = element_array[1]
                else:
                    raise Exception("Invalid element '{}'".format(j))

                if field_name not in element_observables:
                    element_observables[field_name] = {}

                if field_type not in element_observables[field_name]:
                    element_observables[field_name][field_type] = {}

                if field_data not in element_observables[field_name][field_type]:
                    element_observables[field_name][field_type][field_data] = []

                if field_observable not in element_observables[field_name][field_type][field_data]:
                    element_observables[field_name][field_type][field_data].append(field_observable)

    tpx["observable_dictionary_c_array"].append(observable_dict)

    tpx["element_observable_c_array"] = []
    for field, val in element_observables.items():
        for sub_type, val1 in val.items():
            for data, val2 in val1.items():
                e_dict = {}

                e_dict[field] = data

                if sub_type != "_DEFAULT_":
                    e_dict["type_s"] = sub_type

                e_dict["threat_observable_c_map"] = {}
                for observable_id in val2:
                    e_dict["threat_observable_c_map"][observable_id] = {}

                    e_dict["threat_observable_c_map"][observable_id]["occurred_at_t"] = report_data["created_milli"]
                    e_dict["threat_observable_c_map"][observable_id]["occurred_at_s"] = report_data["created_s"]
                    e_dict["threat_observable_c_map"][observable_id]["last_seen_t"] = report_data["last_updated_milli"]
                    e_dict["threat_observable_c_map"][observable_id]["last_seen_s"] = report_data["last_updated_s"]

                tpx["element_observable_c_array"].append(e_dict)
    return tpx
Ejemplo n.º 13
0
def form_to_es(form, report_id):
    logging_prefix = logger_prefix + "form_to_es() - "
    log.info(logging_prefix + "Converting Form to ES for {}".format(report_id))

    doc = {}

    created_t = int(time.mktime(form.report_occurred_at.data.timetuple())) * 1000
    created_s = form.report_occurred_at.data.strftime("%Y-%m-%dT%H:%M:%S")
    now_t = int(time.time()) * 1000
    now_s = datetime.now().strftime("%Y-%m-%dT%H:%M:%S")

    doc["created_milli"] = created_t
    doc["created_s"] = created_s
    doc["last_updated_milli"] = now_t
    doc["last_updated_s"] = now_s

    doc["name"] = escape(form.report_name.data)
    doc["identifier"] = escape(form.report_id.data)
    doc["description"] = escape(form.report_description.data)
    doc["criticality"] = escape(int(form.report_criticality.data))

    doc["classification"] = []
    for sub_form in form.report_class.entries:
        classification_dict = {}
        classification_dict["score"] = int(get_score(sub_form.data["a_family"], sub_form.data["a_id"]))
        classification_dict["id"] = escape(sub_form.data["a_id"])
        classification_dict["family"] = escape(sub_form.data["a_family"])

        if classification_dict not in doc["classification"]:
            doc["classification"].append(classification_dict)

    doc["tlp"] = int(escape(form.report_tlp.data))

    doc["section"] = []
    idx = 1
    for sub_form in form.report_sections.entries:
        section_dict = {}
        section_dict["order"] = idx
        section_dict["title"] = escape(sub_form.title.data)
        section_dict["tlp"] = int(escape(sub_form.tlp.data))
        section_dict["content"] = escape(sub_form.text.data)
        idx += 1

        doc["section"].append(section_dict)

    doc["source_reliability"] = escape(form.report_source_reliability.data)
    doc["info_reliability"] = int(escape(form.report_info_reliability.data))

    doc["source"] = []
    for sub_form in form.report_sources.entries:
        source = sub_form.source.data
        if source not in doc["source"]:
            doc["source"].append(source)

    """
    Related elements
    """

    my_id = report_id

    # Links to actors and reports
    doc["related_actor"] = []
    for sub_form in form.report_actors.entries:
        r_dict = {}
        data = escape(sub_form.data.data)

        if data == "_NONE_":

            continue

        data_array = data.split(":::")

        r_dict["id"] = escape(data_array[0])

        # if the related element's id = my id, dont add to es, this should only apply to current type
        if data_array[0] == my_id:
            flash("A related actor matched this actor, this entry has been discarded", "warning")
            continue

        r_dict["name"] = escape(data_array[1])
        # this is gonna be a nightmare to maintain,
        # but it make sense to have this for searches

        r_dict["elements"] = []
        for sub_sub_form in sub_form.related_elements.entries:
            if sub_sub_form.element.data:
                if sub_sub_form.element_value.data not in r_dict["elements"]:
                    r_dict["elements"].append(sub_sub_form.element_value.data)

        if r_dict not in doc["related_actor"]:
            doc["related_actor"].append(r_dict)

    doc["related_report"] = []
    for sub_form in form.report_reports.entries:
        r_dict = {}
        data = escape(sub_form.data.data)

        if data == "_NONE_":
            continue

        data_array = data.split(":::")

        r_dict["id"] = escape(data_array[0])

        # if the related element's id = my id, dont add to es, this should only apply to current type
        if data_array[0] == my_id:
            flash("A related report matched this report, this entry has been discarded", "warning")
            continue

        r_dict["name"] = escape(data_array[1])
        # this is gonna be a nightmare to maintain,
        # but it make sense to have this for searches

        r_dict["elements"] = []
        for sub_sub_form in sub_form.related_elements.entries:
            if sub_sub_form.element.data:
                if sub_sub_form.element_value.data not in r_dict["elements"]:
                    r_dict["elements"].append(sub_sub_form.element_value.data)

        if r_dict not in doc["related_report"]:
            doc["related_report"].append(r_dict)

    doc["related_ttp"] = []
    for sub_form in form.report_ttps.entries:
        r_dict = {}
        data = escape(sub_form.data.data)

        if data == "_NONE_":
            continue

        data_array = data.split(":::")

        r_dict["id"] = escape(data_array[0])

        # if the related element's id = my id, dont add to es, this should only apply to current type
        if data_array[0] == my_id:
            flash("A related TTP matched this TTP, this entry has been discarded", "warning")
            continue

        r_dict["name"] = escape(data_array[1])
        # this is gonna be a nightmare to maintain,
        # but it make sense to have this for searches

        r_dict["elements"] = []
        for sub_sub_form in sub_form.related_elements.entries:
            if sub_sub_form.element.data:
                if sub_sub_form.element_value.data not in r_dict["elements"]:
                    r_dict["elements"].append(sub_sub_form.element_value.data)

        if r_dict not in doc["related_ttp"]:
            doc["related_ttp"].append(r_dict)

    # store these to make life easier
    doc["related_element_choices"] = []
    rec = json.loads(form.related_element_choices.data)
    for k, v in rec.items():
        # dont include the n/a choice, as this will be first and handled manually
        if v != "_NONE_":
            # this will create a tuple ("display text", "value string")
            doc["related_element_choices"].append({"display_text": k, "value": v})

    """
    Edit Tracking
    """

    doc["editor"] = get_editor_list(
        es=get_es(),
        index=ES_PREFIX + "threat_reports",
        doc_type="report",
        item_id=report_id,
        user_id=session.get("id", None),
    )

    # index the doc
    log.info(logging_prefix + "Start Indexing of {}".format(report_id))
    response = get_es().index(ES_PREFIX + "threat_reports", "report", doc, report_id)
    log.info(logging_prefix + "Done Indexing of {}".format(report_id))

    return response, doc
Ejemplo n.º 14
0
def es_to_form(report_id):
    form = forms.reportForm()

    #get the values from ES
    results = get_es().get(ES_PREFIX + "threat_reports",
                           doc_type="report",
                           id=report_id)

    #store certain fields from ES, so this form can be used in an update
    form.doc_index.data = results['_index']
    form.doc_type.data = results['_type']

    report_data = results['_source']

    form.report_name.data = report_data['name']
    form.report_id.data = report_data['identifier']
    form.report_occurred_at.data = datetime.strptime(report_data['created_s'],
                                                     "%Y-%m-%dT%H:%M:%S")
    form.report_description.data = report_data['description']
    form.report_criticality.data = report_data['criticality']

    idx = 0
    for entry in range(len(form.report_class.entries)):
        form.report_class.pop_entry()
    for i in multikeysort(report_data['classification'], ['family', 'id']):
        report_class_form = forms.TPXClassificationForm()
        report_class_form.a_family = i['family']
        report_class_form.a_id = i['id']

        form.report_class.append_entry(report_class_form)

        #set the options since this select is dynamic
        form.report_class[idx].a_id.choices = fetch_child_data(
            'tpx_classification', i['family'])
        idx += 1

    form.report_tlp.data = int(report_data['tlp'])

    for entry in range(len(form.report_sections.entries)):
        form.report_sections.pop_entry()
    for i in multikeysort(report_data['section'], ['order']):
        report_section_form = forms.ReportSectionsForm()
        report_section_form.title = i['title']
        report_section_form.tlp = i['tlp']
        report_section_form.text = i['content']

        form.report_sections.append_entry(report_section_form)

    form.report_source_reliability.data = report_data["source_reliability"]
    form.report_info_reliability.data = report_data["info_reliability"]

    for entry in range(len(form.report_sources.entries)):
        form.report_sources.pop_entry()
    for i in sorted(report_data['source']):
        report_source_form = forms.ReportSourcesForm()
        report_source_form.source = i

        form.report_sources.append_entry(report_source_form)
    '''
    Related element
    '''

    for entry in range(len(form.report_actors.entries)):
        form.report_actors.pop_entry()
    if report_data['related_actor']:
        idx = 0
        for i in multikeysort(report_data['related_actor'], ['name', 'id']):
            sub_form = forms.RelatedActorsForm()
            sub_form.data = i['id'] + ":::" + i['name']
            has_related_elements = False

            idx2 = 0
            for entry in range(len(sub_form.related_elements.entries)):
                sub_form.related_elements.pop_entry()
            for j in multikeysort(report_data['related_element_choices'],
                                  ['display_text']):
                sub_sub_form = forms.ElementObservablesFrom()

                sub_form.related_elements.append_entry(sub_sub_form)

                is_related = (j['value'] in i['elements'])
                sub_form.related_elements[idx2].element = is_related
                sub_form.related_elements[idx2].element_value = j['value']
                sub_form.related_elements[idx2].element_text = j[
                    'display_text']

                if is_related:
                    has_related_elements = True

                idx2 += 1

            form.report_actors.append_entry(sub_form)

            form.report_actors[
                idx].has_related_elements.data = has_related_elements
            idx += 1
    else:
        sub_form = forms.RelatedActorsForm()
        sub_form.data = "_NONE_"
        form.report_actors.append_entry(sub_form)

    for entry in range(len(form.report_reports.entries)):
        form.report_reports.pop_entry()
    if report_data['related_report']:
        idx = 0
        for i in multikeysort(report_data['related_report'], ['name', 'id']):
            sub_form = forms.RelatedReportsForm()
            sub_form.data = i['id'] + ":::" + i['name']
            has_related_elements = False

            idx2 = 0
            for entry in range(len(sub_form.related_elements.entries)):
                sub_form.related_elements.pop_entry()
            for j in multikeysort(report_data['related_element_choices'],
                                  ['display_text']):
                sub_sub_form = forms.ElementObservablesFrom()

                sub_form.related_elements.append_entry(sub_sub_form)

                is_related = (j['value'] in i['elements'])
                sub_form.related_elements[idx2].element = is_related
                sub_form.related_elements[idx2].element_value = j['value']
                sub_form.related_elements[idx2].element_text = j[
                    'display_text']

                if is_related:
                    has_related_elements = True

                idx2 += 1

            form.report_reports.append_entry(sub_form)

            form.report_reports[
                idx].has_related_elements.data = has_related_elements
            idx += 1
    else:
        sub_form = forms.RelatedReportsForm()
        sub_form.data = "_NONE_"
        form.report_reports.append_entry(sub_form)

    for entry in range(len(form.report_ttps.entries)):
        form.report_ttps.pop_entry()
    if report_data['related_ttp']:
        idx = 0
        for i in multikeysort(report_data['related_ttp'], ['name', 'id']):
            sub_form = forms.RelatedTTPsForm()
            sub_form.data = i['id'] + ":::" + i['name']
            has_related_elements = False

            idx2 = 0
            for entry in range(len(sub_form.related_elements.entries)):
                sub_form.related_elements.pop_entry()
            for j in multikeysort(report_data['related_element_choices'],
                                  ['display_text']):
                sub_sub_form = forms.ElementObservablesFrom()

                sub_form.related_elements.append_entry(sub_sub_form)

                is_related = (j['value'] in i['elements'])
                sub_form.related_elements[idx2].element = is_related
                sub_form.related_elements[idx2].element_value = j['value']
                sub_form.related_elements[idx2].element_text = j[
                    'display_text']

                if is_related:
                    has_related_elements = True

                idx2 += 1

            form.report_ttps.append_entry(sub_form)

            form.report_ttps[
                idx].has_related_elements.data = has_related_elements
            idx += 1
    else:
        sub_form = forms.RelatedTTPsForm()
        sub_form.data = "_NONE_"
        form.report_ttps.append_entry(sub_form)

    #convert editor dictionary of ids and times to names and times
    editors = get_editor_names(get_mysql(), report_data['editor'])

    return form, editors
Ejemplo n.º 15
0
def es_to_form(ttp_id):
    form = forms.ttpForm()

    #get the values from ES
    results = get_es().get(ES_PREFIX + "threat_ttps",
                           doc_type="ttp",
                           id=ttp_id)

    #store certain fields from ES, so this form can be used in an update
    form.doc_index.data = results['_index']
    form.doc_type.data = results['_type']

    ttp_data = results['_source']

    form.ttp_name.data = ttp_data['name']
    form.ttp_first_observed.data = datetime.strptime(ttp_data['created_s'],
                                                     "%Y-%m-%dT%H:%M:%S")
    form.ttp_description.data = ttp_data['description']
    form.ttp_criticality.data = int(ttp_data['criticality'])

    idx = 0
    for entry in range(len(form.ttp_class.entries)):
        form.ttp_class.pop_entry()
    for i in multikeysort(ttp_data['classification'], ['family', 'id']):
        ttp_class_form = forms.TPXClassificationForm()
        ttp_class_form.a_family = i['family']
        ttp_class_form.a_id = i['id']

        form.ttp_class.append_entry(ttp_class_form)

        #set the options since this select is dynamic
        form.ttp_class[idx].a_id.choices = fetch_child_data(
            'tpx_classification', i['family'])
        idx += 1

    if ttp_data['related_actor']:
        for entry in range(len(form.ttp_actors.entries)):
            form.ttp_actors.pop_entry()
        for i in multikeysort(ttp_data['related_actor'], ['name', 'id']):
            sub_form = forms.RelatedActorsForm()
            sub_form.data = i['id'] + ":::" + i['name']

            form.ttp_actors.append_entry(sub_form)

    if ttp_data['related_report']:
        for entry in range(len(form.ttp_reports.entries)):
            form.ttp_reports.pop_entry()
        for i in multikeysort(ttp_data['related_report'], ['name', 'id']):
            sub_form = forms.RelatedReportsForm()
            sub_form.data = i['id'] + ":::" + i['name']

            form.ttp_reports.append_entry(sub_form)

    if ttp_data['related_ttp']:
        for entry in range(len(form.ttp_ttps.entries)):
            form.ttp_ttps.pop_entry()
        for i in multikeysort(ttp_data['related_ttp'], ['name', 'id']):
            sub_form = forms.RelatedTTPsForm()
            sub_form.data = i['id'] + ":::" + i['name']

            form.ttp_ttps.append_entry(sub_form)

    #convert editor dictionary of ids and times to names and times
    editors = get_editor_names(get_mysql(), ttp_data['editor'])

    return form, editors
Ejemplo n.º 16
0
def form_to_es(form, ttp_id):
    logging_prefix = logger_prefix + "form_to_es() - "
    log.info(logging_prefix + "Converting Form to ES for {}".format(ttp_id))

    doc = {}

    created_t = int(time.mktime(
        form.ttp_first_observed.data.timetuple())) * 1000
    created_s = form.ttp_first_observed.data.strftime("%Y-%m-%dT%H:%M:%S")
    now_t = int(time.time()) * 1000
    now_s = datetime.now().strftime("%Y-%m-%dT%H:%M:%S")

    doc["created_milli"] = created_t
    doc["created_s"] = created_s
    doc["last_updated_milli"] = now_t
    doc["last_updated_s"] = now_s

    doc['name'] = escape(form.ttp_name.data)
    doc['description'] = escape(form.ttp_description.data)
    doc['criticality'] = int(escape(form.ttp_criticality.data))

    doc['classification'] = []
    for sub_form in form.ttp_class.entries:
        classification_dict = {}
        classification_dict['score'] = int(
            get_score(sub_form.data['a_family'], sub_form.data['a_id']))
        classification_dict['id'] = escape(sub_form.data['a_id'])
        classification_dict['family'] = escape(sub_form.data['a_family'])

        if classification_dict not in doc['classification']:
            doc['classification'].append(classification_dict)

    #Links to actors and reports
    doc['related_actor'] = []
    for sub_form in form.ttp_actors.entries:
        r_dict = {}
        data = escape(sub_form.data.data)

        if data == "_NONE_":
            continue

        data_array = data.split(":::")

        r_dict['id'] = escape(data_array[0])
        r_dict['name'] = escape(data_array[1])
        #this is gonna be a nightmare to maintain,
        # but it make sense to have this for searches

        if r_dict not in doc['related_actor']:
            doc['related_actor'].append(r_dict)

    doc['related_report'] = []
    for sub_form in form.ttp_reports.entries:
        r_dict = {}
        data = escape(sub_form.data.data)

        if data == "_NONE_":
            continue

        data_array = data.split(":::")

        r_dict['id'] = escape(data_array[0])
        r_dict['name'] = escape(data_array[1])
        #this is gonna be a nightmare to maintain,
        # but it make sense to have this for searches

        if r_dict not in doc['related_report']:
            doc['related_report'].append(r_dict)

    doc['related_ttp'] = []
    for sub_form in form.ttp_ttps.entries:
        r_dict = {}
        data = escape(sub_form.data.data)

        if data == "_NONE_":
            continue

        data_array = data.split(":::")

        r_dict['id'] = escape(data_array[0])
        r_dict['name'] = escape(data_array[1])
        #this is gonna be a nightmare to maintain,
        # but it make sense to have this for searches

        if r_dict not in doc['related_ttp']:
            doc['related_ttp'].append(r_dict)
    '''
    Edit Tracking
    '''

    doc['editor'] = get_editor_list(es=get_es(),
                                    index=ES_PREFIX + "threat_ttps",
                                    doc_type="ttp",
                                    item_id=ttp_id,
                                    user_id=session.get('id', None))

    #print_tpx(doc)

    #index the doc
    log.info(logging_prefix + "Start Indexing of {}".format(ttp_id))
    response = get_es().index(ES_PREFIX + "threat_ttps", "ttp", doc, ttp_id)
    log.info(logging_prefix + "Done Indexing of {}".format(ttp_id))

    return response, doc
Ejemplo n.º 17
0
def es_to_tpx(ttp_id):
    '''
    Build the TPX file from the data stored in Elasticsearch
    '''
    element_observables = {}

    results = get_es().get(ES_PREFIX + "threat_ttps",
                           doc_type="ttp",
                           id=ttp_id)
    ttp_data = results['_source']

    tpx = {}
    tpx["schema_version_s"] = "2.2.0"
    tpx["provider_s"] = "LookingGlass"
    tpx["list_name_s"] = "Threat Actor"
    tpx["created_t"] = ttp_data['created_milli']
    tpx["created_s"] = ttp_data['created_s']
    tpx["last_updated_t"] = ttp_data['last_updated_milli']
    tpx["last_updated_s"] = ttp_data['last_updated_s']
    tpx["score_i"] = 95
    tpx["source_observable_s"] = "Cyveillance Threat Actor"
    tpx["source_description_s"] = "This feed provides threat actor or threat actor group profiles and characterizations created by the LookingGlass Cyber Threat Intelligence Group"

    tpx["observable_dictionary_c_array"] = []

    observable_dict = {}
    observable_dict["ttp_uuid_s"] = ttp_id
    observable_dict["observable_id_s"] = ttp_data['name']
    observable_dict["description_s"] = ttp_data['description']
    observable_dict["criticality_i"] = ttp_data['criticality']

    observable_dict["classification_c_array"] = []

    class_dict = {}
    class_dict["score_i"] = 70
    class_dict["classification_id_s"] = "Intel"
    class_dict["classification_family_s"] = "TTP"
    observable_dict["classification_c_array"].append(class_dict)

    for i in ttp_data['classification']:
        class_dict = {}
        class_dict["score_i"] = i["score"]
        class_dict["classification_id_s"] = i["id"]
        class_dict["classification_family_s"] = i["family"]

        if class_dict not in observable_dict["classification_c_array"]:
            observable_dict["classification_c_array"].append(class_dict)

    observable_dict["related_ttps_c_array"] = []
    for i in ttp_data['related_ttp']:
        if i['name']:
            observable_dict["related_ttps_c_array"].append({
                "name_s": i['name'],
                "uuid_s": i['id']
            })

    observable_dict["related_actors_c_array"] = []
    for i in ttp_data['related_actor']:
        if i['name']:
            observable_dict["related_actors_c_array"].append({
                "name_s":
                i['name'],
                "uuid_s":
                i['id']
            })

    observable_dict["related_reports_c_array"] = []
    for i in ttp_data['related_report']:
        if i['name']:
            observable_dict["related_reports_c_array"].append({
                "name_s":
                i['name'],
                "uuid_s":
                i['id']
            })
    '''
    Related elements
    '''

    relate_element_name_map = {
        "FQDN": "subject_fqdn_s",
        "IPv4": "subject_ipv4_s",
        "TTP": "subject_ttp_s",
        "CommAddr": "subject_address_s"
    }

    tpx["observable_dictionary_c_array"].append(observable_dict)

    return tpx
Ejemplo n.º 18
0
def form_to_es(form, report_id):
    logging_prefix = logger_prefix + "form_to_es() - "
    log.info(logging_prefix + "Converting Form to ES for {}".format(report_id))

    doc = {}

    created_t = int(time.mktime(
        form.report_occurred_at.data.timetuple())) * 1000
    created_s = form.report_occurred_at.data.strftime("%Y-%m-%dT%H:%M:%S")
    now_t = int(time.time()) * 1000
    now_s = datetime.now().strftime("%Y-%m-%dT%H:%M:%S")

    doc["created_milli"] = created_t
    doc["created_s"] = created_s
    doc["last_updated_milli"] = now_t
    doc["last_updated_s"] = now_s

    doc["name"] = escape(form.report_name.data)
    doc["identifier"] = escape(form.report_id.data)
    doc["description"] = escape(form.report_description.data)
    doc["criticality"] = escape(int(form.report_criticality.data))

    doc['classification'] = []
    for sub_form in form.report_class.entries:
        classification_dict = {}
        classification_dict['score'] = int(
            get_score(sub_form.data['a_family'], sub_form.data['a_id']))
        classification_dict['id'] = escape(sub_form.data['a_id'])
        classification_dict['family'] = escape(sub_form.data['a_family'])

        if classification_dict not in doc['classification']:
            doc['classification'].append(classification_dict)

    doc["tlp"] = int(escape(form.report_tlp.data))

    doc['section'] = []
    idx = 1
    for sub_form in form.report_sections.entries:
        section_dict = {}
        section_dict['order'] = idx
        section_dict['title'] = escape(sub_form.title.data)
        section_dict['tlp'] = int(escape(sub_form.tlp.data))
        section_dict['content'] = escape(sub_form.text.data)
        idx += 1

        doc['section'].append(section_dict)

    doc["source_reliability"] = escape(form.report_source_reliability.data)
    doc["info_reliability"] = int(escape(form.report_info_reliability.data))

    doc['source'] = []
    for sub_form in form.report_sources.entries:
        source = sub_form.source.data
        if source not in doc['source']:
            doc['source'].append(source)
    '''
    Related elements
    '''

    my_id = report_id

    #Links to actors and reports
    doc['related_actor'] = []
    for sub_form in form.report_actors.entries:
        r_dict = {}
        data = escape(sub_form.data.data)

        if data == "_NONE_":

            continue

        data_array = data.split(":::")

        r_dict['id'] = escape(data_array[0])

        #if the related element's id = my id, dont add to es, this should only apply to current type
        if data_array[0] == my_id:
            flash(
                "A related actor matched this actor, this entry has been discarded",
                "warning")
            continue

        r_dict['name'] = escape(data_array[1])
        #this is gonna be a nightmare to maintain,
        # but it make sense to have this for searches

        r_dict['elements'] = []
        for sub_sub_form in sub_form.related_elements.entries:
            if sub_sub_form.element.data:
                if sub_sub_form.element_value.data not in r_dict['elements']:
                    r_dict['elements'].append(sub_sub_form.element_value.data)

        if r_dict not in doc['related_actor']:
            doc['related_actor'].append(r_dict)

    doc['related_report'] = []
    for sub_form in form.report_reports.entries:
        r_dict = {}
        data = escape(sub_form.data.data)

        if data == "_NONE_":
            continue

        data_array = data.split(":::")

        r_dict['id'] = escape(data_array[0])

        #if the related element's id = my id, dont add to es, this should only apply to current type
        if data_array[0] == my_id:
            flash(
                "A related report matched this report, this entry has been discarded",
                "warning")
            continue

        r_dict['name'] = escape(data_array[1])
        #this is gonna be a nightmare to maintain,
        # but it make sense to have this for searches

        r_dict['elements'] = []
        for sub_sub_form in sub_form.related_elements.entries:
            if sub_sub_form.element.data:
                if sub_sub_form.element_value.data not in r_dict['elements']:
                    r_dict['elements'].append(sub_sub_form.element_value.data)

        if r_dict not in doc['related_report']:
            doc['related_report'].append(r_dict)

    doc['related_ttp'] = []
    for sub_form in form.report_ttps.entries:
        r_dict = {}
        data = escape(sub_form.data.data)

        if data == "_NONE_":
            continue

        data_array = data.split(":::")

        r_dict['id'] = escape(data_array[0])

        #if the related element's id = my id, dont add to es, this should only apply to current type
        if data_array[0] == my_id:
            flash(
                "A related TTP matched this TTP, this entry has been discarded",
                "warning")
            continue

        r_dict['name'] = escape(data_array[1])
        #this is gonna be a nightmare to maintain,
        # but it make sense to have this for searches

        r_dict['elements'] = []
        for sub_sub_form in sub_form.related_elements.entries:
            if sub_sub_form.element.data:
                if sub_sub_form.element_value.data not in r_dict['elements']:
                    r_dict['elements'].append(sub_sub_form.element_value.data)

        if r_dict not in doc['related_ttp']:
            doc['related_ttp'].append(r_dict)

    #store these to make life easier
    doc['related_element_choices'] = []
    rec = json.loads(form.related_element_choices.data)
    for k, v in rec.items():
        #dont include the n/a choice, as this will be first and handled manually
        if v != "_NONE_":
            #this will create a tuple ("display text", "value string")
            doc['related_element_choices'].append({
                "display_text": k,
                "value": v
            })
    '''
    Edit Tracking
    '''

    doc['editor'] = get_editor_list(es=get_es(),
                                    index=ES_PREFIX + "threat_reports",
                                    doc_type="report",
                                    item_id=report_id,
                                    user_id=session.get('id', None))

    #index the doc
    log.info(logging_prefix + "Start Indexing of {}".format(report_id))
    response = get_es().index(ES_PREFIX + "threat_reports", "report", doc,
                              report_id)
    log.info(logging_prefix + "Done Indexing of {}".format(report_id))

    return response, doc
Ejemplo n.º 19
0
def choices():
    logging_prefix = logger_prefix + "choices() - "
    log.info(logging_prefix + "Starting")

    simple_choices = None
    try:

        body = {
            "query" : {
                "match_all" : {}
            },
            "size" : 1000
        }

        results = get_es().search(ES_PREFIX + 'threat_actor_simple', 'data', body)

        parsed_results = []
        for r in results['hits']['hits']:
            d = {}
            d['type'] = r['_source']['type']
            d['value'] = r['_source']['value']
            d['id'] = r['_id']

            #determine how many actor profiles use this value

            query_string = None
            if d['type'] == "classification":
                query_string = "type:\"" + escape(d['value']) + "\""
            elif d['type'] == "communication":
                query_string = "communication_address.type:\"" + escape(d['value']) + "\""
            elif d['type'] == "country":
                query_string = "country_affiliation:\"" + escape(d['value']) + "\" origin:\"" + escape(d['value']) + "\""

            if query_string:
                body = {
                    "query" : {
                        "query_string" : {
                            "query" : query_string
                        }
                    },
                    "size" : 0
                }

                count = get_es().search(ES_PREFIX + 'threat_actors', 'actor', body)

                d['count'] = count['hits']['total']
                d['q'] = quote_plus(query_string)
            else:
                d['count'] = "-"
                d['q'] = ""

            parsed_results.append(d)

        simple_choices = multikeysort(parsed_results, ['type', 'value'])

    except Exception as e:
        error = "There was an error completing your request. Details: {}".format(e)
        flash(error,'danger')
        log.exception(logging_prefix + error)

    return render_template("admin_choices.html",
                        page_title="Admin",
                        simple_choices = simple_choices
            )
Ejemplo n.º 20
0
def es_to_tpx(actor_id):
    '''
    Build the TPX file from the data stored in Elasticsearch
    '''
    element_observables = {}

    results = get_es().get(ES_PREFIX + "threat_actors", doc_type="actor", id=actor_id)
    actor_data = results['_source']

    tpx = {}
    tpx["schema_version_s"] = "2.2.0"
    tpx["provider_s"] = "LookingGlass"
    tpx["list_name_s"] = "Threat Actor"
    tpx["created_t"] = actor_data['created_milli']
    tpx["created_s"] = actor_data['created_s']
    tpx["last_updated_t"] = actor_data['last_updated_milli']
    tpx["last_updated_s"] = actor_data['last_updated_s']
    tpx["score_i"] = 95
    tpx["source_observable_s"] = "Cyveillance Threat Actor"
    tpx["source_description_s"] = "This feed provides threat actor or threat actor group profiles and characterizations created by the LookingGlass Cyber Threat Intelligence Group"

    tpx["observable_dictionary_c_array"] = []

    observable_dict = {}
    observable_dict["observable_id_s"] = actor_data['name']
    observable_dict["actor_uuid_s"] = actor_id
    observable_dict["criticality_i"] = actor_data['criticality']

    observable_dict["classification_c_array"] = []
    for i in actor_data['classification']:
        class_dict = {}
        class_dict["score_i"] = i["score"]
        #class_dict["score_i"] = get_score(i["family"], i["id"])
        class_dict["classification_id_s"] = i["id"]
        class_dict["classification_family_s"] = i["family"]

        observable_dict["classification_c_array"].append(class_dict)

    observable_dict["description_s"]  = actor_data['description']

    observable_dict["attribute_c_map"]  = {}

    observable_dict["attribute_c_map"]["last_seen_t"]  = actor_data['last_updated_milli']
    observable_dict["attribute_c_map"]["last_seen_s"]  = actor_data['last_updated_s']

    observable_dict["attribute_c_map"]["tlp_i"]  = actor_data['tlp']

    observable_dict["attribute_c_map"]["motivation_s_array"]  = []
    for i in actor_data["motivation"]:
        observable_dict["attribute_c_map"]["motivation_s_array"].append(i)

    observable_dict["attribute_c_map"]["aliases_s_array"]  = []
    for i in actor_data["alias"]:
        if i: #make sure its not blank
            observable_dict["attribute_c_map"]["aliases_s_array"].append(i)

    observable_dict["attribute_c_map"]["communications_c_map"]  = {}
    if "communication_address" in actor_data:
        for i in actor_data["communication_address"]:
            #build the key, this is a bit tricky if the user enters special characters,
            #or has multiple spaces (possible trailing space, should address this when the
            # data is submitted) the key could get messed up.
            # Example: Jabber -> jabber_s_array
            key = i['type'].lower().replace(" ","_") +"_s_array"

            if key not in observable_dict["attribute_c_map"]["communications_c_map"]:
                observable_dict["attribute_c_map"]["communications_c_map"][key] = []

            if i['value']:
                observable_dict["attribute_c_map"]["communications_c_map"][key].append(i['value'])

    observable_dict["attribute_c_map"]["financial_accounts_c_map"]  = {}
    if "financial_account" in actor_data:
        for i in actor_data["financial_account"]:
            #build the key, this is a bit tricky if the user enters special characters,
            #or has multiple spaces (possible trailing space, should address this when the
            # data is submitted) the key could get messed up.
            # Example: Jabber -> jabber_s_array
            key = i['type'].lower().replace(" ","_") +"_s_array"

            if key not in observable_dict["attribute_c_map"]["financial_accounts_c_map"]:
                observable_dict["attribute_c_map"]["financial_accounts_c_map"][key] = []

            if i['value']:
                observable_dict["attribute_c_map"]["financial_accounts_c_map"][key].append(i['value'])

    observable_dict["attribute_c_map"]["frequent_locations_s_array"]  = []
    for i in actor_data["frequented_location"]:
        if i:
            observable_dict["attribute_c_map"]["frequent_locations_s_array"].append(i)

    observable_dict["attribute_c_map"]["affiliations_s_array"]  = []
    for i in actor_data["country_affiliation"]:
        observable_dict["attribute_c_map"]["affiliations_s_array"].append(i)

    observable_dict["attribute_c_map"]["known_targets_s_array"]  = []
    for i in actor_data["known_target"]:
        observable_dict["attribute_c_map"]["known_targets_s_array"].append(i)

    observable_dict["attribute_c_map"]["suspected_point_of_origin_s"]  = actor_data['origin']
    
    observable_dict["attribute_c_map"]["infrastructure_ipv4_s_array"]  = []
    for i in actor_data["infrastructure"]["ipv4"]:
        if i:
            observable_dict["attribute_c_map"]["infrastructure_ipv4_s_array"].append(i)
    
    observable_dict["attribute_c_map"]["infrastructure_fqdn_s_array"]  = []
    for i in actor_data["infrastructure"]["fqdn"]:
        if i:
            observable_dict["attribute_c_map"]["infrastructure_fqdn_s_array"].append(i)

    observable_dict["attribute_c_map"]["infrastructure_action_s"]  = actor_data["infrastructure"]["action"]
    observable_dict["attribute_c_map"]["infrastructure_operation_s"]  = actor_data["infrastructure"]["operation"]
    observable_dict["attribute_c_map"]["infrastructure_status_s"]  = actor_data["infrastructure"]["status"]

    observable_dict["attribute_c_map"]["infrastructure_type_s_array"]  = []
    for i in actor_data["infrastructure"]["type"]:
        observable_dict["attribute_c_map"]["infrastructure_type_s_array"].append(i)

    observable_dict["attribute_c_map"]["detection_c_map"]  = {}
    for i in actor_data["detection_rule"]:
        #build the key, this is a bit tricky if the user enters special characters,
        #or has multiple spaces (possible trailing space, should address this when the
        # data is submitted) the key could get messed up.
        # Example: Snort -> snort_rule_s_map
        key = i['type'].lower().replace(" ","_") +"_s_map"

        if not i['value']:
            continue

        if key not in observable_dict["attribute_c_map"]["detection_c_map"]:
            observable_dict["attribute_c_map"]["detection_c_map"][key] = {}

        observable_dict["attribute_c_map"]["detection_c_map"][key][i['id']] = i['value']

    '''
    Related elements
    '''

    relate_element_name_map = {
            "FQDN" : "subject_fqdn_s",
            "IPv4" : "subject_ipv4_s",
            "TTP" : "subject_ttp_s",
            "CommAddr" : "subject_address_s"
        }

    for i in actor_data['related_element_choices']:
        v = i['value']

        print(v)

        v_array = v.split(":::")

        if len(v_array) == 2:
            field_name = relate_element_name_map[v_array[0]]
            field_type = "_DEFAULT_"
            field_data = v_array[1]
        elif len(v_array) == 3:
            field_name = relate_element_name_map[v_array[1]]
            field_type = v_array[0]
            field_data = v_array[2]
        else:
            raise Exception("Unknown Related Element: '{}'".format(v))
        
        field_observable = actor_data['name']
        
        if field_name not in element_observables:
            element_observables[field_name] = {}

        if field_type not in element_observables[field_name]:
            element_observables[field_name][field_type] = {}

        if field_data not in element_observables[field_name][field_type]:
            element_observables[field_name][field_type][field_data] = []

        if field_observable not in element_observables[field_name][field_type][field_data]:
            element_observables[field_name][field_type][field_data].append(field_observable)

    observable_dict["attribute_c_map"]["related_ttps_c_array"]  = []
    for i in actor_data['related_ttp']:
        if i['name']:
            observable_dict["attribute_c_map"]["related_ttps_c_array"].append({ "name_s" :  i['name'], "uuid_s" : i['id'] })
            field_observable = i['name']

            for j in i['elements']:
                element_array = j.split(":::")

                if len(element_array) == 3:
                    field_type = element_array[0]
                    field_name = relate_element_name_map[element_array[1]]
                    field_data = element_array[2]
                elif len(element_array) == 2:
                    field_type = "_DEFAULT_"
                    field_name = relate_element_name_map[element_array[0]]
                    field_data = element_array[1]
                else:
                    raise Exception("Invalid element '{}'".format(j))

                if field_name not in element_observables:
                    element_observables[field_name] = {}

                if field_type not in element_observables[field_name]:
                    element_observables[field_name][field_type] = {}

                if field_data not in element_observables[field_name][field_type]:
                    element_observables[field_name][field_type][field_data] = []

                if field_observable not in element_observables[field_name][field_type][field_data]:
                    element_observables[field_name][field_type][field_data].append(field_observable)


    observable_dict["attribute_c_map"]["related_actors_c_array"]  = []
    for i in actor_data['related_actor']:
        if i['name']:
            observable_dict["attribute_c_map"]["related_actors_c_array"].append({ "name_s" :  i['name'], "uuid_s" : i['id'] })
            field_observable = i['name']

            for j in i['elements']:
                element_array = j.split(":::")

                if len(element_array) == 3:
                    field_type = element_array[0]
                    field_name = relate_element_name_map[element_array[1]]
                    field_data = element_array[2]
                elif len(element_array) == 2:
                    field_type = "_DEFAULT_"
                    field_name = relate_element_name_map[element_array[0]]
                    field_data = element_array[1]
                else:
                    raise Exception("Invalid element '{}'".format(j))

                if field_name not in element_observables:
                    element_observables[field_name] = {}

                if field_type not in element_observables[field_name]:
                    element_observables[field_name][field_type] = {}

                if field_data not in element_observables[field_name][field_type]:
                    element_observables[field_name][field_type][field_data] = []

                if field_observable not in element_observables[field_name][field_type][field_data]:
                    element_observables[field_name][field_type][field_data].append(field_observable)

    observable_dict["attribute_c_map"]["related_reports_c_array"]  = []
    for i in actor_data['related_report']:
        if i['name']:
            observable_dict["attribute_c_map"]["related_reports_c_array"].append({ "name_s" :  i['name'], "uuid_s" : i['id'] })
            field_observable = i['name']

            for j in i['elements']:
                element_array = j.split(":::")

                if len(element_array) == 3:
                    field_type = element_array[0]
                    field_name = relate_element_name_map[element_array[1]]
                    field_data = element_array[2]
                elif len(element_array) == 2:
                    field_type = "_DEFAULT_"
                    field_name = relate_element_name_map[element_array[0]]
                    field_data = element_array[1]
                else:
                    raise Exception("Invalid element '{}'".format(j))

                if field_name not in element_observables:
                    element_observables[field_name] = {}

                if field_type not in element_observables[field_name]:
                    element_observables[field_name][field_type] = {}

                if field_data not in element_observables[field_name][field_type]:
                    element_observables[field_name][field_type][field_data] = []

                if field_observable not in element_observables[field_name][field_type][field_data]:
                    element_observables[field_name][field_type][field_data].append(field_observable)


    tpx["observable_dictionary_c_array"].append(observable_dict)

    '''
    Element Observables - the reverse linkage
    '''

    tpx["element_observable_c_array"] = []

    for field,val in element_observables.items():
        for sub_type, val1 in val.items():
            for data, val2 in val1.items():
                e_dict = {}

                e_dict[field] = data

                if sub_type != "_DEFAULT_":
                    e_dict['type_s'] = sub_type

                e_dict["threat_observable_c_map"] = {}
                for observable_id in val2:
                     e_dict["threat_observable_c_map"][observable_id] = {}

                     e_dict["threat_observable_c_map"][observable_id]["occurred_at_t"] = actor_data['created_milli']
                     e_dict["threat_observable_c_map"][observable_id]["occurred_at_s"] = actor_data['created_s']
                     e_dict["threat_observable_c_map"][observable_id]["last_seen_t"] = actor_data['last_updated_milli']
                     e_dict["threat_observable_c_map"][observable_id]["last_seen_s"] = actor_data['last_updated_s']

                tpx["element_observable_c_array"].append(e_dict)

    return tpx
Ejemplo n.º 21
0
def es_to_form(report_id):
    form = forms.reportForm()

    # get the values from ES
    results = get_es().get(ES_PREFIX + "threat_reports", doc_type="report", id=report_id)

    # store certain fields from ES, so this form can be used in an update
    form.doc_index.data = results["_index"]
    form.doc_type.data = results["_type"]

    report_data = results["_source"]

    form.report_name.data = report_data["name"]
    form.report_id.data = report_data["identifier"]
    form.report_occurred_at.data = datetime.strptime(report_data["created_s"], "%Y-%m-%dT%H:%M:%S")
    form.report_description.data = report_data["description"]
    form.report_criticality.data = report_data["criticality"]

    idx = 0
    for entry in range(len(form.report_class.entries)):
        form.report_class.pop_entry()
    for i in multikeysort(report_data["classification"], ["family", "id"]):
        report_class_form = forms.TPXClassificationForm()
        report_class_form.a_family = i["family"]
        report_class_form.a_id = i["id"]

        form.report_class.append_entry(report_class_form)

        # set the options since this select is dynamic
        form.report_class[idx].a_id.choices = fetch_child_data("tpx_classification", i["family"])
        idx += 1

    form.report_tlp.data = int(report_data["tlp"])

    for entry in range(len(form.report_sections.entries)):
        form.report_sections.pop_entry()
    for i in multikeysort(report_data["section"], ["order"]):
        report_section_form = forms.ReportSectionsForm()
        report_section_form.title = i["title"]
        report_section_form.tlp = i["tlp"]
        report_section_form.text = i["content"]

        form.report_sections.append_entry(report_section_form)

    form.report_source_reliability.data = report_data["source_reliability"]
    form.report_info_reliability.data = report_data["info_reliability"]

    for entry in range(len(form.report_sources.entries)):
        form.report_sources.pop_entry()
    for i in sorted(report_data["source"]):
        report_source_form = forms.ReportSourcesForm()
        report_source_form.source = i

        form.report_sources.append_entry(report_source_form)

    """
    Related element
    """

    for entry in range(len(form.report_actors.entries)):
        form.report_actors.pop_entry()
    if report_data["related_actor"]:
        idx = 0
        for i in multikeysort(report_data["related_actor"], ["name", "id"]):
            sub_form = forms.RelatedActorsForm()
            sub_form.data = i["id"] + ":::" + i["name"]
            has_related_elements = False

            idx2 = 0
            for entry in range(len(sub_form.related_elements.entries)):
                sub_form.related_elements.pop_entry()
            for j in multikeysort(report_data["related_element_choices"], ["display_text"]):
                sub_sub_form = forms.ElementObservablesFrom()

                sub_form.related_elements.append_entry(sub_sub_form)

                is_related = j["value"] in i["elements"]
                sub_form.related_elements[idx2].element = is_related
                sub_form.related_elements[idx2].element_value = j["value"]
                sub_form.related_elements[idx2].element_text = j["display_text"]

                if is_related:
                    has_related_elements = True

                idx2 += 1

            form.report_actors.append_entry(sub_form)

            form.report_actors[idx].has_related_elements.data = has_related_elements
            idx += 1
    else:
        sub_form = forms.RelatedActorsForm()
        sub_form.data = "_NONE_"
        form.report_actors.append_entry(sub_form)

    for entry in range(len(form.report_reports.entries)):
        form.report_reports.pop_entry()
    if report_data["related_report"]:
        idx = 0
        for i in multikeysort(report_data["related_report"], ["name", "id"]):
            sub_form = forms.RelatedReportsForm()
            sub_form.data = i["id"] + ":::" + i["name"]
            has_related_elements = False

            idx2 = 0
            for entry in range(len(sub_form.related_elements.entries)):
                sub_form.related_elements.pop_entry()
            for j in multikeysort(report_data["related_element_choices"], ["display_text"]):
                sub_sub_form = forms.ElementObservablesFrom()

                sub_form.related_elements.append_entry(sub_sub_form)

                is_related = j["value"] in i["elements"]
                sub_form.related_elements[idx2].element = is_related
                sub_form.related_elements[idx2].element_value = j["value"]
                sub_form.related_elements[idx2].element_text = j["display_text"]

                if is_related:
                    has_related_elements = True

                idx2 += 1

            form.report_reports.append_entry(sub_form)

            form.report_reports[idx].has_related_elements.data = has_related_elements
            idx += 1
    else:
        sub_form = forms.RelatedReportsForm()
        sub_form.data = "_NONE_"
        form.report_reports.append_entry(sub_form)

    for entry in range(len(form.report_ttps.entries)):
        form.report_ttps.pop_entry()
    if report_data["related_ttp"]:
        idx = 0
        for i in multikeysort(report_data["related_ttp"], ["name", "id"]):
            sub_form = forms.RelatedTTPsForm()
            sub_form.data = i["id"] + ":::" + i["name"]
            has_related_elements = False

            idx2 = 0
            for entry in range(len(sub_form.related_elements.entries)):
                sub_form.related_elements.pop_entry()
            for j in multikeysort(report_data["related_element_choices"], ["display_text"]):
                sub_sub_form = forms.ElementObservablesFrom()

                sub_form.related_elements.append_entry(sub_sub_form)

                is_related = j["value"] in i["elements"]
                sub_form.related_elements[idx2].element = is_related
                sub_form.related_elements[idx2].element_value = j["value"]
                sub_form.related_elements[idx2].element_text = j["display_text"]

                if is_related:
                    has_related_elements = True

                idx2 += 1

            form.report_ttps.append_entry(sub_form)

            form.report_ttps[idx].has_related_elements.data = has_related_elements
            idx += 1
    else:
        sub_form = forms.RelatedTTPsForm()
        sub_form.data = "_NONE_"
        form.report_ttps.append_entry(sub_form)

    # convert editor dictionary of ids and times to names and times
    editors = get_editor_names(get_mysql(), report_data["editor"])

    return form, editors
Ejemplo n.º 22
0
def es_to_form(actor_id):
    form = forms.actorForm()
    editors = []

    #get the values from ES
    results = get_es().get(ES_PREFIX + "threat_actors", doc_type="actor", id=actor_id)

    #store certain fields from ES, so this form can be used in an update
    form.doc_index.data = results['_index']
    form.doc_type.data = results['_type']

    actor_data = results['_source']

    form.actor_name.data = actor_data['name']
    form.actor_occurred_at.data = datetime.strptime(actor_data['created_s'],"%Y-%m-%dT%H:%M:%S")
    form.actor_description.data  = actor_data['description']
    form.actor_criticality.data  = actor_data['criticality']

    #blank this field, and then add data
    idx = 0
    for entry in range(len(form.actor_class.entries)): form.actor_class.pop_entry()
    for i in multikeysort(actor_data['classification'], ['family', 'id']):
        actor_class_form = forms.TPXClassificationForm()
        actor_class_form.a_family = i['family']
        actor_class_form.a_id = i['id']

        form.actor_class.append_entry(actor_class_form)

        #set the options since this select is dynamic
        form.actor_class[idx].a_id.choices = fetch_child_data('tpx_classification',i['family'])
        idx += 1

    form.actor_tlp.data = str(actor_data['tlp'])

    for entry in range(len(form.actor_type.entries)): form.actor_type.pop_entry()
    for i in sorted(actor_data['type']):
        actor_type_form = forms.TypeForm()
        actor_type_form.a_type = i
        form.actor_type.append_entry(actor_type_form)


    for entry in range(len(form.actor_motivations.entries)): form.actor_motivations.pop_entry()
    for i in sorted(actor_data['motivation']):
        actor_motivation_form = forms.MotivationForm()
        actor_motivation_form.motivation = i
        actor_motivation_form.motivation_other = ""
        form.actor_motivations.append_entry(actor_motivation_form)

    for entry in range(len(form.actor_aliases.entries)): form.actor_aliases.pop_entry()
    for i in sorted(actor_data['alias']):
        actor_alias_form = forms.AliasForm()
        actor_alias_form.alias = i
        form.actor_aliases.append_entry(actor_alias_form)

    for entry in range(len(form.actor_comms.entries)): form.actor_comms.pop_entry()
    for i in multikeysort(actor_data['communication_address'], ['type', 'value']):
        actor_comms_form = forms.CommunicationsForm()
        actor_comms_form.a_type = i['type']
        actor_comms_form.a_type_other = ""
        actor_comms_form.address = i['value']
        form.actor_comms.append_entry(actor_comms_form)

    for entry in range(len(form.actor_financials.entries)): form.actor_financials.pop_entry()
    for i in multikeysort(actor_data['financial_account'], ['type', 'value']):
        actor_fin_form = forms.FinancialsForm()
        actor_fin_form.f_type = i['type']
        actor_fin_form.f_type_other = ""
        actor_fin_form.account = i['value']
        form.actor_financials.append_entry(actor_fin_form)

    for entry in range(len(form.actor_locations.entries)): form.actor_locations.pop_entry()
    for i in sorted(actor_data['frequented_location']):
        actor_locations_form = forms.LocationsForm()
        actor_locations_form.location = i
        form.actor_locations.append_entry(actor_locations_form)

    for entry in range(len(form.actor_affliations.entries)): form.actor_affliations.pop_entry()
    for i in sorted(actor_data['country_affiliation']):
        actor_affil_form = forms.AffiliationsForm()
        actor_affil_form.affiliation = i
        actor_affil_form.affiliation_other = ""
        form.actor_affliations.append_entry(actor_affil_form)

    for entry in range(len(form.actor_known_targets.entries)): form.actor_known_targets.pop_entry()
    for i in sorted(actor_data['known_target']):
        actor_known_target_form = forms.KnownTargetsForm()
        actor_known_target_form.target = i
        actor_known_target_form.target_other = ""
        form.actor_known_targets.append_entry(actor_known_target_form)

    for entry in range(len(form.actor_origin.entries)): form.actor_origin.pop_entry()
    #theres only one of these, at least there should be
    actor_affil_form = forms.AffiliationsForm()
    actor_affil_form.affiliation = actor_data['origin']
    actor_affil_form.affiliation_other = ""
    form.actor_origin.append_entry(actor_affil_form)

    for entry in range(len(form.actor_infra_ipv4.entries)): form.actor_infra_ipv4.pop_entry()
    for i in sorted(actor_data['infrastructure']['ipv4']):
        actor_infra_ipv4_form = forms.InfrastructureIPv4Form()
        actor_infra_ipv4_form.ipv4 = i
        form.actor_infra_ipv4.append_entry(actor_infra_ipv4_form)

    for entry in range(len(form.actor_infra_fqdn.entries)): form.actor_infra_fqdn.pop_entry()
    for i in sorted(actor_data['infrastructure']['fqdn']):
        actor_infra_fqdn_form = forms.InfrastructureIPv4Form()
        actor_infra_fqdn_form.fqdn = i
        form.actor_infra_fqdn.append_entry(actor_infra_fqdn_form)

    form.actor_infra_action.data = actor_data['infrastructure']['action']
    form.actor_infra_operation.data = actor_data['infrastructure']['operation']
    form.actor_infra_status.data = actor_data['infrastructure']['status']


    for entry in range(len(form.actor_infra_types.entries)): form.actor_infra_types.pop_entry()
    for i in sorted(actor_data['infrastructure']['type']):
        actor_infra_type_form = forms.InfrastructureTypesForm()
        actor_infra_type_form.infra_type = i
        form.actor_infra_types.append_entry(actor_infra_type_form)

    for entry in range(len(form.actor_detections.entries)): form.actor_detections.pop_entry()
    for i in multikeysort(actor_data['detection_rule'], ["type", "id"]):
        actor_detection_form = forms.DetectionsForm()
        actor_detection_form.d_type = i['type']
        actor_detection_form.rule_id = i['id']
        actor_detection_form.rule = i['value']
        form.actor_detections.append_entry(actor_detection_form)

    '''
    Related element
    '''
    
    for entry in range(len(form.actor_actors.entries)): form.actor_actors.pop_entry()
    if actor_data['related_actor']:
        idx = 0
        for i in multikeysort(actor_data['related_actor'], ['name', 'id']):
            sub_form = forms.RelatedActorsForm()
            sub_form.data = i['id'] + ":::" + i['name']
            has_related_elements = False

            idx2=0
            for entry in range(len(sub_form.related_elements.entries)): sub_form.related_elements.pop_entry()
            for j in multikeysort(actor_data['related_element_choices'],['display_text']):
                sub_sub_form = forms.ElementObservablesFrom()

                sub_form.related_elements.append_entry(sub_sub_form)

                is_related = (j['value'] in i['elements'])
                sub_form.related_elements[idx2].element = is_related
                sub_form.related_elements[idx2].element_value = j['value']
                sub_form.related_elements[idx2].element_text = j['display_text']

                if is_related:
                    has_related_elements = True

                idx2 +=1

            form.actor_actors.append_entry(sub_form)

            form.actor_actors[idx].has_related_elements.data = has_related_elements
            idx+=1
    else:
        sub_form = forms.RelatedActorsForm()
        sub_form.data = "_NONE_"
        form.actor_actors.append_entry(sub_form)
            
    for entry in range(len(form.actor_reports.entries)): form.actor_reports.pop_entry()
    if actor_data['related_report']:
        idx = 0
        for i in multikeysort(actor_data['related_report'], ['name', 'id']):
            sub_form = forms.RelatedReportsForm()
            sub_form.data = i['id'] + ":::" + i['name']
            has_related_elements = False

            idx2=0
            for entry in range(len(sub_form.related_elements.entries)): sub_form.related_elements.pop_entry()
            for j in multikeysort(actor_data['related_element_choices'],['display_text']):
                sub_sub_form = forms.ElementObservablesFrom()

                sub_form.related_elements.append_entry(sub_sub_form)

                is_related = (j['value'] in i['elements'])
                sub_form.related_elements[idx2].element = is_related
                sub_form.related_elements[idx2].element_value = j['value']
                sub_form.related_elements[idx2].element_text = j['display_text']

                if is_related:
                    has_related_elements = True

                idx2 +=1

            form.actor_reports.append_entry(sub_form)

            form.actor_reports[idx].has_related_elements.data = has_related_elements
            idx+=1
    else:
        sub_form = forms.RelatedReportsForm()
        sub_form.data = "_NONE_"
        form.actor_reports.append_entry(sub_form)

            
    for entry in range(len(form.actor_ttps.entries)): form.actor_ttps.pop_entry()
    if actor_data['related_ttp']:
        idx = 0
        for i in multikeysort(actor_data['related_ttp'], ['name', 'id']):
            sub_form = forms.RelatedTTPsForm()
            sub_form.data = i['id'] + ":::" + i['name']
            has_related_elements = False

            idx2=0
            for entry in range(len(sub_form.related_elements.entries)): sub_form.related_elements.pop_entry()
            for j in multikeysort(actor_data['related_element_choices'],['display_text']):
                sub_sub_form = forms.ElementObservablesFrom()

                sub_form.related_elements.append_entry(sub_sub_form)

                is_related = (j['value'] in i['elements'])
                sub_form.related_elements[idx2].element = is_related
                sub_form.related_elements[idx2].element_value = j['value']
                sub_form.related_elements[idx2].element_text = j['display_text']

                if is_related:
                    has_related_elements = True

                idx2 +=1

            form.actor_ttps.append_entry(sub_form)

            form.actor_ttps[idx].has_related_elements.data = has_related_elements
            idx+=1

    else:
        sub_form = forms.RelatedTTPsForm()
        sub_form.data = "_NONE_"
        form.actor_ttps.append_entry(sub_form)

    #convert editor dictionary of ids and times to names and times
    editors = get_editor_names(get_mysql(), actor_data['editor'])

    return form, editors
Ejemplo n.º 23
0
def form_to_es(form, actor_id, update=False):
    logging_prefix = logger_prefix + "form_to_es() - "
    log.info(logging_prefix + "Converting Form to ES for {}".format(actor_id))
    
    doc = {}
    element_observables = {}

    doc["approval_state"] = 0   # 0 - newly added
                                # 1 - scores have been approved and this actor is ready for SV

    created_t = int(time.mktime(form.actor_occurred_at.data.timetuple())) * 1000
    created_s = form.actor_occurred_at.data.strftime("%Y-%m-%dT%H:%M:%S")
    now_t = int(time.time()) * 1000
    now_s = datetime.now().strftime("%Y-%m-%dT%H:%M:%S")

    doc["created_milli"] = created_t
    doc["created_s"] = created_s
    doc["last_updated_milli"] = now_t
    doc["last_updated_s"] = now_s

    observable_dictionary = {}

    actor_name = form.actor_name.data

    doc['name'] = escape(actor_name)
    doc['criticality'] = int(escape(form.actor_criticality.data))

    doc['classification'] = []
    for sub_form in form.actor_class.entries:
        classification_dict = {}
        classification_dict['score'] = int(get_score(sub_form.data['a_family'], sub_form.data['a_id']))
        classification_dict['id'] = escape(sub_form.data['a_id'])
        classification_dict['family'] = escape(sub_form.data['a_family'])

        if classification_dict not in doc['classification']:
            doc['classification'].append(classification_dict)

    doc["description"] = escape(form.actor_description.data)
    
    doc["tlp"] = int(form.actor_tlp.data)

    '''
    Types 
    '''
    doc["type"] = [] # ??? should we change this to actor_type_s_array
    for sub_form in form.actor_type.entries:                                      # since the field is labeled Actor Types
        actor_type = sub_form.data['a_type']

        if actor_type not in doc["type"]:
            doc["type"].append(escape(actor_type))

    '''
    Motivations 
    '''
    doc['motivation'] = []
    for sub_form in form.actor_motivations.entries:
        motivation = sub_form.data['motivation']

        if motivation == '_NEW_':
            motivation = sub_form.data['motivation_other']
            add_to_store('motivation', motivation, True)

        if motivation not in doc['motivation']:
            doc['motivation'].append(escape(motivation))

    '''
    Aliases 
    '''
    doc['alias'] = []
    for sub_form in form.actor_aliases.entries:
        alias = sub_form.data['alias']

        if alias not in doc['alias']:
            doc['alias'].append(escape(alias))

    '''
    Communications 
    '''
    doc['communication_address'] = []
    for sub_form in form.actor_comms.entries:
        address = sub_form.data['address']
        comm_type = sub_form.data['a_type']

        if comm_type == "_NEW_":
            comm_type = sub_form.data['a_type_other']
            add_to_store('communication', comm_type, True)

        #since comm_type is used as a key in the tpx file, sanitize it first
        #comm_type = comm_type.lower().replace("  "," ").replace(" ","_") + "_s_array"

        data = {}
        data['type'] = escape(comm_type)
        data['value'] = escape(address)

        if data not in doc['communication_address']:
            doc['communication_address'].append(data)

    '''
    Financials 
    '''
    doc["financial_account"] = []
    for sub_form in form.actor_financials.entries:
        account = sub_form.data['account']
        fin_type = sub_form.data['f_type']

        if fin_type == "_NEW_":
            fin_type = sub_form.data['f_type_other']
            add_to_store('financial', fin_type, True)

        #since fin_type is used as a key in the tpx file, sanitize it first
        #fin_type = fin_type.lower().replace("  "," ").replace(" ","_") + "_s_array"

        data = {}
        data['type'] = escape(fin_type)
        data['value'] = escape(account)

        if data not in doc["financial_account"]:
            doc["financial_account"].append(data)

    '''
    Frequented Locations 
    '''
    doc["frequented_location"] = []
    for sub_form in form.actor_locations.entries:
        location = sub_form.data['location']

        if location not in doc["frequented_location"]:
            doc["frequented_location"].append(escape(location))

    '''
    Country Affliations 
    '''
    doc["country_affiliation"] = []
    for sub_form in form.actor_affliations.entries:
        affiliation = sub_form.data['affiliation']

        if affiliation == '_NEW_':
            affiliation = sub_form.data['affiliation_other']
            add_to_store('country', affiliation, True)

        if affiliation not in doc["country_affiliation"]:
            doc["country_affiliation"].append(escape(affiliation))

    '''
    Known Targets
    '''
    doc["known_target"] = []
    for sub_form in form.actor_known_targets.entries:
        target = sub_form.data['target']

        if target == '_NEW_':
            target = sub_form.data['target_other']
            add_to_store('known_target', target, True)

        if target not in doc["known_target"]:
            doc["known_target"].append(escape(target))

    '''
    Origin
    '''
    #this one is different, because i borrowed from the country_affliation sub form
    affiliation = form.actor_origin.data[0]['affiliation']

    if affiliation == '_NEW_':
        affiliation = form.actor_origin.data[0]['affiliation_other']
        add_to_store('country', affiliation, True)

    doc["origin"] = escape(affiliation)

    '''
    Infrastructure IPv4s
    '''
    doc['infrastructure'] = {}
    doc['infrastructure']['ipv4'] = []
    for sub_form in form.actor_infra_ipv4.entries:
        ip = sub_form.data['ipv4']

        if ip not in doc['infrastructure']['ipv4']:
            doc['infrastructure']['ipv4'].append(escape(ip))

    '''
    Infrastructure FQDNs
    '''
    doc['infrastructure']['fqdn'] = []
    for sub_form in form.actor_infra_fqdn.entries:
        fqdn = sub_form.data['fqdn']

        if fqdn not in doc['infrastructure']['fqdn']:
            doc['infrastructure']['fqdn'].append(escape(fqdn))

    '''
    Infrastructure Action, Operation, Status
    '''
    doc['infrastructure']['action'] = escape(form.actor_infra_action.data)
    doc['infrastructure']['operation'] = escape(form.actor_infra_operation.data)
    doc['infrastructure']['status'] = escape(form.actor_infra_status.data)
     
    '''
    Infrastructure Types
    '''
    doc['infrastructure']['type'] = []
    for sub_form in form.actor_infra_types.entries:
        _type = sub_form.data['infra_type']

        if location not in doc['infrastructure']['type']:
            doc['infrastructure']['type'].append(escape(_type))

    '''
    Detection Rules
    '''
    doc['detection_rule'] = []
    for sub_form in form.actor_detections.entries:
        rule = sub_form.data['rule']
        rule_id = sub_form.data['rule_id']
        rule_type = sub_form.data['d_type']

        #since comm_type is used as a key in the tpx file, sanitize it first
        #rule_type = rule_type.lower().replace("  "," ").replace(" ","_") + "_rule_s_array"
        
        data = {}
        data['type'] = escape(rule_type)
        data['value'] = escape(rule)
        data['id'] = escape(rule_id)

        if data not in doc['detection_rule']:
            doc['detection_rule'].append(data)


    '''
    Related elements
    '''

    my_id = actor_id

    #Links to actors and reports
    doc['related_actor'] = []
    for sub_form in form.actor_actors.entries:
        r_dict = {}
        data = escape(sub_form.data.data)

        if data == "_NONE_":

            continue

        data_array = data.split(":::")

        r_dict['id'] = escape(data_array[0])

        #if the related element's id = my id, dont add to es, this should only apply to current type
        if data_array[0] == my_id:
            flash("A related actor matched this actor, this entry has been discarded", "warning")
            continue

        r_dict['name'] = escape(data_array[1])  
                                        #this is gonna be a nightmare to maintain, 
                                        # but it make sense to have this for searches

        r_dict['elements'] = []
        for sub_sub_form in sub_form.related_elements.entries:
            if sub_sub_form.element.data:
                if sub_sub_form.element_value.data not in r_dict['elements']:
                    r_dict['elements'].append(sub_sub_form.element_value.data)

        if r_dict not in doc['related_actor']:                                
            doc['related_actor'].append(r_dict)

    doc['related_report'] = []
    for sub_form in form.actor_reports.entries:
        r_dict = {}
        data = escape(sub_form.data.data)

        if data == "_NONE_":
            continue

        data_array = data.split(":::")

        r_dict['id'] = escape(data_array[0])

        #if the related element's id = my id, dont add to es, this should only apply to current type
        if data_array[0] == my_id:
            flash("A related report matched this report, this entry has been discarded", "warning")
            continue

        r_dict['name'] = escape(data_array[1])  
                                        #this is gonna be a nightmare to maintain, 
                                        # but it make sense to have this for searches
        
        r_dict['elements'] = []
        for sub_sub_form in sub_form.related_elements.entries:
            if sub_sub_form.element.data:
                if sub_sub_form.element_value.data not in r_dict['elements']:
                    r_dict['elements'].append(sub_sub_form.element_value.data)

        if r_dict not in doc['related_report']:                                
            doc['related_report'].append(r_dict)

    doc['related_ttp'] = []
    for sub_form in form.actor_ttps.entries:
        r_dict = {}
        data = escape(sub_form.data.data)

        if data == "_NONE_":
            continue

        data_array = data.split(":::")

        r_dict['id'] = escape(data_array[0])

        #if the related element's id = my id, dont add to es, this should only apply to current type
        if data_array[0] == my_id:
            flash("A related TTP matched this TTP, this entry has been discarded", "warning")
            continue

        r_dict['name'] = escape(data_array[1])  
                                        #this is gonna be a nightmare to maintain, 
                                        # but it make sense to have this for searches

        r_dict['elements'] = []
        for sub_sub_form in sub_form.related_elements.entries:
            if sub_sub_form.element.data:
                if sub_sub_form.element_value.data not in r_dict['elements']:
                    r_dict['elements'].append(sub_sub_form.element_value.data)

        if r_dict not in doc['related_ttp']:                                
            doc['related_ttp'].append(r_dict)

    #store these to make life easier
    doc['related_element_choices'] = []
    rec = json.loads(form.related_element_choices.data)
    for k,v in rec.items():
        #dont include the n/a choice, as this will be first and handled manually
        if v != "_NONE_":
            #this will create a tuple ("display text", "value string")
            doc['related_element_choices'].append({"display_text" : k, "value" : v})

    '''
    Edit Tracking
    '''

    doc['editor'] = get_editor_list( 
            es=get_es(),
            index=ES_PREFIX + "threat_actors",
            doc_type="actor",
            item_id=actor_id,
            user_id=session.get('id',None)
        )

    #index the doc
    #print_tpx(doc['related_ttp'])
    #print_tpx(doc['related_element_choices'])
    #response = None


    log.info(logging_prefix + "Start Indexing of {}".format(actor_id))
    response = get_es().index(ES_PREFIX + "threat_actors", "actor", doc, actor_id)
    log.info(logging_prefix + "Done Indexing of {}".format(actor_id))

    return response, doc
Ejemplo n.º 24
0
def export_all_the_data():
    logging_prefix = logger_prefix + "export_all_the_data() - "
    log.info(logging_prefix + "Exporting")

    try:
        dump = {}
        dump['actors'] = []
        dump['reports'] = []
        dump['ttps'] = []
        dump['choices'] = {}

        query = {"query": {"match_all": {}}}

        results = scan(get_es(),
                       query=query,
                       index=ES_PREFIX + "threat_actors",
                       doc_type="actor")

        for i in results:
            dump['actors'].append(i)

        results = scan(get_es(),
                       query=query,
                       index=ES_PREFIX + "threat_reports",
                       doc_type="report")

        for i in results:
            dump['reports'].append(i)

        results = scan(get_es(),
                       query=query,
                       index=ES_PREFIX + "threat_ttps",
                       doc_type="ttp")

        for i in results:
            dump['ttps'].append(i)

        results = scan(get_es(),
                       query=query,
                       index=ES_PREFIX + "threat_actor_pc",
                       doc_type="parent")

        dump['choices']['parents'] = []
        for i in results:
            dump['choices']['parents'].append(i)

        results = scan(get_es(),
                       query=query,
                       index=ES_PREFIX + "threat_actor_pc",
                       doc_type="child")

        dump['choices']['children'] = []
        for i in results:
            dump['choices']['children'].append(i)

        results = scan(get_es(),
                       query=query,
                       index=ES_PREFIX + "threat_actor_simple",
                       doc_type="data")

        dump['choices']['simple'] = []
        for i in results:
            dump['choices']['simple'].append(i)

        # We need to modify the response, so the first thing we
        # need to do is create a response out of the Dictionary
        response = make_response(json.dumps(dump))

        # This is the key: Set the right header for the response
        # to be downloaded, instead of just printed on the browser
        response.headers[
            "Content-Disposition"] = "attachment; filename=export.json"
        response.headers["Content-Type"] = "text/json; charset=utf-8"

        return response

    except Exception as e:
        error = "There was an error completing your request. Details: {}".format(
            e)
        log.exception(error)
        flash(error, "danger")
        return redirect("/")