Example #1
0
def get_indicator_type_dropdown(request):
    """
    Get Indicator type dropdown data. Should be an AJAX POST.

    :param request: Django request object (Required)
    :type request: :class:`django.http.HttpRequest`
    :returns: :class:`django.http.HttpResponse`
    """

    if request.method == 'POST':
        if request.is_ajax():
            dd_final = {}
            list_type = request.POST.get('type', None)
            if list_type == 'indicator_type':
                type_list = IndicatorTypes.values(sort=True)
            elif list_type == 'threat_type':
                type_list = IndicatorThreatTypes.values(sort=True)
            elif list_type == 'attack_type':
                type_list = IndicatorAttackTypes.values(sort=True)
            else:
                type_list = []
            for type_ in type_list:
                dd_final[type_] = type_
            result = {'types': dd_final}
            return HttpResponse(json.dumps(result), mimetype="application/json")
        else:
            error = "Expected AJAX"
            return render_to_response("error.html",
                                      {"error" : error },
                                      RequestContext(request))
    else:
        error = "Expected POST"
        return render_to_response("error.html",
                                  {"error" : error },
                                  RequestContext(request))
Example #2
0
def get_indicator_type_dropdown(request):
    """
    Get Indicator type dropdown data. Should be an AJAX POST.

    :param request: Django request object (Required)
    :type request: :class:`django.http.HttpRequest`
    :returns: :class:`django.http.HttpResponse`
    """

    if request.method == 'POST':
        if request.is_ajax():
            dd_final = {}
            list_type = request.POST.get('type', None)
            if list_type == 'indicator_type':
                type_list = IndicatorTypes.values(sort=True)
            elif list_type == 'threat_type':
                type_list = IndicatorThreatTypes.values(sort=True)
            elif list_type == 'attack_type':
                type_list = IndicatorAttackTypes.values(sort=True)
            else:
                type_list = []
            for type_ in type_list:
                dd_final[type_] = type_
            result = {'types': dd_final}
            return HttpResponse(json.dumps(result),
                                content_type="application/json")
        else:
            error = "Expected AJAX"
            return render_to_response("error.html", {"error": error},
                                      RequestContext(request))
    else:
        error = "Expected POST"
        return render_to_response("error.html", {"error": error},
                                  RequestContext(request))
Example #3
0
def set_indicator_threat_type(id_, threat_type, user, **kwargs):
    """
    Set the Indicator threat type.

    :param indicator_id: The ObjectId of the indicator to update.
    :type indicator_id: str
    :param threat_type: The new indicator threat type.
    :type threat_type: str
    :param user: The user updating the indicator.
    :type user: str
    :returns: dict with key "success" (boolean)
    """

    # check to ensure we're not duping an existing indicator
    indicator = Indicator.objects(id=id_).first()
    value = indicator.value
    ind_check = Indicator.objects(threat_type=threat_type, value=value).first()
    if ind_check:
        # we found a dupe
        return {"success": False, "message": "Duplicate would exist making this change."}
    elif threat_type not in IndicatorThreatTypes.values():
        return {"success": False, "message": "Not a valid Threat Type."}
    else:
        try:
            indicator.threat_type = threat_type
            indicator.save(username=user)
            return {"success": True}
        except ValidationError:
            return {"success": False}
Example #4
0
def get_available_threat_types(request):
    """
    Get the available threat types to use.

    :param request: Django request.
    :type request: :class:`django.http.HttpRequest`
    :returns: :class:`django.http.HttpResponse`
    """

    if request.method == "POST" and request.is_ajax():
        return HttpResponse(json.dumps(IndicatorThreatTypes.values(sort=True),
                                       default=json_handler),
                            content_type='application/json')
    return HttpResponse({})
Example #5
0
def get_available_threat_types(request):
    """
    Get the available threat types to use.

    :param request: Django request.
    :type request: :class:`django.http.HttpRequest`
    :returns: :class:`django.http.HttpResponse`
    """

    if request.method == "POST" and request.is_ajax():
        return HttpResponse(
            json.dumps(IndicatorThreatTypes.values(sort=True), default=json_handler), content_type="application/json"
        )
    return HttpResponse({})
Example #6
0
    def __init__(self, username, *args, **kwargs):
        super(UploadIndicatorForm, self).__init__(*args, **kwargs)
        self.fields['source'].choices = [
            (c.name, c.name) for c in get_source_names(True, True, username)]
        self.fields['source'].initial = get_user_organization(username)
        self.fields['status'].choices = [
            (c,c) for c in Status.values()
        ]
        self.fields['indicator_type'].choices = [
            (c,c) for c in IndicatorTypes.values(sort=True)
        ]
        self.fields['threat_type'].choices = [
            (c,c) for c in IndicatorThreatTypes.values(sort=True)
        ]
        self.fields['threat_type'].initial = IndicatorThreatTypes.UNKNOWN
        self.fields['attack_type'].choices = [
            (c,c) for c in IndicatorAttackTypes.values(sort=True)
        ]
        self.fields['attack_type'].initial = IndicatorAttackTypes.UNKNOWN
        self.fields['indicator_type'].widget.attrs = {'class': 'object-types'}
        self.fields['campaign'].choices = [("", "")]
        self.fields['campaign'].choices += [
            (c.name, c.name) for c in get_item_names(Campaign, True)]
        self.fields['campaign_confidence'].choices = [
            ("", ""),
            ("low", "low"),
            ("medium", "medium"),
            ("high", "high")]
        self.fields['confidence'].choices = [
            ("unknown", "unknown"),
            ("benign", "benign"),
            ("low", "low"),
            ("medium", "medium"),
            ("high", "high")]
        self.fields['impact'].choices = [
            ("unknown", "unknown"),
            ("benign", "benign"),
            ("low", "low"),
            ("medium", "medium"),
            ("high", "high")]

        self.fields['relationship_type'].choices = relationship_choices
        self.fields['relationship_type'].initial = RelationshipTypes.RELATED_TO

        add_bucketlist_to_form(self)
        add_ticket_to_form(self)
Example #7
0
    def __init__(self, user, *args, **kwargs):
        super(UploadIndicatorForm, self).__init__(user, *args, **kwargs)
        self.fields['status'].choices = [
            (c,c) for c in Status.values()
        ]
        self.fields['indicator_type'].choices = [
            (c,c) for c in IndicatorTypes.values(sort=True)
        ]
        self.fields['threat_type'].choices = [
            (c,c) for c in IndicatorThreatTypes.values(sort=True)
        ]
        self.fields['threat_type'].initial = IndicatorThreatTypes.UNKNOWN
        self.fields['attack_type'].choices = [
            (c,c) for c in IndicatorAttackTypes.values(sort=True)
        ]
        self.fields['attack_type'].initial = IndicatorAttackTypes.UNKNOWN
        self.fields['indicator_type'].widget.attrs = {'class': 'object-types'}
        if user.has_access_to(Common.CAMPAIGN_READ):
            self.fields['campaign'].choices = [('', '')] + [
                (c.name, c.name) for c in get_item_names(Campaign, True)]

        self.fields['campaign_confidence'].choices = [
            ("", ""),
            ("low", "low"),
            ("medium", "medium"),
            ("high", "high")]
        self.fields['confidence'].choices = [
            ("unknown", "unknown"),
            ("benign", "benign"),
            ("low", "low"),
            ("medium", "medium"),
            ("high", "high")]
        self.fields['impact'].choices = [
            ("unknown", "unknown"),
            ("benign", "benign"),
            ("low", "low"),
            ("medium", "medium"),
            ("high", "high")]

        self.fields['relationship_type'].choices = relationship_choices
        self.fields['relationship_type'].initial = RelationshipTypes.RELATED_TO

        add_bucketlist_to_form(self)
        add_ticket_to_form(self)
Example #8
0
    def __init__(self, username, *args, **kwargs):
        super(UploadIndicatorForm, self).__init__(*args, **kwargs)
        self.fields['source'].choices = [
            (c.name, c.name) for c in get_source_names(True, True, username)]
        self.fields['source'].initial = get_user_organization(username)
        self.fields['indicator_type'].choices = [
            (c,c) for c in IndicatorTypes.values(sort=True)
        ]
        self.fields['threat_type'].choices = [
            (c,c) for c in IndicatorThreatTypes.values(sort=True)
        ]
        self.fields['threat_type'].initial = IndicatorThreatTypes.UNKNOWN
        self.fields['attack_type'].choices = [
            (c,c) for c in IndicatorAttackTypes.values(sort=True)
        ]
        self.fields['attack_type'].initial = IndicatorAttackTypes.UNKNOWN
        self.fields['indicator_type'].widget.attrs = {'class': 'object-types'}
        self.fields['campaign'].choices = [("", "")]
        self.fields['campaign'].choices += [
            (c.name, c.name) for c in get_item_names(Campaign, True)]
        self.fields['campaign_confidence'].choices = [
            ("", ""),
            ("low", "low"),
            ("medium", "medium"),
            ("high", "high")]
        self.fields['confidence'].choices = [
            ("unknown", "unknown"),
            ("benign", "benign"),
            ("low", "low"),
            ("medium", "medium"),
            ("high", "high")]
        self.fields['impact'].choices = [
            ("unknown", "unknown"),
            ("benign", "benign"),
            ("low", "low"),
            ("medium", "medium"),
            ("high", "high")]

        add_bucketlist_to_form(self)
        add_ticket_to_form(self)
Example #9
0
def handle_indicator_insert(
    ind, source, reference="", analyst="", method="", add_domain=False, add_relationship=False, cache={}
):
    """
    Insert an individual indicator into the database.

    NOTE: Setting add_domain to True will always create a relationship as well.
    However, to create a relationship with an object that already exists before
    this function was called, set add_relationship to True. This will assume
    that the domain or IP object to create the relationship with already exists
    and will avoid infinite mutual calls between, for example, add_update_ip
    and this function. add domain/IP objects.

    :param ind: Information about the indicator.
    :type ind: dict
    :param source: The source for this indicator.
    :type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource`
    :param reference: The reference to the data.
    :type reference: str
    :param analyst: The user adding this indicator.
    :type analyst: str
    :param method: Method of acquiring this indicator.
    :type method: str
    :param add_domain: If this indicator is also a top-level object, try to add
                       it.
    :type add_domain: boolean
    :param add_relationship: Attempt to add relationships if applicable.
    :type add_relationship: boolean
    :param cache: Cached data, typically for performance enhancements
                  during bulk uperations.
    :type cache: dict
    :returns: dict with keys:
              "success" (boolean),
              "message" (str) if failed,
              "objectid" (str) if successful,
              "is_new_indicator" (boolean) if successful.
    """

    if ind["type"] not in IndicatorTypes.values():
        return {"success": False, "message": "Not a valid Indicator Type: %s" % ind["type"]}
    if ind["threat_type"] not in IndicatorThreatTypes.values():
        return {"success": False, "message": "Not a valid Indicator Threat Type: %s" % ind["threat_type"]}
    if ind["attack_type"] not in IndicatorAttackTypes.values():
        return {"success": False, "message": "Not a valid Indicator Attack Type: " % ind["attack_type"]}

    (ind["value"], error) = validate_indicator_value(ind["value"], ind["type"])
    if error:
        return {"success": False, "message": error}

    is_new_indicator = False
    dmain = None
    ip = None
    rank = {"unknown": 0, "benign": 1, "low": 2, "medium": 3, "high": 4}

    if ind.get("status", None) is None or len(ind.get("status", "")) < 1:
        ind["status"] = Status.NEW

    indicator = Indicator.objects(
        ind_type=ind["type"], lower=ind["lower"], threat_type=ind["threat_type"], attack_type=ind["attack_type"]
    ).first()
    if not indicator:
        indicator = Indicator()
        indicator.ind_type = ind["type"]
        indicator.threat_type = ind["threat_type"]
        indicator.attack_type = ind["attack_type"]
        indicator.value = ind["value"]
        indicator.lower = ind["lower"]
        indicator.description = ind["description"]
        indicator.created = datetime.datetime.now()
        indicator.confidence = EmbeddedConfidence(analyst=analyst)
        indicator.impact = EmbeddedImpact(analyst=analyst)
        indicator.status = ind["status"]
        is_new_indicator = True
    else:
        if ind["status"] != Status.NEW:
            indicator.status = ind["status"]
        add_desc = "\nSeen on %s as: %s" % (str(datetime.datetime.now()), ind["value"])
        if indicator.description is None:
            indicator.description = add_desc
        else:
            indicator.description += add_desc

    if "campaign" in ind:
        if isinstance(ind["campaign"], basestring) and len(ind["campaign"]) > 0:
            confidence = ind.get("campaign_confidence", "low")
            ind["campaign"] = EmbeddedCampaign(
                name=ind["campaign"],
                confidence=confidence,
                description="",
                analyst=analyst,
                date=datetime.datetime.now(),
            )
        if isinstance(ind["campaign"], EmbeddedCampaign):
            indicator.add_campaign(ind["campaign"])
        elif isinstance(ind["campaign"], list):
            for campaign in ind["campaign"]:
                if isinstance(campaign, EmbeddedCampaign):
                    indicator.add_campaign(campaign)

    if "confidence" in ind and rank.get(ind["confidence"], 0) > rank.get(indicator.confidence.rating, 0):
        indicator.confidence.rating = ind["confidence"]
        indicator.confidence.analyst = analyst

    if "impact" in ind and rank.get(ind["impact"], 0) > rank.get(indicator.impact.rating, 0):
        indicator.impact.rating = ind["impact"]
        indicator.impact.analyst = analyst

    bucket_list = None
    if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind:
        bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME]
        if bucket_list:
            indicator.add_bucket_list(bucket_list, analyst)

    ticket = None
    if form_consts.Common.TICKET_VARIABLE_NAME in ind:
        ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME]
        if ticket:
            indicator.add_ticket(ticket, analyst)

    if isinstance(source, list):
        for s in source:
            indicator.add_source(source_item=s, method=method, reference=reference)
    elif isinstance(source, EmbeddedSource):
        indicator.add_source(source_item=source, method=method, reference=reference)
    elif isinstance(source, basestring):
        s = EmbeddedSource()
        s.name = source
        instance = EmbeddedSource.SourceInstance()
        instance.reference = reference
        instance.method = method
        instance.analyst = analyst
        instance.date = datetime.datetime.now()
        s.instances = [instance]
        indicator.add_source(s)

    if add_domain or add_relationship:
        ind_type = indicator.ind_type
        ind_value = indicator.lower
        url_contains_ip = False
        if ind_type in (IndicatorTypes.DOMAIN, IndicatorTypes.URI):
            if ind_type == IndicatorTypes.URI:
                domain_or_ip = urlparse.urlparse(ind_value).hostname
                try:
                    validate_ipv46_address(domain_or_ip)
                    url_contains_ip = True
                except DjangoValidationError:
                    pass
            else:
                domain_or_ip = ind_value
            if not url_contains_ip:
                success = None
                if add_domain:
                    success = upsert_domain(
                        domain_or_ip,
                        indicator.source,
                        username="******" % analyst,
                        campaign=indicator.campaign,
                        bucket_list=bucket_list,
                        cache=cache,
                    )
                    if not success["success"]:
                        return {"success": False, "message": success["message"]}

                if not success or not "object" in success:
                    dmain = Domain.objects(domain=domain_or_ip).first()
                else:
                    dmain = success["object"]

        if ind_type in IPTypes.values() or url_contains_ip:
            if url_contains_ip:
                ind_value = domain_or_ip
                try:
                    validate_ipv4_address(domain_or_ip)
                    ind_type = IndicatorTypes.IPV4_ADDRESS
                except DjangoValidationError:
                    ind_type = IndicatorTypes.IPV6_ADDRESS
            success = None
            if add_domain:
                success = ip_add_update(
                    ind_value,
                    ind_type,
                    source=indicator.source,
                    campaign=indicator.campaign,
                    analyst=analyst,
                    bucket_list=bucket_list,
                    ticket=ticket,
                    indicator_reference=reference,
                    cache=cache,
                )
                if not success["success"]:
                    return {"success": False, "message": success["message"]}

            if not success or not "object" in success:
                ip = IP.objects(ip=indicator.value).first()
            else:
                ip = success["object"]

    indicator.save(username=analyst)

    if dmain:
        dmain.add_relationship(indicator, RelationshipTypes.RELATED_TO, analyst="%s" % analyst, get_rels=False)
        dmain.save(username=analyst)
    if ip:
        ip.add_relationship(indicator, RelationshipTypes.RELATED_TO, analyst="%s" % analyst, get_rels=False)
        ip.save(username=analyst)

    # run indicator triage
    if is_new_indicator:
        indicator.reload()
        run_triage(indicator, analyst)

    return {"success": True, "objectid": str(indicator.id), "is_new_indicator": is_new_indicator, "object": indicator}
Example #10
0
def handle_indicator_csv(csv_data, source, method, reference, ctype, username, add_domain=False):
    """
    Handle adding Indicators in CSV format (file or blob).

    :param csv_data: The CSV data.
    :type csv_data: str or file handle
    :param source: The name of the source for these indicators.
    :type source: str
    :param method: The method of acquisition of this indicator.
    :type method: str
    :param reference: The reference to this data.
    :type reference: str
    :param ctype: The CSV type.
    :type ctype: str ("file" or "blob")
    :param username: The user adding these indicators.
    :type username: str
    :param add_domain: If the indicators being added are also other top-level
                       objects, add those too.
    :type add_domain: boolean
    :returns: dict with keys "success" (boolean) and "message" (str)
    """

    if ctype == "file":
        cdata = csv_data.read()
    else:
        cdata = csv_data.encode("ascii")
    data = csv.DictReader(BytesIO(cdata), skipinitialspace=True)
    result = {"success": True}
    result_message = ""
    # Compute permitted values in CSV
    valid_ratings = {"unknown": "unknown", "benign": "benign", "low": "low", "medium": "medium", "high": "high"}
    valid_campaign_confidence = {"low": "low", "medium": "medium", "high": "high"}
    valid_campaigns = {}
    for c in Campaign.objects(active="on"):
        valid_campaigns[c["name"].lower().replace(" - ", "-")] = c["name"]
    valid_actions = {}
    for a in Action.objects(active="on"):
        valid_actions[a["name"].lower().replace(" - ", "-")] = a["name"]
    valid_ind_types = {}
    for obj in IndicatorTypes.values(sort=True):
        valid_ind_types[obj.lower().replace(" - ", "-")] = obj

    # Start line-by-line import
    added = 0
    for processed, d in enumerate(data, 1):
        ind = {}
        ind["value"] = d.get("Indicator", "").strip()
        ind["lower"] = d.get("Indicator", "").lower().strip()
        ind["description"] = d.get("Description", "").strip()
        ind["type"] = get_verified_field(d, valid_ind_types, "Type")
        ind["threat_type"] = d.get("Threat Type", IndicatorThreatTypes.UNKNOWN)
        ind["attack_type"] = d.get("Attack Type", IndicatorAttackTypes.UNKNOWN)

        if len(ind["threat_type"]) < 1:
            ind["threat_type"] = IndicatorThreatTypes.UNKNOWN
        if ind["threat_type"] not in IndicatorThreatTypes.values():
            result["success"] = False
            result_message += "Cannot process row %s: Invalid Threat Type<br />" % processed
            continue

        if len(ind["attack_type"]) < 1:
            ind["attack_type"] = IndicatorAttackTypes.UNKNOWN
        if ind["attack_type"] not in IndicatorAttackTypes.values():
            result["success"] = False
            result_message += "Cannot process row %s: Invalid Attack Type<br />" % processed
            continue

        ind["status"] = d.get("Status", Status.NEW)
        if not ind["value"] or not ind["type"]:
            # Mandatory value missing or malformed, cannot process csv row
            i = ""
            result["success"] = False
            if not ind["value"]:
                i += "No valid Indicator value "
            if not ind["type"]:
                i += "No valid Indicator type "
            result_message += "Cannot process row %s: %s<br />" % (processed, i)
            continue
        campaign = get_verified_field(d, valid_campaigns, "Campaign")
        if campaign:
            ind["campaign"] = campaign
            ind["campaign_confidence"] = get_verified_field(
                d, valid_campaign_confidence, "Campaign Confidence", default="low"
            )
        actions = d.get("Action", "")
        if actions:
            actions = get_verified_field(actions.split(","), valid_actions)
            if not actions:
                result["success"] = False
                result_message += "Cannot process row %s: Invalid Action<br />" % processed
                continue
        ind["confidence"] = get_verified_field(d, valid_ratings, "Confidence", default="unknown")
        ind["impact"] = get_verified_field(d, valid_ratings, "Impact", default="unknown")
        ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME] = d.get(form_consts.Common.BUCKET_LIST, "")
        ind[form_consts.Common.TICKET_VARIABLE_NAME] = d.get(form_consts.Common.TICKET, "")
        try:
            response = handle_indicator_insert(
                ind, source, reference, analyst=username, method=method, add_domain=add_domain
            )
        except Exception, e:
            result["success"] = False
            result_message += "Failure processing row %s: %s<br />" % (processed, str(e))
            continue
        if response["success"]:
            if actions:
                action = {
                    "active": "on",
                    "analyst": username,
                    "begin_date": "",
                    "end_date": "",
                    "performed_date": "",
                    "reason": "",
                    "date": datetime.datetime.now(),
                }
                for action_type in actions:
                    action["action_type"] = action_type
                    action_add("Indicator", response.get("objectid"), action, user=username)
        else:
            result["success"] = False
            result_message += "Failure processing row %s: %s<br />" % (processed, response["message"])
            continue
        added += 1