def get_indicator_type_dropdown(request): """ Get Indicator type dropdown data. Should be an AJAX POST. :param request: Django request object (Required) :type request: :class:`django.http.HttpRequest` :returns: :class:`django.http.HttpResponse` """ if request.method == 'POST': if request.is_ajax(): dd_final = {} list_type = request.POST.get('type', None) if list_type == 'indicator_type': type_list = IndicatorTypes.values(sort=True) elif list_type == 'threat_type': type_list = IndicatorThreatTypes.values(sort=True) elif list_type == 'attack_type': type_list = IndicatorAttackTypes.values(sort=True) else: type_list = [] for type_ in type_list: dd_final[type_] = type_ result = {'types': dd_final} return HttpResponse(json.dumps(result), content_type="application/json") else: error = "Expected AJAX" return render_to_response("error.html", {"error": error}, RequestContext(request)) else: error = "Expected POST" return render_to_response("error.html", {"error": error}, RequestContext(request))
def get_indicator_type_dropdown(request): """ Get Indicator type dropdown data. Should be an AJAX POST. :param request: Django request object (Required) :type request: :class:`django.http.HttpRequest` :returns: :class:`django.http.HttpResponse` """ if request.method == 'POST': if request.is_ajax(): dd_final = {} list_type = request.POST.get('type', None) if list_type == 'indicator_type': type_list = IndicatorTypes.values(sort=True) elif list_type == 'threat_type': type_list = IndicatorThreatTypes.values(sort=True) elif list_type == 'attack_type': type_list = IndicatorAttackTypes.values(sort=True) else: type_list = [] for type_ in type_list: dd_final[type_] = type_ result = {'types': dd_final} return HttpResponse(json.dumps(result), mimetype="application/json") else: error = "Expected AJAX" return render_to_response("error.html", {"error" : error }, RequestContext(request)) else: error = "Expected POST" return render_to_response("error.html", {"error" : error }, RequestContext(request))
def is_indicator_type(value): """ Returns True if the value is a valid Indicator Type. :param value: The string to validate as an Indicator Type. :type value: str :returns: bool """ if value in IndicatorTypes.values(): return True else: return False
def __init__(self, username, *args, **kwargs): super(UploadIndicatorForm, self).__init__(*args, **kwargs) self.fields['source'].choices = [ (c.name, c.name) for c in get_source_names(True, True, username)] self.fields['source'].initial = get_user_organization(username) self.fields['status'].choices = [ (c,c) for c in Status.values() ] self.fields['indicator_type'].choices = [ (c,c) for c in IndicatorTypes.values(sort=True) ] self.fields['threat_type'].choices = [ (c,c) for c in IndicatorThreatTypes.values(sort=True) ] self.fields['threat_type'].initial = IndicatorThreatTypes.UNKNOWN self.fields['attack_type'].choices = [ (c,c) for c in IndicatorAttackTypes.values(sort=True) ] self.fields['attack_type'].initial = IndicatorAttackTypes.UNKNOWN self.fields['indicator_type'].widget.attrs = {'class': 'object-types'} self.fields['campaign'].choices = [("", "")] self.fields['campaign'].choices += [ (c.name, c.name) for c in get_item_names(Campaign, True)] self.fields['campaign_confidence'].choices = [ ("", ""), ("low", "low"), ("medium", "medium"), ("high", "high")] self.fields['confidence'].choices = [ ("unknown", "unknown"), ("benign", "benign"), ("low", "low"), ("medium", "medium"), ("high", "high")] self.fields['impact'].choices = [ ("unknown", "unknown"), ("benign", "benign"), ("low", "low"), ("medium", "medium"), ("high", "high")] self.fields['relationship_type'].choices = relationship_choices self.fields['relationship_type'].initial = RelationshipTypes.RELATED_TO add_bucketlist_to_form(self) add_ticket_to_form(self)
def __init__(self, user, *args, **kwargs): super(UploadIndicatorForm, self).__init__(user, *args, **kwargs) self.fields['status'].choices = [ (c,c) for c in Status.values() ] self.fields['indicator_type'].choices = [ (c,c) for c in IndicatorTypes.values(sort=True) ] self.fields['threat_type'].choices = [ (c,c) for c in IndicatorThreatTypes.values(sort=True) ] self.fields['threat_type'].initial = IndicatorThreatTypes.UNKNOWN self.fields['attack_type'].choices = [ (c,c) for c in IndicatorAttackTypes.values(sort=True) ] self.fields['attack_type'].initial = IndicatorAttackTypes.UNKNOWN self.fields['indicator_type'].widget.attrs = {'class': 'object-types'} if user.has_access_to(Common.CAMPAIGN_READ): self.fields['campaign'].choices = [('', '')] + [ (c.name, c.name) for c in get_item_names(Campaign, True)] self.fields['campaign_confidence'].choices = [ ("", ""), ("low", "low"), ("medium", "medium"), ("high", "high")] self.fields['confidence'].choices = [ ("unknown", "unknown"), ("benign", "benign"), ("low", "low"), ("medium", "medium"), ("high", "high")] self.fields['impact'].choices = [ ("unknown", "unknown"), ("benign", "benign"), ("low", "low"), ("medium", "medium"), ("high", "high")] self.fields['relationship_type'].choices = relationship_choices self.fields['relationship_type'].initial = RelationshipTypes.RELATED_TO add_bucketlist_to_form(self) add_ticket_to_form(self)
def __init__(self, username, *args, **kwargs): super(UploadIndicatorForm, self).__init__(*args, **kwargs) self.fields['source'].choices = [ (c.name, c.name) for c in get_source_names(True, True, username)] self.fields['source'].initial = get_user_organization(username) self.fields['indicator_type'].choices = [ (c,c) for c in IndicatorTypes.values(sort=True) ] self.fields['threat_type'].choices = [ (c,c) for c in IndicatorThreatTypes.values(sort=True) ] self.fields['threat_type'].initial = IndicatorThreatTypes.UNKNOWN self.fields['attack_type'].choices = [ (c,c) for c in IndicatorAttackTypes.values(sort=True) ] self.fields['attack_type'].initial = IndicatorAttackTypes.UNKNOWN self.fields['indicator_type'].widget.attrs = {'class': 'object-types'} self.fields['campaign'].choices = [("", "")] self.fields['campaign'].choices += [ (c.name, c.name) for c in get_item_names(Campaign, True)] self.fields['campaign_confidence'].choices = [ ("", ""), ("low", "low"), ("medium", "medium"), ("high", "high")] self.fields['confidence'].choices = [ ("unknown", "unknown"), ("benign", "benign"), ("low", "low"), ("medium", "medium"), ("high", "high")] self.fields['impact'].choices = [ ("unknown", "unknown"), ("benign", "benign"), ("low", "low"), ("medium", "medium"), ("high", "high")] add_bucketlist_to_form(self) add_ticket_to_form(self)
def handle_indicator_insert( ind, source, reference="", analyst="", method="", add_domain=False, add_relationship=False, cache={} ): """ Insert an individual indicator into the database. NOTE: Setting add_domain to True will always create a relationship as well. However, to create a relationship with an object that already exists before this function was called, set add_relationship to True. This will assume that the domain or IP object to create the relationship with already exists and will avoid infinite mutual calls between, for example, add_update_ip and this function. add domain/IP objects. :param ind: Information about the indicator. :type ind: dict :param source: The source for this indicator. :type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource` :param reference: The reference to the data. :type reference: str :param analyst: The user adding this indicator. :type analyst: str :param method: Method of acquiring this indicator. :type method: str :param add_domain: If this indicator is also a top-level object, try to add it. :type add_domain: boolean :param add_relationship: Attempt to add relationships if applicable. :type add_relationship: boolean :param cache: Cached data, typically for performance enhancements during bulk uperations. :type cache: dict :returns: dict with keys: "success" (boolean), "message" (str) if failed, "objectid" (str) if successful, "is_new_indicator" (boolean) if successful. """ if ind["type"] not in IndicatorTypes.values(): return {"success": False, "message": "Not a valid Indicator Type: %s" % ind["type"]} if ind["threat_type"] not in IndicatorThreatTypes.values(): return {"success": False, "message": "Not a valid Indicator Threat Type: %s" % ind["threat_type"]} if ind["attack_type"] not in IndicatorAttackTypes.values(): return {"success": False, "message": "Not a valid Indicator Attack Type: " % ind["attack_type"]} (ind["value"], error) = validate_indicator_value(ind["value"], ind["type"]) if error: return {"success": False, "message": error} is_new_indicator = False dmain = None ip = None rank = {"unknown": 0, "benign": 1, "low": 2, "medium": 3, "high": 4} if ind.get("status", None) is None or len(ind.get("status", "")) < 1: ind["status"] = Status.NEW indicator = Indicator.objects( ind_type=ind["type"], lower=ind["lower"], threat_type=ind["threat_type"], attack_type=ind["attack_type"] ).first() if not indicator: indicator = Indicator() indicator.ind_type = ind["type"] indicator.threat_type = ind["threat_type"] indicator.attack_type = ind["attack_type"] indicator.value = ind["value"] indicator.lower = ind["lower"] indicator.description = ind["description"] indicator.created = datetime.datetime.now() indicator.confidence = EmbeddedConfidence(analyst=analyst) indicator.impact = EmbeddedImpact(analyst=analyst) indicator.status = ind["status"] is_new_indicator = True else: if ind["status"] != Status.NEW: indicator.status = ind["status"] add_desc = "\nSeen on %s as: %s" % (str(datetime.datetime.now()), ind["value"]) if indicator.description is None: indicator.description = add_desc else: indicator.description += add_desc if "campaign" in ind: if isinstance(ind["campaign"], basestring) and len(ind["campaign"]) > 0: confidence = ind.get("campaign_confidence", "low") ind["campaign"] = EmbeddedCampaign( name=ind["campaign"], confidence=confidence, description="", analyst=analyst, date=datetime.datetime.now(), ) if isinstance(ind["campaign"], EmbeddedCampaign): indicator.add_campaign(ind["campaign"]) elif isinstance(ind["campaign"], list): for campaign in ind["campaign"]: if isinstance(campaign, EmbeddedCampaign): indicator.add_campaign(campaign) if "confidence" in ind and rank.get(ind["confidence"], 0) > rank.get(indicator.confidence.rating, 0): indicator.confidence.rating = ind["confidence"] indicator.confidence.analyst = analyst if "impact" in ind and rank.get(ind["impact"], 0) > rank.get(indicator.impact.rating, 0): indicator.impact.rating = ind["impact"] indicator.impact.analyst = analyst bucket_list = None if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind: bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME] if bucket_list: indicator.add_bucket_list(bucket_list, analyst) ticket = None if form_consts.Common.TICKET_VARIABLE_NAME in ind: ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME] if ticket: indicator.add_ticket(ticket, analyst) if isinstance(source, list): for s in source: indicator.add_source(source_item=s, method=method, reference=reference) elif isinstance(source, EmbeddedSource): indicator.add_source(source_item=source, method=method, reference=reference) elif isinstance(source, basestring): s = EmbeddedSource() s.name = source instance = EmbeddedSource.SourceInstance() instance.reference = reference instance.method = method instance.analyst = analyst instance.date = datetime.datetime.now() s.instances = [instance] indicator.add_source(s) if add_domain or add_relationship: ind_type = indicator.ind_type ind_value = indicator.lower url_contains_ip = False if ind_type in (IndicatorTypes.DOMAIN, IndicatorTypes.URI): if ind_type == IndicatorTypes.URI: domain_or_ip = urlparse.urlparse(ind_value).hostname try: validate_ipv46_address(domain_or_ip) url_contains_ip = True except DjangoValidationError: pass else: domain_or_ip = ind_value if not url_contains_ip: success = None if add_domain: success = upsert_domain( domain_or_ip, indicator.source, username="******" % analyst, campaign=indicator.campaign, bucket_list=bucket_list, cache=cache, ) if not success["success"]: return {"success": False, "message": success["message"]} if not success or not "object" in success: dmain = Domain.objects(domain=domain_or_ip).first() else: dmain = success["object"] if ind_type in IPTypes.values() or url_contains_ip: if url_contains_ip: ind_value = domain_or_ip try: validate_ipv4_address(domain_or_ip) ind_type = IndicatorTypes.IPV4_ADDRESS except DjangoValidationError: ind_type = IndicatorTypes.IPV6_ADDRESS success = None if add_domain: success = ip_add_update( ind_value, ind_type, source=indicator.source, campaign=indicator.campaign, analyst=analyst, bucket_list=bucket_list, ticket=ticket, indicator_reference=reference, cache=cache, ) if not success["success"]: return {"success": False, "message": success["message"]} if not success or not "object" in success: ip = IP.objects(ip=indicator.value).first() else: ip = success["object"] indicator.save(username=analyst) if dmain: dmain.add_relationship(indicator, RelationshipTypes.RELATED_TO, analyst="%s" % analyst, get_rels=False) dmain.save(username=analyst) if ip: ip.add_relationship(indicator, RelationshipTypes.RELATED_TO, analyst="%s" % analyst, get_rels=False) ip.save(username=analyst) # run indicator triage if is_new_indicator: indicator.reload() run_triage(indicator, analyst) return {"success": True, "objectid": str(indicator.id), "is_new_indicator": is_new_indicator, "object": indicator}
def handle_indicator_csv(csv_data, source, method, reference, ctype, username, add_domain=False): """ Handle adding Indicators in CSV format (file or blob). :param csv_data: The CSV data. :type csv_data: str or file handle :param source: The name of the source for these indicators. :type source: str :param method: The method of acquisition of this indicator. :type method: str :param reference: The reference to this data. :type reference: str :param ctype: The CSV type. :type ctype: str ("file" or "blob") :param username: The user adding these indicators. :type username: str :param add_domain: If the indicators being added are also other top-level objects, add those too. :type add_domain: boolean :returns: dict with keys "success" (boolean) and "message" (str) """ if ctype == "file": cdata = csv_data.read() else: cdata = csv_data.encode("ascii") data = csv.DictReader(BytesIO(cdata), skipinitialspace=True) result = {"success": True} result_message = "" # Compute permitted values in CSV valid_ratings = {"unknown": "unknown", "benign": "benign", "low": "low", "medium": "medium", "high": "high"} valid_campaign_confidence = {"low": "low", "medium": "medium", "high": "high"} valid_campaigns = {} for c in Campaign.objects(active="on"): valid_campaigns[c["name"].lower().replace(" - ", "-")] = c["name"] valid_actions = {} for a in Action.objects(active="on"): valid_actions[a["name"].lower().replace(" - ", "-")] = a["name"] valid_ind_types = {} for obj in IndicatorTypes.values(sort=True): valid_ind_types[obj.lower().replace(" - ", "-")] = obj # Start line-by-line import added = 0 for processed, d in enumerate(data, 1): ind = {} ind["value"] = d.get("Indicator", "").strip() ind["lower"] = d.get("Indicator", "").lower().strip() ind["description"] = d.get("Description", "").strip() ind["type"] = get_verified_field(d, valid_ind_types, "Type") ind["threat_type"] = d.get("Threat Type", IndicatorThreatTypes.UNKNOWN) ind["attack_type"] = d.get("Attack Type", IndicatorAttackTypes.UNKNOWN) if len(ind["threat_type"]) < 1: ind["threat_type"] = IndicatorThreatTypes.UNKNOWN if ind["threat_type"] not in IndicatorThreatTypes.values(): result["success"] = False result_message += "Cannot process row %s: Invalid Threat Type<br />" % processed continue if len(ind["attack_type"]) < 1: ind["attack_type"] = IndicatorAttackTypes.UNKNOWN if ind["attack_type"] not in IndicatorAttackTypes.values(): result["success"] = False result_message += "Cannot process row %s: Invalid Attack Type<br />" % processed continue ind["status"] = d.get("Status", Status.NEW) if not ind["value"] or not ind["type"]: # Mandatory value missing or malformed, cannot process csv row i = "" result["success"] = False if not ind["value"]: i += "No valid Indicator value " if not ind["type"]: i += "No valid Indicator type " result_message += "Cannot process row %s: %s<br />" % (processed, i) continue campaign = get_verified_field(d, valid_campaigns, "Campaign") if campaign: ind["campaign"] = campaign ind["campaign_confidence"] = get_verified_field( d, valid_campaign_confidence, "Campaign Confidence", default="low" ) actions = d.get("Action", "") if actions: actions = get_verified_field(actions.split(","), valid_actions) if not actions: result["success"] = False result_message += "Cannot process row %s: Invalid Action<br />" % processed continue ind["confidence"] = get_verified_field(d, valid_ratings, "Confidence", default="unknown") ind["impact"] = get_verified_field(d, valid_ratings, "Impact", default="unknown") ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME] = d.get(form_consts.Common.BUCKET_LIST, "") ind[form_consts.Common.TICKET_VARIABLE_NAME] = d.get(form_consts.Common.TICKET, "") try: response = handle_indicator_insert( ind, source, reference, analyst=username, method=method, add_domain=add_domain ) except Exception, e: result["success"] = False result_message += "Failure processing row %s: %s<br />" % (processed, str(e)) continue if response["success"]: if actions: action = { "active": "on", "analyst": username, "begin_date": "", "end_date": "", "performed_date": "", "reason": "", "date": datetime.datetime.now(), } for action_type in actions: action["action_type"] = action_type action_add("Indicator", response.get("objectid"), action, user=username) else: result["success"] = False result_message += "Failure processing row %s: %s<br />" % (processed, response["message"]) continue added += 1