def get_indicator_type_dropdown(request):
"""
Get Indicator type dropdown data. Should be an AJAX POST.
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:returns: :class:`django.http.HttpResponse`
"""
if request.method == 'POST':
if request.is_ajax():
dd_final = {}
list_type = request.POST.get('type', None)
if list_type == 'indicator_type':
type_list = IndicatorTypes.values(sort=True)
elif list_type == 'threat_type':
type_list = IndicatorThreatTypes.values(sort=True)
elif list_type == 'attack_type':
type_list = IndicatorAttackTypes.values(sort=True)
else:
type_list = []
for type_ in type_list:
dd_final[type_] = type_
result = {'types': dd_final}
return HttpResponse(json.dumps(result), mimetype="application/json")
else:
error = "Expected AJAX"
return render_to_response("error.html",
{"error" : error },
RequestContext(request))
else:
error = "Expected POST"
return render_to_response("error.html",
{"error" : error },
RequestContext(request))
def get_indicator_type_dropdown(request):
"""
Get Indicator type dropdown data. Should be an AJAX POST.
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:returns: :class:`django.http.HttpResponse`
"""
if request.method == 'POST':
if request.is_ajax():
dd_final = {}
list_type = request.POST.get('type', None)
if list_type == 'indicator_type':
type_list = IndicatorTypes.values(sort=True)
elif list_type == 'threat_type':
type_list = IndicatorThreatTypes.values(sort=True)
elif list_type == 'attack_type':
type_list = IndicatorAttackTypes.values(sort=True)
else:
type_list = []
for type_ in type_list:
dd_final[type_] = type_
result = {'types': dd_final}
return HttpResponse(json.dumps(result),
content_type="application/json")
else:
error = "Expected AJAX"
return render_to_response("error.html", {"error": error},
RequestContext(request))
else:
error = "Expected POST"
return render_to_response("error.html", {"error": error},
RequestContext(request))
def set_indicator_threat_type(id_, threat_type, user, **kwargs):
"""
Set the Indicator threat type.
:param indicator_id: The ObjectId of the indicator to update.
:type indicator_id: str
:param threat_type: The new indicator threat type.
:type threat_type: str
:param user: The user updating the indicator.
:type user: str
:returns: dict with key "success" (boolean)
"""
# check to ensure we're not duping an existing indicator
indicator = Indicator.objects(id=id_).first()
value = indicator.value
ind_check = Indicator.objects(threat_type=threat_type, value=value).first()
if ind_check:
# we found a dupe
return {"success": False, "message": "Duplicate would exist making this change."}
elif threat_type not in IndicatorThreatTypes.values():
return {"success": False, "message": "Not a valid Threat Type."}
else:
try:
indicator.threat_type = threat_type
indicator.save(username=user)
return {"success": True}
except ValidationError:
return {"success": False}
def get_available_threat_types(request):
"""
Get the available threat types to use.
:param request: Django request.
:type request: :class:`django.http.HttpRequest`
:returns: :class:`django.http.HttpResponse`
"""
if request.method == "POST" and request.is_ajax():
return HttpResponse(json.dumps(IndicatorThreatTypes.values(sort=True),
default=json_handler),
content_type='application/json')
return HttpResponse({})
def get_available_threat_types(request):
"""
Get the available threat types to use.
:param request: Django request.
:type request: :class:`django.http.HttpRequest`
:returns: :class:`django.http.HttpResponse`
"""
if request.method == "POST" and request.is_ajax():
return HttpResponse(
json.dumps(IndicatorThreatTypes.values(sort=True), default=json_handler), content_type="application/json"
)
return HttpResponse({})
def __init__(self, username, *args, **kwargs):
super(UploadIndicatorForm, self).__init__(*args, **kwargs)
self.fields['source'].choices = [
(c.name, c.name) for c in get_source_names(True, True, username)]
self.fields['source'].initial = get_user_organization(username)
self.fields['status'].choices = [
(c,c) for c in Status.values()
]
self.fields['indicator_type'].choices = [
(c,c) for c in IndicatorTypes.values(sort=True)
]
self.fields['threat_type'].choices = [
(c,c) for c in IndicatorThreatTypes.values(sort=True)
]
self.fields['threat_type'].initial = IndicatorThreatTypes.UNKNOWN
self.fields['attack_type'].choices = [
(c,c) for c in IndicatorAttackTypes.values(sort=True)
]
self.fields['attack_type'].initial = IndicatorAttackTypes.UNKNOWN
self.fields['indicator_type'].widget.attrs = {'class': 'object-types'}
self.fields['campaign'].choices = [("", "")]
self.fields['campaign'].choices += [
(c.name, c.name) for c in get_item_names(Campaign, True)]
self.fields['campaign_confidence'].choices = [
("", ""),
("low", "low"),
("medium", "medium"),
("high", "high")]
self.fields['confidence'].choices = [
("unknown", "unknown"),
("benign", "benign"),
("low", "low"),
("medium", "medium"),
("high", "high")]
self.fields['impact'].choices = [
("unknown", "unknown"),
("benign", "benign"),
("low", "low"),
("medium", "medium"),
("high", "high")]
self.fields['relationship_type'].choices = relationship_choices
self.fields['relationship_type'].initial = RelationshipTypes.RELATED_TO
add_bucketlist_to_form(self)
add_ticket_to_form(self)
def __init__(self, user, *args, **kwargs):
super(UploadIndicatorForm, self).__init__(user, *args, **kwargs)
self.fields['status'].choices = [
(c,c) for c in Status.values()
]
self.fields['indicator_type'].choices = [
(c,c) for c in IndicatorTypes.values(sort=True)
]
self.fields['threat_type'].choices = [
(c,c) for c in IndicatorThreatTypes.values(sort=True)
]
self.fields['threat_type'].initial = IndicatorThreatTypes.UNKNOWN
self.fields['attack_type'].choices = [
(c,c) for c in IndicatorAttackTypes.values(sort=True)
]
self.fields['attack_type'].initial = IndicatorAttackTypes.UNKNOWN
self.fields['indicator_type'].widget.attrs = {'class': 'object-types'}
if user.has_access_to(Common.CAMPAIGN_READ):
self.fields['campaign'].choices = [('', '')] + [
(c.name, c.name) for c in get_item_names(Campaign, True)]
self.fields['campaign_confidence'].choices = [
("", ""),
("low", "low"),
("medium", "medium"),
("high", "high")]
self.fields['confidence'].choices = [
("unknown", "unknown"),
("benign", "benign"),
("low", "low"),
("medium", "medium"),
("high", "high")]
self.fields['impact'].choices = [
("unknown", "unknown"),
("benign", "benign"),
("low", "low"),
("medium", "medium"),
("high", "high")]
self.fields['relationship_type'].choices = relationship_choices
self.fields['relationship_type'].initial = RelationshipTypes.RELATED_TO
add_bucketlist_to_form(self)
add_ticket_to_form(self)
def __init__(self, username, *args, **kwargs):
super(UploadIndicatorForm, self).__init__(*args, **kwargs)
self.fields['source'].choices = [
(c.name, c.name) for c in get_source_names(True, True, username)]
self.fields['source'].initial = get_user_organization(username)
self.fields['indicator_type'].choices = [
(c,c) for c in IndicatorTypes.values(sort=True)
]
self.fields['threat_type'].choices = [
(c,c) for c in IndicatorThreatTypes.values(sort=True)
]
self.fields['threat_type'].initial = IndicatorThreatTypes.UNKNOWN
self.fields['attack_type'].choices = [
(c,c) for c in IndicatorAttackTypes.values(sort=True)
]
self.fields['attack_type'].initial = IndicatorAttackTypes.UNKNOWN
self.fields['indicator_type'].widget.attrs = {'class': 'object-types'}
self.fields['campaign'].choices = [("", "")]
self.fields['campaign'].choices += [
(c.name, c.name) for c in get_item_names(Campaign, True)]
self.fields['campaign_confidence'].choices = [
("", ""),
("low", "low"),
("medium", "medium"),
("high", "high")]
self.fields['confidence'].choices = [
("unknown", "unknown"),
("benign", "benign"),
("low", "low"),
("medium", "medium"),
("high", "high")]
self.fields['impact'].choices = [
("unknown", "unknown"),
("benign", "benign"),
("low", "low"),
("medium", "medium"),
("high", "high")]
add_bucketlist_to_form(self)
add_ticket_to_form(self)
def handle_indicator_insert(
ind, source, reference="", analyst="", method="", add_domain=False, add_relationship=False, cache={}
):
"""
Insert an individual indicator into the database.
NOTE: Setting add_domain to True will always create a relationship as well.
However, to create a relationship with an object that already exists before
this function was called, set add_relationship to True. This will assume
that the domain or IP object to create the relationship with already exists
and will avoid infinite mutual calls between, for example, add_update_ip
and this function. add domain/IP objects.
:param ind: Information about the indicator.
:type ind: dict
:param source: The source for this indicator.
:type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param reference: The reference to the data.
:type reference: str
:param analyst: The user adding this indicator.
:type analyst: str
:param method: Method of acquiring this indicator.
:type method: str
:param add_domain: If this indicator is also a top-level object, try to add
it.
:type add_domain: boolean
:param add_relationship: Attempt to add relationships if applicable.
:type add_relationship: boolean
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" (str) if failed,
"objectid" (str) if successful,
"is_new_indicator" (boolean) if successful.
"""
if ind["type"] not in IndicatorTypes.values():
return {"success": False, "message": "Not a valid Indicator Type: %s" % ind["type"]}
if ind["threat_type"] not in IndicatorThreatTypes.values():
return {"success": False, "message": "Not a valid Indicator Threat Type: %s" % ind["threat_type"]}
if ind["attack_type"] not in IndicatorAttackTypes.values():
return {"success": False, "message": "Not a valid Indicator Attack Type: " % ind["attack_type"]}
(ind["value"], error) = validate_indicator_value(ind["value"], ind["type"])
if error:
return {"success": False, "message": error}
is_new_indicator = False
dmain = None
ip = None
rank = {"unknown": 0, "benign": 1, "low": 2, "medium": 3, "high": 4}
if ind.get("status", None) is None or len(ind.get("status", "")) < 1:
ind["status"] = Status.NEW
indicator = Indicator.objects(
ind_type=ind["type"], lower=ind["lower"], threat_type=ind["threat_type"], attack_type=ind["attack_type"]
).first()
if not indicator:
indicator = Indicator()
indicator.ind_type = ind["type"]
indicator.threat_type = ind["threat_type"]
indicator.attack_type = ind["attack_type"]
indicator.value = ind["value"]
indicator.lower = ind["lower"]
indicator.description = ind["description"]
indicator.created = datetime.datetime.now()
indicator.confidence = EmbeddedConfidence(analyst=analyst)
indicator.impact = EmbeddedImpact(analyst=analyst)
indicator.status = ind["status"]
is_new_indicator = True
else:
if ind["status"] != Status.NEW:
indicator.status = ind["status"]
add_desc = "\nSeen on %s as: %s" % (str(datetime.datetime.now()), ind["value"])
if indicator.description is None:
indicator.description = add_desc
else:
indicator.description += add_desc
if "campaign" in ind:
if isinstance(ind["campaign"], basestring) and len(ind["campaign"]) > 0:
confidence = ind.get("campaign_confidence", "low")
ind["campaign"] = EmbeddedCampaign(
name=ind["campaign"],
confidence=confidence,
description="",
analyst=analyst,
date=datetime.datetime.now(),
)
if isinstance(ind["campaign"], EmbeddedCampaign):
indicator.add_campaign(ind["campaign"])
elif isinstance(ind["campaign"], list):
for campaign in ind["campaign"]:
if isinstance(campaign, EmbeddedCampaign):
indicator.add_campaign(campaign)
if "confidence" in ind and rank.get(ind["confidence"], 0) > rank.get(indicator.confidence.rating, 0):
indicator.confidence.rating = ind["confidence"]
indicator.confidence.analyst = analyst
if "impact" in ind and rank.get(ind["impact"], 0) > rank.get(indicator.impact.rating, 0):
indicator.impact.rating = ind["impact"]
indicator.impact.analyst = analyst
bucket_list = None
if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind:
bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME]
if bucket_list:
indicator.add_bucket_list(bucket_list, analyst)
ticket = None
if form_consts.Common.TICKET_VARIABLE_NAME in ind:
ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME]
if ticket:
indicator.add_ticket(ticket, analyst)
if isinstance(source, list):
for s in source:
indicator.add_source(source_item=s, method=method, reference=reference)
elif isinstance(source, EmbeddedSource):
indicator.add_source(source_item=source, method=method, reference=reference)
elif isinstance(source, basestring):
s = EmbeddedSource()
s.name = source
instance = EmbeddedSource.SourceInstance()
instance.reference = reference
instance.method = method
instance.analyst = analyst
instance.date = datetime.datetime.now()
s.instances = [instance]
indicator.add_source(s)
if add_domain or add_relationship:
ind_type = indicator.ind_type
ind_value = indicator.lower
url_contains_ip = False
if ind_type in (IndicatorTypes.DOMAIN, IndicatorTypes.URI):
if ind_type == IndicatorTypes.URI:
domain_or_ip = urlparse.urlparse(ind_value).hostname
try:
validate_ipv46_address(domain_or_ip)
url_contains_ip = True
except DjangoValidationError:
pass
else:
domain_or_ip = ind_value
if not url_contains_ip:
success = None
if add_domain:
success = upsert_domain(
domain_or_ip,
indicator.source,
username="******" % analyst,
campaign=indicator.campaign,
bucket_list=bucket_list,
cache=cache,
)
if not success["success"]:
return {"success": False, "message": success["message"]}
if not success or not "object" in success:
dmain = Domain.objects(domain=domain_or_ip).first()
else:
dmain = success["object"]
if ind_type in IPTypes.values() or url_contains_ip:
if url_contains_ip:
ind_value = domain_or_ip
try:
validate_ipv4_address(domain_or_ip)
ind_type = IndicatorTypes.IPV4_ADDRESS
except DjangoValidationError:
ind_type = IndicatorTypes.IPV6_ADDRESS
success = None
if add_domain:
success = ip_add_update(
ind_value,
ind_type,
source=indicator.source,
campaign=indicator.campaign,
analyst=analyst,
bucket_list=bucket_list,
ticket=ticket,
indicator_reference=reference,
cache=cache,
)
if not success["success"]:
return {"success": False, "message": success["message"]}
if not success or not "object" in success:
ip = IP.objects(ip=indicator.value).first()
else:
ip = success["object"]
indicator.save(username=analyst)
if dmain:
dmain.add_relationship(indicator, RelationshipTypes.RELATED_TO, analyst="%s" % analyst, get_rels=False)
dmain.save(username=analyst)
if ip:
ip.add_relationship(indicator, RelationshipTypes.RELATED_TO, analyst="%s" % analyst, get_rels=False)
ip.save(username=analyst)
# run indicator triage
if is_new_indicator:
indicator.reload()
run_triage(indicator, analyst)
return {"success": True, "objectid": str(indicator.id), "is_new_indicator": is_new_indicator, "object": indicator}
def handle_indicator_csv(csv_data, source, method, reference, ctype, username, add_domain=False):
"""
Handle adding Indicators in CSV format (file or blob).
:param csv_data: The CSV data.
:type csv_data: str or file handle
:param source: The name of the source for these indicators.
:type source: str
:param method: The method of acquisition of this indicator.
:type method: str
:param reference: The reference to this data.
:type reference: str
:param ctype: The CSV type.
:type ctype: str ("file" or "blob")
:param username: The user adding these indicators.
:type username: str
:param add_domain: If the indicators being added are also other top-level
objects, add those too.
:type add_domain: boolean
:returns: dict with keys "success" (boolean) and "message" (str)
"""
if ctype == "file":
cdata = csv_data.read()
else:
cdata = csv_data.encode("ascii")
data = csv.DictReader(BytesIO(cdata), skipinitialspace=True)
result = {"success": True}
result_message = ""
# Compute permitted values in CSV
valid_ratings = {"unknown": "unknown", "benign": "benign", "low": "low", "medium": "medium", "high": "high"}
valid_campaign_confidence = {"low": "low", "medium": "medium", "high": "high"}
valid_campaigns = {}
for c in Campaign.objects(active="on"):
valid_campaigns[c["name"].lower().replace(" - ", "-")] = c["name"]
valid_actions = {}
for a in Action.objects(active="on"):
valid_actions[a["name"].lower().replace(" - ", "-")] = a["name"]
valid_ind_types = {}
for obj in IndicatorTypes.values(sort=True):
valid_ind_types[obj.lower().replace(" - ", "-")] = obj
# Start line-by-line import
added = 0
for processed, d in enumerate(data, 1):
ind = {}
ind["value"] = d.get("Indicator", "").strip()
ind["lower"] = d.get("Indicator", "").lower().strip()
ind["description"] = d.get("Description", "").strip()
ind["type"] = get_verified_field(d, valid_ind_types, "Type")
ind["threat_type"] = d.get("Threat Type", IndicatorThreatTypes.UNKNOWN)
ind["attack_type"] = d.get("Attack Type", IndicatorAttackTypes.UNKNOWN)
if len(ind["threat_type"]) < 1:
ind["threat_type"] = IndicatorThreatTypes.UNKNOWN
if ind["threat_type"] not in IndicatorThreatTypes.values():
result["success"] = False
result_message += "Cannot process row %s: Invalid Threat Type<br />" % processed
continue
if len(ind["attack_type"]) < 1:
ind["attack_type"] = IndicatorAttackTypes.UNKNOWN
if ind["attack_type"] not in IndicatorAttackTypes.values():
result["success"] = False
result_message += "Cannot process row %s: Invalid Attack Type<br />" % processed
continue
ind["status"] = d.get("Status", Status.NEW)
if not ind["value"] or not ind["type"]:
# Mandatory value missing or malformed, cannot process csv row
i = ""
result["success"] = False
if not ind["value"]:
i += "No valid Indicator value "
if not ind["type"]:
i += "No valid Indicator type "
result_message += "Cannot process row %s: %s<br />" % (processed, i)
continue
campaign = get_verified_field(d, valid_campaigns, "Campaign")
if campaign:
ind["campaign"] = campaign
ind["campaign_confidence"] = get_verified_field(
d, valid_campaign_confidence, "Campaign Confidence", default="low"
)
actions = d.get("Action", "")
if actions:
actions = get_verified_field(actions.split(","), valid_actions)
if not actions:
result["success"] = False
result_message += "Cannot process row %s: Invalid Action<br />" % processed
continue
ind["confidence"] = get_verified_field(d, valid_ratings, "Confidence", default="unknown")
ind["impact"] = get_verified_field(d, valid_ratings, "Impact", default="unknown")
ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME] = d.get(form_consts.Common.BUCKET_LIST, "")
ind[form_consts.Common.TICKET_VARIABLE_NAME] = d.get(form_consts.Common.TICKET, "")
try:
response = handle_indicator_insert(
ind, source, reference, analyst=username, method=method, add_domain=add_domain
)
except Exception, e:
result["success"] = False
result_message += "Failure processing row %s: %s<br />" % (processed, str(e))
continue
if response["success"]:
if actions:
action = {
"active": "on",
"analyst": username,
"begin_date": "",
"end_date": "",
"performed_date": "",
"reason": "",
"date": datetime.datetime.now(),
}
for action_type in actions:
action["action_type"] = action_type
action_add("Indicator", response.get("objectid"), action, user=username)
else:
result["success"] = False
result_message += "Failure processing row %s: %s<br />" % (processed, response["message"])
continue
added += 1