def handle_indicator_insert(ind, source, reference='', analyst='', method='',
add_domain=False, add_relationship=False, cache={}):
"""
Insert an individual indicator into the database.
NOTE: Setting add_domain to True will always create a relationship as well.
However, to create a relationship with an object that already exists before
this function was called, set add_relationship to True. This will assume
that the domain or IP object to create the relationship with already exists
and will avoid infinite mutual calls between, for example, add_update_ip
and this function. add domain/IP objects.
:param ind: Information about the indicator.
:type ind: dict
:param source: The source for this indicator.
:type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param reference: The reference to the data.
:type reference: str
:param analyst: The user adding this indicator.
:type analyst: str
:param method: Method of acquiring this indicator.
:type method: str
:param add_domain: If this indicator is also a top-level object, try to add
it.
:type add_domain: boolean
:param add_relationship: Attempt to add relationships if applicable.
:type add_relationship: boolean
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" str) if failed,
"objectid" (str) if successful,
"is_new_indicator" (boolean) if successful.
"""
if ind['type'] == "URI - URL" and "://" not in ind['value'].split('.')[0]:
return {"success": False, "message": "URI - URL must contain protocol prefix (e.g. http://, https://, ftp://) "}
is_new_indicator = False
dmain = None
ip = None
rank = {
'unknown': 0,
'benign': 1,
'low': 2,
'medium': 3,
'high': 4,
}
indicator = Indicator.objects(ind_type=ind['type'],
value=ind['value']).first()
if not indicator:
indicator = Indicator()
indicator.ind_type = ind['type']
indicator.value = ind['value']
indicator.created = datetime.datetime.now()
indicator.confidence = EmbeddedConfidence(analyst=analyst)
indicator.impact = EmbeddedImpact(analyst=analyst)
is_new_indicator = True
if 'campaign' in ind:
if isinstance(ind['campaign'], basestring) and len(ind['campaign']) > 0:
confidence = ind.get('campaign_confidence', 'low')
ind['campaign'] = EmbeddedCampaign(name=ind['campaign'],
confidence=confidence,
description="",
analyst=analyst,
date=datetime.datetime.now())
if isinstance(ind['campaign'], EmbeddedCampaign):
indicator.add_campaign(ind['campaign'])
elif isinstance(ind['campaign'], list):
for campaign in ind['campaign']:
if isinstance(campaign, EmbeddedCampaign):
indicator.add_campaign(campaign)
if 'confidence' in ind and rank.get(ind['confidence'], 0) > rank.get(indicator.confidence.rating, 0):
indicator.confidence.rating = ind['confidence']
indicator.confidence.analyst = analyst
if 'impact' in ind and rank.get(ind['impact'], 0) > rank.get(indicator.impact.rating, 0):
indicator.impact.rating = ind['impact']
indicator.impact.analyst = analyst
bucket_list = None
if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind:
bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME]
if bucket_list:
indicator.add_bucket_list(bucket_list, analyst)
ticket = None
if form_consts.Common.TICKET_VARIABLE_NAME in ind:
ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME]
if ticket:
indicator.add_ticket(ticket, analyst)
if isinstance(source, list):
for s in source:
indicator.add_source(source_item=s, method=method, reference=reference)
elif isinstance(source, EmbeddedSource):
indicator.add_source(source_item=source, method=method, reference=reference)
elif isinstance(source, basestring):
s = EmbeddedSource()
s.name = source
instance = EmbeddedSource.SourceInstance()
instance.reference = reference
instance.method = method
instance.analyst = analyst
instance.date = datetime.datetime.now()
s.instances = [instance]
indicator.add_source(s)
if add_domain or add_relationship:
ind_type = indicator.ind_type
ind_value = indicator.value
url_contains_ip = False
if ind_type in ("URI - Domain Name", "URI - URL"):
if ind_type == "URI - URL":
domain_or_ip = urlparse.urlparse(ind_value).hostname
elif ind_type == "URI - Domain Name":
domain_or_ip = ind_value
(sdomain, fqdn) = get_domain(domain_or_ip)
if sdomain == "no_tld_found_error" and ind_type == "URI - URL":
try:
validate_ipv46_address(domain_or_ip)
url_contains_ip = True
except DjangoValidationError:
pass
if not url_contains_ip:
success = None
if add_domain:
success = upsert_domain(sdomain, fqdn, indicator.source,
'%s' % analyst, None,
bucket_list=bucket_list, cache=cache)
if not success['success']:
return {'success': False, 'message': success['message']}
if not success or not 'object' in success:
dmain = Domain.objects(domain=domain_or_ip).first()
else:
dmain = success['object']
if ind_type.startswith("Address - ip") or ind_type == "Address - cidr" or url_contains_ip:
if url_contains_ip:
ind_value = domain_or_ip
try:
validate_ipv4_address(domain_or_ip)
ind_type = 'Address - ipv4-addr'
except DjangoValidationError:
ind_type = 'Address - ipv6-addr'
success = None
if add_domain:
success = ip_add_update(ind_value,
ind_type,
source=indicator.source,
campaign=indicator.campaign,
analyst=analyst,
bucket_list=bucket_list,
ticket=ticket,
indicator_reference=reference,
cache=cache)
if not success['success']:
return {'success': False, 'message': success['message']}
if not success or not 'object' in success:
ip = IP.objects(ip=indicator.value).first()
else:
ip = success['object']
indicator.save(username=analyst)
if dmain:
dmain.add_relationship(rel_item=indicator,
rel_type='Related_To',
analyst="%s" % analyst,
get_rels=False)
dmain.save(username=analyst)
if ip:
ip.add_relationship(rel_item=indicator,
rel_type='Related_To',
analyst="%s" % analyst,
get_rels=False)
ip.save(username=analyst)
indicator.save(username=analyst)
# run indicator triage
if is_new_indicator:
indicator.reload()
run_triage(indicator, analyst)
return {'success': True, 'objectid': str(indicator.id),
'is_new_indicator': is_new_indicator, 'object': indicator}
def create_indicator_and_ip(type_, id_, ip, analyst):
"""
Add indicators for an IP address.
:param type_: The CRITs top-level object we are getting this IP from.
:type type_: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param id_: The ObjectId of the top-level object to search for.
:type id_: str
:param ip: The IP address to generate an indicator out of.
:type ip: str
:param analyst: The user adding this indicator.
:type analyst: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"value" (str)
"""
obj_class = class_from_id(type_, id_)
if obj_class:
ip_class = IP.objects(ip=ip).first()
ind_type = "Address - ipv4-addr"
ind_class = Indicator.objects(ind_type=ind_type, value=ip).first()
# setup IP
if ip_class:
ip_class.add_relationship(rel_item=obj_class,
rel_type="Related_To",
analyst=analyst)
else:
ip_class = IP()
ip_class.ip = ip
ip_class.source = obj_class.source
ip_class.save(username=analyst)
ip_class.add_relationship(rel_item=obj_class,
rel_type="Related_To",
analyst=analyst)
# setup Indicator
message = ""
if ind_class:
message = ind_class.add_relationship(rel_item=obj_class,
rel_type="Related_To",
analyst=analyst)
ind_class.add_relationship(rel_item=ip_class,
rel_type="Related_To",
analyst=analyst)
else:
ind_class = Indicator()
ind_class.source = obj_class.source
ind_class.ind_type = ind_type
ind_class.value = ip
ind_class.save(username=analyst)
message = ind_class.add_relationship(rel_item=obj_class,
rel_type="Related_To",
analyst=analyst)
ind_class.add_relationship(rel_item=ip_class,
rel_type="Related_To",
analyst=analyst)
# save
try:
obj_class.save(username=analyst)
ip_class.save(username=analyst)
ind_class.save(username=analyst)
if message['success']:
rels = obj_class.sort_relationships("%s" % analyst, meta=True)
return {'success': True, 'message': rels, 'value': obj_class.id}
else:
return {'success': False, 'message': message['message']}
except Exception, e:
return {'success': False, 'message': e}
def create_indicator_and_ip(type_, id_, ip, analyst):
"""
Add indicators for an IP address.
:param type_: The CRITs top-level object we are getting this IP from.
:type type_: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param id_: The ObjectId of the top-level object to search for.
:type id_: str
:param ip: The IP address to generate an indicator out of.
:type ip: str
:param analyst: The user adding this indicator.
:type analyst: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"value" (str)
"""
obj_class = class_from_id(type_, id_)
if obj_class:
ip_class = IP.objects(ip=ip).first()
ind_type = "Address - ipv4-addr"
ind_class = Indicator.objects(ind_type=ind_type, value=ip).first()
# setup IP
if ip_class:
ip_class.add_relationship(rel_item=obj_class,
rel_type="Related_To",
analyst=analyst)
else:
ip_class = IP()
ip_class.ip = ip
ip_class.source = obj_class.source
ip_class.save(username=analyst)
ip_class.add_relationship(rel_item=obj_class,
rel_type="Related_To",
analyst=analyst)
# setup Indicator
message = ""
if ind_class:
message = ind_class.add_relationship(rel_item=obj_class,
rel_type="Related_To",
analyst=analyst)
ind_class.add_relationship(rel_item=ip_class,
rel_type="Related_To",
analyst=analyst)
else:
ind_class = Indicator()
ind_class.source = obj_class.source
ind_class.ind_type = ind_type
ind_class.value = ip
ind_class.save(username=analyst)
message = ind_class.add_relationship(rel_item=obj_class,
rel_type="Related_To",
analyst=analyst)
ind_class.add_relationship(rel_item=ip_class,
rel_type="Related_To",
analyst=analyst)
# save
try:
obj_class.save(username=analyst)
ip_class.save(username=analyst)
ind_class.save(username=analyst)
if message['success']:
rels = obj_class.sort_relationships("%s" % analyst, meta=True)
return {
'success': True,
'message': rels,
'value': obj_class.id
}
else:
return {'success': False, 'message': message['message']}
except Exception, e:
return {'success': False, 'message': e}
def handle_indicator_insert(ind,
source,
reference='',
analyst='',
method='',
add_domain=False,
add_relationship=False,
cache={}):
"""
Insert an individual indicator into the database.
NOTE: Setting add_domain to True will always create a relationship as well.
However, to create a relationship with an object that already exists before
this function was called, set add_relationship to True. This will assume
that the domain or IP object to create the relationship with already exists
and will avoid infinite mutual calls between, for example, add_update_ip
and this function. add domain/IP objects.
:param ind: Information about the indicator.
:type ind: dict
:param source: The source for this indicator.
:type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param reference: The reference to the data.
:type reference: str
:param analyst: The user adding this indicator.
:type analyst: str
:param method: Method of acquiring this indicator.
:type method: str
:param add_domain: If this indicator is also a top-level object, try to add
it.
:type add_domain: boolean
:param add_relationship: Attempt to add relationships if applicable.
:type add_relationship: boolean
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" str) if failed,
"objectid" (str) if successful,
"is_new_indicator" (boolean) if successful.
"""
is_new_indicator = False
rank = {'unknown': 0, 'benign': 1, 'low': 2, 'medium': 3, 'high': 4}
indicator = Indicator.objects(ind_type=ind['type'],
value=ind['value']).first()
if not indicator:
indicator = Indicator()
indicator.ind_type = ind['type']
indicator.value = ind['value']
indicator.created = datetime.datetime.now()
indicator.confidence = EmbeddedConfidence(analyst=analyst)
indicator.impact = EmbeddedImpact(analyst=analyst)
is_new_indicator = True
ec = None
if 'campaign' in ind:
confidence = 'low'
if 'campaign_confidence' in ind:
confidence = ind['campaign_confidence']
ec = EmbeddedCampaign(name=ind['campaign'],
confidence=confidence,
description="",
analyst=analyst,
date=datetime.datetime.now())
if 'confidence' in ind and rank.get(ind['confidence'], 0) > rank.get(
indicator.confidence.rating, 0):
indicator.confidence.rating = ind['confidence']
indicator.confidence.analyst = analyst
if 'impact' in ind and rank.get(ind['impact'], 0) > rank.get(
indicator.impact.rating, 0):
indicator.impact.rating = ind['impact']
indicator.impact.analyst = analyst
bucket_list = None
if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind:
bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME]
indicator.add_bucket_list(bucket_list, analyst)
ticket = None
if form_consts.Common.TICKET_VARIABLE_NAME in ind:
ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME]
indicator.add_ticket(ticket, analyst)
if isinstance(source, list):
for s in source:
indicator.add_source(source_item=s)
elif isinstance(source, EmbeddedSource):
indicator.add_source(source_item=source)
elif isinstance(source, basestring):
s = EmbeddedSource()
s.name = source
instance = EmbeddedSource.SourceInstance()
instance.reference = reference
instance.method = method
instance.analyst = analyst
instance.date = datetime.datetime.now()
s.instances = [instance]
indicator.add_source(s)
if ec:
indicator.add_campaign(ec)
indicator.save(username=analyst)
if add_domain or add_relationship:
ind_type = indicator.ind_type
ind_value = indicator.value
if ind_type in ("URI - Domain Name", "URI - URL"):
if ind_type == "URI - URL":
domain = ind_value.split("/")[2]
elif ind_type == "URI - Domain Name":
domain = ind_value
#try:
(sdomain, fqdn) = get_domain(domain)
success = None
if add_domain:
success = upsert_domain(sdomain,
fqdn,
indicator.source,
'%s' % analyst,
None,
bucket_list=bucket_list,
cache=cache)
if not success['success']:
return {'success': False, 'message': success['message']}
if not success or not 'object' in success:
dmain = Domain.objects(domain=domain).first()
else:
dmain = success['object']
if dmain:
dmain.add_relationship(rel_item=indicator,
rel_type='Related_To',
analyst="%s" % analyst,
get_rels=False)
dmain.save(username=analyst)
indicator.save(username=analyst)
elif ind_type.startswith(
"Address - ip") or ind_type == "Address - cidr":
success = None
if add_domain:
success = ip_add_update(indicator.value,
ind_type,
source=indicator.source,
campaign=indicator.campaign,
analyst=analyst,
bucket_list=bucket_list,
ticket=ticket,
indicator_reference=reference,
cache=cache)
if not success['success']:
return {'success': False, 'message': success['message']}
if not success or not 'object' in success:
ip = IP.objects(ip=indicator.value).first()
else:
ip = success['object']
if ip:
ip.add_relationship(rel_item=indicator,
rel_type='Related_To',
analyst="%s" % analyst,
get_rels=False)
ip.save(username=analyst)
indicator.save(username=analyst)
# run indicator triage
if is_new_indicator:
indicator.reload()
run_triage(None, indicator, analyst)
return {
'success': True,
'objectid': indicator.id,
'is_new_indicator': is_new_indicator,
'object': indicator
}
def handle_indicator_insert(
ind, source, reference="", analyst="", method="", add_domain=False, add_relationship=False, cache={}
):
"""
Insert an individual indicator into the database.
NOTE: Setting add_domain to True will always create a relationship as well.
However, to create a relationship with an object that already exists before
this function was called, set add_relationship to True. This will assume
that the domain or IP object to create the relationship with already exists
and will avoid infinite mutual calls between, for example, add_update_ip
and this function. add domain/IP objects.
:param ind: Information about the indicator.
:type ind: dict
:param source: The source for this indicator.
:type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param reference: The reference to the data.
:type reference: str
:param analyst: The user adding this indicator.
:type analyst: str
:param method: Method of acquiring this indicator.
:type method: str
:param add_domain: If this indicator is also a top-level object, try to add
it.
:type add_domain: boolean
:param add_relationship: Attempt to add relationships if applicable.
:type add_relationship: boolean
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" (str) if failed,
"objectid" (str) if successful,
"is_new_indicator" (boolean) if successful.
"""
if ind["type"] not in IndicatorTypes.values():
return {"success": False, "message": "Not a valid Indicator Type: %s" % ind["type"]}
if ind["threat_type"] not in IndicatorThreatTypes.values():
return {"success": False, "message": "Not a valid Indicator Threat Type: %s" % ind["threat_type"]}
if ind["attack_type"] not in IndicatorAttackTypes.values():
return {"success": False, "message": "Not a valid Indicator Attack Type: " % ind["attack_type"]}
(ind["value"], error) = validate_indicator_value(ind["value"], ind["type"])
if error:
return {"success": False, "message": error}
is_new_indicator = False
dmain = None
ip = None
rank = {"unknown": 0, "benign": 1, "low": 2, "medium": 3, "high": 4}
if ind.get("status", None) is None or len(ind.get("status", "")) < 1:
ind["status"] = Status.NEW
indicator = Indicator.objects(
ind_type=ind["type"], lower=ind["lower"], threat_type=ind["threat_type"], attack_type=ind["attack_type"]
).first()
if not indicator:
indicator = Indicator()
indicator.ind_type = ind["type"]
indicator.threat_type = ind["threat_type"]
indicator.attack_type = ind["attack_type"]
indicator.value = ind["value"]
indicator.lower = ind["lower"]
indicator.description = ind["description"]
indicator.created = datetime.datetime.now()
indicator.confidence = EmbeddedConfidence(analyst=analyst)
indicator.impact = EmbeddedImpact(analyst=analyst)
indicator.status = ind["status"]
is_new_indicator = True
else:
if ind["status"] != Status.NEW:
indicator.status = ind["status"]
add_desc = "\nSeen on %s as: %s" % (str(datetime.datetime.now()), ind["value"])
if indicator.description is None:
indicator.description = add_desc
else:
indicator.description += add_desc
if "campaign" in ind:
if isinstance(ind["campaign"], basestring) and len(ind["campaign"]) > 0:
confidence = ind.get("campaign_confidence", "low")
ind["campaign"] = EmbeddedCampaign(
name=ind["campaign"],
confidence=confidence,
description="",
analyst=analyst,
date=datetime.datetime.now(),
)
if isinstance(ind["campaign"], EmbeddedCampaign):
indicator.add_campaign(ind["campaign"])
elif isinstance(ind["campaign"], list):
for campaign in ind["campaign"]:
if isinstance(campaign, EmbeddedCampaign):
indicator.add_campaign(campaign)
if "confidence" in ind and rank.get(ind["confidence"], 0) > rank.get(indicator.confidence.rating, 0):
indicator.confidence.rating = ind["confidence"]
indicator.confidence.analyst = analyst
if "impact" in ind and rank.get(ind["impact"], 0) > rank.get(indicator.impact.rating, 0):
indicator.impact.rating = ind["impact"]
indicator.impact.analyst = analyst
bucket_list = None
if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind:
bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME]
if bucket_list:
indicator.add_bucket_list(bucket_list, analyst)
ticket = None
if form_consts.Common.TICKET_VARIABLE_NAME in ind:
ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME]
if ticket:
indicator.add_ticket(ticket, analyst)
if isinstance(source, list):
for s in source:
indicator.add_source(source_item=s, method=method, reference=reference)
elif isinstance(source, EmbeddedSource):
indicator.add_source(source_item=source, method=method, reference=reference)
elif isinstance(source, basestring):
s = EmbeddedSource()
s.name = source
instance = EmbeddedSource.SourceInstance()
instance.reference = reference
instance.method = method
instance.analyst = analyst
instance.date = datetime.datetime.now()
s.instances = [instance]
indicator.add_source(s)
if add_domain or add_relationship:
ind_type = indicator.ind_type
ind_value = indicator.lower
url_contains_ip = False
if ind_type in (IndicatorTypes.DOMAIN, IndicatorTypes.URI):
if ind_type == IndicatorTypes.URI:
domain_or_ip = urlparse.urlparse(ind_value).hostname
try:
validate_ipv46_address(domain_or_ip)
url_contains_ip = True
except DjangoValidationError:
pass
else:
domain_or_ip = ind_value
if not url_contains_ip:
success = None
if add_domain:
success = upsert_domain(
domain_or_ip,
indicator.source,
username="******" % analyst,
campaign=indicator.campaign,
bucket_list=bucket_list,
cache=cache,
)
if not success["success"]:
return {"success": False, "message": success["message"]}
if not success or not "object" in success:
dmain = Domain.objects(domain=domain_or_ip).first()
else:
dmain = success["object"]
if ind_type in IPTypes.values() or url_contains_ip:
if url_contains_ip:
ind_value = domain_or_ip
try:
validate_ipv4_address(domain_or_ip)
ind_type = IndicatorTypes.IPV4_ADDRESS
except DjangoValidationError:
ind_type = IndicatorTypes.IPV6_ADDRESS
success = None
if add_domain:
success = ip_add_update(
ind_value,
ind_type,
source=indicator.source,
campaign=indicator.campaign,
analyst=analyst,
bucket_list=bucket_list,
ticket=ticket,
indicator_reference=reference,
cache=cache,
)
if not success["success"]:
return {"success": False, "message": success["message"]}
if not success or not "object" in success:
ip = IP.objects(ip=indicator.value).first()
else:
ip = success["object"]
indicator.save(username=analyst)
if dmain:
dmain.add_relationship(indicator, RelationshipTypes.RELATED_TO, analyst="%s" % analyst, get_rels=False)
dmain.save(username=analyst)
if ip:
ip.add_relationship(indicator, RelationshipTypes.RELATED_TO, analyst="%s" % analyst, get_rels=False)
ip.save(username=analyst)
# run indicator triage
if is_new_indicator:
indicator.reload()
run_triage(indicator, analyst)
return {"success": True, "objectid": str(indicator.id), "is_new_indicator": is_new_indicator, "object": indicator}
