def upsert_domain(sdomain,
domain,
source,
username=None,
campaign=None,
confidence=None,
bucket_list=None,
ticket=None,
cache={}):
"""
Add or update a domain/FQDN. Campaign is assumed to be a list of campaign
dictionary objects.
:param sdomain: Response from parsing the domain for a root domain. Will
either be an error message or the root domain itself.
:type sdomain: str
:param domain: The domain to add/update.
:type domain: str
:param source: The name of the source.
:type source: str
:param username: The user adding/updating the domain.
:type username: str
:param campaign: The campaign to attribute to this domain.
:type campaign: list, str
:param confidence: Confidence for the campaign attribution.
:type confidence: str
:param bucket_list: List of buckets to add to this domain.
:type bucket_list: list, str
:param ticket: The ticket for this domain.
:type ticket: str
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"object" the domain that was added,
"is_domain_new" (boolean)
"""
if sdomain == "no_tld_found_error": #oops...
return {'success': False, 'message': "Invalid domain: %s " % sdomain}
is_fqdn_domain_new = False
is_root_domain_new = False
if not campaign:
campaign = []
# assume it's a list, but check if it's a string
elif isinstance(campaign, basestring):
c = EmbeddedCampaign(name=campaign,
confidence=confidence,
analyst=username)
campaign = [c]
# assume it's a list, but check if it's a string
if isinstance(source, basestring):
s = EmbeddedSource()
s.name = source
instance = EmbeddedSource.SourceInstance()
instance.reference = ''
instance.method = ''
instance.analyst = username
instance.date = datetime.datetime.now()
s.instances = [instance]
source = [s]
fqdn_domain = None
root_domain = None
cached_results = cache.get(form_consts.Domain.CACHED_RESULTS)
if cached_results != None:
if domain != sdomain:
fqdn_domain = cached_results.get(domain)
root_domain = cached_results.get(sdomain)
else:
root_domain = cached_results.get(sdomain)
else:
#first find the domain(s) if it/they already exist
root_domain = Domain.objects(domain=sdomain).first()
if domain != sdomain:
fqdn_domain = Domain.objects(domain=domain).first()
#if they don't exist, create them
if not root_domain:
root_domain = Domain()
root_domain.domain = sdomain
root_domain.source = []
root_domain.record_type = 'A'
is_root_domain_new = True
if cached_results != None:
cached_results[sdomain] = root_domain
if domain != sdomain and not fqdn_domain:
fqdn_domain = Domain()
fqdn_domain.domain = domain
fqdn_domain.source = []
fqdn_domain.record_type = 'A'
is_fqdn_domain_new = True
if cached_results != None:
cached_results[domain] = fqdn_domain
# if new or found, append the new source(s)
for s in source:
if root_domain:
root_domain.add_source(s)
if fqdn_domain:
fqdn_domain.add_source(s)
#campaigns
#both root and fqdn get campaigns updated
for c in campaign:
if root_domain:
root_domain.add_campaign(c)
if fqdn_domain:
fqdn_domain.add_campaign(c)
if username:
if root_domain:
root_domain.analyst = username
if fqdn_domain:
fqdn_domain.analyst = username
if bucket_list:
if root_domain:
root_domain.add_bucket_list(bucket_list, username)
if fqdn_domain:
fqdn_domain.add_bucket_list(bucket_list, username)
if ticket:
if root_domain:
root_domain.add_ticket(ticket, username)
if fqdn_domain:
fqdn_domain.add_ticket(ticket, username)
# save
try:
if root_domain:
root_domain.save(username=username)
if fqdn_domain:
fqdn_domain.save(username=username)
except Exception, e:
return {'success': False, 'message': e}
def handle_indicator_insert(ind, source, reference='', analyst='', method='',
add_domain=False, add_relationship=False, cache={}):
"""
Insert an individual indicator into the database.
NOTE: Setting add_domain to True will always create a relationship as well.
However, to create a relationship with an object that already exists before
this function was called, set add_relationship to True. This will assume
that the domain or IP object to create the relationship with already exists
and will avoid infinite mutual calls between, for example, add_update_ip
and this function. add domain/IP objects.
:param ind: Information about the indicator.
:type ind: dict
:param source: The source for this indicator.
:type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param reference: The reference to the data.
:type reference: str
:param analyst: The user adding this indicator.
:type analyst: str
:param method: Method of acquiring this indicator.
:type method: str
:param add_domain: If this indicator is also a top-level object, try to add
it.
:type add_domain: boolean
:param add_relationship: Attempt to add relationships if applicable.
:type add_relationship: boolean
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" str) if failed,
"objectid" (str) if successful,
"is_new_indicator" (boolean) if successful.
"""
is_new_indicator = False;
rank = {
'unknown': 0,
'benign': 1,
'low': 2,
'medium': 3,
'high': 4
}
indicator = Indicator.objects(ind_type=ind['type'],
value=ind['value']).first()
if not indicator:
indicator = Indicator()
indicator.ind_type = ind['type']
indicator.value = ind['value']
indicator.created = datetime.datetime.now()
indicator.confidence = EmbeddedConfidence(analyst=analyst)
indicator.impact = EmbeddedImpact(analyst=analyst)
is_new_indicator = True;
ec = None
if 'campaign' in ind:
confidence = 'low'
if 'campaign_confidence' in ind:
confidence = ind['campaign_confidence']
ec = EmbeddedCampaign(name=ind['campaign'],
confidence=confidence,
description="",
analyst=analyst,
date=datetime.datetime.now())
if 'confidence' in ind and rank.get(ind['confidence'], 0) > rank.get(indicator.confidence.rating, 0):
indicator.confidence.rating = ind['confidence']
indicator.confidence.analyst = analyst
if 'impact' in ind and rank.get(ind['impact'], 0) > rank.get(indicator.impact.rating, 0):
indicator.impact.rating = ind['impact']
indicator.impact.analyst = analyst
bucket_list = None
if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind:
bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME]
indicator.add_bucket_list(bucket_list, analyst)
ticket = None
if form_consts.Common.TICKET_VARIABLE_NAME in ind:
ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME]
indicator.add_ticket(ticket, analyst)
if isinstance(source, list):
for s in source:
indicator.add_source(source_item=s)
elif isinstance(source, EmbeddedSource):
indicator.add_source(source_item=source)
elif isinstance(source, basestring):
s = EmbeddedSource()
s.name = source
instance = EmbeddedSource.SourceInstance()
instance.reference = reference
instance.method = method
instance.analyst = analyst
instance.date = datetime.datetime.now()
s.instances = [instance]
indicator.add_source(s)
if ec:
indicator.add_campaign(ec)
indicator.save(username=analyst)
if add_domain or add_relationship:
ind_type = indicator.ind_type
ind_value = indicator.value
url_contains_ip = False
if ind_type in ("URI - Domain Name", "URI - URL"):
if ind_type == "URI - URL":
domain_or_ip = urlparse.urlparse(ind_value).hostname
elif ind_type == "URI - Domain Name":
domain_or_ip = ind_value
#try:
(sdomain, fqdn) = get_domain(domain_or_ip)
if sdomain == "no_tld_found_error" and ind_type == "URI - URL":
try:
validate_ipv46_address(domain_or_ip)
url_contains_ip = True
except DjangoValidationError, e:
pass
if not url_contains_ip:
success = None
if add_domain:
success = upsert_domain(sdomain, fqdn, indicator.source,
'%s' % analyst, None,
bucket_list=bucket_list, cache=cache)
if not success['success']:
indicator_link = '<a href=\"%s\">View indicator</a>.</div>' \
% (reverse('crits.indicators.views.indicator', args=[indicator.id]));
message = "Indicator was added, but an error occurred. " + indicator_link + "<br>"
return {'success':False, 'message':message + success['message']}
if not success or not 'object' in success:
dmain = Domain.objects(domain=domain_or_ip).first()
else:
dmain = success['object']
if dmain:
dmain.add_relationship(rel_item=indicator,
rel_type='Related_To',
analyst="%s" % analyst,
get_rels=False)
dmain.save(username=analyst)
indicator.save(username=analyst)
if ind_type.startswith("Address - ip") or ind_type == "Address - cidr" or url_contains_ip:
if url_contains_ip:
ind_value = domain_or_ip
success = None
if add_domain:
success = ip_add_update(ind_value,
ind_type,
source=indicator.source,
campaign=indicator.campaign,
analyst=analyst,
bucket_list=bucket_list,
ticket=ticket,
indicator_reference=reference,
cache=cache)
if not success['success']:
indicator_link = '<a href=\"%s\">View indicator</a>.</div>' \
% (reverse('crits.indicators.views.indicator', args=[indicator.id]));
message = "Indicator was added, but an error occurred. " + indicator_link + "<br>"
return {'success':False, 'message':message + success['message']}
if not success or not 'object' in success:
ip = IP.objects(ip=indicator.value).first()
else:
ip = success['object']
if ip:
ip.add_relationship(rel_item=indicator,
rel_type='Related_To',
analyst="%s" % analyst,
get_rels=False)
ip.save(username=analyst)
indicator.save(username=analyst)
def upsert_domain(domain,
source,
username=None,
campaign=None,
confidence=None,
bucket_list=None,
ticket=None,
cache={},
related_id=None,
related_type=None,
relationship_type=None):
"""
Add or update a domain/FQDN. Campaign is assumed to be a list of campaign
dictionary objects.
:param domain: The domain to add/update.
:type domain: str
:param source: The name of the source.
:type source: str
:param username: The user adding/updating the domain.
:type username: str
:param campaign: The campaign to attribute to this domain.
:type campaign: list, str
:param confidence: Confidence for the campaign attribution.
:type confidence: str
:param bucket_list: List of buckets to add to this domain.
:type bucket_list: list, str
:param ticket: The ticket for this domain.
:type ticket: str
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_id: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:returns: dict with keys:
"success" (boolean),
"object" the domain that was added,
"is_domain_new" (boolean)
"""
# validate domain and grab root domain
(root, domain, error) = get_valid_root_domain(domain)
if error:
return {'success': False, 'message': error}
is_fqdn_domain_new = False
is_root_domain_new = False
if not campaign:
campaign = []
# assume it's a list, but check if it's a string
elif isinstance(campaign, basestring):
c = EmbeddedCampaign(name=campaign,
confidence=confidence,
analyst=username)
campaign = [c]
# assume it's a list, but check if it's a string
if isinstance(source, basestring):
s = EmbeddedSource()
s.name = source
instance = EmbeddedSource.SourceInstance()
instance.reference = ''
instance.method = ''
instance.analyst = username
instance.date = datetime.datetime.now()
s.instances = [instance]
source = [s]
fqdn_domain = None
root_domain = None
cached_results = cache.get(form_consts.Domain.CACHED_RESULTS)
if cached_results != None:
if domain != root:
fqdn_domain = cached_results.get(domain)
root_domain = cached_results.get(root)
else:
root_domain = cached_results.get(root)
else:
#first find the domain(s) if it/they already exist
root_domain = Domain.objects(domain=root).first()
if domain != root:
fqdn_domain = Domain.objects(domain=domain).first()
#if they don't exist, create them
if not root_domain:
root_domain = Domain()
root_domain.domain = root
root_domain.source = []
root_domain.record_type = 'A'
is_root_domain_new = True
if cached_results != None:
cached_results[root] = root_domain
if domain != root and not fqdn_domain:
fqdn_domain = Domain()
fqdn_domain.domain = domain
fqdn_domain.source = []
fqdn_domain.record_type = 'A'
is_fqdn_domain_new = True
if cached_results != None:
cached_results[domain] = fqdn_domain
# if new or found, append the new source(s)
for s in source:
if root_domain:
root_domain.add_source(s)
if fqdn_domain:
fqdn_domain.add_source(s)
#campaigns
#both root and fqdn get campaigns updated
for c in campaign:
if root_domain:
root_domain.add_campaign(c)
if fqdn_domain:
fqdn_domain.add_campaign(c)
if username:
if root_domain:
root_domain.analyst = username
if fqdn_domain:
fqdn_domain.analyst = username
if bucket_list:
if root_domain:
root_domain.add_bucket_list(bucket_list, username)
if fqdn_domain:
fqdn_domain.add_bucket_list(bucket_list, username)
if ticket:
if root_domain:
root_domain.add_ticket(ticket, username)
if fqdn_domain:
fqdn_domain.add_ticket(ticket, username)
related_obj = None
if related_id:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
retVal['success'] = False
retVal['message'] = 'Related Object not found.'
return retVal
# save
try:
if root_domain:
root_domain.save(username=username)
if fqdn_domain:
fqdn_domain.save(username=username)
except Exception, e:
return {'success': False, 'message': e}
