def __init__(self, data, analyst, method):
"""
Instantiation of the STIXParser can take the data to parse, the analyst
doing the parsing, and the method of data aquisition.
:param data: The data to parse.
:type data: str
:param analyst: The analyst parsing the document.
:type analyst: str
:param method: The method of acquiring this data.
:type method: str
"""
self.data = data
self.package = None
self.source = EmbeddedSource(
) # source.name comes from the stix header.
self.source_instance = EmbeddedSource.SourceInstance()
# The reference attribute and appending it to the source is
# done after the TAXII message ID is determined.
self.source_instance.analyst = analyst
self.source_instance.method = method
self.information_source = None
self.event = None # the Event TLO
self.event_rels = {} # track relationships to the event
self.relationships = [] # track other relationships that need forming
self.imported = {} # track items that are imported
self.failed = [] # track STIX/CybOX items that failed import
self.saved_artifacts = {}
def sanitize(self, username=None, sources=None, rels=None):
"""
Sanitize the source list down to only those a user has access to see.
This was sniped from core/crits_mongoengine.
:param username: The user requesting this data.
:type username: str
:param sources: A list of sources the user has access to.
:type sources: list
"""
if username and hasattr(self, 'source'):
length = len(self.source)
if not sources:
sources = user_sources(username)
# use slice to modify in place in case any code is referencing
# the source already will reflect the changes as well
self.source[:] = [s for s in self.source if s.name in sources]
# a bit of a hack but we add a poorly formatted source to the
# source list which has an instances length equal to the amount
# of sources that were sanitized out of the user's list.
# not tested but this has the added benefit of throwing a
# ValidationError if someone were to try and save() this.
new_length = len(self.source)
if length > new_length:
i_length = length - new_length
s = EmbeddedSource()
s.name = "Other"
s.instances = [0] * i_length
self.source.append(s)
def __init__(self, data, analyst, method):
"""
Instantiation of the STIXParser can take the data to parse, the analyst
doing the parsing, and the method of data aquisition.
:param data: The data to parse.
:type data: str
:param analyst: The analyst parsing the document.
:type analyst: str
:param method: The method of acquiring this data.
:type method: str
"""
self.data = data
self.package = None
self.binding = None
self.events = []
self.samples = []
self.emails = []
self.indicators = []
self.saved_artifacts = {}
self.source = EmbeddedSource()
# source.name comes from the stix header.
self.source_instance = EmbeddedSource.SourceInstance()
# The reference attribute and appending it to the source is
# done after the TAXII message ID is determined.
self.source_instance.analyst = analyst
self.source_instance.method = method
self.information_source = None
def sanitize(self, username=None, sources=None, rels=None):
"""
Sanitize the source list down to only those a user has access to see.
This was sniped from core/crits_mongoengine.
:param username: The user requesting this data.
:type username: str
:param sources: A list of sources the user has access to.
:type sources: list
"""
if username and hasattr(self, 'source'):
length = len(self.source)
if not sources:
sources = user_sources(username)
# use slice to modify in place in case any code is referencing
# the source already will reflect the changes as well
self.source[:] = [s for s in self.source if s.name in sources]
# a bit of a hack but we add a poorly formatted source to the
# source list which has an instances length equal to the amount
# of sources that were sanitized out of the user's list.
# not tested but this has the added benefit of throwing a
# ValidationError if someone were to try and save() this.
new_length = len(self.source)
if length > new_length:
i_length = length - new_length
s = EmbeddedSource()
s.name = "Other"
s.instances = [0] * i_length
self.source.append(s)
def __init__(self, data, analyst, method, preview_only=False):
"""
Instantiation of the STIXParser can take the data to parse, the analyst
doing the parsing, and the method of data aquisition.
:param data: The data to parse.
:type data: str
:param analyst: The analyst parsing the document.
:type analyst: str
:param method: The method of acquiring this data.
:type method: str
:param preview_only: If True, nothing is imported and a preview is returned
:type preview_only: boolean
"""
self.data = data
self.preview = preview_only
self.package = None
self.stix_version = None
self.source = EmbeddedSource(
) # source.name comes from the stix header.
self.source_instance = EmbeddedSource.SourceInstance()
# The reference attribute and appending it to the source is
# done after the TAXII message ID is determined.
self.source_instance.analyst = analyst
self.source_instance.method = method
self.information_source = None
self.event = None # the Event TLO
self.event_rels = {} # track relationships to the event
self.relationships = [] # track other relationships that need forming
self.ind2obj = {} # map STIX Indicator IDs to CybOX Object IDs
self.obj2ind = {} # map CybOX Object IDs to STIX Indicator IDs
self.imported = {} # track items that are imported
self.updates = {} # track new/updated CRITs TLOs
self.parsed = [
] # track items that have been parsed, but not necessarily imported
self.failed = [] # track STIX/CybOX items that failed import
self.saved_artifacts = {}
def upsert_domain(sdomain,
domain,
source,
username=None,
campaign=None,
confidence=None,
bucket_list=None,
ticket=None,
cache={}):
"""
Add or update a domain/FQDN. Campaign is assumed to be a list of campaign
dictionary objects.
:param sdomain: Response from parsing the domain for a root domain. Will
either be an error message or the root domain itself.
:type sdomain: str
:param domain: The domain to add/update.
:type domain: str
:param source: The name of the source.
:type source: str
:param username: The user adding/updating the domain.
:type username: str
:param campaign: The campaign to attribute to this domain.
:type campaign: list, str
:param confidence: Confidence for the campaign attribution.
:type confidence: str
:param bucket_list: List of buckets to add to this domain.
:type bucket_list: list, str
:param ticket: The ticket for this domain.
:type ticket: str
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"object" the domain that was added,
"is_domain_new" (boolean)
"""
if sdomain == "no_tld_found_error": #oops...
return {'success': False, 'message': "Invalid domain: %s " % sdomain}
is_fqdn_domain_new = False
is_root_domain_new = False
if not campaign:
campaign = []
# assume it's a list, but check if it's a string
elif isinstance(campaign, basestring):
c = EmbeddedCampaign(name=campaign,
confidence=confidence,
analyst=username)
campaign = [c]
# assume it's a list, but check if it's a string
if isinstance(source, basestring):
s = EmbeddedSource()
s.name = source
instance = EmbeddedSource.SourceInstance()
instance.reference = ''
instance.method = ''
instance.analyst = username
instance.date = datetime.datetime.now()
s.instances = [instance]
source = [s]
fqdn_domain = None
root_domain = None
cached_results = cache.get(form_consts.Domain.CACHED_RESULTS)
if cached_results != None:
if domain != sdomain:
fqdn_domain = cached_results.get(domain)
root_domain = cached_results.get(sdomain)
else:
root_domain = cached_results.get(sdomain)
else:
#first find the domain(s) if it/they already exist
root_domain = Domain.objects(domain=sdomain).first()
if domain != sdomain:
fqdn_domain = Domain.objects(domain=domain).first()
#if they don't exist, create them
if not root_domain:
root_domain = Domain()
root_domain.domain = sdomain.strip()
root_domain.source = []
root_domain.record_type = 'A'
is_root_domain_new = True
if cached_results != None:
cached_results[sdomain] = root_domain
if domain != sdomain and not fqdn_domain:
fqdn_domain = Domain()
fqdn_domain.domain = domain.strip()
fqdn_domain.source = []
fqdn_domain.record_type = 'A'
is_fqdn_domain_new = True
if cached_results != None:
cached_results[domain] = fqdn_domain
# if new or found, append the new source(s)
for s in source:
if root_domain:
root_domain.add_source(s)
if fqdn_domain:
fqdn_domain.add_source(s)
#campaigns
#both root and fqdn get campaigns updated
for c in campaign:
if root_domain:
root_domain.add_campaign(c)
if fqdn_domain:
fqdn_domain.add_campaign(c)
if username:
if root_domain:
root_domain.analyst = username
if fqdn_domain:
fqdn_domain.analyst = username
if bucket_list:
if root_domain:
root_domain.add_bucket_list(bucket_list, username)
if fqdn_domain:
fqdn_domain.add_bucket_list(bucket_list, username)
if ticket:
if root_domain:
root_domain.add_ticket(ticket, username)
if fqdn_domain:
fqdn_domain.add_ticket(ticket, username)
# save
try:
if root_domain:
root_domain.save(username=username)
if fqdn_domain:
fqdn_domain.save(username=username)
except Exception, e:
return {'success': False, 'message': e}
def upsert_domain(domain, source, username=None, campaign=None,
confidence=None, bucket_list=None, ticket=None, cache={}):
"""
Add or update a domain/FQDN. Campaign is assumed to be a list of campaign
dictionary objects.
:param domain: The domain to add/update.
:type domain: str
:param source: The name of the source.
:type source: str
:param username: The user adding/updating the domain.
:type username: str
:param campaign: The campaign to attribute to this domain.
:type campaign: list, str
:param confidence: Confidence for the campaign attribution.
:type confidence: str
:param bucket_list: List of buckets to add to this domain.
:type bucket_list: list, str
:param ticket: The ticket for this domain.
:type ticket: str
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"object" the domain that was added,
"is_domain_new" (boolean)
"""
# validate domain and grab root domain
(root, domain, error) = get_valid_root_domain(domain)
if error:
return {'success': False, 'message': error}
is_fqdn_domain_new = False
is_root_domain_new = False
if not campaign:
campaign = []
# assume it's a list, but check if it's a string
elif isinstance(campaign, basestring):
c = EmbeddedCampaign(name=campaign, confidence=confidence, analyst=username)
campaign = [c]
# assume it's a list, but check if it's a string
if isinstance(source, basestring):
s = EmbeddedSource()
s.name = source
instance = EmbeddedSource.SourceInstance()
instance.reference = ''
instance.method = ''
instance.analyst = username
instance.date = datetime.datetime.now()
s.instances = [instance]
source = [s]
fqdn_domain = None
root_domain = None
cached_results = cache.get(form_consts.Domain.CACHED_RESULTS)
if cached_results != None:
if domain != root:
fqdn_domain = cached_results.get(domain)
root_domain = cached_results.get(root)
else:
root_domain = cached_results.get(root)
else:
#first find the domain(s) if it/they already exist
root_domain = Domain.objects(domain=root).first()
if domain != root:
fqdn_domain = Domain.objects(domain=domain).first()
#if they don't exist, create them
if not root_domain:
root_domain = Domain()
root_domain.domain = root
root_domain.source = []
root_domain.record_type = 'A'
is_root_domain_new = True
if cached_results != None:
cached_results[root] = root_domain
if domain != root and not fqdn_domain:
fqdn_domain = Domain()
fqdn_domain.domain = domain
fqdn_domain.source = []
fqdn_domain.record_type = 'A'
is_fqdn_domain_new = True
if cached_results != None:
cached_results[domain] = fqdn_domain
# if new or found, append the new source(s)
for s in source:
if root_domain:
root_domain.add_source(s)
if fqdn_domain:
fqdn_domain.add_source(s)
#campaigns
#both root and fqdn get campaigns updated
for c in campaign:
if root_domain:
root_domain.add_campaign(c)
if fqdn_domain:
fqdn_domain.add_campaign(c)
if username:
if root_domain:
root_domain.analyst = username
if fqdn_domain:
fqdn_domain.analyst = username
if bucket_list:
if root_domain:
root_domain.add_bucket_list(bucket_list, username)
if fqdn_domain:
fqdn_domain.add_bucket_list(bucket_list, username)
if ticket:
if root_domain:
root_domain.add_ticket(ticket, username)
if fqdn_domain:
fqdn_domain.add_ticket(ticket, username)
# save
try:
if root_domain:
root_domain.save(username=username)
if fqdn_domain:
fqdn_domain.save(username=username)
except Exception, e:
return {'success': False, 'message': e}
def handle_indicator_insert(ind, source, reference='', analyst='', method='',
add_domain=False, add_relationship=False, cache={}):
"""
Insert an individual indicator into the database.
NOTE: Setting add_domain to True will always create a relationship as well.
However, to create a relationship with an object that already exists before
this function was called, set add_relationship to True. This will assume
that the domain or IP object to create the relationship with already exists
and will avoid infinite mutual calls between, for example, add_update_ip
and this function. add domain/IP objects.
:param ind: Information about the indicator.
:type ind: dict
:param source: The source for this indicator.
:type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param reference: The reference to the data.
:type reference: str
:param analyst: The user adding this indicator.
:type analyst: str
:param method: Method of acquiring this indicator.
:type method: str
:param add_domain: If this indicator is also a top-level object, try to add
it.
:type add_domain: boolean
:param add_relationship: Attempt to add relationships if applicable.
:type add_relationship: boolean
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" str) if failed,
"objectid" (str) if successful,
"is_new_indicator" (boolean) if successful.
"""
if ind['type'] == "URI - URL" and "://" not in ind['value'].split('.')[0]:
return {"success": False, "message": "URI - URL must contain protocol prefix (e.g. http://, https://, ftp://) "}
is_new_indicator = False
dmain = None
ip = None
rank = {
'unknown': 0,
'benign': 1,
'low': 2,
'medium': 3,
'high': 4,
}
indicator = Indicator.objects(ind_type=ind['type'],
value=ind['value']).first()
if not indicator:
indicator = Indicator()
indicator.ind_type = ind['type']
indicator.value = ind['value']
indicator.created = datetime.datetime.now()
indicator.confidence = EmbeddedConfidence(analyst=analyst)
indicator.impact = EmbeddedImpact(analyst=analyst)
is_new_indicator = True
if 'campaign' in ind:
if isinstance(ind['campaign'], basestring) and len(ind['campaign']) > 0:
confidence = ind.get('campaign_confidence', 'low')
ind['campaign'] = EmbeddedCampaign(name=ind['campaign'],
confidence=confidence,
description="",
analyst=analyst,
date=datetime.datetime.now())
if isinstance(ind['campaign'], EmbeddedCampaign):
indicator.add_campaign(ind['campaign'])
elif isinstance(ind['campaign'], list):
for campaign in ind['campaign']:
if isinstance(campaign, EmbeddedCampaign):
indicator.add_campaign(campaign)
if 'confidence' in ind and rank.get(ind['confidence'], 0) > rank.get(indicator.confidence.rating, 0):
indicator.confidence.rating = ind['confidence']
indicator.confidence.analyst = analyst
if 'impact' in ind and rank.get(ind['impact'], 0) > rank.get(indicator.impact.rating, 0):
indicator.impact.rating = ind['impact']
indicator.impact.analyst = analyst
bucket_list = None
if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind:
bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME]
if bucket_list:
indicator.add_bucket_list(bucket_list, analyst)
ticket = None
if form_consts.Common.TICKET_VARIABLE_NAME in ind:
ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME]
if ticket:
indicator.add_ticket(ticket, analyst)
if isinstance(source, list):
for s in source:
indicator.add_source(source_item=s, method=method, reference=reference)
elif isinstance(source, EmbeddedSource):
indicator.add_source(source_item=source, method=method, reference=reference)
elif isinstance(source, basestring):
s = EmbeddedSource()
s.name = source
instance = EmbeddedSource.SourceInstance()
instance.reference = reference
instance.method = method
instance.analyst = analyst
instance.date = datetime.datetime.now()
s.instances = [instance]
indicator.add_source(s)
if add_domain or add_relationship:
ind_type = indicator.ind_type
ind_value = indicator.value
url_contains_ip = False
if ind_type in ("URI - Domain Name", "URI - URL"):
if ind_type == "URI - URL":
domain_or_ip = urlparse.urlparse(ind_value).hostname
elif ind_type == "URI - Domain Name":
domain_or_ip = ind_value
(sdomain, fqdn) = get_domain(domain_or_ip)
if sdomain == "no_tld_found_error" and ind_type == "URI - URL":
try:
validate_ipv46_address(domain_or_ip)
url_contains_ip = True
except DjangoValidationError:
pass
if not url_contains_ip:
success = None
if add_domain:
success = upsert_domain(sdomain, fqdn, indicator.source,
'%s' % analyst, None,
bucket_list=bucket_list, cache=cache)
if not success['success']:
return {'success': False, 'message': success['message']}
if not success or not 'object' in success:
dmain = Domain.objects(domain=domain_or_ip).first()
else:
dmain = success['object']
if ind_type.startswith("Address - ip") or ind_type == "Address - cidr" or url_contains_ip:
if url_contains_ip:
ind_value = domain_or_ip
try:
validate_ipv4_address(domain_or_ip)
ind_type = 'Address - ipv4-addr'
except DjangoValidationError:
ind_type = 'Address - ipv6-addr'
success = None
if add_domain:
success = ip_add_update(ind_value,
ind_type,
source=indicator.source,
campaign=indicator.campaign,
analyst=analyst,
bucket_list=bucket_list,
ticket=ticket,
indicator_reference=reference,
cache=cache)
if not success['success']:
return {'success': False, 'message': success['message']}
if not success or not 'object' in success:
ip = IP.objects(ip=indicator.value).first()
else:
ip = success['object']
indicator.save(username=analyst)
if dmain:
dmain.add_relationship(rel_item=indicator,
rel_type='Related_To',
analyst="%s" % analyst,
get_rels=False)
dmain.save(username=analyst)
if ip:
ip.add_relationship(rel_item=indicator,
rel_type='Related_To',
analyst="%s" % analyst,
get_rels=False)
ip.save(username=analyst)
indicator.save(username=analyst)
# run indicator triage
if is_new_indicator:
indicator.reload()
run_triage(indicator, analyst)
return {'success': True, 'objectid': str(indicator.id),
'is_new_indicator': is_new_indicator, 'object': indicator}
def handle_indicator_insert(ind,
source,
reference='',
analyst='',
method='',
add_domain=False,
add_relationship=False,
cache={}):
"""
Insert an individual indicator into the database.
NOTE: Setting add_domain to True will always create a relationship as well.
However, to create a relationship with an object that already exists before
this function was called, set add_relationship to True. This will assume
that the domain or IP object to create the relationship with already exists
and will avoid infinite mutual calls between, for example, add_update_ip
and this function. add domain/IP objects.
:param ind: Information about the indicator.
:type ind: dict
:param source: The source for this indicator.
:type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param reference: The reference to the data.
:type reference: str
:param analyst: The user adding this indicator.
:type analyst: str
:param method: Method of acquiring this indicator.
:type method: str
:param add_domain: If this indicator is also a top-level object, try to add
it.
:type add_domain: boolean
:param add_relationship: Attempt to add relationships if applicable.
:type add_relationship: boolean
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" str) if failed,
"objectid" (str) if successful,
"is_new_indicator" (boolean) if successful.
"""
is_new_indicator = False
rank = {'unknown': 0, 'benign': 1, 'low': 2, 'medium': 3, 'high': 4}
indicator = Indicator.objects(ind_type=ind['type'],
value=ind['value']).first()
if not indicator:
indicator = Indicator()
indicator.ind_type = ind['type']
indicator.value = ind['value']
indicator.created = datetime.datetime.now()
indicator.confidence = EmbeddedConfidence(analyst=analyst)
indicator.impact = EmbeddedImpact(analyst=analyst)
is_new_indicator = True
ec = None
if 'campaign' in ind:
confidence = 'low'
if 'campaign_confidence' in ind:
confidence = ind['campaign_confidence']
ec = EmbeddedCampaign(name=ind['campaign'],
confidence=confidence,
description="",
analyst=analyst,
date=datetime.datetime.now())
if 'confidence' in ind and rank.get(ind['confidence'], 0) > rank.get(
indicator.confidence.rating, 0):
indicator.confidence.rating = ind['confidence']
indicator.confidence.analyst = analyst
if 'impact' in ind and rank.get(ind['impact'], 0) > rank.get(
indicator.impact.rating, 0):
indicator.impact.rating = ind['impact']
indicator.impact.analyst = analyst
bucket_list = None
if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind:
bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME]
indicator.add_bucket_list(bucket_list, analyst)
ticket = None
if form_consts.Common.TICKET_VARIABLE_NAME in ind:
ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME]
indicator.add_ticket(ticket, analyst)
if isinstance(source, list):
for s in source:
indicator.add_source(source_item=s)
elif isinstance(source, EmbeddedSource):
indicator.add_source(source_item=source)
elif isinstance(source, basestring):
s = EmbeddedSource()
s.name = source
instance = EmbeddedSource.SourceInstance()
instance.reference = reference
instance.method = method
instance.analyst = analyst
instance.date = datetime.datetime.now()
s.instances = [instance]
indicator.add_source(s)
if ec:
indicator.add_campaign(ec)
indicator.save(username=analyst)
if add_domain or add_relationship:
ind_type = indicator.ind_type
ind_value = indicator.value
if ind_type in ("URI - Domain Name", "URI - URL"):
if ind_type == "URI - URL":
domain = ind_value.split("/")[2]
elif ind_type == "URI - Domain Name":
domain = ind_value
#try:
(sdomain, fqdn) = get_domain(domain)
success = None
if add_domain:
success = upsert_domain(sdomain,
fqdn,
indicator.source,
'%s' % analyst,
None,
bucket_list=bucket_list,
cache=cache)
if not success['success']:
return {'success': False, 'message': success['message']}
if not success or not 'object' in success:
dmain = Domain.objects(domain=domain).first()
else:
dmain = success['object']
if dmain:
dmain.add_relationship(rel_item=indicator,
rel_type='Related_To',
analyst="%s" % analyst,
get_rels=False)
dmain.save(username=analyst)
indicator.save(username=analyst)
elif ind_type.startswith(
"Address - ip") or ind_type == "Address - cidr":
success = None
if add_domain:
success = ip_add_update(indicator.value,
ind_type,
source=indicator.source,
campaign=indicator.campaign,
analyst=analyst,
bucket_list=bucket_list,
ticket=ticket,
indicator_reference=reference,
cache=cache)
if not success['success']:
return {'success': False, 'message': success['message']}
if not success or not 'object' in success:
ip = IP.objects(ip=indicator.value).first()
else:
ip = success['object']
if ip:
ip.add_relationship(rel_item=indicator,
rel_type='Related_To',
analyst="%s" % analyst,
get_rels=False)
ip.save(username=analyst)
indicator.save(username=analyst)
# run indicator triage
if is_new_indicator:
indicator.reload()
run_triage(None, indicator, analyst)
return {
'success': True,
'objectid': indicator.id,
'is_new_indicator': is_new_indicator,
'object': indicator
}
def upsert_domain(domain, source, username=None, campaign=None,
confidence=None, bucket_list=None, ticket=None, cache={}, related_id=None, related_type=None, relationship_type=None):
"""
Add or update a domain/FQDN. Campaign is assumed to be a list of campaign
dictionary objects.
:param domain: The domain to add/update.
:type domain: str
:param source: The name of the source.
:type source: str
:param username: The user adding/updating the domain.
:type username: str
:param campaign: The campaign to attribute to this domain.
:type campaign: list, str
:param confidence: Confidence for the campaign attribution.
:type confidence: str
:param bucket_list: List of buckets to add to this domain.
:type bucket_list: list, str
:param ticket: The ticket for this domain.
:type ticket: str
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_id: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:returns: dict with keys:
"success" (boolean),
"object" the domain that was added,
"is_domain_new" (boolean)
"""
# validate domain and grab root domain
(root, domain, error) = get_valid_root_domain(domain)
if error:
return {'success': False, 'message': error}
is_fqdn_domain_new = False
is_root_domain_new = False
if not campaign:
campaign = []
# assume it's a list, but check if it's a string
elif isinstance(campaign, basestring):
c = EmbeddedCampaign(name=campaign, confidence=confidence, analyst=username)
campaign = [c]
# assume it's a list, but check if it's a string
if isinstance(source, basestring):
s = EmbeddedSource()
s.name = source
instance = EmbeddedSource.SourceInstance()
instance.reference = ''
instance.method = ''
instance.analyst = username
instance.date = datetime.datetime.now()
s.instances = [instance]
source = [s]
fqdn_domain = None
root_domain = None
cached_results = cache.get(form_consts.Domain.CACHED_RESULTS)
if cached_results != None:
if domain != root:
fqdn_domain = cached_results.get(domain)
root_domain = cached_results.get(root)
else:
root_domain = cached_results.get(root)
else:
#first find the domain(s) if it/they already exist
root_domain = Domain.objects(domain=root).first()
if domain != root:
fqdn_domain = Domain.objects(domain=domain).first()
#if they don't exist, create them
if not root_domain:
root_domain = Domain()
root_domain.domain = root
root_domain.source = []
root_domain.record_type = 'A'
is_root_domain_new = True
if cached_results != None:
cached_results[root] = root_domain
if domain != root and not fqdn_domain:
fqdn_domain = Domain()
fqdn_domain.domain = domain
fqdn_domain.source = []
fqdn_domain.record_type = 'A'
is_fqdn_domain_new = True
if cached_results != None:
cached_results[domain] = fqdn_domain
# if new or found, append the new source(s)
for s in source:
if root_domain:
root_domain.add_source(s)
if fqdn_domain:
fqdn_domain.add_source(s)
#campaigns
#both root and fqdn get campaigns updated
for c in campaign:
if root_domain:
root_domain.add_campaign(c)
if fqdn_domain:
fqdn_domain.add_campaign(c)
if username:
if root_domain:
root_domain.analyst = username
if fqdn_domain:
fqdn_domain.analyst = username
if bucket_list:
if root_domain:
root_domain.add_bucket_list(bucket_list, username)
if fqdn_domain:
fqdn_domain.add_bucket_list(bucket_list, username)
if ticket:
if root_domain:
root_domain.add_ticket(ticket, username)
if fqdn_domain:
fqdn_domain.add_ticket(ticket, username)
related_obj = None
if related_id:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
retVal['success'] = False
retVal['message'] = 'Related Object not found.'
return retVal
# save
try:
if root_domain:
root_domain.save(username=username)
if fqdn_domain:
fqdn_domain.save(username=username)
except Exception, e:
return {'success': False, 'message': e}
def handle_indicator_insert(
ind, source, reference="", analyst="", method="", add_domain=False, add_relationship=False, cache={}
):
"""
Insert an individual indicator into the database.
NOTE: Setting add_domain to True will always create a relationship as well.
However, to create a relationship with an object that already exists before
this function was called, set add_relationship to True. This will assume
that the domain or IP object to create the relationship with already exists
and will avoid infinite mutual calls between, for example, add_update_ip
and this function. add domain/IP objects.
:param ind: Information about the indicator.
:type ind: dict
:param source: The source for this indicator.
:type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param reference: The reference to the data.
:type reference: str
:param analyst: The user adding this indicator.
:type analyst: str
:param method: Method of acquiring this indicator.
:type method: str
:param add_domain: If this indicator is also a top-level object, try to add
it.
:type add_domain: boolean
:param add_relationship: Attempt to add relationships if applicable.
:type add_relationship: boolean
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" (str) if failed,
"objectid" (str) if successful,
"is_new_indicator" (boolean) if successful.
"""
if ind["type"] not in IndicatorTypes.values():
return {"success": False, "message": "Not a valid Indicator Type: %s" % ind["type"]}
if ind["threat_type"] not in IndicatorThreatTypes.values():
return {"success": False, "message": "Not a valid Indicator Threat Type: %s" % ind["threat_type"]}
if ind["attack_type"] not in IndicatorAttackTypes.values():
return {"success": False, "message": "Not a valid Indicator Attack Type: " % ind["attack_type"]}
(ind["value"], error) = validate_indicator_value(ind["value"], ind["type"])
if error:
return {"success": False, "message": error}
is_new_indicator = False
dmain = None
ip = None
rank = {"unknown": 0, "benign": 1, "low": 2, "medium": 3, "high": 4}
if ind.get("status", None) is None or len(ind.get("status", "")) < 1:
ind["status"] = Status.NEW
indicator = Indicator.objects(
ind_type=ind["type"], lower=ind["lower"], threat_type=ind["threat_type"], attack_type=ind["attack_type"]
).first()
if not indicator:
indicator = Indicator()
indicator.ind_type = ind["type"]
indicator.threat_type = ind["threat_type"]
indicator.attack_type = ind["attack_type"]
indicator.value = ind["value"]
indicator.lower = ind["lower"]
indicator.description = ind["description"]
indicator.created = datetime.datetime.now()
indicator.confidence = EmbeddedConfidence(analyst=analyst)
indicator.impact = EmbeddedImpact(analyst=analyst)
indicator.status = ind["status"]
is_new_indicator = True
else:
if ind["status"] != Status.NEW:
indicator.status = ind["status"]
add_desc = "\nSeen on %s as: %s" % (str(datetime.datetime.now()), ind["value"])
if indicator.description is None:
indicator.description = add_desc
else:
indicator.description += add_desc
if "campaign" in ind:
if isinstance(ind["campaign"], basestring) and len(ind["campaign"]) > 0:
confidence = ind.get("campaign_confidence", "low")
ind["campaign"] = EmbeddedCampaign(
name=ind["campaign"],
confidence=confidence,
description="",
analyst=analyst,
date=datetime.datetime.now(),
)
if isinstance(ind["campaign"], EmbeddedCampaign):
indicator.add_campaign(ind["campaign"])
elif isinstance(ind["campaign"], list):
for campaign in ind["campaign"]:
if isinstance(campaign, EmbeddedCampaign):
indicator.add_campaign(campaign)
if "confidence" in ind and rank.get(ind["confidence"], 0) > rank.get(indicator.confidence.rating, 0):
indicator.confidence.rating = ind["confidence"]
indicator.confidence.analyst = analyst
if "impact" in ind and rank.get(ind["impact"], 0) > rank.get(indicator.impact.rating, 0):
indicator.impact.rating = ind["impact"]
indicator.impact.analyst = analyst
bucket_list = None
if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind:
bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME]
if bucket_list:
indicator.add_bucket_list(bucket_list, analyst)
ticket = None
if form_consts.Common.TICKET_VARIABLE_NAME in ind:
ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME]
if ticket:
indicator.add_ticket(ticket, analyst)
if isinstance(source, list):
for s in source:
indicator.add_source(source_item=s, method=method, reference=reference)
elif isinstance(source, EmbeddedSource):
indicator.add_source(source_item=source, method=method, reference=reference)
elif isinstance(source, basestring):
s = EmbeddedSource()
s.name = source
instance = EmbeddedSource.SourceInstance()
instance.reference = reference
instance.method = method
instance.analyst = analyst
instance.date = datetime.datetime.now()
s.instances = [instance]
indicator.add_source(s)
if add_domain or add_relationship:
ind_type = indicator.ind_type
ind_value = indicator.lower
url_contains_ip = False
if ind_type in (IndicatorTypes.DOMAIN, IndicatorTypes.URI):
if ind_type == IndicatorTypes.URI:
domain_or_ip = urlparse.urlparse(ind_value).hostname
try:
validate_ipv46_address(domain_or_ip)
url_contains_ip = True
except DjangoValidationError:
pass
else:
domain_or_ip = ind_value
if not url_contains_ip:
success = None
if add_domain:
success = upsert_domain(
domain_or_ip,
indicator.source,
username="******" % analyst,
campaign=indicator.campaign,
bucket_list=bucket_list,
cache=cache,
)
if not success["success"]:
return {"success": False, "message": success["message"]}
if not success or not "object" in success:
dmain = Domain.objects(domain=domain_or_ip).first()
else:
dmain = success["object"]
if ind_type in IPTypes.values() or url_contains_ip:
if url_contains_ip:
ind_value = domain_or_ip
try:
validate_ipv4_address(domain_or_ip)
ind_type = IndicatorTypes.IPV4_ADDRESS
except DjangoValidationError:
ind_type = IndicatorTypes.IPV6_ADDRESS
success = None
if add_domain:
success = ip_add_update(
ind_value,
ind_type,
source=indicator.source,
campaign=indicator.campaign,
analyst=analyst,
bucket_list=bucket_list,
ticket=ticket,
indicator_reference=reference,
cache=cache,
)
if not success["success"]:
return {"success": False, "message": success["message"]}
if not success or not "object" in success:
ip = IP.objects(ip=indicator.value).first()
else:
ip = success["object"]
indicator.save(username=analyst)
if dmain:
dmain.add_relationship(indicator, RelationshipTypes.RELATED_TO, analyst="%s" % analyst, get_rels=False)
dmain.save(username=analyst)
if ip:
ip.add_relationship(indicator, RelationshipTypes.RELATED_TO, analyst="%s" % analyst, get_rels=False)
ip.save(username=analyst)
# run indicator triage
if is_new_indicator:
indicator.reload()
run_triage(indicator, analyst)
return {"success": True, "objectid": str(indicator.id), "is_new_indicator": is_new_indicator, "object": indicator}
