def obj_create(self, bundle, **kwargs):
"""
Handles creating IPs through the API.
:param bundle: Bundle containing the information to create the IP.
:type bundle: Tastypie Bundle object.
:returns: HttpResponse.
"""
analyst = bundle.request.user.username
data = bundle.data
ip = data.get('ip', None)
name = data.get('source', None)
reference = data.get('reference', None)
method = data.get('method', None)
campaign = data.get('campaign', None)
description = data.get('description', None)
confidence = data.get('confidence', None)
ip_type = data.get('ip_type', None)
add_indicator = data.get('add_indicator', False)
indicator_reference = data.get('indicator_reference', None)
bucket_list = data.get('bucket_list', None)
ticket = data.get('ticket', None)
content = {'return_code': 1, 'type': 'IP'}
if not ip or not name or not ip_type:
content['message'] = "Must provide an IP, IP Type, and Source."
self.crits_response(content)
result = ip_add_update(ip,
ip_type,
source=name,
source_method=method,
source_reference=reference,
campaign=campaign,
confidence=confidence,
description=description,
analyst=analyst,
bucket_list=bucket_list,
ticket=ticket,
is_add_indicator=add_indicator,
indicator_reference=indicator_reference)
if result.get('message'):
content['message'] = result.get('message')
if result.get('object'):
content['id'] = str(result.get('object').id)
if content.get('id'):
url = reverse('api_dispatch_detail',
kwargs={
'resource_name': 'ips',
'api_name': 'v1',
'pk': content.get('id')
})
content['url'] = url
if result['success']:
content['return_code'] = 0
self.crits_response(content)
def obj_create(self, bundle, **kwargs):
"""
Handles creating IPs through the API.
:param bundle: Bundle containing the information to create the IP.
:type bundle: Tastypie Bundle object.
:returns: HttpResponse.
"""
analyst = bundle.request.user.username
data = bundle.data
ip = data.get('ip', None)
name = data.get('source', None)
reference = data.get('reference', None)
method = data.get('method', None)
campaign = data.get('campaign', None)
description = data.get('description', None)
confidence = data.get('confidence', None)
ip_type = data.get('ip_type', None)
add_indicator = data.get('add_indicator', False)
indicator_reference = data.get('indicator_reference', None)
bucket_list = data.get('bucket_list', None)
ticket = data.get('ticket', None)
content = {'return_code': 1,
'type': 'IP'}
if not ip or not name or not ip_type:
content['message'] = "Must provide an IP, IP Type, and Source."
self.crits_response(content)
result = ip_add_update(ip,
ip_type,
source=name,
source_method=method,
source_reference=reference,
campaign=campaign,
confidence=confidence,
description=description,
analyst=analyst,
bucket_list=bucket_list,
ticket=ticket,
is_add_indicator=add_indicator,
indicator_reference=indicator_reference)
if result.get('message'):
content['message'] = result.get('message')
if result.get('object'):
content['id'] = str(result.get('object').id)
if content.get('id'):
url = reverse('api_dispatch_detail',
kwargs={'resource_name': 'ips',
'api_name': 'v1',
'pk': content.get('id')})
content['url'] = url
if result['success']:
content['return_code'] = 0
self.crits_response(content)
def obj_create(self, bundle, **kwargs):
"""
Handles creating IPs through the API.
:param bundle: Bundle containing the information to create the IP.
:type bundle: Tastypie Bundle object.
:returns: Bundle object.
:raises BadRequest: If creation fails.
"""
analyst = bundle.request.user.username
data = bundle.data
ip = data.get('ip', None)
name = data.get('source', None)
reference = data.get('reference', None)
method = data.get('method', None)
campaign = data.get('campaign', None)
confidence = data.get('confidence', None)
ip_type = data.get('ip_type', None)
add_indicator = data.get('add_indicator', False)
indicator_reference = data.get('indicator_reference', None)
bucket_list = data.get('bucket_list', None)
ticket = data.get('ticket', None)
if not ip or not name or not ip_type:
raise BadRequest("Must provide an IP, IP Type, and Source.")
result = ip_add_update(ip,
ip_type,
source=name,
source_method=method,
source_reference=reference,
campaign=campaign,
confidence=confidence,
analyst=analyst,
bucket_list=bucket_list,
ticket=ticket,
is_add_indicator=add_indicator,
indicator_reference=indicator_reference)
if not result['success']:
raise BadRequest(result['message'])
else:
return bundle
def process_ips(self, incident_report):
ips = incident_report.findall("./contacted/ips/ip")
for ip in ips:
ret = ip_add_update(ip.text, self._ip_type(ip.text),
source=get_user_organization(self.current_task.user),
source_method=self.name,
source_tlp=self.obj.tlp,
user=self.current_task.user,
related_id=str(self.obj.id),
related_type=self.obj._meta['crits_type'],
relationship_type=RelationshipTypes.CONNECTED_TO)
if ret['success']:
malicious = ip.get('malicious', 'unknown')
self._add_result("IPs", ip.text, {'malicious': malicious})
else:
self._warning(ret["message"])
self._notify()
def obj_create(self, bundle, **kwargs):
"""
Handles creating IPs through the API.
:param bundle: Bundle containing the information to create the IP.
:type bundle: Tastypie Bundle object.
:returns: Bundle object.
:raises BadRequest: If creation fails.
"""
analyst = bundle.request.user.username
data = bundle.data
ip = data['ip']
name = data['source']
reference = data['reference']
method = data['method']
campaign = data['campaign']
confidence = data['confidence']
ip_type = data['ip_type']
add_indicator = data.get('add_indicator', False)
indicator_reference = data.get('indicator_reference')
bucket_list = data.get('bucket_list', None)
ticket = data.get('ticket', None)
result = ip_add_update(ip,
ip_type,
source=name,
source_method=method,
source_reference=reference,
campaign=campaign,
confidence=confidence,
analyst=analyst,
bucket_list=bucket_list,
ticket=ticket,
is_add_indicator=add_indicator,
indicator_reference=indicator_reference)
if 'message' in result:
raise BadRequest(result['message'])
else:
return bundle
def parse_observables(self, observables):
"""
Parse list of observables in STIX doc.
:param observables: List of STIX observables.
:type observables: List of STIX observables.
"""
analyst = self.source_instance.analyst
for obs in observables: # for each STIX observable
if obs.observable_composition:
object_list = obs.observable_composition.observables
else:
object_list = [obs]
for obs_comp in object_list:
if not obs_comp.object_ or not obs_comp.object_.properties:
self.failed.append(
("No valid object_properties was found!",
type(obs_comp).__name__,
obs_comp.id_)) # note for display in UI
continue
try: # try to create CRITs object from observable
item = obs_comp.object_.properties
if isinstance(item, Address):
if item.category in ('cidr', 'ipv4-addr', 'ipv4-net',
'ipv4-netmask', 'ipv6-addr',
'ipv6-net', 'ipv6-netmask',
'ipv6-subnet'):
imp_type = "IP"
for value in item.address_value.values:
ip = str(value).strip()
iptype = get_crits_ip_type(item.category)
if iptype:
res = ip_add_update(ip,
iptype, [self.source],
analyst=analyst,
id=self.package.id_)
self.parse_res(imp_type, obs, res)
if isinstance(item, DomainName):
imp_type = "Domain"
for value in item.value.values:
res = upsert_domain(str(value), [self.source],
username=analyst,
id=self.package.id_)
self.parse_res(imp_type, obs, res)
elif isinstance(item, Artifact):
# Not sure if this is right, and I believe these can be
# encoded in a couple different ways.
imp_type = "RawData"
rawdata = item.data.decode('utf-8')
description = "None"
# TODO: find out proper ways to determine title, datatype,
# tool_name, tool_version
title = "Artifact for Event: STIX Document %s" % self.package.id_
res = handle_raw_data_file(
rawdata,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="Text",
tool_name="STIX",
tool_version=None,
method=self.source_instance.method,
reference=self.source_instance.reference)
self.parse_res(imp_type, obs, res)
elif (isinstance(item, File) and item.custom_properties
and item.custom_properties[0].name == "crits_type"
and item.custom_properties[0]._value
== "Certificate"):
imp_type = "Certificate"
description = "None"
filename = str(item.file_name)
data = None
for obj in item.parent.related_objects:
if isinstance(obj.properties, Artifact):
data = obj.properties.data
res = handle_cert_file(filename,
data,
self.source,
user=analyst,
description=description)
self.parse_res(imp_type, obs, res)
elif isinstance(item,
File) and self.has_network_artifact(item):
imp_type = "PCAP"
description = "None"
filename = str(item.file_name)
data = None
for obj in item.parent.related_objects:
if (isinstance(obj.properties, Artifact)
and obj.properties.type_
== Artifact.TYPE_NETWORK):
data = obj.properties.data
res = handle_pcap_file(filename,
data,
self.source,
user=analyst,
description=description)
self.parse_res(imp_type, obs, res)
elif isinstance(item, File):
imp_type = "Sample"
filename = str(item.file_name)
md5 = item.md5
data = None
for obj in item.parent.related_objects:
if (isinstance(obj.properties, Artifact)
and obj.properties.type_
== Artifact.TYPE_FILE):
data = obj.properties.data
res = handle_file(filename,
data,
self.source,
user=analyst,
md5_digest=md5,
is_return_only_md5=False,
id=self.package.id_)
self.parse_res(imp_type, obs, res)
if item.extracted_features:
self.parse_filenames(item.extracted_features,
res['object'].id)
elif isinstance(item, EmailMessage):
imp_type = "Email"
data = {}
data['source'] = self.source.name
data['source_method'] = self.source_instance.method
data[
'source_reference'] = self.source_instance.reference
data['raw_body'] = str(item.raw_body)
data['raw_header'] = str(item.raw_header)
data['helo'] = str(item.email_server)
if item.header:
data['message_id'] = str(item.header.message_id)
data['subject'] = str(item.header.subject)
data['sender'] = str(item.header.sender)
data['reply_to'] = str(item.header.reply_to)
data['x_originating_ip'] = str(
item.header.x_originating_ip)
data['x_mailer'] = str(item.header.x_mailer)
data['boundary'] = str(item.header.boundary)
data['from_address'] = str(item.header.from_)
data['date'] = item.header.date.value
if item.header.to:
data['to'] = [str(r) for r in item.header.to]
if item.header.cc:
data['cc'] = [str(r) for r in item.header.cc]
res = handle_email_fields(data,
analyst,
"STIX",
id=self.package.id_)
# Should check for attachments and add them here.
self.parse_res(imp_type, obs, res)
if res.get('status') and item.attachments:
for attach in item.attachments:
rel_id = attach.to_dict()['object_reference']
self.relationships.append(
(obs.id_, "Contains", rel_id, "High"))
else: # try to parse all other possibilities as Indicator
imp_type = "Indicator"
obj = make_crits_object(item)
if obj.object_type == 'Address':
# This was already caught above
continue
else:
ind_type = obj.object_type
for value in obj.value:
if value and ind_type:
res = handle_indicator_ind(
value.strip(),
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True)
self.parse_res(imp_type, obs, res)
except Exception, e: # probably caused by cybox object we don't handle
self.failed.append(
(e.message, type(item).__name__,
item.parent.id_)) # note for display in UI
def add_new_domain(data,
request,
errors,
rowData=None,
is_validate_only=False,
cache={}):
"""
Add a new domain to CRITs.
:param data: The data about the domain.
:type data: dict
:param request: The Django request.
:type request: :class:`django.http.HttpRequest`
:param errors: A list of current errors to append to.
:type errors: list
:param rowData: Any objects that need to be added to the domain.
:type rowData: dict
:param is_validate_only: Only validate the data and return any errors.
:type is_validate_only: boolean
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:returns: tuple (<result>, <errors>, <retVal>)
"""
result = False
retVal = {}
domain = data['domain']
add_ip = data.get('add_ip')
ip = data.get('ip')
ip_type = data.get('ip_type')
if add_ip:
error = validate_and_normalize_ip(ip, ip_type)[1]
if error:
errors.append(error)
if is_validate_only:
error = get_valid_root_domain(domain)[2]
if error:
errors.append(error)
# check for duplicate domains
fqdn_domain = retrieve_domain(domain, cache)
if fqdn_domain:
if isinstance(fqdn_domain, Domain):
resp_url = reverse('crits.domains.views.domain_detail',
args=[domain])
message = ('Warning: Domain already exists: '
'<a href="%s">%s</a>' % (resp_url, domain))
retVal['message'] = message
retVal['status'] = form_consts.Status.DUPLICATE
retVal['warning'] = message
else:
result_cache = cache.get(form_consts.Domain.CACHED_RESULTS)
result_cache[domain.lower()] = True
elif not errors:
username = request.user.username
reference = data.get('domain_reference')
source_name = data.get('domain_source')
method = data.get('domain_method')
source = [
create_embedded_source(source_name,
reference=reference,
method=method,
analyst=username)
]
bucket_list = data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
ticket = data.get(form_consts.Common.TICKET_VARIABLE_NAME)
if data.get('campaign') and data.get('confidence'):
campaign = [
EmbeddedCampaign(name=data.get('campaign'),
confidence=data.get('confidence'),
analyst=username)
]
else:
campaign = []
retVal = upsert_domain(domain,
source,
username,
campaign,
bucket_list=bucket_list,
ticket=ticket,
cache=cache)
if not retVal['success']:
errors.append(retVal.get('message'))
retVal['message'] = ""
else:
new_domain = retVal['object']
ip_result = {}
if add_ip:
if data.get('same_source'):
ip_source = source_name
ip_method = method
ip_reference = reference
else:
ip_source = data.get('ip_source')
ip_method = data.get('ip_method')
ip_reference = data.get('ip_reference')
from crits.ips.handlers import ip_add_update
ip_result = ip_add_update(ip,
ip_type,
ip_source,
ip_method,
ip_reference,
campaign=campaign,
analyst=username,
bucket_list=bucket_list,
ticket=ticket,
cache=cache)
if not ip_result['success']:
errors.append(ip_result['message'])
else:
#add a relationship with the new IP address
new_ip = ip_result['object']
if new_domain and new_ip:
new_domain.add_relationship(rel_item=new_ip,
rel_type='Resolved_To',
analyst=username,
get_rels=False)
new_domain.save(username=username)
new_ip.save(username=username)
#set the URL for viewing the new data
resp_url = reverse('crits.domains.views.domain_detail',
args=[domain])
if retVal['is_domain_new'] == True:
retVal['message'] = (
'Success! Click here to view the new domain: '
'<a href="%s">%s</a>' % (resp_url, domain))
else:
message = ('Updated existing domain: <a href="%s">%s</a>' %
(resp_url, domain))
retVal['message'] = message
retVal[form_consts.Status.
STATUS_FIELD] = form_consts.Status.DUPLICATE
retVal['warning'] = message
#add indicators
if data.get('add_indicators'):
from crits.indicators.handlers import create_indicator_from_tlo
# If we have an IP object, add an indicator for that.
if ip_result.get('success'):
ip = ip_result['object']
result = create_indicator_from_tlo('IP',
ip,
username,
ip_source,
add_domain=False)
ip_ind = result.get('indicator')
if not result['success']:
errors.append(result['message'])
# Add an indicator for the domain.
result = create_indicator_from_tlo('Domain',
new_domain,
username,
source_name,
add_domain=False)
if not result['success']:
errors.append(result['message'])
elif ip_result.get('success') and ip_ind:
forge_relationship(left_class=result['indicator'],
right_class=ip_ind,
rel_type='Resolved_To',
analyst=username)
result = True
# This block validates, and may also add, objects to the Domain
if retVal.get('success') or is_validate_only == True:
if rowData:
objectsData = rowData.get(form_consts.Common.OBJECTS_DATA)
# add new objects if they exist
if objectsData:
objectsData = json.loads(objectsData)
current_domain = retrieve_domain(domain, cache)
for object_row_counter, objectData in enumerate(
objectsData, 1):
if current_domain != None:
# if the domain exists then try to add objects to it
if isinstance(current_domain, Domain) == True:
objectDict = object_array_to_dict(
objectData, "Domain", current_domain.id)
else:
objectDict = object_array_to_dict(
objectData, "Domain", "")
current_domain = None
else:
objectDict = object_array_to_dict(
objectData, "Domain", "")
(obj_result, errors,
obj_retVal) = validate_and_add_new_handler_object(
None,
objectDict,
request,
errors,
object_row_counter,
is_validate_only=is_validate_only,
cache=cache,
obj=current_domain)
if not obj_result:
retVal['success'] = False
return result, errors, retVal
def add_update_ip(request, method):
"""
Add/update an IP address. Should be an AJAX POST.
:param request: Django request.
:type request: :class:`django.http.HttpRequest`
:param method: If this is an "add" or an "update".
:type method: str
:returns: :class:`django.http.HttpResponse`
"""
if request.method == "POST" and request.is_ajax():
request.user._setup()
data = request.POST
form = AddIPForm(request.user, None, data)
if form.is_valid():
cleaned_data = form.cleaned_data
ip = cleaned_data['ip']
source_name = cleaned_data['source_name']
source_reference = cleaned_data['source_reference']
source_method = cleaned_data['source_method']
source_tlp = cleaned_data['source_tlp']
campaign = cleaned_data['campaign']
confidence = cleaned_data['confidence']
ip_type = cleaned_data['ip_type']
add_indicator = False
if cleaned_data.get('add_indicator'):
add_indicator = True
indicator_reference = cleaned_data.get('indicator_reference')
bucket_list = cleaned_data.get(
form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
ticket = cleaned_data.get(form_consts.Common.TICKET_VARIABLE_NAME)
related_id = cleaned_data['related_id']
related_type = cleaned_data['related_type']
relationship_type = cleaned_data['relationship_type']
result = ip_add_update(ip,
ip_type,
source=source_name,
source_method=source_method,
source_reference=source_reference,
source_tlp=source_tlp,
campaign=campaign,
confidence=confidence,
user=request.user,
bucket_list=bucket_list,
ticket=ticket,
is_add_indicator=add_indicator,
indicator_reference=indicator_reference,
related_id=related_id,
related_type=related_type,
relationship_type=relationship_type)
if 'message' in result:
if not isinstance(result['message'], list):
result['message'] = [result['message']]
else:
result['message'] = []
message = (
'<div>Success! Click here to view the new IP: <a '
'href="%s">%s</a></div>' %
(reverse('crits.ips.views.ip_detail', args=[ip]), ip))
result['message'].insert(0, message)
return HttpResponse(json.dumps(result, default=json_handler),
content_type="application/json")
return HttpResponse(json.dumps({
'success': False,
'form': form.as_table()
}),
content_type="application/json")
return render_to_response("error.html", {'error': 'Expected AJAX/POST'},
RequestContext(request))
def add_new_domain(data, request, errors, rowData=None, is_validate_only=False, cache={}):
"""
Add a new domain to CRITs.
:param data: The data about the domain.
:type data: dict
:param request: The Django request.
:type request: :class:`django.http.HttpRequest`
:param errors: A list of current errors to append to.
:type errors: list
:param rowData: Any objects that need to be added to the domain.
:type rowData: dict
:param is_validate_only: Only validate the data and return any errors.
:type is_validate_only: boolean
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:returns: tuple (<result>, <errors>, <retVal>)
"""
result = False
retVal = {}
domain = data['domain']
add_ip = data.get('add_ip')
ip = data.get('ip')
ip_type = data.get('ip_type')
if add_ip:
error = validate_and_normalize_ip(ip, ip_type)[1]
if error:
errors.append(error)
if is_validate_only:
error = get_valid_root_domain(domain)[2]
if error:
errors.append(error)
# check for duplicate domains
fqdn_domain = retrieve_domain(domain, cache)
if fqdn_domain:
if isinstance(fqdn_domain, Domain):
resp_url = reverse('crits.domains.views.domain_detail', args=[domain])
message = ('Warning: Domain already exists: '
'<a href="%s">%s</a>' % (resp_url, domain))
retVal['message'] = message
retVal['status'] = form_consts.Status.DUPLICATE
retVal['warning'] = message
else:
result_cache = cache.get(form_consts.Domain.CACHED_RESULTS);
result_cache[domain.lower()] = True
elif not errors:
username = request.user.username
reference = data.get('domain_reference')
source_name = data.get('domain_source')
method = data.get('domain_method')
source = [create_embedded_source(source_name, reference=reference,
method=method, analyst=username)]
bucket_list = data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
ticket = data.get(form_consts.Common.TICKET_VARIABLE_NAME)
if data.get('campaign') and data.get('confidence'):
campaign = [EmbeddedCampaign(name=data.get('campaign'),
confidence=data.get('confidence'),
analyst=username)]
else:
campaign = []
retVal = upsert_domain(domain, source, username, campaign,
bucket_list=bucket_list, ticket=ticket, cache=cache)
if not retVal['success']:
errors.append(retVal.get('message'))
retVal['message'] = ""
else:
new_domain = retVal['object']
ip_result = {}
if add_ip:
if data.get('same_source'):
ip_source = source_name
ip_method = method
ip_reference = reference
else:
ip_source = data.get('ip_source')
ip_method = data.get('ip_method')
ip_reference = data.get('ip_reference')
from crits.ips.handlers import ip_add_update
ip_result = ip_add_update(ip,
ip_type,
ip_source,
ip_method,
ip_reference,
campaign=campaign,
analyst=username,
bucket_list=bucket_list,
ticket=ticket,
cache=cache)
if not ip_result['success']:
errors.append(ip_result['message'])
else:
#add a relationship with the new IP address
new_ip = ip_result['object']
if new_domain and new_ip:
new_domain.add_relationship(new_ip,
'Resolved_To',
analyst=username,
get_rels=False)
new_domain.save(username=username)
#set the URL for viewing the new data
resp_url = reverse('crits.domains.views.domain_detail', args=[domain])
if retVal['is_domain_new'] == True:
retVal['message'] = ('Success! Click here to view the new domain: '
'<a href="%s">%s</a>' % (resp_url, domain))
else:
message = ('Updated existing domain: <a href="%s">%s</a>' % (resp_url, domain))
retVal['message'] = message
retVal[form_consts.Status.STATUS_FIELD] = form_consts.Status.DUPLICATE
retVal['warning'] = message
#add indicators
if data.get('add_indicators'):
from crits.indicators.handlers import create_indicator_from_tlo
# If we have an IP object, add an indicator for that.
if ip_result.get('success'):
ip = ip_result['object']
result = create_indicator_from_tlo('IP',
ip,
username,
ip_source,
add_domain=False)
ip_ind = result.get('indicator')
if not result['success']:
errors.append(result['message'])
# Add an indicator for the domain.
result = create_indicator_from_tlo('Domain',
new_domain,
username,
source_name,
add_domain=False)
if not result['success']:
errors.append(result['message'])
elif ip_result.get('success') and ip_ind:
forge_relationship(left_class=result['indicator'],
right_class=ip_ind,
rel_type='Resolved_To',
analyst=username)
result = True
# This block validates, and may also add, objects to the Domain
if retVal.get('success') or is_validate_only == True:
if rowData:
objectsData = rowData.get(form_consts.Common.OBJECTS_DATA)
# add new objects if they exist
if objectsData:
objectsData = json.loads(objectsData)
current_domain = retrieve_domain(domain, cache)
for object_row_counter, objectData in enumerate(objectsData, 1):
if current_domain != None:
# if the domain exists then try to add objects to it
if isinstance(current_domain, Domain) == True:
objectDict = object_array_to_dict(objectData,
"Domain",
current_domain.id)
else:
objectDict = object_array_to_dict(objectData,
"Domain",
"")
current_domain = None;
else:
objectDict = object_array_to_dict(objectData,
"Domain",
"")
(obj_result,
errors,
obj_retVal) = validate_and_add_new_handler_object(
None, objectDict, request, errors, object_row_counter,
is_validate_only=is_validate_only,
cache=cache, obj=current_domain)
if not obj_result:
retVal['success'] = False
return result, errors, retVal
def handle_indicator_insert(ind, source, reference='', analyst='', method='',
add_domain=False, add_relationship=False, cache={}):
"""
Insert an individual indicator into the database.
NOTE: Setting add_domain to True will always create a relationship as well.
However, to create a relationship with an object that already exists before
this function was called, set add_relationship to True. This will assume
that the domain or IP object to create the relationship with already exists
and will avoid infinite mutual calls between, for example, add_update_ip
and this function. add domain/IP objects.
:param ind: Information about the indicator.
:type ind: dict
:param source: The source for this indicator.
:type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param reference: The reference to the data.
:type reference: str
:param analyst: The user adding this indicator.
:type analyst: str
:param method: Method of acquiring this indicator.
:type method: str
:param add_domain: If this indicator is also a top-level object, try to add
it.
:type add_domain: boolean
:param add_relationship: Attempt to add relationships if applicable.
:type add_relationship: boolean
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" str) if failed,
"objectid" (str) if successful,
"is_new_indicator" (boolean) if successful.
"""
if ind['type'] == "URI - URL" and "://" not in ind['value'].split('.')[0]:
return {"success": False, "message": "URI - URL must contain protocol prefix (e.g. http://, https://, ftp://) "}
is_new_indicator = False
dmain = None
ip = None
rank = {
'unknown': 0,
'benign': 1,
'low': 2,
'medium': 3,
'high': 4,
}
indicator = Indicator.objects(ind_type=ind['type'],
value=ind['value']).first()
if not indicator:
indicator = Indicator()
indicator.ind_type = ind['type']
indicator.value = ind['value']
indicator.created = datetime.datetime.now()
indicator.confidence = EmbeddedConfidence(analyst=analyst)
indicator.impact = EmbeddedImpact(analyst=analyst)
is_new_indicator = True
if 'campaign' in ind:
if isinstance(ind['campaign'], basestring) and len(ind['campaign']) > 0:
confidence = ind.get('campaign_confidence', 'low')
ind['campaign'] = EmbeddedCampaign(name=ind['campaign'],
confidence=confidence,
description="",
analyst=analyst,
date=datetime.datetime.now())
if isinstance(ind['campaign'], EmbeddedCampaign):
indicator.add_campaign(ind['campaign'])
elif isinstance(ind['campaign'], list):
for campaign in ind['campaign']:
if isinstance(campaign, EmbeddedCampaign):
indicator.add_campaign(campaign)
if 'confidence' in ind and rank.get(ind['confidence'], 0) > rank.get(indicator.confidence.rating, 0):
indicator.confidence.rating = ind['confidence']
indicator.confidence.analyst = analyst
if 'impact' in ind and rank.get(ind['impact'], 0) > rank.get(indicator.impact.rating, 0):
indicator.impact.rating = ind['impact']
indicator.impact.analyst = analyst
bucket_list = None
if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind:
bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME]
if bucket_list:
indicator.add_bucket_list(bucket_list, analyst)
ticket = None
if form_consts.Common.TICKET_VARIABLE_NAME in ind:
ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME]
if ticket:
indicator.add_ticket(ticket, analyst)
if isinstance(source, list):
for s in source:
indicator.add_source(source_item=s, method=method, reference=reference)
elif isinstance(source, EmbeddedSource):
indicator.add_source(source_item=source, method=method, reference=reference)
elif isinstance(source, basestring):
s = EmbeddedSource()
s.name = source
instance = EmbeddedSource.SourceInstance()
instance.reference = reference
instance.method = method
instance.analyst = analyst
instance.date = datetime.datetime.now()
s.instances = [instance]
indicator.add_source(s)
if add_domain or add_relationship:
ind_type = indicator.ind_type
ind_value = indicator.value
url_contains_ip = False
if ind_type in ("URI - Domain Name", "URI - URL"):
if ind_type == "URI - URL":
domain_or_ip = urlparse.urlparse(ind_value).hostname
elif ind_type == "URI - Domain Name":
domain_or_ip = ind_value
(sdomain, fqdn) = get_domain(domain_or_ip)
if sdomain == "no_tld_found_error" and ind_type == "URI - URL":
try:
validate_ipv46_address(domain_or_ip)
url_contains_ip = True
except DjangoValidationError:
pass
if not url_contains_ip:
success = None
if add_domain:
success = upsert_domain(sdomain, fqdn, indicator.source,
'%s' % analyst, None,
bucket_list=bucket_list, cache=cache)
if not success['success']:
return {'success': False, 'message': success['message']}
if not success or not 'object' in success:
dmain = Domain.objects(domain=domain_or_ip).first()
else:
dmain = success['object']
if ind_type.startswith("Address - ip") or ind_type == "Address - cidr" or url_contains_ip:
if url_contains_ip:
ind_value = domain_or_ip
try:
validate_ipv4_address(domain_or_ip)
ind_type = 'Address - ipv4-addr'
except DjangoValidationError:
ind_type = 'Address - ipv6-addr'
success = None
if add_domain:
success = ip_add_update(ind_value,
ind_type,
source=indicator.source,
campaign=indicator.campaign,
analyst=analyst,
bucket_list=bucket_list,
ticket=ticket,
indicator_reference=reference,
cache=cache)
if not success['success']:
return {'success': False, 'message': success['message']}
if not success or not 'object' in success:
ip = IP.objects(ip=indicator.value).first()
else:
ip = success['object']
indicator.save(username=analyst)
if dmain:
dmain.add_relationship(rel_item=indicator,
rel_type='Related_To',
analyst="%s" % analyst,
get_rels=False)
dmain.save(username=analyst)
if ip:
ip.add_relationship(rel_item=indicator,
rel_type='Related_To',
analyst="%s" % analyst,
get_rels=False)
ip.save(username=analyst)
indicator.save(username=analyst)
# run indicator triage
if is_new_indicator:
indicator.reload()
run_triage(indicator, analyst)
return {'success': True, 'objectid': str(indicator.id),
'is_new_indicator': is_new_indicator, 'object': indicator}
def parse_observables(self, observables):
"""
Parse list of observables in STIX doc.
:param observables: List of STIX observables.
:type observables: List of STIX observables.
"""
analyst = self.source_instance.analyst
for obs in observables: # for each STIX observable
if not obs.object_ or not obs.object_.properties:
self.failed.append(("No valid object_properties was found!",
type(obs).__name__,
obs.id_)) # note for display in UI
continue
try: # try to create CRITs object from observable
item = obs.object_.properties
if isinstance(item, Address):
if item.category in ('cidr', 'ipv4-addr', 'ipv4-net',
'ipv4-netmask', 'ipv6-addr',
'ipv6-net', 'ipv6-netmask'):
imp_type = "IP"
for value in item.address_value.values:
ip = str(value).strip()
iptype = get_crits_ip_type(item.category)
if iptype:
res = ip_add_update(ip,
iptype,
[self.source],
analyst=analyst,
is_add_indicator=True)
self.parse_res(imp_type, obs, res)
if isinstance(item, DomainName):
imp_type = "Domain"
for value in item.value.values:
res = upsert_domain(str(value),
[self.source],
username=analyst)
self.parse_res(imp_type, obs, res)
elif isinstance(item, Artifact):
# Not sure if this is right, and I believe these can be
# encoded in a couple different ways.
imp_type = "RawData"
rawdata = item.data.decode('utf-8')
description = "None"
# TODO: find out proper ways to determine title, datatype,
# tool_name, tool_version
title = "Artifact for Event: STIX Document %s" % self.package.id_
res = handle_raw_data_file(rawdata,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="Text",
tool_name="STIX",
tool_version=None,
method=self.source_instance.method,
reference=self.source_instance.reference)
self.parse_res(imp_type, obs, res)
elif (isinstance(item, File) and
item.custom_properties and
item.custom_properties[0].name == "crits_type" and
item.custom_properties[0]._value == "Certificate"):
imp_type = "Certificate"
description = "None"
filename = str(item.file_name)
data = None
for obj in item.parent.related_objects:
if isinstance(obj.properties, Artifact):
data = obj.properties.data
res = handle_cert_file(filename,
data,
self.source,
user=analyst,
description=description)
self.parse_res(imp_type, obs, res)
elif isinstance(item, File) and self.has_network_artifact(item):
imp_type = "PCAP"
description = "None"
filename = str(item.file_name)
data = None
for obj in item.parent.related_objects:
if (isinstance(obj.properties, Artifact) and
obj.properties.type_ == Artifact.TYPE_NETWORK):
data = obj.properties.data
res = handle_pcap_file(filename,
data,
self.source,
user=analyst,
description=description)
self.parse_res(imp_type, obs, res)
elif isinstance(item, File):
imp_type = "Sample"
filename = str(item.file_name)
md5 = item.md5
data = None
for obj in item.parent.related_objects:
if (isinstance(obj.properties, Artifact) and
obj.properties.type_ == Artifact.TYPE_FILE):
data = obj.properties.data
res = handle_file(filename,
data,
self.source,
user=analyst,
md5_digest=md5,
is_return_only_md5=False)
self.parse_res(imp_type, obs, res)
elif isinstance(item, EmailMessage):
imp_type = "Email"
data = {}
data['source'] = self.source.name
data['source_method'] = self.source_instance.method
data['source_reference'] = self.source_instance.reference
data['raw_body'] = str(item.raw_body)
data['raw_header'] = str(item.raw_header)
data['helo'] = str(item.email_server)
if item.header:
data['message_id'] = str(item.header.message_id)
data['subject'] = str(item.header.subject)
data['sender'] = str(item.header.sender)
data['reply_to'] = str(item.header.reply_to)
data['x_originating_ip'] = str(item.header.x_originating_ip)
data['x_mailer'] = str(item.header.x_mailer)
data['boundary'] = str(item.header.boundary)
data['from_address'] = str(item.header.from_)
data['date'] = item.header.date.value
if item.header.to:
data['to'] = [str(r) for r in item.header.to.to_list()]
res = handle_email_fields(data,
analyst,
"STIX")
# Should check for attachments and add them here.
self.parse_res(imp_type, obs, res)
if res.get('status') and item.attachments:
for attach in item.attachments:
rel_id = attach.to_dict()['object_reference']
self.relationships.append((obs.id_,
"Contains",
rel_id, "High"))
else: # try to parse all other possibilities as Indicator
imp_type = "Indicator"
obj = make_crits_object(item)
if obj.object_type == 'Address':
# This was already caught above
continue
else:
ind_type = obj.object_type
for value in obj.value:
if value and ind_type:
res = handle_indicator_ind(value.strip(),
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True)
self.parse_res(imp_type, obs, res)
except Exception, e: # probably caused by cybox object we don't handle
self.failed.append((e.message,
type(item).__name__,
item.parent.id_)) # note for display in UI
def handle_indicator_insert(
ind, source, reference="", analyst="", method="", add_domain=False, add_relationship=False, cache={}
):
"""
Insert an individual indicator into the database.
NOTE: Setting add_domain to True will always create a relationship as well.
However, to create a relationship with an object that already exists before
this function was called, set add_relationship to True. This will assume
that the domain or IP object to create the relationship with already exists
and will avoid infinite mutual calls between, for example, add_update_ip
and this function. add domain/IP objects.
:param ind: Information about the indicator.
:type ind: dict
:param source: The source for this indicator.
:type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param reference: The reference to the data.
:type reference: str
:param analyst: The user adding this indicator.
:type analyst: str
:param method: Method of acquiring this indicator.
:type method: str
:param add_domain: If this indicator is also a top-level object, try to add
it.
:type add_domain: boolean
:param add_relationship: Attempt to add relationships if applicable.
:type add_relationship: boolean
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" (str) if failed,
"objectid" (str) if successful,
"is_new_indicator" (boolean) if successful.
"""
if ind["type"] not in IndicatorTypes.values():
return {"success": False, "message": "Not a valid Indicator Type: %s" % ind["type"]}
if ind["threat_type"] not in IndicatorThreatTypes.values():
return {"success": False, "message": "Not a valid Indicator Threat Type: %s" % ind["threat_type"]}
if ind["attack_type"] not in IndicatorAttackTypes.values():
return {"success": False, "message": "Not a valid Indicator Attack Type: " % ind["attack_type"]}
(ind["value"], error) = validate_indicator_value(ind["value"], ind["type"])
if error:
return {"success": False, "message": error}
is_new_indicator = False
dmain = None
ip = None
rank = {"unknown": 0, "benign": 1, "low": 2, "medium": 3, "high": 4}
if ind.get("status", None) is None or len(ind.get("status", "")) < 1:
ind["status"] = Status.NEW
indicator = Indicator.objects(
ind_type=ind["type"], lower=ind["lower"], threat_type=ind["threat_type"], attack_type=ind["attack_type"]
).first()
if not indicator:
indicator = Indicator()
indicator.ind_type = ind["type"]
indicator.threat_type = ind["threat_type"]
indicator.attack_type = ind["attack_type"]
indicator.value = ind["value"]
indicator.lower = ind["lower"]
indicator.description = ind["description"]
indicator.created = datetime.datetime.now()
indicator.confidence = EmbeddedConfidence(analyst=analyst)
indicator.impact = EmbeddedImpact(analyst=analyst)
indicator.status = ind["status"]
is_new_indicator = True
else:
if ind["status"] != Status.NEW:
indicator.status = ind["status"]
add_desc = "\nSeen on %s as: %s" % (str(datetime.datetime.now()), ind["value"])
if indicator.description is None:
indicator.description = add_desc
else:
indicator.description += add_desc
if "campaign" in ind:
if isinstance(ind["campaign"], basestring) and len(ind["campaign"]) > 0:
confidence = ind.get("campaign_confidence", "low")
ind["campaign"] = EmbeddedCampaign(
name=ind["campaign"],
confidence=confidence,
description="",
analyst=analyst,
date=datetime.datetime.now(),
)
if isinstance(ind["campaign"], EmbeddedCampaign):
indicator.add_campaign(ind["campaign"])
elif isinstance(ind["campaign"], list):
for campaign in ind["campaign"]:
if isinstance(campaign, EmbeddedCampaign):
indicator.add_campaign(campaign)
if "confidence" in ind and rank.get(ind["confidence"], 0) > rank.get(indicator.confidence.rating, 0):
indicator.confidence.rating = ind["confidence"]
indicator.confidence.analyst = analyst
if "impact" in ind and rank.get(ind["impact"], 0) > rank.get(indicator.impact.rating, 0):
indicator.impact.rating = ind["impact"]
indicator.impact.analyst = analyst
bucket_list = None
if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind:
bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME]
if bucket_list:
indicator.add_bucket_list(bucket_list, analyst)
ticket = None
if form_consts.Common.TICKET_VARIABLE_NAME in ind:
ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME]
if ticket:
indicator.add_ticket(ticket, analyst)
if isinstance(source, list):
for s in source:
indicator.add_source(source_item=s, method=method, reference=reference)
elif isinstance(source, EmbeddedSource):
indicator.add_source(source_item=source, method=method, reference=reference)
elif isinstance(source, basestring):
s = EmbeddedSource()
s.name = source
instance = EmbeddedSource.SourceInstance()
instance.reference = reference
instance.method = method
instance.analyst = analyst
instance.date = datetime.datetime.now()
s.instances = [instance]
indicator.add_source(s)
if add_domain or add_relationship:
ind_type = indicator.ind_type
ind_value = indicator.lower
url_contains_ip = False
if ind_type in (IndicatorTypes.DOMAIN, IndicatorTypes.URI):
if ind_type == IndicatorTypes.URI:
domain_or_ip = urlparse.urlparse(ind_value).hostname
try:
validate_ipv46_address(domain_or_ip)
url_contains_ip = True
except DjangoValidationError:
pass
else:
domain_or_ip = ind_value
if not url_contains_ip:
success = None
if add_domain:
success = upsert_domain(
domain_or_ip,
indicator.source,
username="******" % analyst,
campaign=indicator.campaign,
bucket_list=bucket_list,
cache=cache,
)
if not success["success"]:
return {"success": False, "message": success["message"]}
if not success or not "object" in success:
dmain = Domain.objects(domain=domain_or_ip).first()
else:
dmain = success["object"]
if ind_type in IPTypes.values() or url_contains_ip:
if url_contains_ip:
ind_value = domain_or_ip
try:
validate_ipv4_address(domain_or_ip)
ind_type = IndicatorTypes.IPV4_ADDRESS
except DjangoValidationError:
ind_type = IndicatorTypes.IPV6_ADDRESS
success = None
if add_domain:
success = ip_add_update(
ind_value,
ind_type,
source=indicator.source,
campaign=indicator.campaign,
analyst=analyst,
bucket_list=bucket_list,
ticket=ticket,
indicator_reference=reference,
cache=cache,
)
if not success["success"]:
return {"success": False, "message": success["message"]}
if not success or not "object" in success:
ip = IP.objects(ip=indicator.value).first()
else:
ip = success["object"]
indicator.save(username=analyst)
if dmain:
dmain.add_relationship(indicator, RelationshipTypes.RELATED_TO, analyst="%s" % analyst, get_rels=False)
dmain.save(username=analyst)
if ip:
ip.add_relationship(indicator, RelationshipTypes.RELATED_TO, analyst="%s" % analyst, get_rels=False)
ip.save(username=analyst)
# run indicator triage
if is_new_indicator:
indicator.reload()
run_triage(indicator, analyst)
return {"success": True, "objectid": str(indicator.id), "is_new_indicator": is_new_indicator, "object": indicator}
def handle_indicator_insert(ind,
source,
reference='',
analyst='',
method='',
add_domain=False,
add_relationship=False,
cache={}):
"""
Insert an individual indicator into the database.
NOTE: Setting add_domain to True will always create a relationship as well.
However, to create a relationship with an object that already exists before
this function was called, set add_relationship to True. This will assume
that the domain or IP object to create the relationship with already exists
and will avoid infinite mutual calls between, for example, add_update_ip
and this function. add domain/IP objects.
:param ind: Information about the indicator.
:type ind: dict
:param source: The source for this indicator.
:type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param reference: The reference to the data.
:type reference: str
:param analyst: The user adding this indicator.
:type analyst: str
:param method: Method of acquiring this indicator.
:type method: str
:param add_domain: If this indicator is also a top-level object, try to add
it.
:type add_domain: boolean
:param add_relationship: Attempt to add relationships if applicable.
:type add_relationship: boolean
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" str) if failed,
"objectid" (str) if successful,
"is_new_indicator" (boolean) if successful.
"""
is_new_indicator = False
rank = {'unknown': 0, 'benign': 1, 'low': 2, 'medium': 3, 'high': 4}
indicator = Indicator.objects(ind_type=ind['type'],
value=ind['value']).first()
if not indicator:
indicator = Indicator()
indicator.ind_type = ind['type']
indicator.value = ind['value']
indicator.created = datetime.datetime.now()
indicator.confidence = EmbeddedConfidence(analyst=analyst)
indicator.impact = EmbeddedImpact(analyst=analyst)
is_new_indicator = True
ec = None
if 'campaign' in ind:
confidence = 'low'
if 'campaign_confidence' in ind:
confidence = ind['campaign_confidence']
ec = EmbeddedCampaign(name=ind['campaign'],
confidence=confidence,
description="",
analyst=analyst,
date=datetime.datetime.now())
if 'confidence' in ind and rank.get(ind['confidence'], 0) > rank.get(
indicator.confidence.rating, 0):
indicator.confidence.rating = ind['confidence']
indicator.confidence.analyst = analyst
if 'impact' in ind and rank.get(ind['impact'], 0) > rank.get(
indicator.impact.rating, 0):
indicator.impact.rating = ind['impact']
indicator.impact.analyst = analyst
bucket_list = None
if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind:
bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME]
indicator.add_bucket_list(bucket_list, analyst)
ticket = None
if form_consts.Common.TICKET_VARIABLE_NAME in ind:
ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME]
indicator.add_ticket(ticket, analyst)
if isinstance(source, list):
for s in source:
indicator.add_source(source_item=s)
elif isinstance(source, EmbeddedSource):
indicator.add_source(source_item=source)
elif isinstance(source, basestring):
s = EmbeddedSource()
s.name = source
instance = EmbeddedSource.SourceInstance()
instance.reference = reference
instance.method = method
instance.analyst = analyst
instance.date = datetime.datetime.now()
s.instances = [instance]
indicator.add_source(s)
if ec:
indicator.add_campaign(ec)
indicator.save(username=analyst)
if add_domain or add_relationship:
ind_type = indicator.ind_type
ind_value = indicator.value
if ind_type in ("URI - Domain Name", "URI - URL"):
if ind_type == "URI - URL":
domain = ind_value.split("/")[2]
elif ind_type == "URI - Domain Name":
domain = ind_value
#try:
(sdomain, fqdn) = get_domain(domain)
success = None
if add_domain:
success = upsert_domain(sdomain,
fqdn,
indicator.source,
'%s' % analyst,
None,
bucket_list=bucket_list,
cache=cache)
if not success['success']:
return {'success': False, 'message': success['message']}
if not success or not 'object' in success:
dmain = Domain.objects(domain=domain).first()
else:
dmain = success['object']
if dmain:
dmain.add_relationship(rel_item=indicator,
rel_type='Related_To',
analyst="%s" % analyst,
get_rels=False)
dmain.save(username=analyst)
indicator.save(username=analyst)
elif ind_type.startswith(
"Address - ip") or ind_type == "Address - cidr":
success = None
if add_domain:
success = ip_add_update(indicator.value,
ind_type,
source=indicator.source,
campaign=indicator.campaign,
analyst=analyst,
bucket_list=bucket_list,
ticket=ticket,
indicator_reference=reference,
cache=cache)
if not success['success']:
return {'success': False, 'message': success['message']}
if not success or not 'object' in success:
ip = IP.objects(ip=indicator.value).first()
else:
ip = success['object']
if ip:
ip.add_relationship(rel_item=indicator,
rel_type='Related_To',
analyst="%s" % analyst,
get_rels=False)
ip.save(username=analyst)
indicator.save(username=analyst)
# run indicator triage
if is_new_indicator:
indicator.reload()
run_triage(None, indicator, analyst)
return {
'success': True,
'objectid': indicator.id,
'is_new_indicator': is_new_indicator,
'object': indicator
}
def add_new_domain(data, request, errors, rowData=None, is_validate_only=False, cache={}):
"""
Add a new domain to CRITs.
:param data: The data about the domain.
:type data: dict
:param request: The Django request.
:type request: :class:`django.http.HttpRequest`
:param errors: A list of current errors to append to.
:type errors: list
:param rowData: Any objects that need to be added to the domain.
:type rowData: dict
:param is_validate_only: Only validate the data and return any errors.
:type is_validate_only: boolean
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:returns: tuple
"""
username = request.user.username
result = False
retVal = {}
reference = data.get('domain_reference')
name = data.get('domain_source')
method = data.get('domain_method')
source = [create_embedded_source(name, reference=reference, method=method,
analyst=username)]
bucket_list = data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
ticket = data.get(form_consts.Common.TICKET_VARIABLE_NAME)
if data.get('campaign') and data.get('confidence'):
campaign = [EmbeddedCampaign(name=data.get('campaign'),
confidence=data.get('confidence'),
analyst=username)]
else:
campaign = []
(sdomain, fqdn) = get_domain(data['domain'])
if sdomain == "no_tld_found_error":
errors.append(u"Error: Invalid domain: " + data['domain'])
elif is_validate_only == False:
retVal = upsert_domain(sdomain, fqdn, source, username, campaign,
bucket_list=bucket_list, ticket=ticket, cache=cache)
ip_result = None
if retVal['success']:
add_ip = data.get('add_ip')
if add_ip:
ip = data.get('ip')
if data.get('same_source'):
ip_source = data.get('domain_source')
ip_method = data.get('domain_method')
ip_reference = reference
else:
ip_source = data.get('ip_source')
ip_method = data.get('ip_method')
ip_reference = data.get('ip_reference')
from crits.ips.handlers import ip_add_update
ip_result = ip_add_update(ip,
None,
ip_source,
ip_method,
ip_reference,
campaign=campaign,
analyst=username,
bucket_list=bucket_list,
ticket=ticket,
cache=cache)
if not ip_result['success']:
errors.append(ip_result['message'])
else:
#add a relationship with the new IP address
new_domain = retVal['object']
new_ip = ip_result['object']
if new_domain and new_ip:
new_domain.add_relationship(rel_item=new_ip,
rel_type='Resolved_To',
analyst=username,
get_rels=False)
new_domain.save(username=username)
new_ip.save(username=username)
#set the URL for viewing the new data
resp_url = reverse('crits.domains.views.domain_detail', args=[fqdn])
if retVal['is_domain_new'] == True:
retVal['message'] = ('Success! Click here to view the new domain: '
'<a href="%s">%s</a>' % (resp_url, fqdn))
else:
message = ('Updated existing domain: <a href="%s">%s</a>' % (resp_url, fqdn))
retVal['message'] = message
retVal[form_consts.Status.STATUS_FIELD] = form_consts.Status.DUPLICATE
retVal['warning'] = message
#add indicators
# add_indicators_for_domain handles adding relationships as well
if data.get('add_indicators'):
analyst = username
from crits.indicators.handlers import add_indicators_for_domain
if ip_result and ip_result['success']:
obj = ip_result['object']
errors += add_indicators_for_domain(sdomain,
fqdn,
source,
analyst,
reference,
obj,
bucket_list=bucket_list,
ticket=ticket)['errors']
else:
errors += add_indicators_for_domain(sdomain,
fqdn,
source,
analyst,
reference,
bucket_list=bucket_list,
ticket=ticket)['errors']
result = True
elif 'message' in retVal: #database error? (!c_dom)
errors.append(retVal['message']) #u"Unknown error: unable to add domain")
elif is_validate_only == True:
domain = data['domain']
fqdn_domain = retrieve_domain(domain, cache);
if fqdn_domain:
if isinstance(fqdn_domain, Domain):
resp_url = reverse('crits.domains.views.domain_detail', args=[fqdn])
message = ('Warning: Domain already exists: '
'<a href="%s">%s</a>' % (resp_url, fqdn))
retVal['message'] = message;
retVal['status'] = form_consts.Status.DUPLICATE;
retVal['warning'] = message;
else:
result_cache = cache.get(form_consts.Domain.CACHED_RESULTS);
result_cache[domain.lower()] = True;
# This block tries to add objects to the item
if retVal.get('success') or is_validate_only == True:
if rowData:
objectsData = rowData.get(form_consts.Common.OBJECTS_DATA)
# add new objects if they exist
if objectsData:
objectsData = json.loads(objectsData)
object_row_counter = 1
current_domain = retrieve_domain(fqdn, cache)
for objectData in objectsData:
if current_domain != None:
# if the domain exists then try to add objects to it
if isinstance(current_domain, Domain) == True:
objectDict = object_array_to_dict(objectData, "Domain", current_domain.id)
else:
objectDict = object_array_to_dict(objectData, "Domain", "")
current_domain = None;
else:
objectDict = object_array_to_dict(objectData, "Domain", "")
(object_result, object_errors, object_retVal) = validate_and_add_new_handler_object(
None, objectDict, request, errors, object_row_counter,
is_validate_only=is_validate_only,
cache=cache, obj=current_domain)
if object_retVal.get('success') == False:
retVal['success'] = False
if object_retVal.get('message'):
errors.append(object_retVal['message'])
object_row_counter += 1
return result, errors, retVal
def add_update_ip(request, method):
"""
Add/update an IP address. Should be an AJAX POST.
:param request: Django request.
:type request: :class:`django.http.HttpRequest`
:param method: If this is an "add" or an "update".
:type method: str
:returns: :class:`django.http.HttpResponse`
"""
if request.method == "POST" and request.is_ajax():
request.user._setup()
data = request.POST
form = AddIPForm(request.user, None, data)
if form.is_valid():
cleaned_data = form.cleaned_data
ip = cleaned_data['ip']
source_name = cleaned_data['source_name']
source_reference = cleaned_data['source_reference']
source_method = cleaned_data['source_method']
source_tlp= cleaned_data['source_tlp']
campaign = cleaned_data['campaign']
confidence = cleaned_data['confidence']
ip_type = cleaned_data['ip_type']
add_indicator = False
if cleaned_data.get('add_indicator'):
add_indicator = True
indicator_reference = cleaned_data.get('indicator_reference')
bucket_list = cleaned_data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
ticket = cleaned_data.get(form_consts.Common.TICKET_VARIABLE_NAME)
related_id = cleaned_data['related_id']
related_type = cleaned_data['related_type']
relationship_type = cleaned_data['relationship_type']
result = ip_add_update(ip,
ip_type,
source=source_name,
source_method=source_method,
source_reference=source_reference,
source_tlp=source_tlp,
campaign=campaign,
confidence=confidence,
user=request.user,
bucket_list=bucket_list,
ticket=ticket,
is_add_indicator=add_indicator,
indicator_reference=indicator_reference,
related_id=related_id,
related_type=related_type,
relationship_type=relationship_type)
if 'message' in result:
if not isinstance(result['message'], list):
result['message'] = [result['message']]
else:
result['message'] = []
message = ('<div>Success! Click here to view the new IP: <a '
'href="%s">%s</a></div>'
% (reverse('crits.ips.views.ip_detail',
args=[ip]),
ip))
result['message'].insert(0, message)
return HttpResponse(json.dumps(result,
default=json_handler),
content_type="application/json")
return HttpResponse(json.dumps({'success': False,
'form':form.as_table()}),
content_type="application/json")
return render_to_response("error.html",
{'error': 'Expected AJAX/POST'},
RequestContext(request))
def parse_cybox_object(self, cbx_obj, description='', ind_id=None):
"""
Parse a CybOX object form a STIX doc. An object can contain
multiple related_objects, which in turn can have their own
related_objects, so this handles those recursively.
:param cbx_obj: The CybOX object to parse.
:type cbx_obj: A CybOX object.
:param description: Parent-level (e.g. Observable) description.
:type description: str
:param ind_id: The ID of a parent STIX Indicator.
:type ind_id: str
"""
# check for missing attributes
if not cbx_obj or not cbx_obj.properties:
if cbx_obj.idref: # just a reference, so nothing to parse
return
else:
cbx_id = getattr(cbx_obj, 'id_', 'None')
self.failed.append(("No valid object_properties was found!",
"Observable (%s)" % cbx_id,
cbx_id)) # note for display in UI
return
# Don't parse if already been parsed
# This is for artifacts that are related to CybOX File Objects
if cbx_obj.id_ in self.parsed:
return
try: # try to create CRITs object from Cybox Object
analyst = self.source_instance.analyst
item = cbx_obj.properties
val = cbx_obj.id_
if isinstance(item, Address) and not ind_id:
if item.category in ('cidr', 'ipv4-addr', 'ipv4-net',
'ipv4-netmask', 'ipv6-addr', 'ipv6-net',
'ipv6-netmask'):
imp_type = "IP"
for value in item.address_value.values:
val = str(value).strip()
if self.preview:
res = None
else:
iptype = get_crits_ip_type(item.category)
if iptype:
res = ip_add_update(val,
iptype, [self.source],
analyst=analyst,
is_add_indicator=True)
else:
res = {
'success': False,
'reason': 'No IP Type'
}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
if (not ind_id and
(isinstance(item, DomainName) or
(isinstance(item, URI) and item.type_ == 'Domain Name'))):
imp_type = "Domain"
for val in item.value.values:
if self.preview:
res = None
else:
res = upsert_domain(str(val), [self.source],
username=analyst)
self.parse_res(imp_type, str(val), cbx_obj, res, ind_id)
elif isinstance(item, HTTPSession):
imp_type = "RawData"
val = cbx_obj.id_
try:
c_req = item.http_request_response[0].http_client_request
hdr = c_req.http_request_header
if hdr.raw_header:
data = hdr.raw_header.value
title = "HTTP Header from STIX: %s" % self.package.id_
method = self.source_instance.method
ref = self.source_instance.reference
if self.preview:
res = None
val = title
else:
res = handle_raw_data_file(data,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="HTTP Header",
tool_name="STIX",
tool_version=None,
method=method,
reference=ref)
else:
imp_type = "Indicator"
ind_type = "HTTP Request Header Fields - User-Agent"
val = hdr.parsed_header.user_agent.value
val = ','.join(val) if isinstance(val, list) else val
if self.preview:
res = None
else:
res = handle_indicator_ind(
val,
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_relationship=True,
description=description)
except:
msg = "Unsupported use of 'HTTPSession' object."
res = {'success': False, 'reason': msg}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, WhoisEntry):
# No sure where else to put this
imp_type = "RawData"
val = cbx_obj.id_
if item.remarks:
data = item.remarks.value
title = "WHOIS Entry from STIX: %s" % self.package.id_
if self.preview:
res = None
val = title
else:
res = handle_raw_data_file(
data,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="Text",
tool_name="WHOIS",
tool_version=None,
method=self.source_instance.method,
reference=self.source_instance.reference)
else:
msg = "Unsupported use of 'WhoisEntry' object."
res = {'success': False, 'reason': msg}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, Artifact):
# Not sure if this is right, and I believe these can be
# encoded in a couple different ways.
imp_type = "RawData"
val = cbx_obj.id_
rawdata = item.data.decode('utf-8')
# TODO: find out proper ways to determine title, datatype,
# tool_name, tool_version
title = "Artifact for Event: STIX Document %s" % self.package.id_
if self.preview:
res = None
val = title
else:
res = handle_raw_data_file(
rawdata,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="Text",
tool_name="STIX",
tool_version=None,
method=self.source_instance.method,
reference=self.source_instance.reference)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif (isinstance(item, File) and item.custom_properties
and item.custom_properties[0].name == "crits_type"
and item.custom_properties[0]._value == "Certificate"):
imp_type = "Certificate"
val = str(item.file_name)
data = None
if self.preview:
res = None
else:
for rel_obj in item.parent.related_objects:
if isinstance(rel_obj.properties, Artifact):
data = rel_obj.properties.data
self.parsed.append(rel_obj.id_)
res = handle_cert_file(val,
data,
self.source,
user=analyst,
description=description)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, File) and self.has_network_artifact(item):
imp_type = "PCAP"
val = str(item.file_name)
data = None
if self.preview:
res = None
else:
for rel_obj in item.parent.related_objects:
if (isinstance(rel_obj.properties, Artifact)
and rel_obj.properties.type_
== Artifact.TYPE_NETWORK):
data = rel_obj.properties.data
self.parsed.append(rel_obj.id_)
res = handle_pcap_file(val,
data,
self.source,
user=analyst,
description=description)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, File):
imp_type = "Sample"
md5 = item.md5
if md5:
md5 = md5.lower()
val = str(item.file_name or md5)
# add sha1/sha256/ssdeep once handle_file supports it
size = item.size_in_bytes
data = None
if item.file_path:
path = "File Path: " + str(item.file_path)
description += "\n" + path
for rel_obj in item.parent.related_objects:
if (isinstance(rel_obj.properties, Artifact) and
rel_obj.properties.type_ == Artifact.TYPE_FILE):
data = rel_obj.properties.data
self.parsed.append(rel_obj.id_)
if not md5 and not data and val and val != "None":
imp_type = "Indicator"
if self.preview:
res = None
else:
res = handle_indicator_ind(
val,
self.source,
"Win File",
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True,
description=description)
elif md5 or data:
if self.preview:
res = None
else:
res = handle_file(val,
data,
self.source,
user=analyst,
md5_digest=md5,
is_return_only_md5=False,
size=size,
description=description)
else:
val = cbx_obj.id_
msg = "CybOX 'File' object has no MD5, data, or filename"
res = {'success': False, 'reason': msg}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, EmailMessage):
imp_type = 'Email'
id_list = []
data = {}
val = cbx_obj.id_
get_attach = False
data['raw_body'] = str(item.raw_body)
data['raw_header'] = str(item.raw_header)
data['helo'] = str(item.email_server)
if item.header:
data['subject'] = str(item.header.subject)
if item.header.date:
data['date'] = item.header.date.value
val = "Date: %s, Subject: %s" % (data.get(
'date', 'None'), data['subject'])
data['message_id'] = str(item.header.message_id)
data['sender'] = str(item.header.sender)
data['reply_to'] = str(item.header.reply_to)
data['x_originating_ip'] = str(
item.header.x_originating_ip)
data['x_mailer'] = str(item.header.x_mailer)
data['boundary'] = str(item.header.boundary)
data['from_address'] = str(item.header.from_)
if item.header.to:
data['to'] = [str(r) for r in item.header.to.to_list()]
if data.get('date'): # Email TLOs must have a date
data['source'] = self.source.name
data['source_method'] = self.source_instance.method
data['source_reference'] = self.source_instance.reference
if self.preview:
res = None
else:
res = handle_email_fields(data, analyst, "STIX")
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
if not self.preview and res.get('status'):
id_list.append(cbx_obj.id_) # save ID for atchmnt rels
get_attach = True
else: # Can't be an Email TLO, so save fields
for x, key in enumerate(data):
if data[key] and data[key] != "None":
if key in ('raw_header', 'raw_body'):
if key == 'raw_header':
title = "Email Header from STIX Email: %s"
d_type = "Email Header"
else:
title = "Email Body from STIX Email: %s"
d_type = "Email Body"
imp_type = 'RawData'
title = title % cbx_obj.id_
if self.preview:
res = None
else:
res = handle_raw_data_file(
data[key], self.source, analyst,
description, title, d_type, "STIX",
self.stix_version)
self.parse_res(imp_type, title, cbx_obj, res,
ind_id)
elif key == 'to':
imp_type = 'Target'
for y, addr in enumerate(data[key]):
tgt_dict = {'email_address': addr}
if self.preview:
res = None
else:
res = upsert_target(tgt_dict, analyst)
if res['success']:
get_attach = True
tmp_obj = copy(cbx_obj)
tmp_obj.id_ = '%s-%s-%s' % (cbx_obj.id_, x,
y)
self.parse_res(imp_type, addr, tmp_obj,
res, ind_id)
self.ind2obj.setdefault(
cbx_obj.id_, []).append(tmp_obj.id_)
id_list.append(tmp_obj.id_)
else:
imp_type = 'Indicator'
if key in ('sender', 'reply_to',
'from_address'):
ind_type = "Address - e-mail"
elif 'ip' in key:
ind_type = "Address - ipv4-addr"
elif key == 'raw_body':
ind_type = "Email Message"
else:
ind_type = "String"
if self.preview:
res = None
else:
res = handle_indicator_ind(
data[key],
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True,
description=description)
if res['success']:
get_attach = True
tmp_obj = copy(cbx_obj)
tmp_obj.id_ = '%s-%s' % (cbx_obj.id_, x)
self.parse_res(imp_type, data[key], tmp_obj,
res, ind_id)
self.ind2obj.setdefault(cbx_obj.id_,
[]).append(tmp_obj.id_)
id_list.append(tmp_obj.id_)
if not self.preview:
# Setup relationships between all Email attributes
for oid in id_list:
for oid2 in id_list:
if oid != oid2:
self.relationships.append(
(oid, RelationshipTypes.RELATED_TO, oid2,
"High"))
# Should check for attachments and add them here.
if get_attach and item.attachments:
for attach in item.attachments:
rel_id = attach.to_dict()['object_reference']
for oid in id_list:
self.relationships.append(
(oid, RelationshipTypes.CONTAINS, rel_id,
"High"))
else: # try to parse all other possibilities as Indicator
imp_type = "Indicator"
val = cbx_obj.id_
c_obj = make_crits_object(item)
# Ignore what was already caught above
if (ind_id or c_obj.object_type not in IPTypes.values()):
ind_type = c_obj.object_type
for val in [str(v).strip() for v in c_obj.value if v]:
if ind_type:
# handle domains mislabeled as URLs
if c_obj.object_type == 'URI' and '/' not in val:
ind_type = "Domain"
if self.preview:
res = None
else:
res = handle_indicator_ind(
val,
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True,
description=description)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
except Exception, e: # probably caused by cybox object we don't handle
self.failed.append((e.message, "%s (%s)" % (imp_type, val),
cbx_obj.id_)) # note for display in UI
def add_new_domain(data,
request,
errors,
rowData=None,
is_validate_only=False,
cache={}):
"""
Add a new domain to CRITs.
:param data: The data about the domain.
:type data: dict
:param request: The Django request.
:type request: :class:`django.http.HttpRequest`
:param errors: A list of current errors to append to.
:type errors: list
:param rowData: Any objects that need to be added to the domain.
:type rowData: dict
:param is_validate_only: Only validate the data and return any errors.
:type is_validate_only: boolean
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:returns: tuple
"""
username = request.user.username
result = False
retVal = {}
reference = data.get('domain_reference')
name = data.get('domain_source')
method = data.get('domain_method')
source = [
create_embedded_source(name,
reference=reference,
method=method,
analyst=username)
]
bucket_list = data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
ticket = data.get(form_consts.Common.TICKET_VARIABLE_NAME)
if data.get('campaign') and data.get('confidence'):
campaign = [
EmbeddedCampaign(name=data.get('campaign'),
confidence=data.get('confidence'),
analyst=username)
]
else:
campaign = []
(sdomain, fqdn) = get_domain(data['domain'])
if sdomain == "no_tld_found_error":
errors.append(u"Error: Invalid domain: " + data['domain'])
elif is_validate_only == False:
retVal = upsert_domain(sdomain,
fqdn,
source,
username,
campaign,
bucket_list=bucket_list,
ticket=ticket,
cache=cache)
ip_result = None
if retVal['success']:
new_domain = retVal['object']
add_ip = data.get('add_ip')
if add_ip:
ip = data.get('ip')
ip_type = data.get('ip_type')
if data.get('same_source'):
ip_source = data.get('domain_source')
ip_method = data.get('domain_method')
ip_reference = reference
else:
ip_source = data.get('ip_source')
ip_method = data.get('ip_method')
ip_reference = data.get('ip_reference')
from crits.ips.handlers import ip_add_update
ip_result = ip_add_update(ip,
ip_type,
ip_source,
ip_method,
ip_reference,
campaign=campaign,
analyst=username,
bucket_list=bucket_list,
ticket=ticket,
cache=cache)
if not ip_result['success']:
errors.append(ip_result['message'])
else:
#add a relationship with the new IP address
new_ip = ip_result['object']
if new_domain and new_ip:
new_domain.add_relationship(rel_item=new_ip,
rel_type='Resolved_To',
analyst=username,
get_rels=False)
new_domain.save(username=username)
new_ip.save(username=username)
#set the URL for viewing the new data
resp_url = reverse('crits.domains.views.domain_detail',
args=[fqdn])
if retVal['is_domain_new'] == True:
retVal['message'] = (
'Success! Click here to view the new domain: '
'<a href="%s">%s</a>' % (resp_url, fqdn))
else:
message = ('Updated existing domain: <a href="%s">%s</a>' %
(resp_url, fqdn))
retVal['message'] = message
retVal[form_consts.Status.
STATUS_FIELD] = form_consts.Status.DUPLICATE
retVal['warning'] = message
#add indicators
if data.get('add_indicators'):
from crits.indicators.handlers import create_indicator_from_obj
# If we have an IP object, add an indicator for that.
if ip_result and ip_result['success']:
obj = ip_result['object']
result = create_indicator_from_obj('Address - ipv4-addr',
'IP', obj.id, obj.ip,
username)
if result['success'] == False:
errors.append(result['message'])
# Add an indicator for the domain.
result = create_indicator_from_obj('URI - Domain Name',
'Domain', new_domain.id,
sdomain, username)
if result['success'] == False:
errors.append(result['message'])
# If we have an FQDN (ie: it is not the same as sdomain)
# then add that also.
if fqdn != sdomain:
result = create_indicator_from_obj('URI - Domain Name',
'Domain', new_domain.id,
fqdn, username)
if result['success'] == False:
errors.append(result['message'])
result = True
elif 'message' in retVal: #database error? (!c_dom)
errors.append(
retVal['message']) #u"Unknown error: unable to add domain")
elif is_validate_only == True:
domain = data['domain']
fqdn_domain = retrieve_domain(domain, cache)
if fqdn_domain:
if isinstance(fqdn_domain, Domain):
resp_url = reverse('crits.domains.views.domain_detail',
args=[fqdn])
message = ('Warning: Domain already exists: '
'<a href="%s">%s</a>' % (resp_url, fqdn))
retVal['message'] = message
retVal['status'] = form_consts.Status.DUPLICATE
retVal['warning'] = message
else:
result_cache = cache.get(form_consts.Domain.CACHED_RESULTS)
result_cache[domain.lower()] = True
# This block tries to add objects to the item
if retVal.get('success') or is_validate_only == True:
if rowData:
objectsData = rowData.get(form_consts.Common.OBJECTS_DATA)
# add new objects if they exist
if objectsData:
objectsData = json.loads(objectsData)
current_domain = retrieve_domain(fqdn, cache)
for object_row_counter, objectData in enumerate(
objectsData, 1):
if current_domain != None:
# if the domain exists then try to add objects to it
if isinstance(current_domain, Domain) == True:
objectDict = object_array_to_dict(
objectData, "Domain", current_domain.id)
else:
objectDict = object_array_to_dict(
objectData, "Domain", "")
current_domain = None
else:
objectDict = object_array_to_dict(
objectData, "Domain", "")
(object_result, object_errors,
object_retVal) = validate_and_add_new_handler_object(
None,
objectDict,
request,
errors,
object_row_counter,
is_validate_only=is_validate_only,
cache=cache,
obj=current_domain)
if object_retVal.get('success') == False:
retVal['success'] = False
if object_retVal.get('message'):
errors.append(object_retVal['message'])
return result, errors, retVal
def parse_cybox_object(self, cbx_obj, description='', ind_id=None):
"""
Parse a CybOX object form a STIX doc. An object can contain
multiple related_objects, which in turn can have their own
related_objects, so this handles those recursively.
:param cbx_obj: The CybOX object to parse.
:type cbx_obj: A CybOX object.
:param description: Parent-level (e.g. Observable) description.
:type description: str
:param ind_id: The ID of a parent STIX Indicator.
:type ind_id: str
"""
# check for missing attributes
if not cbx_obj or not cbx_obj.properties:
if cbx_obj.idref: # just a reference, so nothing to parse
return
else:
cbx_id = getattr(cbx_obj, 'id_', 'None')
self.failed.append(("No valid object_properties was found!",
"Observable (%s)" % cbx_id,
cbx_id)) # note for display in UI
return
# Don't parse if already been parsed
# This is for artifacts that are related to CybOX File Objects
if cbx_obj.id_ in self.parsed:
return
try: # try to create CRITs object from Cybox Object
analyst = self.source_instance.analyst
item = cbx_obj.properties
val = cbx_obj.id_
if isinstance(item, Address) and not ind_id:
if item.category in ('cidr', 'ipv4-addr', 'ipv4-net',
'ipv4-netmask', 'ipv6-addr',
'ipv6-net', 'ipv6-netmask'):
imp_type = "IP"
for value in item.address_value.values:
val = str(value).strip()
if self.preview:
res = None
else:
iptype = get_crits_ip_type(item.category)
if iptype:
res = ip_add_update(val,
iptype,
[self.source],
analyst=analyst,
is_add_indicator=True)
else:
res = {'success': False, 'reason': 'No IP Type'}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
if (not ind_id and (isinstance(item, DomainName) or
(isinstance(item, URI) and item.type_ == 'Domain Name'))):
imp_type = "Domain"
for val in item.value.values:
if self.preview:
res = None
else:
res = upsert_domain(str(val),
[self.source],
username=analyst)
self.parse_res(imp_type, str(val), cbx_obj, res, ind_id)
elif isinstance(item, HTTPSession):
imp_type = "RawData"
val = cbx_obj.id_
try:
c_req = item.http_request_response[0].http_client_request
hdr = c_req.http_request_header
if hdr.raw_header:
data = hdr.raw_header.value
title = "HTTP Header from STIX: %s" % self.package.id_
method = self.source_instance.method
ref = self.source_instance.reference
if self.preview:
res = None
val = title
else:
res = handle_raw_data_file(data,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="HTTP Header",
tool_name="STIX",
tool_version=None,
method=method,
reference=ref)
else:
imp_type = "Indicator"
ind_type = "HTTP Request Header Fields - User-Agent"
val = hdr.parsed_header.user_agent.value
val = ','.join(val) if isinstance(val, list) else val
if self.preview:
res = None
else:
res = handle_indicator_ind(val,
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_relationship=True,
description=description)
except:
msg = "Unsupported use of 'HTTPSession' object."
res = {'success': False, 'reason': msg}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, WhoisEntry):
# No sure where else to put this
imp_type = "RawData"
val = cbx_obj.id_
if item.remarks:
data = item.remarks.value
title = "WHOIS Entry from STIX: %s" % self.package.id_
if self.preview:
res = None
val = title
else:
res = handle_raw_data_file(data,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="Text",
tool_name="WHOIS",
tool_version=None,
method=self.source_instance.method,
reference=self.source_instance.reference)
else:
msg = "Unsupported use of 'WhoisEntry' object."
res = {'success': False, 'reason': msg}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, Artifact):
# Not sure if this is right, and I believe these can be
# encoded in a couple different ways.
imp_type = "RawData"
val = cbx_obj.id_
rawdata = item.data.decode('utf-8')
# TODO: find out proper ways to determine title, datatype,
# tool_name, tool_version
title = "Artifact for Event: STIX Document %s" % self.package.id_
if self.preview:
res = None
val = title
else:
res = handle_raw_data_file(rawdata,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="Text",
tool_name="STIX",
tool_version=None,
method=self.source_instance.method,
reference=self.source_instance.reference)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif (isinstance(item, File) and
item.custom_properties and
item.custom_properties[0].name == "crits_type" and
item.custom_properties[0]._value == "Certificate"):
imp_type = "Certificate"
val = str(item.file_name)
data = None
if self.preview:
res = None
else:
for rel_obj in item.parent.related_objects:
if isinstance(rel_obj.properties, Artifact):
data = rel_obj.properties.data
self.parsed.append(rel_obj.id_)
res = handle_cert_file(val,
data,
self.source,
user=analyst,
description=description)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, File) and self.has_network_artifact(item):
imp_type = "PCAP"
val = str(item.file_name)
data = None
if self.preview:
res = None
else:
for rel_obj in item.parent.related_objects:
if (isinstance(rel_obj.properties, Artifact) and
rel_obj.properties.type_ == Artifact.TYPE_NETWORK):
data = rel_obj.properties.data
self.parsed.append(rel_obj.id_)
res = handle_pcap_file(val,
data,
self.source,
user=analyst,
description=description)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, File):
imp_type = "Sample"
md5 = item.md5
if md5:
md5 = md5.lower()
val = str(item.file_name or md5)
# add sha1/sha256/ssdeep once handle_file supports it
size = item.size_in_bytes
data = None
if item.file_path:
path = "File Path: " + str(item.file_path)
description += "\n" + path
for rel_obj in item.parent.related_objects:
if (isinstance(rel_obj.properties, Artifact) and
rel_obj.properties.type_ == Artifact.TYPE_FILE):
data = rel_obj.properties.data
self.parsed.append(rel_obj.id_)
if not md5 and not data and val and val != "None":
imp_type = "Indicator"
if self.preview:
res = None
else:
res = handle_indicator_ind(val,
self.source,
"Win File",
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True,
description=description)
elif md5 or data:
if self.preview:
res = None
else:
res = handle_file(val,
data,
self.source,
user=analyst,
md5_digest=md5,
is_return_only_md5=False,
size=size,
description=description)
else:
val = cbx_obj.id_
msg = "CybOX 'File' object has no MD5, data, or filename"
res = {'success': False, 'reason': msg}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, EmailMessage):
imp_type = 'Email'
id_list = []
data = {}
val = cbx_obj.id_
get_attach = False
data['raw_body'] = str(item.raw_body)
data['raw_header'] = str(item.raw_header)
data['helo'] = str(item.email_server)
if item.header:
data['subject'] = str(item.header.subject)
if item.header.date:
data['date'] = item.header.date.value
val = "Date: %s, Subject: %s" % (data.get('date', 'None'),
data['subject'])
data['message_id'] = str(item.header.message_id)
data['sender'] = str(item.header.sender)
data['reply_to'] = str(item.header.reply_to)
data['x_originating_ip'] = str(item.header.x_originating_ip)
data['x_mailer'] = str(item.header.x_mailer)
data['boundary'] = str(item.header.boundary)
data['from_address'] = str(item.header.from_)
if item.header.to:
data['to'] = [str(r) for r in item.header.to.to_list()]
if data.get('date'): # Email TLOs must have a date
data['source'] = self.source.name
data['source_method'] = self.source_instance.method
data['source_reference'] = self.source_instance.reference
if self.preview:
res = None
else:
res = handle_email_fields(data,
analyst,
"STIX")
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
if not self.preview and res.get('status'):
id_list.append(cbx_obj.id_) # save ID for atchmnt rels
get_attach = True
else: # Can't be an Email TLO, so save fields
for x, key in enumerate(data):
if data[key] and data[key] != "None":
if key in ('raw_header', 'raw_body'):
if key == 'raw_header':
title = "Email Header from STIX Email: %s"
d_type = "Email Header"
else:
title = "Email Body from STIX Email: %s"
d_type = "Email Body"
imp_type = 'RawData'
title = title % cbx_obj.id_
if self.preview:
res = None
else:
res = handle_raw_data_file(data[key],
self.source,
analyst,
description,
title,
d_type,
"STIX",
self.stix_version)
self.parse_res(imp_type, title, cbx_obj,
res, ind_id)
elif key == 'to':
imp_type = 'Target'
for y, addr in enumerate(data[key]):
tgt_dict = {'email_address': addr}
if self.preview:
res = None
else:
res = upsert_target(tgt_dict, analyst)
if res['success']:
get_attach = True
tmp_obj = copy(cbx_obj)
tmp_obj.id_ = '%s-%s-%s' % (cbx_obj.id_,
x, y)
self.parse_res(imp_type, addr, tmp_obj,
res, ind_id)
self.ind2obj.setdefault(cbx_obj.id_,
[]).append(tmp_obj.id_)
id_list.append(tmp_obj.id_)
else:
imp_type = 'Indicator'
if key in ('sender', 'reply_to', 'from_address'):
ind_type = "Address - e-mail"
elif 'ip' in key:
ind_type = "Address - ipv4-addr"
elif key == 'raw_body':
ind_type = "Email Message"
else:
ind_type = "String"
if self.preview:
res = None
else:
res = handle_indicator_ind(data[key],
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True,
description=description)
if res['success']:
get_attach = True
tmp_obj = copy(cbx_obj)
tmp_obj.id_ = '%s-%s' % (cbx_obj.id_, x)
self.parse_res(imp_type, data[key], tmp_obj,
res, ind_id)
self.ind2obj.setdefault(cbx_obj.id_,
[]).append(tmp_obj.id_)
id_list.append(tmp_obj.id_)
if not self.preview:
# Setup relationships between all Email attributes
for oid in id_list:
for oid2 in id_list:
if oid != oid2:
self.relationships.append((oid,
RelationshipTypes.RELATED_TO,
oid2, "High"))
# Should check for attachments and add them here.
if get_attach and item.attachments:
for attach in item.attachments:
rel_id = attach.to_dict()['object_reference']
for oid in id_list:
self.relationships.append((oid,
RelationshipTypes.CONTAINS,
rel_id, "High"))
else: # try to parse all other possibilities as Indicator
imp_type = "Indicator"
val = cbx_obj.id_
c_obj = make_crits_object(item)
# Ignore what was already caught above
if (ind_id or c_obj.object_type not in IPTypes.values()):
ind_type = c_obj.object_type
for val in [str(v).strip() for v in c_obj.value if v]:
if ind_type:
# handle domains mislabeled as URLs
if c_obj.object_type == 'URI' and '/' not in val:
ind_type = "Domain"
if self.preview:
res = None
else:
res = handle_indicator_ind(val,
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True,
description=description)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
except Exception, e: # probably caused by cybox object we don't handle
self.failed.append((e.message,
"%s (%s)" % (imp_type, val),
cbx_obj.id_)) # note for display in UI
