def obj_create(self, bundle, **kwargs):
"""
Handles creating Indicators through the API.
:param bundle: Bundle containing the information to create the Indicator.
:type bundle: Tastypie Bundle object.
:returns: HttpResponse.
"""
analyst = bundle.request.user.username
value = bundle.data.get('value', None)
ctype = bundle.data.get('type', None)
threat_type = bundle.data.get('threat_type', None)
attack_type = bundle.data.get('attack_type', None)
source = bundle.data.get('source', None)
reference = bundle.data.get('reference', None)
method = bundle.data.get('method', None)
add_domain = bundle.data.get('add_domain', False)
add_relationship = bundle.data.get('add_relationship', False)
campaign = bundle.data.get('campaign', None)
campaign_confidence = bundle.data.get('confidence', None)
confidence = bundle.data.get('indicator_confidence', None)
description = bundle.data.get('description', None)
impact = bundle.data.get('indicator_impact', None)
bucket_list = bundle.data.get('bucket_list', None)
ticket = bundle.data.get('ticket', None)
result = handle_indicator_ind(value,
source,
ctype,
threat_type,
attack_type,
analyst,
method,
reference,
add_domain,
add_relationship,
campaign,
campaign_confidence,
confidence,
description,
impact,
bucket_list,
ticket)
content = {'return_code': 0,
'type': 'Indicator'}
if result.get('message'):
content['message'] = result.get('message')
if result.get('objectid'):
url = reverse('api_dispatch_detail',
kwargs={'resource_name': 'indicators',
'api_name': 'v1',
'pk': result.get('objectid')})
content['id'] = result.get('objectid')
content['url'] = url
if not result['success']:
content['return_code'] = 1
self.crits_response(content)
def import_object(request, type_, id_):
setup_access()
if type_ == "Threat Descriptors":
obj = ThreatDescriptor(id=id_)
obj.details(
fields=[f for f in ThreatDescriptor._default_fields if f not in
(td.PRIVACY_MEMBERS, td.SUBMITTER_COUNT, td.METADATA)]
)
itype = getattr(IndicatorTypes, obj.get(td.TYPE))
ithreat_type = getattr(IndicatorThreatTypes, obj.get(td.THREAT_TYPE))
results = handle_indicator_ind(
obj.get(td.RAW_INDICATOR),
"ThreatExchange",
itype,
ithreat_type,
None,
request.user.username,
method="ThreatExchange Service",
reference="id: %s, owner: %s, share_level: %s" % (obj.get(td.ID),
obj.get(td.OWNER)['name'],
obj.get(td.SHARE_LEVEL)),
add_domain=True,
add_relationship=True,
confidence=build_ci(obj.get(td.CONFIDENCE)),
description=obj.get(td.DESCRIPTION)
)
return results
elif type_ == "Malware Analyses":
obj = Malware(id=id_)
obj.details(
fields=[f for f in Malware._fields if f not in
(td.PRIVACY_MEMBERS, td.METADATA)]
)
filename = obj.get(m.MD5)
try:
data = obj.rf
except:
data = None
results = handle_file(
filename,
data,
"ThreatExchange",
method="ThreatExchange Service",
reference="id: %s, share_level: %s" % (obj.get(td.ID),
obj.get(td.SHARE_LEVEL)),
user=request.user.username,
md5_digest = obj.get(m.MD5),
sha1_digest = obj.get(m.SHA1),
sha256_digest = obj.get(m.SHA256),
size = obj.get(m.SAMPLE_SIZE),
mimetype = obj.get(m.SAMPLE_TYPE),
)
return {'success': True,
'md5': results}
else:
return {'success': False,
'message': "Invalid Type"}
return {'success': True}
def parse_indicators(self, indicators):
"""
Parse list of indicators.
:param indicators: List of STIX indicators.
:type indicators: List of STIX indicators.
"""
analyst = self.source_instance.analyst
for indicator in indicators: # for each STIX indicator
# handled indicator-wrapped observable
if getattr(indicator, 'title', ""):
if "Top-Level Object" in indicator.title:
self.parse_observables(indicator.observables)
result = self.imported.pop(
indicator.observables[0].id_,
None) #result[0]: Crits type | result[1]: crits obj
if result:
self.imported[indicator.id_] = result
continue
elif "MARTI Campaign" in indicator.title:
return False
for observable in indicator.observables: # get each observable from indicator (expecting only 1)
if not observable.object_ or not observable.object_.properties:
self.failed.append(
("No valid object_properties was found!",
type(obs).__name__,
obs.id_)) # note for display in UI
continue
try: # create CRITs Indicator from observable
item = observable.object_.properties
obj = make_crits_object(item)
ind_type = obj.object_type
for value in obj.value:
if value and ind_type:
res = handle_indicator_ind(
value.strip(),
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True)
if res['success']:
self.imported[indicator.id_] = (
Indicator._meta['crits_type'],
res['object'])
else:
self.failed.append((
res['message'], type(item).__name__,
item.parent.id_)) # note for display in UI
except Exception, e: # probably caused by cybox object we don't handle
self.failed.append(
(e.message, type(item).__name__,
item.parent.id_)) # note for display in UI
def import_object(request, type_, id_):
setup_access()
if type_ == "Threat Descriptors":
obj = ThreatDescriptor(id=id_)
obj.details(fields=[
f for f in ThreatDescriptor._default_fields
if f not in (td.PRIVACY_MEMBERS, td.METADATA)
])
itype = get_mapped_itype(obj.get(td.TYPE))
if itype is None:
return {
'success': False,
'message': "Descriptor type is not supported by CRITs"
}
ithreat_type = getattr(IndicatorThreatTypes, obj.get(td.THREAT_TYPE))
results = handle_indicator_ind(
obj.get(td.RAW_INDICATOR),
"ThreatExchange",
itype,
ithreat_type,
None,
request.user.username,
method="ThreatExchange Service",
reference="id: %s, owner: %s, share_level: %s" % (obj.get(
td.ID), obj.get(td.OWNER)['name'], obj.get(td.SHARE_LEVEL)),
add_domain=True,
add_relationship=True,
confidence=build_ci(obj.get(td.CONFIDENCE)),
description=obj.get(td.DESCRIPTION))
return results
elif type_ == "Malware Analyses":
obj = Malware(id=id_)
obj.details(
fields=[f for f in Malware._fields if f not in (m.METADATA)])
filename = obj.get(m.MD5)
try:
data = obj.rf
except:
data = None
results = handle_file(
filename,
data,
"ThreatExchange",
method="ThreatExchange Service",
reference="id: %s, share_level: %s" %
(obj.get(td.ID), obj.get(td.SHARE_LEVEL)),
user=request.user.username,
md5_digest=obj.get(m.MD5),
sha1_digest=obj.get(m.SHA1),
sha256_digest=obj.get(m.SHA256),
size=obj.get(m.SAMPLE_SIZE),
mimetype=obj.get(m.SAMPLE_TYPE),
)
return {'success': True, 'md5': results}
else:
return {'success': False, 'message': "Invalid Type"}
return {'success': True}
def parse_indicators(self, indicators):
"""
Parse list of indicators.
:param indicators: List of STIX indicators.
:type indicators: List of STIX indicators.
"""
analyst = self.source_instance.analyst
for indicator in indicators: # for each STIX indicator
# store relationships
for rel in getattr(indicator, 'related_indicators', ()):
self.relationships.append((indicator.id_,
rel.relationship.value,
rel.item.idref,
rel.confidence.value.value))
# handled indicator-wrapped observable
if getattr(indicator, 'title', ""):
if "Top-Level Object" in indicator.title:
self.parse_observables(indicator.observables)
result = self.imported.pop(indicator.observables[0].id_, None)
if result:
self.imported[indicator.id_] = result
continue
for observable in indicator.observables: # get each observable from indicator (expecting only 1)
if not observable.object_ or not observable.object_.properties:
self.failed.append(("No valid object_properties was found!",
type(obs).__name__,
obs.id_)) # note for display in UI
continue
try: # create CRITs Indicator from observable
item = observable.object_.properties
obj = make_crits_object(item)
ind_type = obj.object_type
for value in obj.value:
if value and ind_type:
res = handle_indicator_ind(value.strip(),
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True)
if res['success']:
self.imported[indicator.id_] = (Indicator._meta['crits_type'],
res['object'])
else:
self.failed.append((res['message'],
type(item).__name__,
item.parent.id_)) # note for display in UI
except Exception, e: # probably caused by cybox object we don't handle
self.failed.append((e.message, type(item).__name__, item.parent.id_)) # note for display in UI
def parse_indicators(self, indicators):
"""
Parse list of indicators.
:param indicators: List of STIX indicators.
:type indicators: List of STIX indicators.
"""
analyst = self.source_instance.analyst
for indicator in indicators: # for each STIX indicator
# store relationships
for rel in getattr(indicator, 'related_indicators', ()):
self.relationships.append(
(indicator.id_, rel.relationship.value, rel.item.idref,
rel.confidence.value.value))
# handled indicator-wrapped observable
if getattr(indicator, 'title', ""):
if "Top-Level Object" in indicator.title:
self.parse_observables(indicator.observables)
result = self.imported.pop(indicator.observables[0].id_,
None)
if result:
self.imported[indicator.id_] = result
continue
for observable in indicator.observables: # get each observable from indicator (expecting only 1)
try: # create CRITs Indicator from observable
item = observable.object_.properties
obj = make_crits_object(item)
if obj.name and obj.name != obj.object_type:
ind_type = "%s - %s" % (obj.object_type, obj.name)
else:
ind_type = obj.object_type
for value in obj.value:
if value and ind_type:
res = handle_indicator_ind(value.strip(),
self.source,
ind_type,
analyst,
add_domain=True,
add_relationship=True)
if res['success']:
self.imported[indicator.id_] = (
Indicator._meta['crits_type'],
res['object'])
else:
self.failed.append((
res['message'], type(item).__name__,
item.parent.id_)) # note for display in UI
except Exception, e: # probably caused by cybox object we don't handle
self.failed.append(
(e.message, type(item).__name__,
item.parent.id_)) # note for display in UI
def obj_create(self, bundle, **kwargs):
"""
Handles creating Indicators through the API.
:param bundle: Bundle containing the information to create the Indicator.
:type bundle: Tastypie Bundle object.
:returns: HttpResponse.
"""
analyst = bundle.request.user.username
value = bundle.data.get('value', None)
ctype = bundle.data.get('type', None)
threat_type = bundle.data.get('threat_type', None)
attack_type = bundle.data.get('attack_type', None)
source = bundle.data.get('source', None)
reference = bundle.data.get('reference', None)
method = bundle.data.get('method', None)
add_domain = bundle.data.get('add_domain', False)
add_relationship = bundle.data.get('add_relationship', False)
campaign = bundle.data.get('campaign', None)
campaign_confidence = bundle.data.get('confidence', None)
confidence = bundle.data.get('indicator_confidence', None)
description = bundle.data.get('description', None)
impact = bundle.data.get('indicator_impact', None)
bucket_list = bundle.data.get('bucket_list', None)
ticket = bundle.data.get('ticket', None)
result = handle_indicator_ind(value, source, ctype, threat_type,
attack_type, analyst, method, reference,
add_domain, add_relationship, campaign,
campaign_confidence, confidence,
description, impact, bucket_list, ticket)
content = {'return_code': 0, 'type': 'Indicator'}
if result.get('message'):
content['message'] = result.get('message')
if result.get('objectid'):
url = reverse('api_dispatch_detail',
kwargs={
'resource_name': 'indicators',
'api_name': 'v1',
'pk': result.get('objectid')
})
content['id'] = result.get('objectid')
content['url'] = url
if not result['success']:
content['return_code'] = 1
self.crits_response(content)
def parse_indicators(self, indicators):
"""
Parse list of indicators.
:param indicators: List of STIX indicators.
:type indicators: List of STIX indicators.
"""
analyst = self.source_instance.analyst
for indicator in indicators: # for each STIX indicator
# store relationships
for rel in getattr(indicator, "related_indicators", ()):
self.relationships.append(
(indicator.id_, rel.relationship.value, rel.item.idref, rel.confidence.value.value)
)
# handled indicator-wrapped observable
if getattr(indicator, "title", ""):
if "Top-Level Object" in indicator.title:
self.parse_observables(indicator.observables)
result = self.imported.pop(indicator.observables[0].id_, None)
if result:
self.imported[indicator.id_] = result
continue
for observable in indicator.observables: # get each observable from indicator (expecting only 1)
try: # create CRITs Indicator from observable
item = observable.object_.properties
obj = make_crits_object(item)
if obj.name and obj.name != obj.object_type:
ind_type = "%s - %s" % (obj.object_type, obj.name)
else:
ind_type = obj.object_type
for value in obj.value:
if value and ind_type:
res = handle_indicator_ind(
value.strip(), self.source, ind_type, analyst, add_domain=True, add_relationship=True
)
if res["success"]:
self.imported[indicator.id_] = (Indicator._meta["crits_type"], res["object"])
else:
self.failed.append(
(res["message"], type(item).__name__, item.parent.id_)
) # note for display in UI
except Exception, e: # probably caused by cybox object we don't handle
self.failed.append((e.message, type(item).__name__, item.parent.id_)) # note for display in UI
def obj_create(self, bundle, **kwargs):
"""
Handles creating Indicators through the API.
:param bundle: Bundle containing the information to create the Indicator.
:type bundle: Tastypie Bundle object.
:returns: Bundle object.
:raises BadRequest: If a campaign name is not provided or creation fails.
"""
analyst = bundle.request.user.username
value = bundle.data.get('value', None)
ctype = bundle.data.get('type', None)
source = bundle.data.get('source', None)
reference = bundle.data.get('reference', None)
method = bundle.data.get('method', None)
add_domain = bundle.data.get('add_domain', False)
add_relationship = bundle.data.get('add_relationship', False)
campaign = bundle.data.get('campaign', None)
campaign_confidence = bundle.data.get('confidence', None)
confidence = bundle.data.get('indicator_confidence', None)
impact = bundle.data.get('indicator_impact', None)
bucket_list = bundle.data.get('bucket_list', None)
ticket = bundle.data.get('ticket', None)
if analyst:
result = handle_indicator_ind(value,
source,
reference,
ctype,
analyst,
method,
add_domain,
add_relationship,
campaign,
campaign_confidence,
confidence,
impact,
bucket_list,
ticket)
if not result['success']:
raise BadRequest(result['message'])
else:
return bundle
else:
raise BadRequest('You must be an authenticated user!')
def parse_indicators(self, indicators):
"""
Parse list of indicators.
:param indicators: List of STIX indicators.
:type indicators: List of STIX indicators.
"""
analyst = self.source_instance.analyst
for indicator in indicators: # for each STIX indicator
for observable in indicator.observables: # get each observable from indicator (expecting only 1)
try: # create CRITs Indicator from observable
item = observable.object_.properties
obj = make_crits_object(item)
if obj.name and obj.name != obj.object_type:
ind_type = "%s - %s" % (obj.object_type, obj.name)
else:
ind_type = obj.object_type
for value in obj.value:
if value and ind_type:
res = handle_indicator_ind(value.strip(),
self.source,
None,
ind_type,
analyst,
add_domain=True,
add_relationship=True)
if res['success']:
self.imported.append(
(Indicator._meta['crits_type'],
res['object']))
else:
self.failed.append((
res['message'], type(item).__name__,
item.parent.id_)) # note for display in UI
except Exception, e: # probably caused by cybox object we don't handle
self.failed.append(
(e.message, type(item).__name__,
item.parent.id_)) # note for display in UI
def obj_create(self, bundle, **kwargs):
"""
Handles creating Indicators through the API.
:param bundle: Bundle containing the information to create the Indicator.
:type bundle: Tastypie Bundle object.
:returns: Bundle object.
:raises BadRequest: If a campaign name is not provided or creation fails.
"""
analyst = bundle.request.user.username
value = bundle.data.get('value', None)
ctype = bundle.data.get('type', None)
source = bundle.data.get('source', None)
reference = bundle.data.get('reference', None)
method = bundle.data.get('method', None)
add_domain = bundle.data.get('add_domain', False)
add_relationship = bundle.data.get('add_relationship', False)
campaign = bundle.data.get('campaign', None)
campaign_confidence = bundle.data.get('confidence', None)
confidence = bundle.data.get('indicator_confidence', None)
impact = bundle.data.get('indicator_impact', None)
bucket_list = bundle.data.get('bucket_list', None)
ticket = bundle.data.get('ticket', None)
if analyst:
result = handle_indicator_ind(value, source, reference, ctype,
analyst, method, add_domain,
add_relationship, campaign,
campaign_confidence, confidence,
impact, bucket_list, ticket)
if not result['success']:
raise BadRequest(result['message'])
else:
return bundle
else:
raise BadRequest('You must be an authenticated user!')
def parse_indicators(self, indicators):
"""
Parse list of indicators.
:param indicators: List of STIX indicators.
:type indicators: List of STIX indicators.
"""
analyst = self.source_instance.analyst
for indicator in indicators: # for each STIX indicator
for observable in indicator.observables: # get each observable from indicator (expecting only 1)
try: # create CRITs Indicator from observable
item = observable.object_.properties
obj = make_crits_object(item)
if obj.name and obj.name != obj.object_type:
ind_type = "%s - %s" % (obj.object_type, obj.name)
else:
ind_type = obj.object_type
for value in obj.value:
if value and ind_type:
res = handle_indicator_ind(
value.strip(),
self.source,
None,
ind_type,
analyst,
add_domain=True,
add_relationship=True,
)
if res["success"]:
self.imported.append((Indicator._meta["crits_type"], res["object"]))
else:
self.failed.append(
(res["message"], type(item).__name__, item.parent.id_)
) # note for display in UI
except Exception, e: # probably caused by cybox object we don't handle
self.failed.append((e.message, type(item).__name__, item.parent.id_)) # note for display in UI
def create_indicator_from_object(rel_type, rel_id, ind_type, value,
source_name, method, reference, analyst, request):
"""
Create an indicator out of this object.
:param rel_type: The top-level object type this object is for.
:type rel_type: str
:param rel_id: The ObjectId of the top-level object.
:param ind_type: The indicator type to use.
:type ind_type: str
:param value: The indicator value.
:type value: str
:param source_name: The source name for the indicator.
:type source_name: str
:param method: The source method for the indicator.
:type method: str
:param reference: The source reference for the indicator.
:type reference: str
:param analyst: The user creating this indicator.
:type analyst: str
:param request: The Django request.
:type request: :class:`django.http.HttpRequest`
:returns: dict with keys "success" (bool) and "message" (str)
"""
result = None
me = class_from_id(rel_type, rel_id)
if not me:
result = {'success': False,
'message': "Could not find %s" % rel_type}
elif value == None or value.strip() == "":
result = {'success': False,
'message': "Can't create indicator with an empty value field"}
elif ind_type == None or ind_type.strip() == "":
result = {'success': False,
'message': "Can't create indicator with an empty type field"}
elif source_name == None or source_name.strip() == "":
result = {'success': False,
'message': "Can't create indicator with an empty source field"}
else:
value = value.lower().strip()
ind_type = ind_type.strip()
source_name = source_name.strip()
create_indicator_result = {}
ind_tlist = ind_type.split(" - ")
if ind_tlist[0] == ind_tlist[1]:
ind_type = ind_tlist[0]
from crits.indicators.handlers import handle_indicator_ind
campaign = me.campaign if hasattr(me, 'campaign') else None
create_indicator_result = handle_indicator_ind(value,
source_name,
reference,
ind_type,
analyst,
method,
add_domain=True,
campaign=campaign)
# Check if an error occurred, if it did then return the error result
if create_indicator_result.get('success', True) == False:
return create_indicator_result
indicator = Indicator.objects(ind_type=ind_type,
value=value).first()
if not indicator:
result = {'success': False,
'message': "Could not create indicator"}
else:
results = me.add_relationship(rel_item=indicator,
rel_type="Related_To",
analyst=analyst,
get_rels=True)
if results['success']:
me.save(username=analyst)
indicator.save(username=analyst)
relationship= {'type': rel_type, 'value': rel_id}
message = render_to_string('relationships_listing_widget.html',
{'relationship': relationship,
'nohide': True,
'relationships': results['message']},
RequestContext(request))
result = {'success': True, 'message': message}
else:
message = "Indicator created. Could not create relationship"
result = {'success': False,
'message': message}
return result
def upload_indicator(request):
"""
Upload new indicators (individual, blob, or CSV file).
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:returns: :class:`django.http.HttpResponse`
:class:`django.http.HttpResponseRedirect`
"""
if request.method == "POST":
username = request.user.username
failed_msg = ''
result = None;
if request.POST['svalue'] == "Upload CSV":
form = UploadIndicatorCSVForm(username,
request.POST,
request.FILES)
if form.is_valid():
result = handle_indicator_csv(request.FILES['filedata'],
request.POST['source'],
request.POST['reference'],
"file",
username,
add_domain=True)
if result['success']:
message = {'message': ('<div>CSV added successfully! <a '
'href="%s">Go to all indicators</a>'
'.</div>'
% reverse('crits.indicators.views.indicators_listing'))}
else:
failed_msg = '<div>%s</div>' % result['message']
if request.POST['svalue'] == "Upload Text":
form = UploadIndicatorTextForm(username,
request.POST)
if form.is_valid():
result = handle_indicator_csv(request.POST['data'],
request.POST['source'],
request.POST['reference'],
"ti",
username,
add_domain=True)
if result['success']:
message = {'message': ('<div>Indicators added successfully! '
'<a href="%s">Go to all indicators</a>'
'.</div>' % reverse('crits.indicators.views.indicators_listing'))}
else:
failed_msg = '<div>%s</div>' % result['message']
if request.POST['svalue'] == "Upload Indicator":
all_ind_type_choices = [(c[0],
c[0],
{'datatype': c[1].keys()[0],
'datatype_value': c[1].values()[0]}) for c in get_object_types(active=False, query={'datatype.file':{'$exists':0}})]
form = UploadIndicatorForm(username,
all_ind_type_choices,
request.POST)
if form.is_valid():
if request.POST['indicator_type'] == "URI - URL" and "://" not in request.POST['value'].split('.')[0]:
result = {"success" : False, "message" : "URI - URL must contain protocol prefix (e.g. http://, https://, ftp://)"}
else:
result = handle_indicator_ind(request.POST['value'],
request.POST['source'],
request.POST['reference'],
request.POST['indicator_type'],
username,
add_domain=True,
campaign=request.POST['campaign'],
campaign_confidence=request.POST['campaign_confidence'],
confidence=request.POST['confidence'],
impact=request.POST['impact'],
bucket_list=request.POST[form_consts.Common.BUCKET_LIST_VARIABLE_NAME],
ticket=request.POST[form_consts.Common.TICKET_VARIABLE_NAME])
if result['success']:
indicator_link = '<a href=\"%s\">Go to this indicator</a> or <a href="%s">all indicators</a>.</div>' % (reverse('crits.indicators.views.indicator', args=[result['objectid']]), reverse('crits.indicators.views.indicators_listing'));
if result.get('is_new_indicator', False) == False:
message = {'message': ('<div>Warning: Updated indicator since indicator already exists! ' + indicator_link)}
else:
message = {'message': ('<div>Indicator added successfully! ' + indicator_link)}
else:
failed_msg = result['message']
if result == None or not result['success']:
failed_msg += ('<a href="%s">Go to all indicators</a>'
'.</div>' % reverse('crits.indicators.views.indicators_listing'))
message = {'message': failed_msg, 'form': form.as_table()}
elif result != None:
message['success'] = result['success']
if request.is_ajax():
return HttpResponse(json.dumps(message),
mimetype="application/json")
else: #file upload
return render_to_response('file_upload_response.html',
{'response': json.dumps(message)},
RequestContext(request))
def add_object(type_, id_, object_type, source, method, reference, tlp, user,
value=None, file_=None, add_indicator=False, get_objects=True,
tlo=None, is_sort_relationships=False, is_validate_only=False,
is_validate_locally=False, cache={}, **kwargs):
"""
Add an object to the database.
:param type_: The top-level object type.
:type type_: str
:param id_: The ObjectId of the top-level object.
:type id_: str
:param object_type: The type of the ObjectType being added.
:type object_type: str
:param source: The name of the source adding this object.
:type source: str
:param method: The method for this object.
:type method: str
:param reference: The reference for this object.
:type reference: str
:param user: The user adding this object.
:type user: str
:param value: The value of the object.
:type value: str
:param file_: The file if the object is a file upload.
:type file_: file handle.
:param add_indicator: Also add an indicator for this object.
:type add_indicator: bool
:param get_objects: Return the formatted list of objects when completed.
:type get_objects: bool
:param tlo: The CRITs top-level object we are adding objects to.
This is an optional parameter used mainly for performance
reasons (by not querying mongo if we already have the
top level-object).
:type tlo: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param is_sort_relationships: Return all relationships and meta, sorted
:type is_sort_relationships: bool
:param is_validate_only: Validate, but do not add to TLO.
:type is_validate_only: bool
:param is_validate_locally: Validate, but do not add b/c there is no TLO.
:type is_validate_locally: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" (str),
"objects" (list),
"relationships" (list)
"""
# if object_type is a validated indicator type, then validate value
if value:
from crits.indicators.handlers import validate_indicator_value
(value, error) = validate_indicator_value(value, object_type)
if error:
return {"success": False, "message": error}
if is_validate_locally: # no TLO provided
return {"success": True}
if not tlo:
if type_ and id_:
tlo = class_from_id(type_, id_)
if not tlo:
return {'success': False, 'message': "Failed to find TLO"}
try:
if file_:
data = file_.read()
filename = file_.name
md5sum = md5(data).hexdigest()
value = md5sum
reference = filename
ret = tlo.add_object(object_type, value,
source, method, reference, user)
if not ret['success']:
msg = '%s! [Type: "%s"][Value: "%s"]'
return {"success": False,
"message": msg % (ret['message'], object_type, value)}
else:
results = {'success': True}
if not is_validate_only: # save the object
tlo.update(add_to_set__obj=ret['object'])
results['message'] = "Object added successfully"
if file_:
# do we have a pcap?
if detect_pcap(data):
handle_pcap_file(filename,
data,
source,
user=user,
related_id=id_,
related_type=type_)
else:
#XXX: MongoEngine provides no direct GridFS access so we
# need to use pymongo directly.
col = settings.COL_OBJECTS
grid = mongo_connector("%s.files" % col)
if grid.find({'md5': md5sum}).count() == 0:
put_file(filename, data, collection=col)
if add_indicator and not is_validate_only:
campaign = tlo.campaign if hasattr(tlo, 'campaign') else None
from crits.indicators.handlers import handle_indicator_ind
ind_res = handle_indicator_ind(value,
source,
object_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
user,
source_method=method,
source_reference=reference,
source_tlp=tlp,
add_domain=True,
campaign=campaign,
cache=cache)
if ind_res['success']:
forge_relationship(class_=tlo,
right_class=ind_res['object'],
rel_type=RelationshipTypes.RELATED_TO,
user=user)
else:
msg = "Object added, but failed to add Indicator.<br>Error: %s"
results['message'] = msg % ind_res.get('message')
if is_sort_relationships == True:
results['relationships'] = tlo.sort_relationships(user, meta=True)
if get_objects:
results['objects'] = tlo.sort_objects()
results['id'] = str(tlo.id)
return results
except ValidationError as e:
return {'success': False, 'message': str(e)}
def add_object(type_,
id_,
object_type,
source,
method,
reference,
tlp,
user,
value=None,
file_=None,
add_indicator=False,
get_objects=True,
tlo=None,
is_sort_relationships=False,
is_validate_only=False,
is_validate_locally=False,
cache={},
**kwargs):
"""
Add an object to the database.
:param type_: The top-level object type.
:type type_: str
:param id_: The ObjectId of the top-level object.
:type id_: str
:param object_type: The type of the ObjectType being added.
:type object_type: str
:param source: The name of the source adding this object.
:type source: str
:param method: The method for this object.
:type method: str
:param reference: The reference for this object.
:type reference: str
:param user: The user adding this object.
:type user: str
:param value: The value of the object.
:type value: str
:param file_: The file if the object is a file upload.
:type file_: file handle.
:param add_indicator: Also add an indicator for this object.
:type add_indicator: bool
:param get_objects: Return the formatted list of objects when completed.
:type get_objects: bool
:param tlo: The CRITs top-level object we are adding objects to.
This is an optional parameter used mainly for performance
reasons (by not querying mongo if we already have the
top level-object).
:type tlo: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param is_sort_relationships: Return all relationships and meta, sorted
:type is_sort_relationships: bool
:param is_validate_only: Validate, but do not add to TLO.
:type is_validate_only: bool
:param is_validate_locally: Validate, but do not add b/c there is no TLO.
:type is_validate_locally: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" (str),
"objects" (list),
"relationships" (list)
"""
# if object_type is a validated indicator type, then validate value
if value:
from crits.indicators.handlers import validate_indicator_value
(value, error) = validate_indicator_value(value, object_type)
if error:
return {"success": False, "message": error}
if is_validate_locally: # no TLO provided
return {"success": True}
if not tlo:
if type_ and id_:
tlo = class_from_id(type_, id_)
if not tlo:
return {'success': False, 'message': "Failed to find TLO"}
try:
if file_:
data = file_.read()
filename = file_.name
md5sum = md5(data).hexdigest()
value = md5sum
reference = filename
ret = tlo.add_object(object_type, value, source, method, reference,
user)
if not ret['success']:
msg = '%s! [Type: "%s"][Value: "%s"]'
return {
"success": False,
"message": msg % (ret['message'], object_type, value)
}
else:
results = {'success': True}
if not is_validate_only: # save the object
tlo.update(add_to_set__obj=ret['object'])
results['message'] = "Object added successfully"
if file_:
# do we have a pcap?
if detect_pcap(data):
handle_pcap_file(filename,
data,
source,
user=user,
related_id=id_,
related_type=type_)
else:
#XXX: MongoEngine provides no direct GridFS access so we
# need to use pymongo directly.
col = settings.COL_OBJECTS
grid = mongo_connector("%s.files" % col)
if grid.find({'md5': md5sum}).count() == 0:
put_file(filename, data, collection=col)
if add_indicator and not is_validate_only:
campaign = tlo.campaign if hasattr(tlo, 'campaign') else None
from crits.indicators.handlers import handle_indicator_ind
ind_res = handle_indicator_ind(value,
source,
object_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
user,
source_method=method,
source_reference=reference,
source_tlp=tlp,
add_domain=True,
campaign=campaign,
cache=cache)
if ind_res['success']:
forge_relationship(class_=tlo,
right_class=ind_res['object'],
rel_type=RelationshipTypes.RELATED_TO,
user=user)
else:
msg = "Object added, but failed to add Indicator.<br>Error: %s"
results['message'] = msg % ind_res.get('message')
if is_sort_relationships == True:
results['relationships'] = tlo.sort_relationships(user, meta=True)
if get_objects:
results['objects'] = tlo.sort_objects()
results['id'] = str(tlo.id)
return results
except ValidationError as e:
return {'success': False, 'message': str(e)}
def ip_add_update(ip_address,
ip_type,
source=None,
source_method='',
source_reference='',
source_tlp=None,
campaign=None,
confidence='low',
user=None,
is_add_indicator=False,
indicator_reference='',
bucket_list=None,
ticket=None,
is_validate_only=False,
cache={},
related_id=None,
related_type=None,
relationship_type=None,
description=''):
"""
Add/update an IP address.
:param ip_address: The IP to add/update.
:type ip_address: str
:param ip_type: The type of IP this is.
:type ip_type: str
:param source: Name of the source which provided this information.
:type source: str
:param source_method: Method of acquiring this data.
:type source_method: str
:param source_reference: A reference to this data.
:type source_reference: str
:param campaign: A campaign to attribute to this IP address.
:type campaign: str
:param confidence: Confidence level in the campaign attribution.
:type confidence: str ("low", "medium", "high")
:param user: The user adding/updating this IP.
:type user: str
:param is_add_indicator: Also add an Indicator for this IP.
:type is_add_indicator: bool
:param indicator_reference: Reference for the indicator.
:type indicator_reference: str
:param bucket_list: Buckets to assign to this IP.
:type bucket_list: str
:param ticket: Ticket to assign to this IP.
:type ticket: str
:param is_validate_only: Only validate, do not add/update.
:type is_validate_only: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_type: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:param description: A description for this IP
:type description: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"object" (if successful) :class:`crits.ips.ip.IP`
"""
if not source:
return {"success": False, "message": "Missing source information."}
source_name = source
(ip_address, error) = validate_and_normalize_ip(ip_address, ip_type)
if error:
return {"success": False, "message": error}
retVal = {}
is_item_new = False
ip_object = None
cached_results = cache.get(form_consts.IP.CACHED_RESULTS)
if cached_results != None:
ip_object = cached_results.get(ip_address)
else:
ip_object = IP.objects(ip=ip_address).first()
if not ip_object:
ip_object = IP()
ip_object.ip = ip_address
ip_object.ip_type = ip_type
is_item_new = True
if cached_results != None:
cached_results[ip_address] = ip_object
if not ip_object.description:
ip_object.description = description or ''
elif ip_object.description != description:
ip_object.description += "\n" + (description or '')
if isinstance(source_name, basestring):
if user.check_source_write(source):
source = [
create_embedded_source(source,
reference=source_reference,
method=source_method,
tlp=source_tlp,
analyst=user.username)
]
else:
return {
"success":
False,
"message":
"User does not have permission to add object \
using source %s." % source
}
if isinstance(campaign, basestring):
c = EmbeddedCampaign(name=campaign,
confidence=confidence,
analyst=user.username)
campaign = [c]
if campaign:
for camp in campaign:
ip_object.add_campaign(camp)
if source:
for s in source:
ip_object.add_source(s)
else:
return {"success": False, "message": "Missing source information."}
if bucket_list:
ip_object.add_bucket_list(bucket_list, user.username)
if ticket:
ip_object.add_ticket(ticket, user.username)
related_obj = None
if related_id:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
retVal['success'] = False
retVal['message'] = 'Related Object not found.'
return retVal
resp_url = reverse('crits.ips.views.ip_detail', args=[ip_object.ip])
if is_validate_only == False:
ip_object.save(username=user.username)
#set the URL for viewing the new data
if is_item_new == True:
retVal['message'] = ('Success! Click here to view the new IP: '
'<a href="%s">%s</a>' %
(resp_url, ip_object.ip))
else:
message = ('Updated existing IP: '
'<a href="%s">%s</a>' % (resp_url, ip_object.ip))
retVal['message'] = message
retVal['status'] = form_consts.Status.DUPLICATE
retVal['warning'] = message
elif is_validate_only == True:
if ip_object.id != None and is_item_new == False:
message = ('Warning: IP already exists: '
'<a href="%s">%s</a>' % (resp_url, ip_object.ip))
retVal['message'] = message
retVal['status'] = form_consts.Status.DUPLICATE
retVal['warning'] = message
if is_add_indicator:
from crits.indicators.handlers import handle_indicator_ind
result = handle_indicator_ind(ip_address,
source_name,
ip_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
user,
source_method=source_method,
source_reference=indicator_reference,
source_tlp=source_tlp,
add_domain=False,
add_relationship=True,
bucket_list=bucket_list,
ticket=ticket,
cache=cache)
if related_obj and ip_object and relationship_type:
relationship_type = RelationshipTypes.inverse(
relationship=relationship_type)
ip_object.add_relationship(related_obj,
relationship_type,
analyst=user.username,
get_rels=False)
ip_object.save(username=user.username)
# run ip triage
if is_item_new and is_validate_only == False:
ip_object.reload()
run_triage(ip_object, user)
retVal['success'] = True
retVal['object'] = ip_object
return retVal
def obj_create(self, bundle, **kwargs):
"""
Handles creating Indicators through the API.
:param bundle: Bundle containing the information to create the Indicator.
:type bundle: Tastypie Bundle object.
:returns: HttpResponse.
"""
user = bundle.request.user
value = bundle.data.get('value', None)
ctype = bundle.data.get('type', None)
threat_type = bundle.data.get('threat_type', None)
attack_type = bundle.data.get('attack_type', None)
source = bundle.data.get('source', None)
status = bundle.data.get('status', None)
reference = bundle.data.get('reference', None)
method = bundle.data.get('method', None)
tlp = bundle.data.get('tlp', 'amber')
add_domain = bundle.data.get('add_domain', False)
add_relationship = bundle.data.get('add_relationship', False)
campaign = bundle.data.get('campaign', None)
campaign_confidence = bundle.data.get('confidence', None)
confidence = bundle.data.get('indicator_confidence', None)
description = bundle.data.get('description', None)
impact = bundle.data.get('indicator_impact', None)
bucket_list = bundle.data.get('bucket_list', None)
ticket = bundle.data.get('ticket', None)
if user.has_access_to(IndicatorACL.WRITE):
result = handle_indicator_ind(
value,
source,
ctype,
threat_type,
attack_type,
user,
status=status,
source_method=method,
source_reference=reference,
source_tlp=tlp,
add_domain=add_domain,
add_relationship=add_relationship,
campaign=campaign,
campaign_confidence=campaign_confidence,
confidence=confidence,
description=description,
impact=impact,
bucket_list=bucket_list,
ticket=ticket)
else:
result = {
'success': False,
'message': 'User does not have permission to create Object.'
}
content = {'return_code': 0, 'type': 'Indicator'}
if result.get('message'):
content['message'] = result.get('message')
if result.get('objectid'):
url = reverse('api_dispatch_detail',
kwargs={
'resource_name': 'indicators',
'api_name': 'v1',
'pk': result.get('objectid')
})
content['id'] = result.get('objectid')
content['url'] = url
if not result['success']:
content['return_code'] = 1
self.crits_response(content)
def upload_indicator(request):
"""
Upload new indicators (individual, blob, or CSV file).
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:returns: :class:`django.http.HttpResponse`
:class:`django.http.HttpResponseRedirect`
"""
if request.method == "POST":
username = request.user.username
failed_msg = ''
result = None
if request.POST['svalue'] == "Upload CSV":
form = UploadIndicatorCSVForm(username, request.POST,
request.FILES)
if form.is_valid():
result = handle_indicator_csv(request.FILES['filedata'],
request.POST['source'],
request.POST['reference'],
"file",
username,
add_domain=True)
if result['success']:
message = {
'message':
('<div>CSV added successfully! <a '
'href="%s">Go to all indicators</a>'
'.</div>' %
reverse('crits.indicators.views.indicators_listing'))
}
else:
failed_msg = '<div>%s</div>' % result['message']
if request.POST['svalue'] == "Upload Text":
form = UploadIndicatorTextForm(username, request.POST)
if form.is_valid():
result = handle_indicator_csv(request.POST['data'],
request.POST['source'],
request.POST['reference'],
"ti",
username,
add_domain=True)
if result['success']:
message = {
'message':
('<div>Indicators added successfully! '
'<a href="%s">Go to all indicators</a>'
'.</div>' %
reverse('crits.indicators.views.indicators_listing'))
}
else:
failed_msg = '<div>%s</div>' % result['message']
if request.POST['svalue'] == "Upload Indicator":
all_ind_type_choices = [(c[0], c[0], {
'datatype': c[1].keys()[0],
'datatype_value': c[1].values()[0]
}) for c in get_object_types(
active=False, query={'datatype.file': {
'$exists': 0
}})]
form = UploadIndicatorForm(username, all_ind_type_choices,
request.POST)
if form.is_valid():
if request.POST[
'indicator_type'] == "URI - URL" and "://" not in request.POST[
'value'].split('.')[0]:
result = {
"success":
False,
"message":
"URI - URL must contain protocol prefix (e.g. http://, https://, ftp://)"
}
else:
result = handle_indicator_ind(
request.POST['value'],
request.POST['source'],
request.POST['reference'],
request.POST['indicator_type'],
username,
add_domain=True,
campaign=request.POST['campaign'],
campaign_confidence=request.
POST['campaign_confidence'],
confidence=request.POST['confidence'],
impact=request.POST['impact'],
bucket_list=request.POST[
form_consts.Common.BUCKET_LIST_VARIABLE_NAME],
ticket=request.POST[
form_consts.Common.TICKET_VARIABLE_NAME])
if result['success']:
indicator_link = '<a href=\"%s\">Go to this indicator</a> or <a href="%s">all indicators</a>.</div>' % (
reverse('crits.indicators.views.indicator',
args=[result['objectid']]),
reverse('crits.indicators.views.indicators_listing'))
if result.get('is_new_indicator', False) == False:
message = {
'message':
('<div>Warning: Updated indicator since indicator already exists! '
+ indicator_link)
}
else:
message = {
'message': ('<div>Indicator added successfully! ' +
indicator_link)
}
else:
failed_msg = result['message']
if result == None or not result['success']:
failed_msg += (
'<a href="%s">Go to all indicators</a>'
'.</div>' %
reverse('crits.indicators.views.indicators_listing'))
message = {'message': failed_msg, 'form': form.as_table()}
elif result != None:
message['success'] = result['success']
if request.is_ajax():
return HttpResponse(json.dumps(message),
mimetype="application/json")
else: #file upload
return render_to_response('file_upload_response.html',
{'response': json.dumps(message)},
RequestContext(request))
def parse_result(self, result_extract, acl_write, md5_parent):
stream_md5 = None
if type(result_extract) is dict:
#make reccursion extract each file embbed
if 'FileMD5' in result_extract and result_extract['FileMD5']:
tmp_dict = {}
b_yara = False
b_ioc = False
#extract info
no_info = [
'ExtractInfo', 'ContainedObjects', 'Yara', 'PathFile',
'FileMD5', 'RootFileType', 'TempDirExtract'
]
for key, value in result_extract.iteritems():
if not key in no_info:
self._add_result(
'File: ' + result_extract['FileMD5'] + ' - Info', key,
{'value': str(value)})
#extract yara match
if result_extract['Yara']:
for item_v in result_extract['Yara']:
for key, value in item_v.iteritems():
self._add_result(
'File: ' + result_extract['FileMD5'] +
' - Signatures yara matched', key, value)
b_yara = True
#extract IOC
if result_extract['ExtractInfo']:
for item_v in result_extract['ExtractInfo']:
for key, value in item_v.iteritems():
self._add_result(
'File: ' + result_extract['FileMD5'] +
' - Extract potential IOC', key,
{'value': str(value)})
b_ioc = True
#add_sample
if 'PathFile' in result_extract and type(
result_extract['PathFile']) is list and len(
result_extract['PathFile']) > 0:
if os.path.isfile(str(result_extract['PathFile'][0])):
with open(str(result_extract['PathFile'][0]),
'r') as content_file_tmp:
content_tmp = content_file_tmp.read()
stream_md5 = hashlib.md5(content_tmp).hexdigest()
name = str(stream_md5).decode('ascii', errors='ignore')
id_ = Sample.objects(md5=stream_md5).only('id').first()
if id_:
self._info(
'Add relationship with sample existed:' +
str(stream_md5))
#make relationship
id_.add_relationship(
rel_item=self.obj,
rel_type=RelationshipTypes.CONTAINED_WITHIN,
rel_date=datetime.now(),
analyst=self.current_task.user.username)
else:
if acl_write and (
self.config['import_file'] or
(self.config['import_file_yara'] and b_yara) or
(self.config['import_file_ioc'] and b_ioc)):
obj_parent = None
if md5_parent:
obj_parent = Sample.objects(
md5=md5_parent).only('id').first()
if not obj_parent:
sample = handle_file(
name,
content_tmp,
self.obj.source,
related_id=str(self.obj.id),
related_type=str(
self.obj._meta['crits_type']),
campaign=self.obj.campaign,
source_method=self.name,
relationship=RelationshipTypes.
CONTAINED_WITHIN,
user=self.current_task.user)
else:
sample = handle_file(
name,
content_tmp,
obj_parent.source,
related_id=str(obj_parent.id),
related_type=str(
obj_parent._meta['crits_type']),
campaign=obj_parent.campaign,
source_method=self.name,
relationship=RelationshipTypes.
CONTAINED_WITHIN,
user=self.current_task.user)
self._info('Add sample ' + str(stream_md5))
else:
#add IOC if not add sample
if self.current_task.user.has_access_to(
IndicatorACL.WRITE) and b_yara:
res = handle_indicator_ind(
stream_md5,
self.obj.source,
IndicatorTypes.MD5,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
self.current_task.user,
add_relationship=True,
source_method=self.name,
campaign=self.obj.campaign,
description='Extracted by service ' +
self.name)
self._info('Add indicator md5:' +
str(stream_md5) + ' -- id: ' +
str(res))
#contains file
if 'ContainedObjects' in result_extract and type(
result_extract['ContainedObjects']
) is list and result_extract['ContainedObjects']:
for item_v in result_extract['ContainedObjects']:
if item_v['FileMD5'] and item_v['FileType'] and item_v[
'FileSize']:
#search if file exist
id_ = Sample.objects(
md5=str(item_v['FileMD5'])).only('id').first()
sample_exist = False
ioc_exist = False
if id_:
sample_exist = True
id_ = Indicator.objects(
value=str(item_v['FileMD5'])).only('id').first()
if id_:
ioc_exist = True
self._add_result(
'File: ' + result_extract['FileMD5'] +
' - Contains md5 files', item_v['FileMD5'], {
'type': str(item_v['FileType']),
'size': str(item_v['FileSize']),
'Exists Sample': str(sample_exist),
'Exists IOC md5': str(ioc_exist)
})
for item_v in result_extract['ContainedObjects']:
#re do loop for best display result
parse_result(self, item_v, acl_write, stream_md5)
def upload_indicator(request):
"""
Upload new indicators (individual, blob, or CSV file).
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:returns: :class:`django.http.HttpResponse`
:class:`django.http.HttpResponseRedirect`
"""
if request.method == "POST":
username = request.user.username
failed_msg = ""
result = None
if request.POST["svalue"] == "Upload CSV":
form = UploadIndicatorCSVForm(username, request.POST, request.FILES)
if form.is_valid():
result = handle_indicator_csv(
request.FILES["filedata"],
request.POST["source"],
request.POST["method"],
request.POST["reference"],
"file",
username,
add_domain=True,
related_id=request.POST["related_id"],
related_type=request.POST["related_type"],
relationship_type=request.POST["relationship_type"],
)
if result["success"]:
message = {
"message": (
'<div>%s <a href="%s">Go to all'
" indicators</a></div>"
% (result["message"], reverse("crits.indicators.views.indicators_listing"))
)
}
else:
failed_msg = "<div>%s</div>" % result["message"]
if request.POST["svalue"] == "Upload Text":
form = UploadIndicatorTextForm(username, request.POST)
if form.is_valid():
result = handle_indicator_csv(
request.POST["data"],
request.POST["source"],
request.POST["method"],
request.POST["reference"],
"ti",
username,
add_domain=True,
related_id=request.POST["related_id"],
related_type=request.POST["related_type"],
relationship_type=request.POST["relationship_type"],
)
if result["success"]:
message = {
"message": (
'<div>%s <a href="%s">Go to all'
" indicators</a></div>"
% (result["message"], reverse("crits.indicators.views.indicators_listing"))
)
}
else:
failed_msg = "<div>%s</div>" % result["message"]
if request.POST["svalue"] == "Upload Indicator":
form = UploadIndicatorForm(username, request.POST)
if form.is_valid():
result = handle_indicator_ind(
request.POST["value"],
request.POST["source"],
request.POST["indicator_type"],
request.POST["threat_type"],
request.POST["attack_type"],
username,
status=request.POST["status"],
method=request.POST["method"],
reference=request.POST["reference"],
add_domain=True,
description=request.POST["description"],
campaign=request.POST["campaign"],
campaign_confidence=request.POST["campaign_confidence"],
confidence=request.POST["confidence"],
impact=request.POST["impact"],
bucket_list=request.POST[form_consts.Common.BUCKET_LIST_VARIABLE_NAME],
ticket=request.POST[form_consts.Common.TICKET_VARIABLE_NAME],
related_id=request.POST["related_id"],
related_type=request.POST["related_type"],
relationship_type=request.POST["relationship_type"],
)
if result["success"]:
indicator_link = (
' - <a href="%s">Go to this ' 'indicator</a> or <a href="%s">all ' "indicators</a>.</div>"
) % (
reverse("crits.indicators.views.indicator", args=[result["objectid"]]),
reverse("crits.indicators.views.indicators_listing"),
)
if result.get("is_new_indicator", False) == False:
message = {"message": ("<div>Warning: Updated existing" " Indicator!" + indicator_link)}
else:
message = {"message": ("<div>Indicator added " "successfully!" + indicator_link)}
else:
failed_msg = result["message"] + " - "
if result == None or not result["success"]:
failed_msg += '<a href="%s"> Go to all indicators</a></div>' % reverse(
"crits.indicators.views.indicators_listing"
)
message = {"message": failed_msg, "form": form.as_table()}
elif result != None:
message["success"] = result["success"]
if request.is_ajax():
return HttpResponse(json.dumps(message), content_type="application/json")
else: # file upload
return render_to_response(
"file_upload_response.html", {"response": json.dumps(message)}, RequestContext(request)
)
def ip_add_update(ip_address, ip_type, source=None, source_method=None,
source_reference=None, campaign=None, confidence='low', analyst=None,
is_add_indicator=False, indicator_reference=None,
bucket_list=None, ticket=None, is_validate_only=False, cache={}):
"""
Add/update an IP address.
:param ip_address: The IP to add/update.
:type ip_address: str
:param ip_type: The type of IP this is.
:type ip_type: str
:param source: Name of the source which provided this information.
:type source: str
:param source_method: Method of acquiring this data.
:type source_method: str
:param source_reference: A reference to this data.
:type source_reference: str
:param campaign: A campaign to attribute to this IP address.
:type campaign: str
:param confidence: Confidence level in the campaign attribution.
:type confidence: str ("low", "medium", "high")
:param analyst: The user adding/updating this IP.
:type analyst: str
:param is_add_indicator: Also add an Indicator for this IP.
:type is_add_indicator: bool
:param indicator_reference: Reference for the indicator.
:type indicator_reference: str
:param bucket_list: Buckets to assign to this IP.
:type bucket_list: str
:param ticket: Ticket to assign to this IP.
:type ticket: str
:param is_validate_only: Only validate, do not add/update.
:type is_validate_only: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" (str),
"object" (if successful) :class:`crits.ips.ip.IP`
"""
retVal = {}
is_item_new = False
ip_object = None
cached_results = cache.get(form_consts.IP.CACHED_RESULTS)
if cached_results != None:
ip_object = cached_results.get(ip_address)
else:
ip_object = IP.objects(ip=ip_address).first()
if not ip_object:
ip_object = IP()
ip_object.ip = ip_address
ip_object.ip_type = ip_type
is_item_new = True
if cached_results != None:
cached_results[ip_address] = ip_object
if isinstance(source, basestring):
source = [create_embedded_source(source,
reference=source_reference,
method=source_method,
analyst=analyst)]
if isinstance(campaign, basestring):
c = EmbeddedCampaign(name=campaign, confidence=confidence, analyst=analyst)
campaign = [c]
if campaign:
for camp in campaign:
ip_object.add_campaign(camp)
if source:
for s in source:
ip_object.add_source(s)
if bucket_list:
ip_object.add_bucket_list(bucket_list, analyst)
if ticket:
ip_object.add_ticket(ticket, analyst)
resp_url = reverse('crits.ips.views.ip_detail', args=[ip_object.ip])
if is_validate_only == False:
ip_object.save(username=analyst)
#set the URL for viewing the new data
if is_item_new == True:
retVal['message'] = ('Success! Click here to view the new IP: '
'<a href="%s">%s</a>' % (resp_url, ip_object.ip))
else:
message = ('Updated existing IP: '
'<a href="%s">%s</a>' % (resp_url, ip_object.ip))
retVal['message'] = message
retVal['status'] = form_consts.Status.DUPLICATE
retVal['warning'] = message
elif is_validate_only == True:
if ip_object.id != None and is_item_new == False:
message = ('Warning: IP already exists: '
'<a href="%s">%s</a>' % (resp_url, ip_object.ip))
retVal['message'] = message
retVal['status'] = form_consts.Status.DUPLICATE
retVal['warning'] = message
if is_add_indicator:
from crits.indicators.handlers import handle_indicator_ind
handle_indicator_ind(ip_address,
source,
indicator_reference,
ip_type,
analyst,
source_method,
add_domain=False,
add_relationship=True,
bucket_list=bucket_list,
ticket=ticket,
cache=cache)
# run ip triage
if is_item_new and is_validate_only == False:
ip_object.reload()
run_triage(None, ip_object, analyst)
retVal['success'] = True
retVal['object'] = ip_object
return retVal
def obj_create(self, bundle, **kwargs):
"""
Handles creating Indicators through the API.
:param bundle: Bundle containing the information to create the Indicator.
:type bundle: Tastypie Bundle object.
:returns: HttpResponse.
"""
user = bundle.request.user
value = bundle.data.get('value', None)
ctype = bundle.data.get('type', None)
threat_type = bundle.data.get('threat_type', None)
attack_type = bundle.data.get('attack_type', None)
source = bundle.data.get('source', None)
status = bundle.data.get('status', None)
reference = bundle.data.get('reference', None)
method = bundle.data.get('method', None)
tlp = bundle.data.get('tlp', 'amber')
add_domain = bundle.data.get('add_domain', False)
add_relationship = bundle.data.get('add_relationship', False)
campaign = bundle.data.get('campaign', None)
campaign_confidence = bundle.data.get('confidence', None)
confidence = bundle.data.get('indicator_confidence', None)
description = bundle.data.get('description', None)
impact = bundle.data.get('indicator_impact', None)
bucket_list = bundle.data.get('bucket_list', None)
ticket = bundle.data.get('ticket', None)
if user.has_access_to(IndicatorACL.WRITE):
result = handle_indicator_ind(value,
source,
ctype,
threat_type,
attack_type,
user,
status=status,
source_method=method,
source_reference=reference,
source_tlp=tlp,
add_domain=add_domain,
add_relationship=add_relationship,
campaign=campaign,
campaign_confidence=campaign_confidence,
confidence=confidence,
description=description,
impact=impact,
bucket_list=bucket_list,
ticket=ticket)
else:
result = {'success':False,
'message':'User does not have permission to create Object.'}
content = {'return_code': 0,
'type': 'Indicator'}
if result.get('message'):
content['message'] = result.get('message')
if result.get('objectid'):
url = reverse('api_dispatch_detail',
kwargs={'resource_name': 'indicators',
'api_name': 'v1',
'pk': result.get('objectid')})
content['id'] = result.get('objectid')
content['url'] = url
if not result['success']:
content['return_code'] = 1
self.crits_response(content)
def add_object(type_,
oid,
object_type,
name,
source,
method,
reference,
analyst,
value=None,
file_=None,
add_indicator=False,
get_objects=True,
indicator_campaign=None,
indicator_campaign_confidence=None,
obj=None,
is_sort_relationships=False,
is_validate_only=False,
is_validate_locally=False,
cache={}):
"""
Add an object to the database.
:param type_: The top-level object type.
:type type_: str
:param oid: The ObjectId of the top-level object.
:type oid: str
:param object_type: The type of the ObjectType being added.
:type object_type: str
:param name: The name of the ObjectType being added.
:type name: str
:param source: The name of the source adding this object.
:type source: str
:param method: The method for this object.
:type method: str
:param reference: The reference for this object.
:type reference: str
:param analyst: The user adding this object.
:type analyst: str
:param value: The value of the object.
:type value: str
:param file_: The file if the object is a file upload.
:type file_: file handle.
:param add_indicator: Also add an indicator for this object.
:type add_indicator: bool
:param get_objects: Return the formatted list of objects when completed.
:type get_object: bool
:param is_validate_only: Only validate, do not add.
:type is_validate_only: bool
:param is_validate_locally: Only validate, do not add.
:type is_validate_locally: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:param obj: The CRITs top-level object we are adding objects to.
This is an optional parameter used mainly for performance
reasons (by not querying mongo if we already have the
top level-object).
:type obj: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:returns: dict with keys:
"success" (boolean),
"message" (str),
"objects" (list),
"relationships" (list)
"""
results = {}
if oid == None:
oid = ""
if obj == None:
obj = class_from_id(type_, oid)
if not obj:
if is_validate_locally == True:
# TODO: Perform some form of validation
results['success'] = True
return results
else:
results['message'] = "Could not find item to add object to."
results['success'] = False
return results
try:
cur_len = len(obj.obj)
if file_:
data = file_.read()
filename = file_.name
md5sum = md5(data).hexdigest()
value = md5sum
reference = filename
obj.add_object(object_type, name, value, source, method, reference,
analyst)
if is_validate_only == False:
obj.save(username=analyst)
new_len = len(obj.obj)
if new_len > cur_len:
results['message'] = "Object added successfully!"
results['success'] = True
if file_:
# do we have a pcap?
if data[:4] in ('\xa1\xb2\xc3\xd4', '\xd4\xc3\xb2\xa1',
'\x0a\x0d\x0d\x0a'):
handle_pcap_file(filename,
data,
source,
user=analyst,
related_id=oid,
related_type=type_)
else:
#XXX: MongoEngine provides no direct GridFS access so we
# need to use pymongo directly.
col = settings.COL_OBJECTS
grid = mongo_connector("%s.files" % col)
if grid.find({'md5': md5sum}).count() == 0:
put_file(filename, data, collection=col)
if add_indicator and is_validate_only == False:
from crits.indicators.handlers import handle_indicator_ind
if object_type != name:
object_type = "%s - %s" % (object_type, name)
ind_res = handle_indicator_ind(
value,
source,
reference,
object_type,
analyst,
method=method,
add_domain=True,
campaign=indicator_campaign,
campaign_confidence=indicator_campaign_confidence,
cache=cache)
if ind_res['success']:
ind = ind_res['object']
# Inherit campaigns from top level item when creating
# an indicator from an object if no campaigns were specified
if indicator_campaign == None and ind != None:
for campaign in obj.campaign:
ec = EmbeddedCampaign(
name=campaign.name,
confidence=campaign.confidence,
description="",
analyst=analyst,
date=datetime.datetime.now())
ind.add_campaign(ec)
ind.save(username=analyst)
forge_relationship(left_class=obj,
right_class=ind,
rel_type="Related_To",
analyst=analyst,
get_rels=is_sort_relationships)
if is_sort_relationships == True:
if file_ or add_indicator:
# does this line need to be here?
# obj.reload()
results['relationships'] = obj.sort_relationships(
analyst, meta=True)
else:
results['relationships'] = obj.sort_relationships(
analyst, meta=True)
else:
results[
'message'] = "Object already exists! [Type: " + object_type + "][Value: " + value + "] "
results['success'] = False
if (get_objects):
results['objects'] = obj.sort_objects()
return results
except ValidationError, e:
return {'success': False, 'message': e}
def add_object(type_, id_, object_type, source, method,
reference, user, value=None, file_=None,
add_indicator=False, get_objects=True,
obj=None, is_sort_relationships=False,
is_validate_only=False, is_validate_locally=False, cache={},
**kwargs):
"""
Add an object to the database.
:param type_: The top-level object type.
:type type_: str
:param id_: The ObjectId of the top-level object.
:type id_: str
:param object_type: The type of the ObjectType being added.
:type object_type: str
:param source: The name of the source adding this object.
:type source: str
:param method: The method for this object.
:type method: str
:param reference: The reference for this object.
:type reference: str
:param user: The user adding this object.
:type user: str
:param value: The value of the object.
:type value: str
:param file_: The file if the object is a file upload.
:type file_: file handle.
:param add_indicator: Also add an indicator for this object.
:type add_indicator: bool
:param get_objects: Return the formatted list of objects when completed.
:type get_object: bool
:param obj: The CRITs top-level object we are adding objects to.
This is an optional parameter used mainly for performance
reasons (by not querying mongo if we already have the
top level-object).
:type obj: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param is_validate_only: Validate, but do not add to TLO.
:type is_validate_only: bool
:param is_validate_locally: Validate, but do not add b/c there is no TLO.
:type is_validate_locally: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" (str),
"objects" (list),
"relationships" (list)
"""
results = {}
if id_ == None:
id_ = ""
if obj == None:
obj = class_from_id(type_, id_)
from crits.indicators.handlers import validate_indicator_value
if value is not None:
(value, error) = validate_indicator_value(value, object_type)
if error:
return {"success": False, "message": error}
if is_validate_locally: # no obj provided
results['success'] = True
return results
if not obj:
results['message'] = "TLO could not be found"
results['success'] = False
return results
try:
cur_len = len(obj.obj)
if file_:
data = file_.read()
filename = file_.name
md5sum = md5(data).hexdigest()
value = md5sum
reference = filename
obj.add_object(object_type,
value,
source,
method,
reference,
user)
if is_validate_only == False:
obj.save(username=user)
new_len = len(obj.obj)
if new_len > cur_len:
if not is_validate_only:
results['message'] = "Object added successfully!"
results['success'] = True
if file_:
# do we have a pcap?
if data[:4] in ('\xa1\xb2\xc3\xd4',
'\xd4\xc3\xb2\xa1',
'\x0a\x0d\x0d\x0a'):
handle_pcap_file(filename,
data,
source,
user=user,
related_id=id_,
related_type=type_)
else:
#XXX: MongoEngine provides no direct GridFS access so we
# need to use pymongo directly.
col = settings.COL_OBJECTS
grid = mongo_connector("%s.files" % col)
if grid.find({'md5': md5sum}).count() == 0:
put_file(filename, data, collection=col)
if add_indicator and is_validate_only == False:
from crits.indicators.handlers import handle_indicator_ind
campaign = obj.campaign if hasattr(obj, 'campaign') else None
ind_res = handle_indicator_ind(value,
source,
object_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
user,
method,
reference,
add_domain=True,
campaign=campaign,
cache=cache)
if ind_res['success']:
ind = ind_res['object']
forge_relationship(class_=obj,
right_class=ind,
rel_type=RelationshipTypes.RELATED_TO,
user=user,
get_rels=is_sort_relationships)
else:
results['message'] = "Object was added, but failed to add Indicator." \
"<br>Error: " + ind_res.get('message')
if is_sort_relationships == True:
if file_ or add_indicator:
# does this line need to be here?
# obj.reload()
results['relationships'] = obj.sort_relationships(user, meta=True)
else:
results['relationships'] = obj.sort_relationships(user, meta=True)
else:
results['message'] = "Object already exists! [Type: " + object_type + "][Value: " + value + "] "
results['success'] = False
if (get_objects):
results['objects'] = obj.sort_objects()
results['id'] = str(obj.id)
return results
except ValidationError, e:
return {'success': False,
'message': str(e)}
def parse_cybox_object(self, cbx_obj, description='', ind_id=None):
"""
Parse a CybOX object form a STIX doc. An object can contain
multiple related_objects, which in turn can have their own
related_objects, so this handles those recursively.
:param cbx_obj: The CybOX object to parse.
:type cbx_obj: A CybOX object.
:param description: Parent-level (e.g. Observable) description.
:type description: str
:param ind_id: The ID of a parent STIX Indicator.
:type ind_id: str
"""
# check for missing attributes
if not cbx_obj or not cbx_obj.properties:
if cbx_obj.idref: # just a reference, so nothing to parse
return
else:
cbx_id = getattr(cbx_obj, 'id_', 'None')
self.failed.append(("No valid object_properties was found!",
"Observable (%s)" % cbx_id,
cbx_id)) # note for display in UI
return
# Don't parse if already been parsed
# This is for artifacts that are related to CybOX File Objects
if cbx_obj.id_ in self.parsed:
return
try: # try to create CRITs object from Cybox Object
analyst = self.source_instance.analyst
item = cbx_obj.properties
val = cbx_obj.id_
if isinstance(item, Address) and not ind_id:
if item.category in ('cidr', 'ipv4-addr', 'ipv4-net',
'ipv4-netmask', 'ipv6-addr', 'ipv6-net',
'ipv6-netmask'):
imp_type = "IP"
for value in item.address_value.values:
val = str(value).strip()
if self.preview:
res = None
else:
iptype = get_crits_ip_type(item.category)
if iptype:
res = ip_add_update(val,
iptype, [self.source],
analyst=analyst,
is_add_indicator=True)
else:
res = {
'success': False,
'reason': 'No IP Type'
}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
if (not ind_id and
(isinstance(item, DomainName) or
(isinstance(item, URI) and item.type_ == 'Domain Name'))):
imp_type = "Domain"
for val in item.value.values:
if self.preview:
res = None
else:
res = upsert_domain(str(val), [self.source],
username=analyst)
self.parse_res(imp_type, str(val), cbx_obj, res, ind_id)
elif isinstance(item, HTTPSession):
imp_type = "RawData"
val = cbx_obj.id_
try:
c_req = item.http_request_response[0].http_client_request
hdr = c_req.http_request_header
if hdr.raw_header:
data = hdr.raw_header.value
title = "HTTP Header from STIX: %s" % self.package.id_
method = self.source_instance.method
ref = self.source_instance.reference
if self.preview:
res = None
val = title
else:
res = handle_raw_data_file(data,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="HTTP Header",
tool_name="STIX",
tool_version=None,
method=method,
reference=ref)
else:
imp_type = "Indicator"
ind_type = "HTTP Request Header Fields - User-Agent"
val = hdr.parsed_header.user_agent.value
val = ','.join(val) if isinstance(val, list) else val
if self.preview:
res = None
else:
res = handle_indicator_ind(
val,
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_relationship=True,
description=description)
except:
msg = "Unsupported use of 'HTTPSession' object."
res = {'success': False, 'reason': msg}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, WhoisEntry):
# No sure where else to put this
imp_type = "RawData"
val = cbx_obj.id_
if item.remarks:
data = item.remarks.value
title = "WHOIS Entry from STIX: %s" % self.package.id_
if self.preview:
res = None
val = title
else:
res = handle_raw_data_file(
data,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="Text",
tool_name="WHOIS",
tool_version=None,
method=self.source_instance.method,
reference=self.source_instance.reference)
else:
msg = "Unsupported use of 'WhoisEntry' object."
res = {'success': False, 'reason': msg}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, Artifact):
# Not sure if this is right, and I believe these can be
# encoded in a couple different ways.
imp_type = "RawData"
val = cbx_obj.id_
rawdata = item.data.decode('utf-8')
# TODO: find out proper ways to determine title, datatype,
# tool_name, tool_version
title = "Artifact for Event: STIX Document %s" % self.package.id_
if self.preview:
res = None
val = title
else:
res = handle_raw_data_file(
rawdata,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="Text",
tool_name="STIX",
tool_version=None,
method=self.source_instance.method,
reference=self.source_instance.reference)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif (isinstance(item, File) and item.custom_properties
and item.custom_properties[0].name == "crits_type"
and item.custom_properties[0]._value == "Certificate"):
imp_type = "Certificate"
val = str(item.file_name)
data = None
if self.preview:
res = None
else:
for rel_obj in item.parent.related_objects:
if isinstance(rel_obj.properties, Artifact):
data = rel_obj.properties.data
self.parsed.append(rel_obj.id_)
res = handle_cert_file(val,
data,
self.source,
user=analyst,
description=description)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, File) and self.has_network_artifact(item):
imp_type = "PCAP"
val = str(item.file_name)
data = None
if self.preview:
res = None
else:
for rel_obj in item.parent.related_objects:
if (isinstance(rel_obj.properties, Artifact)
and rel_obj.properties.type_
== Artifact.TYPE_NETWORK):
data = rel_obj.properties.data
self.parsed.append(rel_obj.id_)
res = handle_pcap_file(val,
data,
self.source,
user=analyst,
description=description)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, File):
imp_type = "Sample"
md5 = item.md5
if md5:
md5 = md5.lower()
val = str(item.file_name or md5)
# add sha1/sha256/ssdeep once handle_file supports it
size = item.size_in_bytes
data = None
if item.file_path:
path = "File Path: " + str(item.file_path)
description += "\n" + path
for rel_obj in item.parent.related_objects:
if (isinstance(rel_obj.properties, Artifact) and
rel_obj.properties.type_ == Artifact.TYPE_FILE):
data = rel_obj.properties.data
self.parsed.append(rel_obj.id_)
if not md5 and not data and val and val != "None":
imp_type = "Indicator"
if self.preview:
res = None
else:
res = handle_indicator_ind(
val,
self.source,
"Win File",
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True,
description=description)
elif md5 or data:
if self.preview:
res = None
else:
res = handle_file(val,
data,
self.source,
user=analyst,
md5_digest=md5,
is_return_only_md5=False,
size=size,
description=description)
else:
val = cbx_obj.id_
msg = "CybOX 'File' object has no MD5, data, or filename"
res = {'success': False, 'reason': msg}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, EmailMessage):
imp_type = 'Email'
id_list = []
data = {}
val = cbx_obj.id_
get_attach = False
data['raw_body'] = str(item.raw_body)
data['raw_header'] = str(item.raw_header)
data['helo'] = str(item.email_server)
if item.header:
data['subject'] = str(item.header.subject)
if item.header.date:
data['date'] = item.header.date.value
val = "Date: %s, Subject: %s" % (data.get(
'date', 'None'), data['subject'])
data['message_id'] = str(item.header.message_id)
data['sender'] = str(item.header.sender)
data['reply_to'] = str(item.header.reply_to)
data['x_originating_ip'] = str(
item.header.x_originating_ip)
data['x_mailer'] = str(item.header.x_mailer)
data['boundary'] = str(item.header.boundary)
data['from_address'] = str(item.header.from_)
if item.header.to:
data['to'] = [str(r) for r in item.header.to.to_list()]
if data.get('date'): # Email TLOs must have a date
data['source'] = self.source.name
data['source_method'] = self.source_instance.method
data['source_reference'] = self.source_instance.reference
if self.preview:
res = None
else:
res = handle_email_fields(data, analyst, "STIX")
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
if not self.preview and res.get('status'):
id_list.append(cbx_obj.id_) # save ID for atchmnt rels
get_attach = True
else: # Can't be an Email TLO, so save fields
for x, key in enumerate(data):
if data[key] and data[key] != "None":
if key in ('raw_header', 'raw_body'):
if key == 'raw_header':
title = "Email Header from STIX Email: %s"
d_type = "Email Header"
else:
title = "Email Body from STIX Email: %s"
d_type = "Email Body"
imp_type = 'RawData'
title = title % cbx_obj.id_
if self.preview:
res = None
else:
res = handle_raw_data_file(
data[key], self.source, analyst,
description, title, d_type, "STIX",
self.stix_version)
self.parse_res(imp_type, title, cbx_obj, res,
ind_id)
elif key == 'to':
imp_type = 'Target'
for y, addr in enumerate(data[key]):
tgt_dict = {'email_address': addr}
if self.preview:
res = None
else:
res = upsert_target(tgt_dict, analyst)
if res['success']:
get_attach = True
tmp_obj = copy(cbx_obj)
tmp_obj.id_ = '%s-%s-%s' % (cbx_obj.id_, x,
y)
self.parse_res(imp_type, addr, tmp_obj,
res, ind_id)
self.ind2obj.setdefault(
cbx_obj.id_, []).append(tmp_obj.id_)
id_list.append(tmp_obj.id_)
else:
imp_type = 'Indicator'
if key in ('sender', 'reply_to',
'from_address'):
ind_type = "Address - e-mail"
elif 'ip' in key:
ind_type = "Address - ipv4-addr"
elif key == 'raw_body':
ind_type = "Email Message"
else:
ind_type = "String"
if self.preview:
res = None
else:
res = handle_indicator_ind(
data[key],
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True,
description=description)
if res['success']:
get_attach = True
tmp_obj = copy(cbx_obj)
tmp_obj.id_ = '%s-%s' % (cbx_obj.id_, x)
self.parse_res(imp_type, data[key], tmp_obj,
res, ind_id)
self.ind2obj.setdefault(cbx_obj.id_,
[]).append(tmp_obj.id_)
id_list.append(tmp_obj.id_)
if not self.preview:
# Setup relationships between all Email attributes
for oid in id_list:
for oid2 in id_list:
if oid != oid2:
self.relationships.append(
(oid, RelationshipTypes.RELATED_TO, oid2,
"High"))
# Should check for attachments and add them here.
if get_attach and item.attachments:
for attach in item.attachments:
rel_id = attach.to_dict()['object_reference']
for oid in id_list:
self.relationships.append(
(oid, RelationshipTypes.CONTAINS, rel_id,
"High"))
else: # try to parse all other possibilities as Indicator
imp_type = "Indicator"
val = cbx_obj.id_
c_obj = make_crits_object(item)
# Ignore what was already caught above
if (ind_id or c_obj.object_type not in IPTypes.values()):
ind_type = c_obj.object_type
for val in [str(v).strip() for v in c_obj.value if v]:
if ind_type:
# handle domains mislabeled as URLs
if c_obj.object_type == 'URI' and '/' not in val:
ind_type = "Domain"
if self.preview:
res = None
else:
res = handle_indicator_ind(
val,
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True,
description=description)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
except Exception, e: # probably caused by cybox object we don't handle
self.failed.append((e.message, "%s (%s)" % (imp_type, val),
cbx_obj.id_)) # note for display in UI
def parse_observables(self, observables):
"""
Parse list of observables in STIX doc.
:param observables: List of STIX observables.
:type observables: List of STIX observables.
"""
analyst = self.source_instance.analyst
for obs in observables: # for each STIX observable
if not obs.object_ or not obs.object_.properties:
self.failed.append(("No valid object_properties was found!",
type(obs).__name__,
obs.id_)) # note for display in UI
continue
try: # try to create CRITs object from observable
item = obs.object_.properties
if isinstance(item, Address):
if item.category in ('cidr', 'ipv4-addr', 'ipv4-net',
'ipv4-netmask', 'ipv6-addr',
'ipv6-net', 'ipv6-netmask'):
imp_type = "IP"
for value in item.address_value.values:
ip = str(value).strip()
iptype = get_crits_ip_type(item.category)
if iptype:
res = ip_add_update(ip,
iptype,
[self.source],
analyst=analyst,
is_add_indicator=True)
self.parse_res(imp_type, obs, res)
if isinstance(item, DomainName):
imp_type = "Domain"
for value in item.value.values:
res = upsert_domain(str(value),
[self.source],
username=analyst)
self.parse_res(imp_type, obs, res)
elif isinstance(item, Artifact):
# Not sure if this is right, and I believe these can be
# encoded in a couple different ways.
imp_type = "RawData"
rawdata = item.data.decode('utf-8')
description = "None"
# TODO: find out proper ways to determine title, datatype,
# tool_name, tool_version
title = "Artifact for Event: STIX Document %s" % self.package.id_
res = handle_raw_data_file(rawdata,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="Text",
tool_name="STIX",
tool_version=None,
method=self.source_instance.method,
reference=self.source_instance.reference)
self.parse_res(imp_type, obs, res)
elif (isinstance(item, File) and
item.custom_properties and
item.custom_properties[0].name == "crits_type" and
item.custom_properties[0]._value == "Certificate"):
imp_type = "Certificate"
description = "None"
filename = str(item.file_name)
data = None
for obj in item.parent.related_objects:
if isinstance(obj.properties, Artifact):
data = obj.properties.data
res = handle_cert_file(filename,
data,
self.source,
user=analyst,
description=description)
self.parse_res(imp_type, obs, res)
elif isinstance(item, File) and self.has_network_artifact(item):
imp_type = "PCAP"
description = "None"
filename = str(item.file_name)
data = None
for obj in item.parent.related_objects:
if (isinstance(obj.properties, Artifact) and
obj.properties.type_ == Artifact.TYPE_NETWORK):
data = obj.properties.data
res = handle_pcap_file(filename,
data,
self.source,
user=analyst,
description=description)
self.parse_res(imp_type, obs, res)
elif isinstance(item, File):
imp_type = "Sample"
filename = str(item.file_name)
md5 = item.md5
data = None
for obj in item.parent.related_objects:
if (isinstance(obj.properties, Artifact) and
obj.properties.type_ == Artifact.TYPE_FILE):
data = obj.properties.data
res = handle_file(filename,
data,
self.source,
user=analyst,
md5_digest=md5,
is_return_only_md5=False)
self.parse_res(imp_type, obs, res)
elif isinstance(item, EmailMessage):
imp_type = "Email"
data = {}
data['source'] = self.source.name
data['source_method'] = self.source_instance.method
data['source_reference'] = self.source_instance.reference
data['raw_body'] = str(item.raw_body)
data['raw_header'] = str(item.raw_header)
data['helo'] = str(item.email_server)
if item.header:
data['message_id'] = str(item.header.message_id)
data['subject'] = str(item.header.subject)
data['sender'] = str(item.header.sender)
data['reply_to'] = str(item.header.reply_to)
data['x_originating_ip'] = str(item.header.x_originating_ip)
data['x_mailer'] = str(item.header.x_mailer)
data['boundary'] = str(item.header.boundary)
data['from_address'] = str(item.header.from_)
data['date'] = item.header.date.value
if item.header.to:
data['to'] = [str(r) for r in item.header.to.to_list()]
res = handle_email_fields(data,
analyst,
"STIX")
# Should check for attachments and add them here.
self.parse_res(imp_type, obs, res)
if res.get('status') and item.attachments:
for attach in item.attachments:
rel_id = attach.to_dict()['object_reference']
self.relationships.append((obs.id_,
"Contains",
rel_id, "High"))
else: # try to parse all other possibilities as Indicator
imp_type = "Indicator"
obj = make_crits_object(item)
if obj.object_type == 'Address':
# This was already caught above
continue
else:
ind_type = obj.object_type
for value in obj.value:
if value and ind_type:
res = handle_indicator_ind(value.strip(),
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True)
self.parse_res(imp_type, obs, res)
except Exception, e: # probably caused by cybox object we don't handle
self.failed.append((e.message,
type(item).__name__,
item.parent.id_)) # note for display in UI
def create_indicator_from_object(rel_type, rel_id, ind_type, value,
source_name, method, reference, tlp, analyst,
request):
"""
Create an indicator out of this object.
:param rel_type: The top-level object type this object is for.
:type rel_type: str
:param rel_id: The ObjectId of the top-level object.
:param ind_type: The indicator type to use.
:type ind_type: str
:param value: The indicator value.
:type value: str
:param source_name: The source name for the indicator.
:type source_name: str
:param method: The source method for the indicator.
:type method: str
:param reference: The source reference for the indicator.
:type reference: str
:param analyst: The user creating this indicator.
:type analyst: str
:param request: The Django request.
:type request: :class:`django.http.HttpRequest`
:returns: dict with keys "success" (bool) and "message" (str)
"""
result = None
me = class_from_id(rel_type, rel_id)
if not me:
result = {'success': False, 'message': "Could not find %s" % rel_type}
elif value == None or value.strip() == "":
result = {
'success': False,
'message': "Can't create indicator with an empty value field"
}
elif ind_type == None or ind_type.strip() == "":
result = {
'success': False,
'message': "Can't create indicator with an empty type field"
}
elif source_name == None or source_name.strip() == "":
result = {
'success': False,
'message': "Can't create indicator with an empty source field"
}
else:
value = value.lower().strip()
ind_type = ind_type.strip()
source_name = source_name.strip()
create_indicator_result = {}
from crits.indicators.handlers import handle_indicator_ind
campaign = me.campaign if hasattr(me, 'campaign') else None
create_indicator_result = handle_indicator_ind(
value,
source_name,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
source_method=method,
source_reference=reference,
source_tlp=tlp,
add_domain=True,
campaign=campaign)
# Check if an error occurred, if it did then return the error result
if create_indicator_result.get('success', True) == False:
return create_indicator_result
indicator = Indicator.objects(ind_type=ind_type, value=value).first()
if not indicator:
result = {
'success': False,
'message': "Could not create indicator"
}
else:
results = me.add_relationship(indicator,
RelationshipTypes.RELATED_TO,
analyst=analyst,
get_rels=True)
if results['success']:
me.save(username=analyst)
relationship = {'type': rel_type, 'value': rel_id}
message = render_to_string(
'relationships_listing_widget.html', {
'relationship': relationship,
'nohide': True,
'relationships': results['message']
}, RequestContext(request))
result = {'success': True, 'message': message}
else:
message = "Indicator created. Could not create relationship"
result = {'success': False, 'message': message}
return result
def upload_indicator(request):
"""
Upload new indicators (individual, blob, or CSV file).
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:returns: :class:`django.http.HttpResponse`
:class:`django.http.HttpResponseRedirect`
"""
if request.method == "POST":
username = request.user.username
failed_msg = ''
result = None
if request.POST['svalue'] == "Upload CSV":
form = UploadIndicatorCSVForm(
username,
request.POST,
request.FILES)
if form.is_valid():
result = handle_indicator_csv(request.FILES['filedata'],
request.POST['source'],
request.POST['method'],
request.POST['reference'],
"file",
username, add_domain=True)
if result['success']:
message = {'message': ('<div>%s <a href="%s">Go to all'
' indicators</a></div>' %
(result['message'],
reverse('crits.indicators.views.indicators_listing')))}
else:
failed_msg = '<div>%s</div>' % result['message']
if request.POST['svalue'] == "Upload Text":
form = UploadIndicatorTextForm(username, request.POST)
if form.is_valid():
result = handle_indicator_csv(request.POST['data'],
request.POST['source'],
request.POST['method'],
request.POST['reference'],
"ti",
username,
add_domain=True)
if result['success']:
message = {'message': ('<div>%s <a href="%s">Go to all'
' indicators</a></div>' %
(result['message'],
reverse('crits.indicators.views.indicators_listing')))}
else:
failed_msg = '<div>%s</div>' % result['message']
if request.POST['svalue'] == "Upload Indicator":
form = UploadIndicatorForm(username,
request.POST)
if form.is_valid():
result = handle_indicator_ind(
request.POST['value'],
request.POST['source'],
request.POST['indicator_type'],
request.POST['threat_type'],
request.POST['attack_type'],
username,
request.POST['method'],
request.POST['reference'],
add_domain=True,
description=request.POST['description'],
campaign=request.POST['campaign'],
campaign_confidence=request.POST['campaign_confidence'],
confidence=request.POST['confidence'],
impact=request.POST['impact'],
bucket_list=request.POST[form_consts.Common.BUCKET_LIST_VARIABLE_NAME],
ticket=request.POST[form_consts.Common.TICKET_VARIABLE_NAME])
if result['success']:
indicator_link = ((' - <a href=\"%s\">Go to this '
'indicator</a> or <a href="%s">all '
'indicators</a>.</div>') %
(reverse('crits.indicators.views.indicator',
args=[result['objectid']]),
reverse('crits.indicators.views.indicators_listing')))
if result.get('is_new_indicator', False) == False:
message = {'message': ('<div>Warning: Updated existing'
' Indicator!' + indicator_link)}
else:
message = {'message': ('<div>Indicator added '
'successfully!' + indicator_link)}
else:
failed_msg = result['message'] + ' - '
if result == None or not result['success']:
failed_msg += ('<a href="%s"> Go to all indicators</a></div>'
% reverse('crits.indicators.views.indicators_listing'))
message = {'message': failed_msg, 'form': form.as_table()}
elif result != None:
message['success'] = result['success']
if request.is_ajax():
return HttpResponse(json.dumps(message),
mimetype="application/json")
else: #file upload
return render_to_response('file_upload_response.html',
{'response': json.dumps(message)},
RequestContext(request))
def parse_observables(self, observables):
"""
Parse list of observables in STIX doc.
:param observables: List of STIX observables.
:type observables: List of STIX observables.
"""
analyst = self.source_instance.analyst
for obs in observables: # for each STIX observable
if obs.observable_composition:
object_list = obs.observable_composition.observables
else:
object_list = [obs]
for obs_comp in object_list:
if not obs_comp.object_ or not obs_comp.object_.properties:
self.failed.append(
("No valid object_properties was found!",
type(obs_comp).__name__,
obs_comp.id_)) # note for display in UI
continue
try: # try to create CRITs object from observable
item = obs_comp.object_.properties
if isinstance(item, Address):
if item.category in ('cidr', 'ipv4-addr', 'ipv4-net',
'ipv4-netmask', 'ipv6-addr',
'ipv6-net', 'ipv6-netmask',
'ipv6-subnet'):
imp_type = "IP"
for value in item.address_value.values:
ip = str(value).strip()
iptype = get_crits_ip_type(item.category)
if iptype:
res = ip_add_update(ip,
iptype, [self.source],
analyst=analyst,
id=self.package.id_)
self.parse_res(imp_type, obs, res)
if isinstance(item, DomainName):
imp_type = "Domain"
for value in item.value.values:
res = upsert_domain(str(value), [self.source],
username=analyst,
id=self.package.id_)
self.parse_res(imp_type, obs, res)
elif isinstance(item, Artifact):
# Not sure if this is right, and I believe these can be
# encoded in a couple different ways.
imp_type = "RawData"
rawdata = item.data.decode('utf-8')
description = "None"
# TODO: find out proper ways to determine title, datatype,
# tool_name, tool_version
title = "Artifact for Event: STIX Document %s" % self.package.id_
res = handle_raw_data_file(
rawdata,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="Text",
tool_name="STIX",
tool_version=None,
method=self.source_instance.method,
reference=self.source_instance.reference)
self.parse_res(imp_type, obs, res)
elif (isinstance(item, File) and item.custom_properties
and item.custom_properties[0].name == "crits_type"
and item.custom_properties[0]._value
== "Certificate"):
imp_type = "Certificate"
description = "None"
filename = str(item.file_name)
data = None
for obj in item.parent.related_objects:
if isinstance(obj.properties, Artifact):
data = obj.properties.data
res = handle_cert_file(filename,
data,
self.source,
user=analyst,
description=description)
self.parse_res(imp_type, obs, res)
elif isinstance(item,
File) and self.has_network_artifact(item):
imp_type = "PCAP"
description = "None"
filename = str(item.file_name)
data = None
for obj in item.parent.related_objects:
if (isinstance(obj.properties, Artifact)
and obj.properties.type_
== Artifact.TYPE_NETWORK):
data = obj.properties.data
res = handle_pcap_file(filename,
data,
self.source,
user=analyst,
description=description)
self.parse_res(imp_type, obs, res)
elif isinstance(item, File):
imp_type = "Sample"
filename = str(item.file_name)
md5 = item.md5
data = None
for obj in item.parent.related_objects:
if (isinstance(obj.properties, Artifact)
and obj.properties.type_
== Artifact.TYPE_FILE):
data = obj.properties.data
res = handle_file(filename,
data,
self.source,
user=analyst,
md5_digest=md5,
is_return_only_md5=False,
id=self.package.id_)
self.parse_res(imp_type, obs, res)
if item.extracted_features:
self.parse_filenames(item.extracted_features,
res['object'].id)
elif isinstance(item, EmailMessage):
imp_type = "Email"
data = {}
data['source'] = self.source.name
data['source_method'] = self.source_instance.method
data[
'source_reference'] = self.source_instance.reference
data['raw_body'] = str(item.raw_body)
data['raw_header'] = str(item.raw_header)
data['helo'] = str(item.email_server)
if item.header:
data['message_id'] = str(item.header.message_id)
data['subject'] = str(item.header.subject)
data['sender'] = str(item.header.sender)
data['reply_to'] = str(item.header.reply_to)
data['x_originating_ip'] = str(
item.header.x_originating_ip)
data['x_mailer'] = str(item.header.x_mailer)
data['boundary'] = str(item.header.boundary)
data['from_address'] = str(item.header.from_)
data['date'] = item.header.date.value
if item.header.to:
data['to'] = [str(r) for r in item.header.to]
if item.header.cc:
data['cc'] = [str(r) for r in item.header.cc]
res = handle_email_fields(data,
analyst,
"STIX",
id=self.package.id_)
# Should check for attachments and add them here.
self.parse_res(imp_type, obs, res)
if res.get('status') and item.attachments:
for attach in item.attachments:
rel_id = attach.to_dict()['object_reference']
self.relationships.append(
(obs.id_, "Contains", rel_id, "High"))
else: # try to parse all other possibilities as Indicator
imp_type = "Indicator"
obj = make_crits_object(item)
if obj.object_type == 'Address':
# This was already caught above
continue
else:
ind_type = obj.object_type
for value in obj.value:
if value and ind_type:
res = handle_indicator_ind(
value.strip(),
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True)
self.parse_res(imp_type, obs, res)
except Exception, e: # probably caused by cybox object we don't handle
self.failed.append(
(e.message, type(item).__name__,
item.parent.id_)) # note for display in UI
def create_indicator_from_object(rel_type, rel_id, ind_type, value, analyst,
request):
"""
Create an indicator out of this object.
:param rel_type: The top-level object type this object is for.
:type rel_type: str
:param rel_id: The ObjectId of the top-level object.
:param ind_type: The indicator type to use.
:type ind_type: str
:param value: The indicator value.
:type value: str
:param analyst: The user creating this indicator.
:type analyst: str
:param request: The Django request.
:type request: :class:`django.http.HttpRequest`
:returns: dict with keys "success" (bool) and "message" (str)
"""
result = None
me = class_from_id(rel_type, rel_id)
if not me:
result = {'success': False, 'message': "Could not find %s" % rel_type}
elif value == None or value.strip() == "":
result = {
'success': False,
'message': "Can't create indicator with an empty value field"
}
elif ind_type == None or ind_type.strip() == "":
result = {
'success': False,
'message': "Can't create indicator with an empty type field"
}
else:
value = value.lower().strip()
ind_type = ind_type.strip()
create_indicator_result = {}
ind_tlist = ind_type.split(" - ")
if ind_tlist[0] == ind_tlist[1]:
ind_type = ind_tlist[0]
from crits.indicators.handlers import handle_indicator_ind
if hasattr(me, 'source'):
create_indicator_result = handle_indicator_ind(value,
me.source,
'',
ind_type,
analyst=analyst,
add_domain=True)
else:
# In case the top level item doesn't have sources (such as campaign)...
# then just default to the user's organization
create_indicator_result = handle_indicator_ind(
value,
get_user_organization(analyst),
'',
ind_type,
analyst=analyst,
add_domain=True)
# Check if an error occurred, if it did then return the error result
if create_indicator_result.get('success', True) == False:
return result
indicator = Indicator.objects(ind_type=ind_type, value=value).first()
if not indicator:
result = {
'success': False,
'message': "Could not create indicator"
}
else:
results = me.add_relationship(rel_item=indicator,
rel_type="Related_To",
analyst=analyst,
get_rels=True)
if results['success']:
me.save(username=analyst)
indicator.save(username=analyst)
relationship = {'type': rel_type, 'value': rel_id}
message = render_to_string(
'relationships_listing_widget.html', {
'relationship': relationship,
'nohide': True,
'relationships': results['message']
}, RequestContext(request))
result = {'success': True, 'message': message}
else:
message = "Indicator created. Could not create relationship"
result = {'success': False, 'message': message}
return result
def upload_indicator(request):
"""
Upload new indicators (individual, blob, or CSV file).
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:returns: :class:`django.http.HttpResponse`
:class:`django.http.HttpResponseRedirect`
"""
if request.method == "POST":
username = request.user.username
failed_msg = ''
result = None
if request.POST['svalue'] == "Upload CSV":
form = UploadIndicatorCSVForm(username, request.POST,
request.FILES)
if form.is_valid():
result = handle_indicator_csv(
request.FILES['filedata'],
request.POST['source'],
request.POST['method'],
request.POST['reference'],
"file",
username,
add_domain=True,
related_id=request.POST['related_id'],
related_type=request.POST['related_type'],
relationship_type=request.POST['relationship_type'])
if result['success']:
message = {
'message':
('<div>%s <a href="%s">Go to all'
' indicators</a></div>' %
(result['message'],
reverse('crits.indicators.views.indicators_listing'))
)
}
else:
failed_msg = '<div>%s</div>' % result['message']
if request.POST['svalue'] == "Upload Text":
form = UploadIndicatorTextForm(username, request.POST)
if form.is_valid():
result = handle_indicator_csv(
request.POST['data'],
request.POST['source'],
request.POST['method'],
request.POST['reference'],
"ti",
username,
add_domain=True,
related_id=request.POST['related_id'],
related_type=request.POST['related_type'],
relationship_type=request.POST['relationship_type'])
if result['success']:
message = {
'message':
('<div>%s <a href="%s">Go to all'
' indicators</a></div>' %
(result['message'],
reverse('crits.indicators.views.indicators_listing'))
)
}
else:
failed_msg = '<div>%s</div>' % result['message']
if request.POST['svalue'] == "Upload Indicator":
form = UploadIndicatorForm(username, request.POST)
if form.is_valid():
result = handle_indicator_ind(
request.POST['value'],
request.POST['source'],
request.POST['indicator_type'],
request.POST['threat_type'],
request.POST['attack_type'],
username,
request.POST['method'],
request.POST['reference'],
add_domain=True,
description=request.POST['description'],
campaign=request.POST['campaign'],
campaign_confidence=request.POST['campaign_confidence'],
confidence=request.POST['confidence'],
impact=request.POST['impact'],
bucket_list=request.POST[
form_consts.Common.BUCKET_LIST_VARIABLE_NAME],
ticket=request.POST[
form_consts.Common.TICKET_VARIABLE_NAME],
related_id=request.POST['related_id'],
related_type=request.POST['related_type'],
relationship_type=request.POST['relationship_type'])
if result['success']:
indicator_link = (
(' - <a href=\"%s\">Go to this '
'indicator</a> or <a href="%s">all '
'indicators</a>.</div>') %
(reverse('crits.indicators.views.indicator',
args=[result['objectid']]),
reverse('crits.indicators.views.indicators_listing')))
if result.get('is_new_indicator', False) == False:
message = {
'message': ('<div>Warning: Updated existing'
' Indicator!' + indicator_link)
}
else:
message = {
'message': ('<div>Indicator added '
'successfully!' + indicator_link)
}
else:
failed_msg = result['message'] + ' - '
if result == None or not result['success']:
failed_msg += (
'<a href="%s"> Go to all indicators</a></div>' %
reverse('crits.indicators.views.indicators_listing'))
message = {'message': failed_msg, 'form': form.as_table()}
elif result != None:
message['success'] = result['success']
if request.is_ajax():
return HttpResponse(json.dumps(message),
content_type="application/json")
else: #file upload
return render_to_response('file_upload_response.html',
{'response': json.dumps(message)},
RequestContext(request))
def add_object(type_,
oid,
object_type,
name,
source,
method,
reference,
analyst,
value=None,
file_=None,
add_indicator=False,
get_objects=True,
obj=None,
is_sort_relationships=False,
is_validate_only=False,
is_validate_locally=False,
cache={}):
"""
Add an object to the database.
:param type_: The top-level object type.
:type type_: str
:param oid: The ObjectId of the top-level object.
:type oid: str
:param object_type: The type of the ObjectType being added.
:type object_type: str
:param name: The name of the ObjectType being added.
:type name: str
:param source: The name of the source adding this object.
:type source: str
:param method: The method for this object.
:type method: str
:param reference: The reference for this object.
:type reference: str
:param analyst: The user adding this object.
:type analyst: str
:param value: The value of the object.
:type value: str
:param file_: The file if the object is a file upload.
:type file_: file handle.
:param add_indicator: Also add an indicator for this object.
:type add_indicator: bool
:param get_objects: Return the formatted list of objects when completed.
:type get_object: bool
:param obj: The CRITs top-level object we are adding objects to.
This is an optional parameter used mainly for performance
reasons (by not querying mongo if we already have the
top level-object).
:type obj: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param is_validate_only: Only validate, do not add.
:type is_validate_only: bool
:param is_validate_locally: Only validate, do not add.
:type is_validate_locally: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" (str),
"objects" (list),
"relationships" (list)
"""
results = {}
if oid == None:
oid = ""
if obj == None:
obj = class_from_id(type_, oid)
if not obj:
if is_validate_locally == True:
# TODO: Perform some form of validation
results['success'] = True
return results
else:
results['message'] = "Could not find item to add object to."
results['success'] = False
return results
if name == "URL" and "://" not in value.split('.')[0]:
return {
"success":
False,
"message":
"URI - URL must contain protocol prefix (e.g. http://, https://, ftp://)"
}
elif object_type == "Address":
if "ipv4" in name:
try:
validate_ipv4_address(value)
except DjangoValidationError:
return {"success": False, "message": "Invalid IPv4 address. "}
elif "ipv6" in name:
try:
validate_ipv6_address(value)
except DjangoValidationError:
return {"success": False, "message": "Invalid IPv6 address. "}
elif "cidr" in name:
try:
if '/' not in value:
raise ValidationError("")
cidr_parts = value.split('/')
if int(cidr_parts[1]) < 0 or int(cidr_parts[1]) > 128:
raise ValidationError("")
if ':' not in cidr_parts[0] and int(cidr_parts[1]) > 32:
raise ValidationError("")
validate_ipv46_address(cidr_parts[0])
except (ValidationError, ValueError) as cidr_error:
return {"success": False, "message": "Invalid CIDR address. "}
try:
cur_len = len(obj.obj)
if file_:
data = file_.read()
filename = file_.name
md5sum = md5(data).hexdigest()
value = md5sum
reference = filename
obj.add_object(object_type, name, value, source, method, reference,
analyst)
if is_validate_only == False:
obj.save(username=analyst)
new_len = len(obj.obj)
if new_len > cur_len:
results['message'] = "Object added successfully!"
results['success'] = True
if file_:
# do we have a pcap?
if data[:4] in ('\xa1\xb2\xc3\xd4', '\xd4\xc3\xb2\xa1',
'\x0a\x0d\x0d\x0a'):
handle_pcap_file(filename,
data,
source,
user=analyst,
related_id=oid,
related_type=type_)
else:
#XXX: MongoEngine provides no direct GridFS access so we
# need to use pymongo directly.
col = settings.COL_OBJECTS
grid = mongo_connector("%s.files" % col)
if grid.find({'md5': md5sum}).count() == 0:
put_file(filename, data, collection=col)
if add_indicator and is_validate_only == False:
from crits.indicators.handlers import handle_indicator_ind
if object_type != name:
object_type = "%s - %s" % (object_type, name)
campaign = obj.campaign if hasattr(obj, 'campaign') else None
ind_res = handle_indicator_ind(value,
source,
reference,
object_type,
analyst,
method,
add_domain=True,
campaign=campaign,
cache=cache)
if ind_res['success']:
ind = ind_res['object']
forge_relationship(left_class=obj,
right_class=ind,
rel_type="Related_To",
analyst=analyst,
get_rels=is_sort_relationships)
else:
results['message'] = "Object was added, but failed to add Indicator." \
"<br>Error: " + ind_res.get('message')
if is_sort_relationships == True:
if file_ or add_indicator:
# does this line need to be here?
# obj.reload()
results['relationships'] = obj.sort_relationships(
analyst, meta=True)
else:
results['relationships'] = obj.sort_relationships(
analyst, meta=True)
else:
results[
'message'] = "Object already exists! [Type: " + object_type + "][Value: " + value + "] "
results['success'] = False
if (get_objects):
results['objects'] = obj.sort_objects()
results['id'] = str(obj.id)
return results
except ValidationError, e:
return {'success': False, 'message': str(e)}
def add_object(type_,
id_,
object_type,
source,
method,
reference,
user,
value=None,
file_=None,
add_indicator=False,
get_objects=True,
tlo=None,
is_sort_relationships=False,
is_validate_only=False,
is_validate_locally=False,
cache={},
**kwargs):
"""
Add an object to the database.
:param type_: The top-level object type.
:type type_: str
:param id_: The ObjectId of the top-level object.
:type id_: str
:param object_type: The type of the ObjectType being added.
:type object_type: str
:param source: The name of the source adding this object.
:type source: str
:param method: The method for this object.
:type method: str
:param reference: The reference for this object.
:type reference: str
:param user: The user adding this object.
:type user: str
:param value: The value of the object.
:type value: str
:param file_: The file if the object is a file upload.
:type file_: file handle.
:param add_indicator: Also add an indicator for this object.
:type add_indicator: bool
:param get_objects: Return the formatted list of objects when completed.
:type get_object: bool
:param tlo: The CRITs top-level object we are adding objects to.
This is an optional parameter used mainly for performance
reasons (by not querying mongo if we already have the
top level-object).
:type tlo: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param is_validate_only: Validate, but do not add to TLO.
:type is_validate_only: bool
:param is_validate_locally: Validate, but do not add b/c there is no TLO.
:type is_validate_locally: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" (str),
"objects" (list),
"relationships" (list)
"""
results = {}
obj = tlo
if id_ == None:
id_ = ""
if obj == None:
obj = class_from_id(type_, id_)
from crits.indicators.handlers import validate_indicator_value
if value is not None:
(value, error) = validate_indicator_value(value, object_type)
if error:
return {"success": False, "message": error}
if is_validate_locally: # no obj provided
results['success'] = True
return results
if not obj:
results['message'] = "TLO could not be found"
results['success'] = False
return results
try:
cur_len = len(obj.obj)
if file_:
data = file_.read()
filename = file_.name
md5sum = md5(data).hexdigest()
value = md5sum
reference = filename
obj.add_object(object_type, value, source, method, reference, user)
if is_validate_only == False:
obj.save(username=user)
new_len = len(obj.obj)
if new_len > cur_len:
if not is_validate_only:
results['message'] = "Object added successfully!"
results['success'] = True
if file_:
# do we have a pcap?
if data[:4] in ('\xa1\xb2\xc3\xd4', '\xd4\xc3\xb2\xa1',
'\x0a\x0d\x0d\x0a'):
handle_pcap_file(filename,
data,
source,
user=user,
related_id=id_,
related_type=type_)
else:
#XXX: MongoEngine provides no direct GridFS access so we
# need to use pymongo directly.
col = settings.COL_OBJECTS
grid = mongo_connector("%s.files" % col)
if grid.find({'md5': md5sum}).count() == 0:
put_file(filename, data, collection=col)
if add_indicator and is_validate_only == False:
from crits.indicators.handlers import handle_indicator_ind
campaign = obj.campaign if hasattr(obj, 'campaign') else None
ind_res = handle_indicator_ind(value,
source,
object_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
user,
method,
reference,
add_domain=True,
campaign=campaign,
cache=cache)
if ind_res['success']:
ind = ind_res['object']
forge_relationship(class_=obj,
right_class=ind,
rel_type=RelationshipTypes.RELATED_TO,
user=user,
get_rels=is_sort_relationships)
else:
results['message'] = "Object was added, but failed to add Indicator." \
"<br>Error: " + ind_res.get('message')
if is_sort_relationships == True:
if file_ or add_indicator:
# does this line need to be here?
# obj.reload()
results['relationships'] = obj.sort_relationships(
user, meta=True)
else:
results['relationships'] = obj.sort_relationships(
user, meta=True)
else:
results[
'message'] = "Object already exists! [Type: " + object_type + "][Value: " + value + "] "
results['success'] = False
if (get_objects):
results['objects'] = obj.sort_objects()
results['id'] = str(obj.id)
return results
except ValidationError, e:
return {'success': False, 'message': str(e)}
def import_object(request, type_, id_):
setup_access()
user = request.user
if type_ == "Threat Descriptors":
obj = ThreatDescriptor(id=id_)
obj.details(
fields=[f for f in ThreatDescriptor._default_fields if f not in
(td.PRIVACY_MEMBERS, td.METADATA)]
)
itype = get_mapped_itype(obj.get(td.TYPE))
tags = obj.get(td.TAGS)
if itype is None:
return {'success': False,
'message': "Descriptor type is not supported by CRITs"}
if not user.has_access_to(IndicatorACL.WRITE):
return {'success': False,
'message': "User does not have permission to add Indicators to CRITs"}
ithreat_type = getattr(IndicatorThreatTypes, obj.get(td.THREAT_TYPE))
results = handle_indicator_ind(
obj.get(td.RAW_INDICATOR),
"ThreatExchange",
itype,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
request.user.username,
method="ThreatExchange Service",
reference="id: %s, owner: %s, share_level: %s" % (obj.get(td.ID),
obj.get(td.OWNER)['name'],
obj.get(td.SHARE_LEVEL)),
add_domain=True,
add_relationship=True,
confidence=build_ci(obj.get(td.CONFIDENCE)),
description=obj.get(td.DESCRIPTION),
bucket_list=tags
)
return results
elif type_ == "Malware Analyses":
if not user.has_access_to(SampleACL.WRITE):
return {'success': False,
'message': "User does not have permission to add Sample to CRITs"}
obj = Malware(id=id_)
obj.details(
fields=[f for f in Malware._fields if f not in
(m.METADATA)]
)
filename = obj.get(m.MD5)
tags = obj.get(m.TAGS)
try:
data = obj.rf
except:
data = None
results = handle_file(
filename,
data,
"ThreatExchange",
method="ThreatExchange Service",
reference="id: %s, share_level: %s" % (obj.get(td.ID),
obj.get(td.SHARE_LEVEL)),
user=request.user.username,
md5_digest = obj.get(m.MD5),
sha1_digest = obj.get(m.SHA1),
sha256_digest = obj.get(m.SHA256),
size = obj.get(m.SAMPLE_SIZE),
mimetype = obj.get(m.SAMPLE_TYPE),
bucket_list=tags,
)
return {'success': True,
'md5': results}
else:
return {'success': False,
'message': "Invalid Type"}
return {'success': True}
def add_object(type_, oid, object_type, name, source, method,
reference, analyst, value=None, file_=None,
add_indicator=False, get_objects=True,
obj=None, is_sort_relationships=False,
is_validate_only=False, is_validate_locally=False, cache={}):
"""
Add an object to the database.
:param type_: The top-level object type.
:type type_: str
:param oid: The ObjectId of the top-level object.
:type oid: str
:param object_type: The type of the ObjectType being added.
:type object_type: str
:param name: The name of the ObjectType being added.
:type name: str
:param source: The name of the source adding this object.
:type source: str
:param method: The method for this object.
:type method: str
:param reference: The reference for this object.
:type reference: str
:param analyst: The user adding this object.
:type analyst: str
:param value: The value of the object.
:type value: str
:param file_: The file if the object is a file upload.
:type file_: file handle.
:param add_indicator: Also add an indicator for this object.
:type add_indicator: bool
:param get_objects: Return the formatted list of objects when completed.
:type get_object: bool
:param obj: The CRITs top-level object we are adding objects to.
This is an optional parameter used mainly for performance
reasons (by not querying mongo if we already have the
top level-object).
:type obj: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param is_validate_only: Only validate, do not add.
:type is_validate_only: bool
:param is_validate_locally: Only validate, do not add.
:type is_validate_locally: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" (str),
"objects" (list),
"relationships" (list)
"""
results = {}
if oid == None:
oid = ""
if obj == None:
obj = class_from_id(type_, oid)
if not obj:
if is_validate_locally == True:
# TODO: Perform some form of validation
results['success'] = True
return results
else:
results['message'] = "Could not find item to add object to."
results['success'] = False
return results
if name == "URL" and "://" not in value.split('.')[0]:
return {"success" : False, "message" : "URI - URL must contain protocol prefix (e.g. http://, https://, ftp://)"}
elif object_type == "Address":
if "ipv4" in name:
try:
validate_ipv4_address(value)
except DjangoValidationError:
return {"success" : False, "message" : "Invalid IPv4 address. "}
elif "ipv6" in name:
try:
validate_ipv6_address(value)
except DjangoValidationError:
return {"success" : False, "message" : "Invalid IPv6 address. "}
elif "cidr" in name:
try:
if '/' not in value:
raise ValidationError("")
cidr_parts = value.split('/')
if int(cidr_parts[1]) < 0 or int(cidr_parts[1]) > 128:
raise ValidationError("")
if ':' not in cidr_parts[0] and int(cidr_parts[1]) > 32:
raise ValidationError("")
validate_ipv46_address(cidr_parts[0])
except (ValidationError, ValueError) as cidr_error:
return {"success" : False, "message" : "Invalid CIDR address. "}
try:
cur_len = len(obj.obj)
if file_:
data = file_.read()
filename = file_.name
md5sum = md5(data).hexdigest()
value = md5sum
reference = filename
obj.add_object(object_type,
name,
value,
source,
method,
reference,
analyst)
if is_validate_only == False:
obj.save(username=analyst)
new_len = len(obj.obj)
if new_len > cur_len:
results['message'] = "Object added successfully!"
results['success'] = True
if file_:
# do we have a pcap?
if data[:4] in ('\xa1\xb2\xc3\xd4',
'\xd4\xc3\xb2\xa1',
'\x0a\x0d\x0d\x0a'):
handle_pcap_file(filename,
data,
source,
user=analyst,
related_id=oid,
related_type=type_)
else:
#XXX: MongoEngine provides no direct GridFS access so we
# need to use pymongo directly.
col = settings.COL_OBJECTS
grid = mongo_connector("%s.files" % col)
if grid.find({'md5': md5sum}).count() == 0:
put_file(filename, data, collection=col)
if add_indicator and is_validate_only == False:
from crits.indicators.handlers import handle_indicator_ind
if object_type != name:
object_type = "%s - %s" % (object_type, name)
campaign = obj.campaign if hasattr(obj, 'campaign') else None
ind_res = handle_indicator_ind(value,
source,
reference,
object_type,
analyst,
method,
add_domain=True,
campaign=campaign,
cache=cache)
if ind_res['success']:
ind = ind_res['object']
forge_relationship(left_class=obj,
right_class=ind,
rel_type="Related_To",
analyst=analyst,
get_rels=is_sort_relationships)
else:
results['message'] = "Object was added, but failed to add Indicator." \
"<br>Error: " + ind_res.get('message')
if is_sort_relationships == True:
if file_ or add_indicator:
# does this line need to be here?
# obj.reload()
results['relationships'] = obj.sort_relationships(analyst, meta=True)
else:
results['relationships'] = obj.sort_relationships(analyst, meta=True)
else:
results['message'] = "Object already exists! [Type: " + object_type + "][Value: " + value + "] "
results['success'] = False
if (get_objects):
results['objects'] = obj.sort_objects()
results['id'] = str(obj.id)
return results
except ValidationError, e:
return {'success': False,
'message': str(e)}
def add_object(type_, oid, object_type, name, source, method,
reference, analyst, value=None, file_=None,
add_indicator=False, get_objects=True,
indicator_campaign=None, indicator_campaign_confidence=None,
obj=None, is_sort_relationships=False,
is_validate_only=False, is_validate_locally=False, cache={}):
"""
Add an object to the database.
:param type_: The top-level object type.
:type type_: str
:param oid: The ObjectId of the top-level object.
:type oid: str
:param object_type: The type of the ObjectType being added.
:type object_type: str
:param name: The name of the ObjectType being added.
:type name: str
:param source: The name of the source adding this object.
:type source: str
:param method: The method for this object.
:type method: str
:param reference: The reference for this object.
:type reference: str
:param analyst: The user adding this object.
:type analyst: str
:param value: The value of the object.
:type value: str
:param file_: The file if the object is a file upload.
:type file_: file handle.
:param add_indicator: Also add an indicator for this object.
:type add_indicator: bool
:param get_objects: Return the formatted list of objects when completed.
:type get_object: bool
:param is_validate_only: Only validate, do not add.
:type is_validate_only: bool
:param is_validate_locally: Only validate, do not add.
:type is_validate_locally: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:param obj: The CRITs top-level object we are adding objects to.
This is an optional parameter used mainly for performance
reasons (by not querying mongo if we already have the
top level-object).
:type obj: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:returns: dict with keys:
"success" (boolean),
"message" (str),
"objects" (list),
"relationships" (list)
"""
results = {}
if oid == None:
oid = ""
if obj == None:
obj = class_from_id(type_, oid)
if not obj:
if is_validate_locally == True:
# TODO: Perform some form of validation
results['success'] = True
return results
else:
results['message'] = "Could not find item to add object to."
results['success'] = False
return results
try:
cur_len = len(obj.obj)
if file_:
data = file_.read()
filename = file_.name
md5sum = md5(data).hexdigest()
value = md5sum
reference = filename
obj.add_object(object_type,
name,
value,
source,
method,
reference,
analyst)
if is_validate_only == False:
obj.save(username=analyst)
new_len = len(obj.obj)
if new_len > cur_len:
results['message'] = "Object added successfully!"
results['success'] = True
if file_:
# do we have a pcap?
if data[:4] in ('\xa1\xb2\xc3\xd4',
'\xd4\xc3\xb2\xa1',
'\x0a\x0d\x0d\x0a'):
handle_pcap_file(filename,
data,
source,
user=analyst,
parent_id=oid,
parent_type=type_)
else:
#XXX: MongoEngine provides no direct GridFS access so we
# need to use pymongo directly.
col = settings.COL_OBJECTS
grid = mongo_connector("%s.files" % col)
if grid.find({'md5': md5sum}).count() == 0:
put_file(filename, data, collection=col)
if add_indicator and is_validate_only == False:
from crits.indicators.handlers import handle_indicator_ind
if object_type != name:
object_type = "%s - %s" % (object_type, name)
ind_res = handle_indicator_ind(value,
source,
reference,
object_type,
analyst,
method=method,
add_domain=True,
campaign=indicator_campaign,
campaign_confidence=indicator_campaign_confidence,
cache=cache)
if ind_res['success']:
ind = ind_res['object']
# Inherit campaigns from top level item when creating
# an indicator from an object if no campaigns were specified
if indicator_campaign == None and ind != None:
for campaign in obj.campaign:
ec = EmbeddedCampaign(name=campaign.name,
confidence=campaign.confidence,
description="",
analyst=analyst,
date=datetime.datetime.now())
ind.add_campaign(ec)
ind.save(username=analyst)
forge_relationship(left_class=obj,
right_class=ind,
rel_type="Related_To",
analyst=analyst,
get_rels=is_sort_relationships)
if is_sort_relationships == True:
if file_ or add_indicator:
# does this line need to be here?
# obj.reload()
results['relationships'] = obj.sort_relationships(analyst, meta=True)
else:
results['relationships'] = obj.sort_relationships(analyst, meta=True)
else:
results['message'] = "Object already exists! [Type: " + object_type + "][Value: " + value + "] "
results['success'] = False
if (get_objects):
results['objects'] = obj.sort_objects()
return results
except ValidationError, e:
return {'success': False,
'message': e}
def ip_add_update(ip_address, ip_type, source=None, source_method='',
source_reference='', source_tlp=None, campaign=None,
confidence='low', user=None, is_add_indicator=False,
indicator_reference='', bucket_list=None, ticket=None,
is_validate_only=False, cache={}, related_id=None,
related_type=None, relationship_type=None, description=''):
"""
Add/update an IP address.
:param ip_address: The IP to add/update.
:type ip_address: str
:param ip_type: The type of IP this is.
:type ip_type: str
:param source: Name of the source which provided this information.
:type source: str
:param source_method: Method of acquiring this data.
:type source_method: str
:param source_reference: A reference to this data.
:type source_reference: str
:param campaign: A campaign to attribute to this IP address.
:type campaign: str
:param confidence: Confidence level in the campaign attribution.
:type confidence: str ("low", "medium", "high")
:param user: The user adding/updating this IP.
:type user: str
:param is_add_indicator: Also add an Indicator for this IP.
:type is_add_indicator: bool
:param indicator_reference: Reference for the indicator.
:type indicator_reference: str
:param bucket_list: Buckets to assign to this IP.
:type bucket_list: str
:param ticket: Ticket to assign to this IP.
:type ticket: str
:param is_validate_only: Only validate, do not add/update.
:type is_validate_only: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_type: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:param description: A description for this IP
:type description: str
:returns: dict with keys:
"success" (boolean),
"message" (str),
"object" (if successful) :class:`crits.ips.ip.IP`
"""
if not source:
return {"success" : False, "message" : "Missing source information."}
source_name = source
(ip_address, error) = validate_and_normalize_ip(ip_address, ip_type)
if error:
return {"success": False, "message": error}
retVal = {}
is_item_new = False
ip_object = None
cached_results = cache.get(form_consts.IP.CACHED_RESULTS)
if cached_results != None:
ip_object = cached_results.get(ip_address)
else:
ip_object = IP.objects(ip=ip_address).first()
if not ip_object:
ip_object = IP()
ip_object.ip = ip_address
ip_object.ip_type = ip_type
is_item_new = True
if cached_results != None:
cached_results[ip_address] = ip_object
if not ip_object.description:
ip_object.description = description or ''
elif ip_object.description != description:
ip_object.description += "\n" + (description or '')
if isinstance(source_name, basestring):
if user.check_source_write(source):
source = [create_embedded_source(source,
reference=source_reference,
method=source_method,
tlp=source_tlp,
analyst=user.username)]
else:
return {"success":False,
"message": "User does not have permission to add object \
using source %s." % source}
if isinstance(campaign, basestring):
c = EmbeddedCampaign(name=campaign, confidence=confidence, analyst=user.username)
campaign = [c]
if campaign:
for camp in campaign:
ip_object.add_campaign(camp)
if source:
for s in source:
ip_object.add_source(s)
else:
return {"success" : False, "message" : "Missing source information."}
if bucket_list:
ip_object.add_bucket_list(bucket_list, user.username)
if ticket:
ip_object.add_ticket(ticket, user.username)
related_obj = None
if related_id:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
retVal['success'] = False
retVal['message'] = 'Related Object not found.'
return retVal
resp_url = reverse('crits.ips.views.ip_detail', args=[ip_object.ip])
if is_validate_only == False:
ip_object.save(username=user.username)
#set the URL for viewing the new data
if is_item_new == True:
retVal['message'] = ('Success! Click here to view the new IP: '
'<a href="%s">%s</a>' % (resp_url, ip_object.ip))
else:
message = ('Updated existing IP: '
'<a href="%s">%s</a>' % (resp_url, ip_object.ip))
retVal['message'] = message
retVal['status'] = form_consts.Status.DUPLICATE
retVal['warning'] = message
elif is_validate_only == True:
if ip_object.id != None and is_item_new == False:
message = ('Warning: IP already exists: '
'<a href="%s">%s</a>' % (resp_url, ip_object.ip))
retVal['message'] = message
retVal['status'] = form_consts.Status.DUPLICATE
retVal['warning'] = message
if is_add_indicator:
from crits.indicators.handlers import handle_indicator_ind
result = handle_indicator_ind(ip_address,
source_name,
ip_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
user,
source_method=source_method,
source_reference = indicator_reference,
source_tlp = source_tlp,
add_domain=False,
add_relationship=True,
bucket_list=bucket_list,
ticket=ticket,
cache=cache)
if related_obj and ip_object and relationship_type:
relationship_type=RelationshipTypes.inverse(relationship=relationship_type)
ip_object.add_relationship(related_obj,
relationship_type,
analyst=user.username,
get_rels=False)
ip_object.save(username=user.username)
# run ip triage
if is_item_new and is_validate_only == False:
ip_object.reload()
run_triage(ip_object, user)
retVal['success'] = True
retVal['object'] = ip_object
return retVal
def parse_cybox_object(self, cbx_obj, description='', ind_id=None):
"""
Parse a CybOX object form a STIX doc. An object can contain
multiple related_objects, which in turn can have their own
related_objects, so this handles those recursively.
:param cbx_obj: The CybOX object to parse.
:type cbx_obj: A CybOX object.
:param description: Parent-level (e.g. Observable) description.
:type description: str
:param ind_id: The ID of a parent STIX Indicator.
:type ind_id: str
"""
# check for missing attributes
if not cbx_obj or not cbx_obj.properties:
if cbx_obj.idref: # just a reference, so nothing to parse
return
else:
cbx_id = getattr(cbx_obj, 'id_', 'None')
self.failed.append(("No valid object_properties was found!",
"Observable (%s)" % cbx_id,
cbx_id)) # note for display in UI
return
# Don't parse if already been parsed
# This is for artifacts that are related to CybOX File Objects
if cbx_obj.id_ in self.parsed:
return
try: # try to create CRITs object from Cybox Object
analyst = self.source_instance.analyst
item = cbx_obj.properties
val = cbx_obj.id_
if isinstance(item, Address) and not ind_id:
if item.category in ('cidr', 'ipv4-addr', 'ipv4-net',
'ipv4-netmask', 'ipv6-addr',
'ipv6-net', 'ipv6-netmask'):
imp_type = "IP"
for value in item.address_value.values:
val = str(value).strip()
if self.preview:
res = None
else:
iptype = get_crits_ip_type(item.category)
if iptype:
res = ip_add_update(val,
iptype,
[self.source],
analyst=analyst,
is_add_indicator=True)
else:
res = {'success': False, 'reason': 'No IP Type'}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
if (not ind_id and (isinstance(item, DomainName) or
(isinstance(item, URI) and item.type_ == 'Domain Name'))):
imp_type = "Domain"
for val in item.value.values:
if self.preview:
res = None
else:
res = upsert_domain(str(val),
[self.source],
username=analyst)
self.parse_res(imp_type, str(val), cbx_obj, res, ind_id)
elif isinstance(item, HTTPSession):
imp_type = "RawData"
val = cbx_obj.id_
try:
c_req = item.http_request_response[0].http_client_request
hdr = c_req.http_request_header
if hdr.raw_header:
data = hdr.raw_header.value
title = "HTTP Header from STIX: %s" % self.package.id_
method = self.source_instance.method
ref = self.source_instance.reference
if self.preview:
res = None
val = title
else:
res = handle_raw_data_file(data,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="HTTP Header",
tool_name="STIX",
tool_version=None,
method=method,
reference=ref)
else:
imp_type = "Indicator"
ind_type = "HTTP Request Header Fields - User-Agent"
val = hdr.parsed_header.user_agent.value
val = ','.join(val) if isinstance(val, list) else val
if self.preview:
res = None
else:
res = handle_indicator_ind(val,
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_relationship=True,
description=description)
except:
msg = "Unsupported use of 'HTTPSession' object."
res = {'success': False, 'reason': msg}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, WhoisEntry):
# No sure where else to put this
imp_type = "RawData"
val = cbx_obj.id_
if item.remarks:
data = item.remarks.value
title = "WHOIS Entry from STIX: %s" % self.package.id_
if self.preview:
res = None
val = title
else:
res = handle_raw_data_file(data,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="Text",
tool_name="WHOIS",
tool_version=None,
method=self.source_instance.method,
reference=self.source_instance.reference)
else:
msg = "Unsupported use of 'WhoisEntry' object."
res = {'success': False, 'reason': msg}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, Artifact):
# Not sure if this is right, and I believe these can be
# encoded in a couple different ways.
imp_type = "RawData"
val = cbx_obj.id_
rawdata = item.data.decode('utf-8')
# TODO: find out proper ways to determine title, datatype,
# tool_name, tool_version
title = "Artifact for Event: STIX Document %s" % self.package.id_
if self.preview:
res = None
val = title
else:
res = handle_raw_data_file(rawdata,
self.source.name,
user=analyst,
description=description,
title=title,
data_type="Text",
tool_name="STIX",
tool_version=None,
method=self.source_instance.method,
reference=self.source_instance.reference)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif (isinstance(item, File) and
item.custom_properties and
item.custom_properties[0].name == "crits_type" and
item.custom_properties[0]._value == "Certificate"):
imp_type = "Certificate"
val = str(item.file_name)
data = None
if self.preview:
res = None
else:
for rel_obj in item.parent.related_objects:
if isinstance(rel_obj.properties, Artifact):
data = rel_obj.properties.data
self.parsed.append(rel_obj.id_)
res = handle_cert_file(val,
data,
self.source,
user=analyst,
description=description)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, File) and self.has_network_artifact(item):
imp_type = "PCAP"
val = str(item.file_name)
data = None
if self.preview:
res = None
else:
for rel_obj in item.parent.related_objects:
if (isinstance(rel_obj.properties, Artifact) and
rel_obj.properties.type_ == Artifact.TYPE_NETWORK):
data = rel_obj.properties.data
self.parsed.append(rel_obj.id_)
res = handle_pcap_file(val,
data,
self.source,
user=analyst,
description=description)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, File):
imp_type = "Sample"
md5 = item.md5
if md5:
md5 = md5.lower()
val = str(item.file_name or md5)
# add sha1/sha256/ssdeep once handle_file supports it
size = item.size_in_bytes
data = None
if item.file_path:
path = "File Path: " + str(item.file_path)
description += "\n" + path
for rel_obj in item.parent.related_objects:
if (isinstance(rel_obj.properties, Artifact) and
rel_obj.properties.type_ == Artifact.TYPE_FILE):
data = rel_obj.properties.data
self.parsed.append(rel_obj.id_)
if not md5 and not data and val and val != "None":
imp_type = "Indicator"
if self.preview:
res = None
else:
res = handle_indicator_ind(val,
self.source,
"Win File",
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True,
description=description)
elif md5 or data:
if self.preview:
res = None
else:
res = handle_file(val,
data,
self.source,
user=analyst,
md5_digest=md5,
is_return_only_md5=False,
size=size,
description=description)
else:
val = cbx_obj.id_
msg = "CybOX 'File' object has no MD5, data, or filename"
res = {'success': False, 'reason': msg}
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
elif isinstance(item, EmailMessage):
imp_type = 'Email'
id_list = []
data = {}
val = cbx_obj.id_
get_attach = False
data['raw_body'] = str(item.raw_body)
data['raw_header'] = str(item.raw_header)
data['helo'] = str(item.email_server)
if item.header:
data['subject'] = str(item.header.subject)
if item.header.date:
data['date'] = item.header.date.value
val = "Date: %s, Subject: %s" % (data.get('date', 'None'),
data['subject'])
data['message_id'] = str(item.header.message_id)
data['sender'] = str(item.header.sender)
data['reply_to'] = str(item.header.reply_to)
data['x_originating_ip'] = str(item.header.x_originating_ip)
data['x_mailer'] = str(item.header.x_mailer)
data['boundary'] = str(item.header.boundary)
data['from_address'] = str(item.header.from_)
if item.header.to:
data['to'] = [str(r) for r in item.header.to.to_list()]
if data.get('date'): # Email TLOs must have a date
data['source'] = self.source.name
data['source_method'] = self.source_instance.method
data['source_reference'] = self.source_instance.reference
if self.preview:
res = None
else:
res = handle_email_fields(data,
analyst,
"STIX")
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
if not self.preview and res.get('status'):
id_list.append(cbx_obj.id_) # save ID for atchmnt rels
get_attach = True
else: # Can't be an Email TLO, so save fields
for x, key in enumerate(data):
if data[key] and data[key] != "None":
if key in ('raw_header', 'raw_body'):
if key == 'raw_header':
title = "Email Header from STIX Email: %s"
d_type = "Email Header"
else:
title = "Email Body from STIX Email: %s"
d_type = "Email Body"
imp_type = 'RawData'
title = title % cbx_obj.id_
if self.preview:
res = None
else:
res = handle_raw_data_file(data[key],
self.source,
analyst,
description,
title,
d_type,
"STIX",
self.stix_version)
self.parse_res(imp_type, title, cbx_obj,
res, ind_id)
elif key == 'to':
imp_type = 'Target'
for y, addr in enumerate(data[key]):
tgt_dict = {'email_address': addr}
if self.preview:
res = None
else:
res = upsert_target(tgt_dict, analyst)
if res['success']:
get_attach = True
tmp_obj = copy(cbx_obj)
tmp_obj.id_ = '%s-%s-%s' % (cbx_obj.id_,
x, y)
self.parse_res(imp_type, addr, tmp_obj,
res, ind_id)
self.ind2obj.setdefault(cbx_obj.id_,
[]).append(tmp_obj.id_)
id_list.append(tmp_obj.id_)
else:
imp_type = 'Indicator'
if key in ('sender', 'reply_to', 'from_address'):
ind_type = "Address - e-mail"
elif 'ip' in key:
ind_type = "Address - ipv4-addr"
elif key == 'raw_body':
ind_type = "Email Message"
else:
ind_type = "String"
if self.preview:
res = None
else:
res = handle_indicator_ind(data[key],
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True,
description=description)
if res['success']:
get_attach = True
tmp_obj = copy(cbx_obj)
tmp_obj.id_ = '%s-%s' % (cbx_obj.id_, x)
self.parse_res(imp_type, data[key], tmp_obj,
res, ind_id)
self.ind2obj.setdefault(cbx_obj.id_,
[]).append(tmp_obj.id_)
id_list.append(tmp_obj.id_)
if not self.preview:
# Setup relationships between all Email attributes
for oid in id_list:
for oid2 in id_list:
if oid != oid2:
self.relationships.append((oid,
RelationshipTypes.RELATED_TO,
oid2, "High"))
# Should check for attachments and add them here.
if get_attach and item.attachments:
for attach in item.attachments:
rel_id = attach.to_dict()['object_reference']
for oid in id_list:
self.relationships.append((oid,
RelationshipTypes.CONTAINS,
rel_id, "High"))
else: # try to parse all other possibilities as Indicator
imp_type = "Indicator"
val = cbx_obj.id_
c_obj = make_crits_object(item)
# Ignore what was already caught above
if (ind_id or c_obj.object_type not in IPTypes.values()):
ind_type = c_obj.object_type
for val in [str(v).strip() for v in c_obj.value if v]:
if ind_type:
# handle domains mislabeled as URLs
if c_obj.object_type == 'URI' and '/' not in val:
ind_type = "Domain"
if self.preview:
res = None
else:
res = handle_indicator_ind(val,
self.source,
ind_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
analyst,
add_domain=True,
add_relationship=True,
description=description)
self.parse_res(imp_type, val, cbx_obj, res, ind_id)
except Exception, e: # probably caused by cybox object we don't handle
self.failed.append((e.message,
"%s (%s)" % (imp_type, val),
cbx_obj.id_)) # note for display in UI
