def setUp(self):
prep_db()
self.user = CRITsUser.objects(username=TUSER_NAME).first()
self.user2 = CRITsUser.objects(username=TUSER2_NAME).first()
self.campaign1 = Campaign.objects(name=TCAMPAIGN1).first()
self.campaign2 = Campaign.objects(name=TCAMPAIGN2).first()
forge_relationship(left_class=self.campaign1,
right_class=self.campaign2,
rel_type=TRELATIONSHIP_TYPE,
analyst=self.user.username,
rel_confidence=TRELATIONSHIP_CONFIDENCE)
def setUp(self):
prep_db()
self.user = CRITsUser.objects(username=TUSER_NAME).first()
self.user2 = CRITsUser.objects(username=TUSER2_NAME).first()
self.campaign1 = Campaign.objects(name=TCAMPAIGN1).first()
self.campaign2 = Campaign.objects(name=TCAMPAIGN2).first()
forge_relationship(class_=self.campaign1,
right_class=self.campaign2,
rel_type=TRELATIONSHIP_TYPE,
user=self.user.username,
rel_confidence=TRELATIONSHIP_CONFIDENCE)
def obj_create(self, bundle, **kwargs):
"""
Handles forging relationships through the API.
:param bundle: Bundle containing the relationship information.
:type bundle: Tastypie Bundle object.
:returns: Bundle object.
:raises BadRequest: If necessary data is not provided or creation fails.
"""
analyst = bundle.request.user.username
left_type = bundle.data.get('left_type', None)
left_id = bundle.data.get('left_id', None)
right_type = bundle.data.get('right_type', None)
right_id = bundle.data.get('right_id', None)
rel_type = bundle.data.get('rel_type', None)
rel_date = bundle.data.get('rel_date', None)
if (not left_type or not left_id or not right_type or not right_id
or not rel_type):
raise BadRequest('Need all of the relationship information.')
result = forge_relationship(left_type=left_type,
left_id=left_id,
right_type=right_type,
right_id=right_id,
rel_type=rel_type,
rel_date=rel_date,
analyst=analyst,
get_rels=False)
if not result['success']:
raise BadRequest(result['message'])
else:
return bundle
def obj_create(self, bundle, **kwargs):
"""
Handles forging relationships through the API.
:param bundle: Bundle containing the relationship information.
:type bundle: Tastypie Bundle object.
:returns: HttpResponse.
"""
analyst = bundle.request.user.username
left_type = bundle.data.get('left_type', None)
left_id = bundle.data.get('left_id', None)
right_type = bundle.data.get('right_type', None)
right_id = bundle.data.get('right_id', None)
rel_type = bundle.data.get('rel_type', None)
rel_date = bundle.data.get('rel_date', None)
rel_confidence = bundle.data.get('rel_confidence', 'unknown')
rel_reason = bundle.data.get('rel_reason', 'N/A')
content = {'return_code': 1,
'type': left_type}
if (not left_type
or not left_id
or not right_type
or not right_id
or not rel_type):
content['message'] = 'Need all of the relationship information.'
self.crits_response(content)
if rel_confidence not in ('unknown', 'low', 'medium', 'high'):
content['message'] = 'Bad relationship confidence.'
self.crits_response(content)
result = forge_relationship(left_type=left_type,
left_id=left_id,
right_type=right_type,
right_id=right_id,
rel_type=rel_type,
rel_date=rel_date,
analyst=analyst,
rel_confidence=rel_confidence,
rel_reason=rel_reason,
get_rels=False)
if result.get('message'):
content['message'] = result.get('message')
content['id'] = left_id
rname = self.resource_name_from_type(left_type)
url = reverse('api_dispatch_detail',
kwargs={'resource_name': rname,
'api_name': 'v1',
'pk': left_id})
content['url'] = url
if result['success']:
content['return_code'] = 0
self.crits_response(content)
def obj_create(self, bundle, **kwargs):
"""
Handles forging relationships through the API.
:param bundle: Bundle containing the relationship information.
:type bundle: Tastypie Bundle object.
:returns: HttpResponse.
"""
analyst = bundle.request.user.username
left_type = bundle.data.get('left_type', None)
left_id = bundle.data.get('left_id', None)
right_type = bundle.data.get('right_type', None)
right_id = bundle.data.get('right_id', None)
rel_type = bundle.data.get('rel_type', None)
rel_date = bundle.data.get('rel_date', None)
rel_confidence = bundle.data.get('rel_confidence', 'unknown')
rel_reason = bundle.data.get('rel_reason', 'N/A')
content = {'return_code': 1, 'type': left_type}
if (not left_type or not left_id or not right_type or not right_id
or not rel_type):
content['message'] = 'Need all of the relationship information.'
self.crits_response(content)
if rel_confidence not in ('unknown', 'low', 'medium', 'high'):
content['message'] = 'Bad relationship confidence.'
self.crits_response(content)
result = forge_relationship(left_type=left_type,
left_id=left_id,
right_type=right_type,
right_id=right_id,
rel_type=rel_type,
rel_date=rel_date,
analyst=analyst,
rel_confidence=rel_confidence,
rel_reason=rel_reason,
get_rels=False)
if result.get('message'):
content['message'] = result.get('message')
content['id'] = left_id
rname = self.resource_name_from_type(left_type)
url = reverse('api_dispatch_detail',
kwargs={
'resource_name': rname,
'api_name': 'v1',
'pk': left_id
})
content['url'] = url
if result['success']:
content['return_code'] = 0
self.crits_response(content)
def add_new_relationship(request):
"""
Add a new relationship. Should be an AJAX POST.
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:returns: :class:`django.http.HttpResponse`
"""
if request.method == 'POST' and request.is_ajax():
form = ForgeRelationshipForm(request.POST)
user = request.user
choices = [(c,c) for c in RelationshipTypes.values(sort=True)]
form.fields['forward_relationship'].choices = choices
if form.is_valid():
cleaned_data = form.cleaned_data
# Get user permission to verify the user can forge relationships...
# Should we check permission on both the forward and reverse TLO for this?
acl = get_acl_object(cleaned_data.get('forward_type'))
if user.has_access_to(acl.RELATIONSHIPS_ADD):
results = forge_relationship(type_=cleaned_data.get('forward_type'),
id_=cleaned_data.get('forward_value'),
right_type=cleaned_data.get('reverse_type'),
right_id=cleaned_data.get('dest_id'),
rel_type=cleaned_data.get('forward_relationship'),
rel_date=cleaned_data.get('relationship_date'),
user=request.user.username,
rel_reason=cleaned_data.get('rel_reason'),
rel_confidence=cleaned_data.get('rel_confidence'),
get_rels=True)
else:
results = {"success":False,
"message":"User does not have permission to forge relationships"}
if results['success'] == True:
relationship = {'type': cleaned_data.get('forward_type'),
'value': cleaned_data.get('forward_value')}
message = render_to_string('relationships_listing_widget.html',
{'relationship': relationship,
'nohide': True,
'relationships': results['relationships']},
RequestContext(request))
result = {'success': True, 'message': message}
else:
message = "Error adding relationship: %s" % results['message']
result = {'success': False, 'message': message}
else:
message = "Invalid Form: %s" % form.errors
form = form.as_table()
result = {'success': False, 'form': form, 'message': message}
return HttpResponse(json.dumps(result), content_type="application/json")
else:
error = "Expected AJAX POST"
return render_to_response("error.html",
{"error" : error },
RequestContext(request))
def add_new_relationship(request):
"""
Add a new relationship. Should be an AJAX POST.
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:returns: :class:`django.http.HttpResponse`
"""
if request.method == 'POST' and request.is_ajax():
form = ForgeRelationshipForm(request.POST)
choices = [(c, c) for c in RelationshipTypes.values(sort=True)]
form.fields['forward_relationship'].choices = choices
if form.is_valid():
cleaned_data = form.cleaned_data
results = forge_relationship(
type_=cleaned_data.get('forward_type'),
id_=cleaned_data.get('forward_value'),
right_type=cleaned_data.get('reverse_type'),
right_id=cleaned_data.get('dest_id'),
rel_type=cleaned_data.get('forward_relationship'),
rel_date=cleaned_data.get('relationship_date'),
user=request.user.username,
rel_reason=cleaned_data.get('rel_reason'),
rel_confidence=cleaned_data.get('rel_confidence'),
get_rels=True)
if results['success'] == True:
set_releasability_flag(cleaned_data.get('forward_type'),
cleaned_data.get('forward_value'),
request.user.username)
relationship = {
'type': cleaned_data.get('forward_type'),
'url_key': cleaned_data.get('forward_value')
}
message = render_to_string(
'relationships_listing_widget.html', {
'relationship': relationship,
'nohide': True,
'relationships': results['relationships']
}, RequestContext(request))
result = {'success': True, 'message': message}
else:
message = "Error adding relationship: %s" % results['message']
result = {'success': False, 'message': message}
else:
message = "Invalid Form: %s" % form.errors
form = form.as_table()
result = {'success': False, 'form': form, 'message': message}
return HttpResponse(json.dumps(result), mimetype="application/json")
else:
error = "Expected AJAX POST"
return render_to_response("error.html", {"error": error},
RequestContext(request))
def add_new_relationship(request):
"""
Add a new relationship. Should be an AJAX POST.
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:returns: :class:`django.http.HttpResponse`
"""
if request.method == 'POST' and request.is_ajax():
form = ForgeRelationshipForm(request.POST)
choices = [(c,c) for c in RelationshipTypes.values(sort=True)]
form.fields['forward_relationship'].choices = choices
if form.is_valid():
cleaned_data = form.cleaned_data;
results = forge_relationship(type_=cleaned_data.get('forward_type'),
id_=cleaned_data.get('forward_value'),
right_type=cleaned_data.get('reverse_type'),
right_id=cleaned_data.get('dest_id'),
rel_type=cleaned_data.get('forward_relationship'),
rel_date=cleaned_data.get('relationship_date'),
user=request.user.username,
rel_reason=cleaned_data.get('rel_reason'),
rel_confidence=cleaned_data.get('rel_confidence'),
get_rels=True)
if results['success'] == True:
relationship = {'type': cleaned_data.get('forward_type'),
'value': cleaned_data.get('forward_value')}
message = render_to_string('relationships_listing_widget.html',
{'relationship': relationship,
'nohide': True,
'relationships': results['relationships']},
RequestContext(request))
result = {'success': True, 'message': message}
else:
message = "Error adding relationship: %s" % results['message']
result = {'success': False, 'message': message}
else:
message = "Invalid Form: %s" % form.errors
form = form.as_table()
result = {'success': False, 'form': form, 'message': message}
return HttpResponse(json.dumps(result), mimetype="application/json")
else:
error = "Expected AJAX POST"
return render_to_response("error.html",
{"error" : error },
RequestContext(request))
def add_new_relationship(request):
"""
Add a new relationship. Should be an AJAX POST.
:param request: Django request object (Required)
:type request: :class:`django.http.HttpRequest`
:returns: :class:`django.http.HttpResponse`
"""
if request.method == "POST" and request.is_ajax():
form = ForgeRelationshipForm(request.POST)
choices = [(c, c) for c in get_relationship_types(False)]
form.fields["forward_relationship"].choices = choices
if form.is_valid():
cleaned_data = form.cleaned_data
results = forge_relationship(
left_type=cleaned_data.get("forward_type"),
left_id=cleaned_data.get("forward_value"),
right_type=cleaned_data.get("reverse_type"),
right_id=cleaned_data.get("dest_id"),
rel_type=cleaned_data.get("forward_relationship"),
rel_date=cleaned_data.get("relationship_date"),
analyst=request.user.username,
get_rels=True,
)
if results["success"] == True:
relationship = {"type": cleaned_data.get("forward_type"), "value": cleaned_data.get("forward_value")}
message = render_to_string(
"relationships_listing_widget.html",
{"relationship": relationship, "nohide": True, "relationships": results["relationships"]},
RequestContext(request),
)
result = {"success": True, "message": message}
else:
message = "Error adding relationship: %s" % results["message"]
result = {"success": False, "message": message}
else:
message = "Invalid Form: %s" % form.errors
form = form.as_table()
result = {"success": False, "form": form, "message": message}
return HttpResponse(json.dumps(result), mimetype="application/json")
else:
error = "Expected AJAX POST"
return render_to_response("error.html", {"error": error}, RequestContext(request))
def parse_relationship(self, indicators):
from crits.relationships.handlers import forge_relationship
for indicator in indicators:
if self.was_saved(indicator):
obj = class_from_id(str(self.imported[indicator.id_][0]),
str(self.imported[indicator.id_][1].id))
for rel in getattr(indicator, 'related_indicators', ()):
if rel.item.title in 'MARTI Relation':
result = forge_relationship(
type_=obj._meta['crits_type'],
id_=obj.get_url_key(),
right_type=self.get_marti_type(
rel.item.indicator_types),
right_id=str(rel.item.id_),
rel_type=str(rel.item.short_description),
rel_date=rel.item.timestamp,
user='******',
rel_reason=str(rel.item.description),
rel_confidence=str(rel.item.confidence.value),
get_rels=True)
def obj_create(self, bundle, **kwargs):
"""
Handles forging relationships through the API.
:param bundle: Bundle containing the relationship information.
:type bundle: Tastypie Bundle object.
:returns: Bundle object.
:raises BadRequest: If necessary data is not provided or creation fails.
"""
analyst = bundle.request.user.username
left_type = bundle.data.get('left_type', None)
left_id = bundle.data.get('left_id', None)
right_type = bundle.data.get('right_type', None)
right_id = bundle.data.get('right_id', None)
rel_type = bundle.data.get('rel_type', None)
rel_date = bundle.data.get('rel_date', None)
if (not left_type
or not left_id
or not right_type
or not right_id
or not rel_type):
raise BadRequest('Need all of the relationship information.')
result = forge_relationship(left_type=left_type,
left_id=left_id,
right_type=right_type,
right_id=right_id,
rel_type=rel_type,
rel_date=rel_date,
analyst=analyst,
get_rels=False)
if not result['success']:
raise BadRequest(result['message'])
else:
return bundle
def add_object(type_, oid, object_type, name, source, method,
reference, analyst, value=None, file_=None,
add_indicator=False, get_objects=True,
obj=None, is_sort_relationships=False,
is_validate_only=False, is_validate_locally=False, cache={}):
"""
Add an object to the database.
:param type_: The top-level object type.
:type type_: str
:param oid: The ObjectId of the top-level object.
:type oid: str
:param object_type: The type of the ObjectType being added.
:type object_type: str
:param name: The name of the ObjectType being added.
:type name: str
:param source: The name of the source adding this object.
:type source: str
:param method: The method for this object.
:type method: str
:param reference: The reference for this object.
:type reference: str
:param analyst: The user adding this object.
:type analyst: str
:param value: The value of the object.
:type value: str
:param file_: The file if the object is a file upload.
:type file_: file handle.
:param add_indicator: Also add an indicator for this object.
:type add_indicator: bool
:param get_objects: Return the formatted list of objects when completed.
:type get_object: bool
:param obj: The CRITs top-level object we are adding objects to.
This is an optional parameter used mainly for performance
reasons (by not querying mongo if we already have the
top level-object).
:type obj: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param is_validate_only: Only validate, do not add.
:type is_validate_only: bool
:param is_validate_locally: Only validate, do not add.
:type is_validate_locally: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" (str),
"objects" (list),
"relationships" (list)
"""
results = {}
if oid == None:
oid = ""
if obj == None:
obj = class_from_id(type_, oid)
if not obj:
if is_validate_locally == True:
# TODO: Perform some form of validation
results['success'] = True
return results
else:
results['message'] = "Could not find item to add object to."
results['success'] = False
return results
if name == "URL" and "://" not in value.split('.')[0]:
return {"success" : False, "message" : "URI - URL must contain protocol prefix (e.g. http://, https://, ftp://)"}
elif object_type == "Address":
if "ipv4" in name:
try:
validate_ipv4_address(value)
except DjangoValidationError:
return {"success" : False, "message" : "Invalid IPv4 address. "}
elif "ipv6" in name:
try:
validate_ipv6_address(value)
except DjangoValidationError:
return {"success" : False, "message" : "Invalid IPv6 address. "}
elif "cidr" in name:
try:
if '/' not in value:
raise ValidationError("")
cidr_parts = value.split('/')
if int(cidr_parts[1]) < 0 or int(cidr_parts[1]) > 128:
raise ValidationError("")
if ':' not in cidr_parts[0] and int(cidr_parts[1]) > 32:
raise ValidationError("")
validate_ipv46_address(cidr_parts[0])
except (ValidationError, ValueError) as cidr_error:
return {"success" : False, "message" : "Invalid CIDR address. "}
try:
cur_len = len(obj.obj)
if file_:
data = file_.read()
filename = file_.name
md5sum = md5(data).hexdigest()
value = md5sum
reference = filename
obj.add_object(object_type,
name,
value,
source,
method,
reference,
analyst)
if is_validate_only == False:
obj.save(username=analyst)
new_len = len(obj.obj)
if new_len > cur_len:
results['message'] = "Object added successfully!"
results['success'] = True
if file_:
# do we have a pcap?
if data[:4] in ('\xa1\xb2\xc3\xd4',
'\xd4\xc3\xb2\xa1',
'\x0a\x0d\x0d\x0a'):
handle_pcap_file(filename,
data,
source,
user=analyst,
related_id=oid,
related_type=type_)
else:
#XXX: MongoEngine provides no direct GridFS access so we
# need to use pymongo directly.
col = settings.COL_OBJECTS
grid = mongo_connector("%s.files" % col)
if grid.find({'md5': md5sum}).count() == 0:
put_file(filename, data, collection=col)
if add_indicator and is_validate_only == False:
from crits.indicators.handlers import handle_indicator_ind
if object_type != name:
object_type = "%s - %s" % (object_type, name)
campaign = obj.campaign if hasattr(obj, 'campaign') else None
ind_res = handle_indicator_ind(value,
source,
reference,
object_type,
analyst,
method,
add_domain=True,
campaign=campaign,
cache=cache)
if ind_res['success']:
ind = ind_res['object']
forge_relationship(left_class=obj,
right_class=ind,
rel_type="Related_To",
analyst=analyst,
get_rels=is_sort_relationships)
else:
results['message'] = "Object was added, but failed to add Indicator." \
"<br>Error: " + ind_res.get('message')
if is_sort_relationships == True:
if file_ or add_indicator:
# does this line need to be here?
# obj.reload()
results['relationships'] = obj.sort_relationships(analyst, meta=True)
else:
results['relationships'] = obj.sort_relationships(analyst, meta=True)
else:
results['message'] = "Object already exists! [Type: " + object_type + "][Value: " + value + "] "
results['success'] = False
if (get_objects):
results['objects'] = obj.sort_objects()
results['id'] = str(obj.id)
return results
except ValidationError, e:
return {'success': False,
'message': str(e)}
def add_new_domain(data,
request,
errors,
rowData=None,
is_validate_only=False,
cache={}):
"""
Add a new domain to CRITs.
:param data: The data about the domain.
:type data: dict
:param request: The Django request.
:type request: :class:`django.http.HttpRequest`
:param errors: A list of current errors to append to.
:type errors: list
:param rowData: Any objects that need to be added to the domain.
:type rowData: dict
:param is_validate_only: Only validate the data and return any errors.
:type is_validate_only: boolean
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:returns: tuple (<result>, <errors>, <retVal>)
"""
result = False
retVal = {}
domain = data['domain']
add_ip = data.get('add_ip')
ip = data.get('ip')
ip_type = data.get('ip_type')
if add_ip:
error = validate_and_normalize_ip(ip, ip_type)[1]
if error:
errors.append(error)
if is_validate_only:
error = get_valid_root_domain(domain)[2]
if error:
errors.append(error)
# check for duplicate domains
fqdn_domain = retrieve_domain(domain, cache)
if fqdn_domain:
if isinstance(fqdn_domain, Domain):
resp_url = reverse('crits.domains.views.domain_detail',
args=[domain])
message = ('Warning: Domain already exists: '
'<a href="%s">%s</a>' % (resp_url, domain))
retVal['message'] = message
retVal['status'] = form_consts.Status.DUPLICATE
retVal['warning'] = message
else:
result_cache = cache.get(form_consts.Domain.CACHED_RESULTS)
result_cache[domain.lower()] = True
elif not errors:
username = request.user.username
reference = data.get('domain_reference')
source_name = data.get('domain_source')
method = data.get('domain_method')
source = [
create_embedded_source(source_name,
reference=reference,
method=method,
analyst=username)
]
bucket_list = data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
ticket = data.get(form_consts.Common.TICKET_VARIABLE_NAME)
if data.get('campaign') and data.get('confidence'):
campaign = [
EmbeddedCampaign(name=data.get('campaign'),
confidence=data.get('confidence'),
analyst=username)
]
else:
campaign = []
retVal = upsert_domain(domain,
source,
username,
campaign,
bucket_list=bucket_list,
ticket=ticket,
cache=cache)
if not retVal['success']:
errors.append(retVal.get('message'))
retVal['message'] = ""
else:
new_domain = retVal['object']
ip_result = {}
if add_ip:
if data.get('same_source'):
ip_source = source_name
ip_method = method
ip_reference = reference
else:
ip_source = data.get('ip_source')
ip_method = data.get('ip_method')
ip_reference = data.get('ip_reference')
from crits.ips.handlers import ip_add_update
ip_result = ip_add_update(ip,
ip_type,
ip_source,
ip_method,
ip_reference,
campaign=campaign,
analyst=username,
bucket_list=bucket_list,
ticket=ticket,
cache=cache)
if not ip_result['success']:
errors.append(ip_result['message'])
else:
#add a relationship with the new IP address
new_ip = ip_result['object']
if new_domain and new_ip:
new_domain.add_relationship(rel_item=new_ip,
rel_type='Resolved_To',
analyst=username,
get_rels=False)
new_domain.save(username=username)
new_ip.save(username=username)
#set the URL for viewing the new data
resp_url = reverse('crits.domains.views.domain_detail',
args=[domain])
if retVal['is_domain_new'] == True:
retVal['message'] = (
'Success! Click here to view the new domain: '
'<a href="%s">%s</a>' % (resp_url, domain))
else:
message = ('Updated existing domain: <a href="%s">%s</a>' %
(resp_url, domain))
retVal['message'] = message
retVal[form_consts.Status.
STATUS_FIELD] = form_consts.Status.DUPLICATE
retVal['warning'] = message
#add indicators
if data.get('add_indicators'):
from crits.indicators.handlers import create_indicator_from_tlo
# If we have an IP object, add an indicator for that.
if ip_result.get('success'):
ip = ip_result['object']
result = create_indicator_from_tlo('IP',
ip,
username,
ip_source,
add_domain=False)
ip_ind = result.get('indicator')
if not result['success']:
errors.append(result['message'])
# Add an indicator for the domain.
result = create_indicator_from_tlo('Domain',
new_domain,
username,
source_name,
add_domain=False)
if not result['success']:
errors.append(result['message'])
elif ip_result.get('success') and ip_ind:
forge_relationship(left_class=result['indicator'],
right_class=ip_ind,
rel_type='Resolved_To',
analyst=username)
result = True
# This block validates, and may also add, objects to the Domain
if retVal.get('success') or is_validate_only == True:
if rowData:
objectsData = rowData.get(form_consts.Common.OBJECTS_DATA)
# add new objects if they exist
if objectsData:
objectsData = json.loads(objectsData)
current_domain = retrieve_domain(domain, cache)
for object_row_counter, objectData in enumerate(
objectsData, 1):
if current_domain != None:
# if the domain exists then try to add objects to it
if isinstance(current_domain, Domain) == True:
objectDict = object_array_to_dict(
objectData, "Domain", current_domain.id)
else:
objectDict = object_array_to_dict(
objectData, "Domain", "")
current_domain = None
else:
objectDict = object_array_to_dict(
objectData, "Domain", "")
(obj_result, errors,
obj_retVal) = validate_and_add_new_handler_object(
None,
objectDict,
request,
errors,
object_row_counter,
is_validate_only=is_validate_only,
cache=cache,
obj=current_domain)
if not obj_result:
retVal['success'] = False
return result, errors, retVal
def add_new_domain(data, request, errors, rowData=None, is_validate_only=False, cache={}):
"""
Add a new domain to CRITs.
:param data: The data about the domain.
:type data: dict
:param request: The Django request.
:type request: :class:`django.http.HttpRequest`
:param errors: A list of current errors to append to.
:type errors: list
:param rowData: Any objects that need to be added to the domain.
:type rowData: dict
:param is_validate_only: Only validate the data and return any errors.
:type is_validate_only: boolean
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:returns: tuple (<result>, <errors>, <retVal>)
"""
result = False
retVal = {}
domain = data['domain']
add_ip = data.get('add_ip')
ip = data.get('ip')
ip_type = data.get('ip_type')
if add_ip:
error = validate_and_normalize_ip(ip, ip_type)[1]
if error:
errors.append(error)
if is_validate_only:
error = get_valid_root_domain(domain)[2]
if error:
errors.append(error)
# check for duplicate domains
fqdn_domain = retrieve_domain(domain, cache)
if fqdn_domain:
if isinstance(fqdn_domain, Domain):
resp_url = reverse('crits.domains.views.domain_detail', args=[domain])
message = ('Warning: Domain already exists: '
'<a href="%s">%s</a>' % (resp_url, domain))
retVal['message'] = message
retVal['status'] = form_consts.Status.DUPLICATE
retVal['warning'] = message
else:
result_cache = cache.get(form_consts.Domain.CACHED_RESULTS);
result_cache[domain.lower()] = True
elif not errors:
username = request.user.username
reference = data.get('domain_reference')
source_name = data.get('domain_source')
method = data.get('domain_method')
source = [create_embedded_source(source_name, reference=reference,
method=method, analyst=username)]
bucket_list = data.get(form_consts.Common.BUCKET_LIST_VARIABLE_NAME)
ticket = data.get(form_consts.Common.TICKET_VARIABLE_NAME)
if data.get('campaign') and data.get('confidence'):
campaign = [EmbeddedCampaign(name=data.get('campaign'),
confidence=data.get('confidence'),
analyst=username)]
else:
campaign = []
retVal = upsert_domain(domain, source, username, campaign,
bucket_list=bucket_list, ticket=ticket, cache=cache)
if not retVal['success']:
errors.append(retVal.get('message'))
retVal['message'] = ""
else:
new_domain = retVal['object']
ip_result = {}
if add_ip:
if data.get('same_source'):
ip_source = source_name
ip_method = method
ip_reference = reference
else:
ip_source = data.get('ip_source')
ip_method = data.get('ip_method')
ip_reference = data.get('ip_reference')
from crits.ips.handlers import ip_add_update
ip_result = ip_add_update(ip,
ip_type,
ip_source,
ip_method,
ip_reference,
campaign=campaign,
analyst=username,
bucket_list=bucket_list,
ticket=ticket,
cache=cache)
if not ip_result['success']:
errors.append(ip_result['message'])
else:
#add a relationship with the new IP address
new_ip = ip_result['object']
if new_domain and new_ip:
new_domain.add_relationship(new_ip,
'Resolved_To',
analyst=username,
get_rels=False)
new_domain.save(username=username)
#set the URL for viewing the new data
resp_url = reverse('crits.domains.views.domain_detail', args=[domain])
if retVal['is_domain_new'] == True:
retVal['message'] = ('Success! Click here to view the new domain: '
'<a href="%s">%s</a>' % (resp_url, domain))
else:
message = ('Updated existing domain: <a href="%s">%s</a>' % (resp_url, domain))
retVal['message'] = message
retVal[form_consts.Status.STATUS_FIELD] = form_consts.Status.DUPLICATE
retVal['warning'] = message
#add indicators
if data.get('add_indicators'):
from crits.indicators.handlers import create_indicator_from_tlo
# If we have an IP object, add an indicator for that.
if ip_result.get('success'):
ip = ip_result['object']
result = create_indicator_from_tlo('IP',
ip,
username,
ip_source,
add_domain=False)
ip_ind = result.get('indicator')
if not result['success']:
errors.append(result['message'])
# Add an indicator for the domain.
result = create_indicator_from_tlo('Domain',
new_domain,
username,
source_name,
add_domain=False)
if not result['success']:
errors.append(result['message'])
elif ip_result.get('success') and ip_ind:
forge_relationship(left_class=result['indicator'],
right_class=ip_ind,
rel_type='Resolved_To',
analyst=username)
result = True
# This block validates, and may also add, objects to the Domain
if retVal.get('success') or is_validate_only == True:
if rowData:
objectsData = rowData.get(form_consts.Common.OBJECTS_DATA)
# add new objects if they exist
if objectsData:
objectsData = json.loads(objectsData)
current_domain = retrieve_domain(domain, cache)
for object_row_counter, objectData in enumerate(objectsData, 1):
if current_domain != None:
# if the domain exists then try to add objects to it
if isinstance(current_domain, Domain) == True:
objectDict = object_array_to_dict(objectData,
"Domain",
current_domain.id)
else:
objectDict = object_array_to_dict(objectData,
"Domain",
"")
current_domain = None;
else:
objectDict = object_array_to_dict(objectData,
"Domain",
"")
(obj_result,
errors,
obj_retVal) = validate_and_add_new_handler_object(
None, objectDict, request, errors, object_row_counter,
is_validate_only=is_validate_only,
cache=cache, obj=current_domain)
if not obj_result:
retVal['success'] = False
return result, errors, retVal
def add_object(type_, id_, object_type, source, method, reference, tlp, user,
value=None, file_=None, add_indicator=False, get_objects=True,
tlo=None, is_sort_relationships=False, is_validate_only=False,
is_validate_locally=False, cache={}, **kwargs):
"""
Add an object to the database.
:param type_: The top-level object type.
:type type_: str
:param id_: The ObjectId of the top-level object.
:type id_: str
:param object_type: The type of the ObjectType being added.
:type object_type: str
:param source: The name of the source adding this object.
:type source: str
:param method: The method for this object.
:type method: str
:param reference: The reference for this object.
:type reference: str
:param user: The user adding this object.
:type user: str
:param value: The value of the object.
:type value: str
:param file_: The file if the object is a file upload.
:type file_: file handle.
:param add_indicator: Also add an indicator for this object.
:type add_indicator: bool
:param get_objects: Return the formatted list of objects when completed.
:type get_objects: bool
:param tlo: The CRITs top-level object we are adding objects to.
This is an optional parameter used mainly for performance
reasons (by not querying mongo if we already have the
top level-object).
:type tlo: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param is_sort_relationships: Return all relationships and meta, sorted
:type is_sort_relationships: bool
:param is_validate_only: Validate, but do not add to TLO.
:type is_validate_only: bool
:param is_validate_locally: Validate, but do not add b/c there is no TLO.
:type is_validate_locally: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" (str),
"objects" (list),
"relationships" (list)
"""
# if object_type is a validated indicator type, then validate value
if value:
from crits.indicators.handlers import validate_indicator_value
(value, error) = validate_indicator_value(value, object_type)
if error:
return {"success": False, "message": error}
if is_validate_locally: # no TLO provided
return {"success": True}
if not tlo:
if type_ and id_:
tlo = class_from_id(type_, id_)
if not tlo:
return {'success': False, 'message': "Failed to find TLO"}
try:
if file_:
data = file_.read()
filename = file_.name
md5sum = md5(data).hexdigest()
value = md5sum
reference = filename
ret = tlo.add_object(object_type, value,
source, method, reference, user)
if not ret['success']:
msg = '%s! [Type: "%s"][Value: "%s"]'
return {"success": False,
"message": msg % (ret['message'], object_type, value)}
else:
results = {'success': True}
if not is_validate_only: # save the object
tlo.update(add_to_set__obj=ret['object'])
results['message'] = "Object added successfully"
if file_:
# do we have a pcap?
if detect_pcap(data):
handle_pcap_file(filename,
data,
source,
user=user,
related_id=id_,
related_type=type_)
else:
#XXX: MongoEngine provides no direct GridFS access so we
# need to use pymongo directly.
col = settings.COL_OBJECTS
grid = mongo_connector("%s.files" % col)
if grid.find({'md5': md5sum}).count() == 0:
put_file(filename, data, collection=col)
if add_indicator and not is_validate_only:
campaign = tlo.campaign if hasattr(tlo, 'campaign') else None
from crits.indicators.handlers import handle_indicator_ind
ind_res = handle_indicator_ind(value,
source,
object_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
user,
source_method=method,
source_reference=reference,
source_tlp=tlp,
add_domain=True,
campaign=campaign,
cache=cache)
if ind_res['success']:
forge_relationship(class_=tlo,
right_class=ind_res['object'],
rel_type=RelationshipTypes.RELATED_TO,
user=user)
else:
msg = "Object added, but failed to add Indicator.<br>Error: %s"
results['message'] = msg % ind_res.get('message')
if is_sort_relationships == True:
results['relationships'] = tlo.sort_relationships(user, meta=True)
if get_objects:
results['objects'] = tlo.sort_objects()
results['id'] = str(tlo.id)
return results
except ValidationError as e:
return {'success': False, 'message': str(e)}
def add_object(type_,
id_,
object_type,
source,
method,
reference,
tlp,
user,
value=None,
file_=None,
add_indicator=False,
get_objects=True,
tlo=None,
is_sort_relationships=False,
is_validate_only=False,
is_validate_locally=False,
cache={},
**kwargs):
"""
Add an object to the database.
:param type_: The top-level object type.
:type type_: str
:param id_: The ObjectId of the top-level object.
:type id_: str
:param object_type: The type of the ObjectType being added.
:type object_type: str
:param source: The name of the source adding this object.
:type source: str
:param method: The method for this object.
:type method: str
:param reference: The reference for this object.
:type reference: str
:param user: The user adding this object.
:type user: str
:param value: The value of the object.
:type value: str
:param file_: The file if the object is a file upload.
:type file_: file handle.
:param add_indicator: Also add an indicator for this object.
:type add_indicator: bool
:param get_objects: Return the formatted list of objects when completed.
:type get_objects: bool
:param tlo: The CRITs top-level object we are adding objects to.
This is an optional parameter used mainly for performance
reasons (by not querying mongo if we already have the
top level-object).
:type tlo: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param is_sort_relationships: Return all relationships and meta, sorted
:type is_sort_relationships: bool
:param is_validate_only: Validate, but do not add to TLO.
:type is_validate_only: bool
:param is_validate_locally: Validate, but do not add b/c there is no TLO.
:type is_validate_locally: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" (str),
"objects" (list),
"relationships" (list)
"""
# if object_type is a validated indicator type, then validate value
if value:
from crits.indicators.handlers import validate_indicator_value
(value, error) = validate_indicator_value(value, object_type)
if error:
return {"success": False, "message": error}
if is_validate_locally: # no TLO provided
return {"success": True}
if not tlo:
if type_ and id_:
tlo = class_from_id(type_, id_)
if not tlo:
return {'success': False, 'message': "Failed to find TLO"}
try:
if file_:
data = file_.read()
filename = file_.name
md5sum = md5(data).hexdigest()
value = md5sum
reference = filename
ret = tlo.add_object(object_type, value, source, method, reference,
user)
if not ret['success']:
msg = '%s! [Type: "%s"][Value: "%s"]'
return {
"success": False,
"message": msg % (ret['message'], object_type, value)
}
else:
results = {'success': True}
if not is_validate_only: # save the object
tlo.update(add_to_set__obj=ret['object'])
results['message'] = "Object added successfully"
if file_:
# do we have a pcap?
if detect_pcap(data):
handle_pcap_file(filename,
data,
source,
user=user,
related_id=id_,
related_type=type_)
else:
#XXX: MongoEngine provides no direct GridFS access so we
# need to use pymongo directly.
col = settings.COL_OBJECTS
grid = mongo_connector("%s.files" % col)
if grid.find({'md5': md5sum}).count() == 0:
put_file(filename, data, collection=col)
if add_indicator and not is_validate_only:
campaign = tlo.campaign if hasattr(tlo, 'campaign') else None
from crits.indicators.handlers import handle_indicator_ind
ind_res = handle_indicator_ind(value,
source,
object_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
user,
source_method=method,
source_reference=reference,
source_tlp=tlp,
add_domain=True,
campaign=campaign,
cache=cache)
if ind_res['success']:
forge_relationship(class_=tlo,
right_class=ind_res['object'],
rel_type=RelationshipTypes.RELATED_TO,
user=user)
else:
msg = "Object added, but failed to add Indicator.<br>Error: %s"
results['message'] = msg % ind_res.get('message')
if is_sort_relationships == True:
results['relationships'] = tlo.sort_relationships(user, meta=True)
if get_objects:
results['objects'] = tlo.sort_objects()
results['id'] = str(tlo.id)
return results
except ValidationError as e:
return {'success': False, 'message': str(e)}
def add_object(type_,
oid,
object_type,
name,
source,
method,
reference,
analyst,
value=None,
file_=None,
add_indicator=False,
get_objects=True,
indicator_campaign=None,
indicator_campaign_confidence=None,
obj=None,
is_sort_relationships=False,
is_validate_only=False,
is_validate_locally=False,
cache={}):
"""
Add an object to the database.
:param type_: The top-level object type.
:type type_: str
:param oid: The ObjectId of the top-level object.
:type oid: str
:param object_type: The type of the ObjectType being added.
:type object_type: str
:param name: The name of the ObjectType being added.
:type name: str
:param source: The name of the source adding this object.
:type source: str
:param method: The method for this object.
:type method: str
:param reference: The reference for this object.
:type reference: str
:param analyst: The user adding this object.
:type analyst: str
:param value: The value of the object.
:type value: str
:param file_: The file if the object is a file upload.
:type file_: file handle.
:param add_indicator: Also add an indicator for this object.
:type add_indicator: bool
:param get_objects: Return the formatted list of objects when completed.
:type get_object: bool
:param is_validate_only: Only validate, do not add.
:type is_validate_only: bool
:param is_validate_locally: Only validate, do not add.
:type is_validate_locally: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:param obj: The CRITs top-level object we are adding objects to.
This is an optional parameter used mainly for performance
reasons (by not querying mongo if we already have the
top level-object).
:type obj: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:returns: dict with keys:
"success" (boolean),
"message" (str),
"objects" (list),
"relationships" (list)
"""
results = {}
if oid == None:
oid = ""
if obj == None:
obj = class_from_id(type_, oid)
if not obj:
if is_validate_locally == True:
# TODO: Perform some form of validation
results['success'] = True
return results
else:
results['message'] = "Could not find item to add object to."
results['success'] = False
return results
try:
cur_len = len(obj.obj)
if file_:
data = file_.read()
filename = file_.name
md5sum = md5(data).hexdigest()
value = md5sum
reference = filename
obj.add_object(object_type, name, value, source, method, reference,
analyst)
if is_validate_only == False:
obj.save(username=analyst)
new_len = len(obj.obj)
if new_len > cur_len:
results['message'] = "Object added successfully!"
results['success'] = True
if file_:
# do we have a pcap?
if data[:4] in ('\xa1\xb2\xc3\xd4', '\xd4\xc3\xb2\xa1',
'\x0a\x0d\x0d\x0a'):
handle_pcap_file(filename,
data,
source,
user=analyst,
related_id=oid,
related_type=type_)
else:
#XXX: MongoEngine provides no direct GridFS access so we
# need to use pymongo directly.
col = settings.COL_OBJECTS
grid = mongo_connector("%s.files" % col)
if grid.find({'md5': md5sum}).count() == 0:
put_file(filename, data, collection=col)
if add_indicator and is_validate_only == False:
from crits.indicators.handlers import handle_indicator_ind
if object_type != name:
object_type = "%s - %s" % (object_type, name)
ind_res = handle_indicator_ind(
value,
source,
reference,
object_type,
analyst,
method=method,
add_domain=True,
campaign=indicator_campaign,
campaign_confidence=indicator_campaign_confidence,
cache=cache)
if ind_res['success']:
ind = ind_res['object']
# Inherit campaigns from top level item when creating
# an indicator from an object if no campaigns were specified
if indicator_campaign == None and ind != None:
for campaign in obj.campaign:
ec = EmbeddedCampaign(
name=campaign.name,
confidence=campaign.confidence,
description="",
analyst=analyst,
date=datetime.datetime.now())
ind.add_campaign(ec)
ind.save(username=analyst)
forge_relationship(left_class=obj,
right_class=ind,
rel_type="Related_To",
analyst=analyst,
get_rels=is_sort_relationships)
if is_sort_relationships == True:
if file_ or add_indicator:
# does this line need to be here?
# obj.reload()
results['relationships'] = obj.sort_relationships(
analyst, meta=True)
else:
results['relationships'] = obj.sort_relationships(
analyst, meta=True)
else:
results[
'message'] = "Object already exists! [Type: " + object_type + "][Value: " + value + "] "
results['success'] = False
if (get_objects):
results['objects'] = obj.sort_objects()
return results
except ValidationError, e:
return {'success': False, 'message': e}
def add_object(type_, id_, object_type, source, method,
reference, user, value=None, file_=None,
add_indicator=False, get_objects=True,
obj=None, is_sort_relationships=False,
is_validate_only=False, is_validate_locally=False, cache={},
**kwargs):
"""
Add an object to the database.
:param type_: The top-level object type.
:type type_: str
:param id_: The ObjectId of the top-level object.
:type id_: str
:param object_type: The type of the ObjectType being added.
:type object_type: str
:param source: The name of the source adding this object.
:type source: str
:param method: The method for this object.
:type method: str
:param reference: The reference for this object.
:type reference: str
:param user: The user adding this object.
:type user: str
:param value: The value of the object.
:type value: str
:param file_: The file if the object is a file upload.
:type file_: file handle.
:param add_indicator: Also add an indicator for this object.
:type add_indicator: bool
:param get_objects: Return the formatted list of objects when completed.
:type get_object: bool
:param obj: The CRITs top-level object we are adding objects to.
This is an optional parameter used mainly for performance
reasons (by not querying mongo if we already have the
top level-object).
:type obj: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param is_validate_only: Validate, but do not add to TLO.
:type is_validate_only: bool
:param is_validate_locally: Validate, but do not add b/c there is no TLO.
:type is_validate_locally: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" (str),
"objects" (list),
"relationships" (list)
"""
results = {}
if id_ == None:
id_ = ""
if obj == None:
obj = class_from_id(type_, id_)
from crits.indicators.handlers import validate_indicator_value
if value is not None:
(value, error) = validate_indicator_value(value, object_type)
if error:
return {"success": False, "message": error}
if is_validate_locally: # no obj provided
results['success'] = True
return results
if not obj:
results['message'] = "TLO could not be found"
results['success'] = False
return results
try:
cur_len = len(obj.obj)
if file_:
data = file_.read()
filename = file_.name
md5sum = md5(data).hexdigest()
value = md5sum
reference = filename
obj.add_object(object_type,
value,
source,
method,
reference,
user)
if is_validate_only == False:
obj.save(username=user)
new_len = len(obj.obj)
if new_len > cur_len:
if not is_validate_only:
results['message'] = "Object added successfully!"
results['success'] = True
if file_:
# do we have a pcap?
if data[:4] in ('\xa1\xb2\xc3\xd4',
'\xd4\xc3\xb2\xa1',
'\x0a\x0d\x0d\x0a'):
handle_pcap_file(filename,
data,
source,
user=user,
related_id=id_,
related_type=type_)
else:
#XXX: MongoEngine provides no direct GridFS access so we
# need to use pymongo directly.
col = settings.COL_OBJECTS
grid = mongo_connector("%s.files" % col)
if grid.find({'md5': md5sum}).count() == 0:
put_file(filename, data, collection=col)
if add_indicator and is_validate_only == False:
from crits.indicators.handlers import handle_indicator_ind
campaign = obj.campaign if hasattr(obj, 'campaign') else None
ind_res = handle_indicator_ind(value,
source,
object_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
user,
method,
reference,
add_domain=True,
campaign=campaign,
cache=cache)
if ind_res['success']:
ind = ind_res['object']
forge_relationship(class_=obj,
right_class=ind,
rel_type=RelationshipTypes.RELATED_TO,
user=user,
get_rels=is_sort_relationships)
else:
results['message'] = "Object was added, but failed to add Indicator." \
"<br>Error: " + ind_res.get('message')
if is_sort_relationships == True:
if file_ or add_indicator:
# does this line need to be here?
# obj.reload()
results['relationships'] = obj.sort_relationships(user, meta=True)
else:
results['relationships'] = obj.sort_relationships(user, meta=True)
else:
results['message'] = "Object already exists! [Type: " + object_type + "][Value: " + value + "] "
results['success'] = False
if (get_objects):
results['objects'] = obj.sort_objects()
results['id'] = str(obj.id)
return results
except ValidationError, e:
return {'success': False,
'message': str(e)}
def add_object(type_, oid, object_type, name, source, method,
reference, analyst, value=None, file_=None,
add_indicator=False, get_objects=True,
indicator_campaign=None, indicator_campaign_confidence=None,
obj=None, is_sort_relationships=False,
is_validate_only=False, is_validate_locally=False, cache={}):
"""
Add an object to the database.
:param type_: The top-level object type.
:type type_: str
:param oid: The ObjectId of the top-level object.
:type oid: str
:param object_type: The type of the ObjectType being added.
:type object_type: str
:param name: The name of the ObjectType being added.
:type name: str
:param source: The name of the source adding this object.
:type source: str
:param method: The method for this object.
:type method: str
:param reference: The reference for this object.
:type reference: str
:param analyst: The user adding this object.
:type analyst: str
:param value: The value of the object.
:type value: str
:param file_: The file if the object is a file upload.
:type file_: file handle.
:param add_indicator: Also add an indicator for this object.
:type add_indicator: bool
:param get_objects: Return the formatted list of objects when completed.
:type get_object: bool
:param is_validate_only: Only validate, do not add.
:type is_validate_only: bool
:param is_validate_locally: Only validate, do not add.
:type is_validate_locally: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:param obj: The CRITs top-level object we are adding objects to.
This is an optional parameter used mainly for performance
reasons (by not querying mongo if we already have the
top level-object).
:type obj: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:returns: dict with keys:
"success" (boolean),
"message" (str),
"objects" (list),
"relationships" (list)
"""
results = {}
if oid == None:
oid = ""
if obj == None:
obj = class_from_id(type_, oid)
if not obj:
if is_validate_locally == True:
# TODO: Perform some form of validation
results['success'] = True
return results
else:
results['message'] = "Could not find item to add object to."
results['success'] = False
return results
try:
cur_len = len(obj.obj)
if file_:
data = file_.read()
filename = file_.name
md5sum = md5(data).hexdigest()
value = md5sum
reference = filename
obj.add_object(object_type,
name,
value,
source,
method,
reference,
analyst)
if is_validate_only == False:
obj.save(username=analyst)
new_len = len(obj.obj)
if new_len > cur_len:
results['message'] = "Object added successfully!"
results['success'] = True
if file_:
# do we have a pcap?
if data[:4] in ('\xa1\xb2\xc3\xd4',
'\xd4\xc3\xb2\xa1',
'\x0a\x0d\x0d\x0a'):
handle_pcap_file(filename,
data,
source,
user=analyst,
parent_id=oid,
parent_type=type_)
else:
#XXX: MongoEngine provides no direct GridFS access so we
# need to use pymongo directly.
col = settings.COL_OBJECTS
grid = mongo_connector("%s.files" % col)
if grid.find({'md5': md5sum}).count() == 0:
put_file(filename, data, collection=col)
if add_indicator and is_validate_only == False:
from crits.indicators.handlers import handle_indicator_ind
if object_type != name:
object_type = "%s - %s" % (object_type, name)
ind_res = handle_indicator_ind(value,
source,
reference,
object_type,
analyst,
method=method,
add_domain=True,
campaign=indicator_campaign,
campaign_confidence=indicator_campaign_confidence,
cache=cache)
if ind_res['success']:
ind = ind_res['object']
# Inherit campaigns from top level item when creating
# an indicator from an object if no campaigns were specified
if indicator_campaign == None and ind != None:
for campaign in obj.campaign:
ec = EmbeddedCampaign(name=campaign.name,
confidence=campaign.confidence,
description="",
analyst=analyst,
date=datetime.datetime.now())
ind.add_campaign(ec)
ind.save(username=analyst)
forge_relationship(left_class=obj,
right_class=ind,
rel_type="Related_To",
analyst=analyst,
get_rels=is_sort_relationships)
if is_sort_relationships == True:
if file_ or add_indicator:
# does this line need to be here?
# obj.reload()
results['relationships'] = obj.sort_relationships(analyst, meta=True)
else:
results['relationships'] = obj.sort_relationships(analyst, meta=True)
else:
results['message'] = "Object already exists! [Type: " + object_type + "][Value: " + value + "] "
results['success'] = False
if (get_objects):
results['objects'] = obj.sort_objects()
return results
except ValidationError, e:
return {'success': False,
'message': e}
def add_object(type_,
id_,
object_type,
source,
method,
reference,
user,
value=None,
file_=None,
add_indicator=False,
get_objects=True,
tlo=None,
is_sort_relationships=False,
is_validate_only=False,
is_validate_locally=False,
cache={},
**kwargs):
"""
Add an object to the database.
:param type_: The top-level object type.
:type type_: str
:param id_: The ObjectId of the top-level object.
:type id_: str
:param object_type: The type of the ObjectType being added.
:type object_type: str
:param source: The name of the source adding this object.
:type source: str
:param method: The method for this object.
:type method: str
:param reference: The reference for this object.
:type reference: str
:param user: The user adding this object.
:type user: str
:param value: The value of the object.
:type value: str
:param file_: The file if the object is a file upload.
:type file_: file handle.
:param add_indicator: Also add an indicator for this object.
:type add_indicator: bool
:param get_objects: Return the formatted list of objects when completed.
:type get_object: bool
:param tlo: The CRITs top-level object we are adding objects to.
This is an optional parameter used mainly for performance
reasons (by not querying mongo if we already have the
top level-object).
:type tlo: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param is_validate_only: Validate, but do not add to TLO.
:type is_validate_only: bool
:param is_validate_locally: Validate, but do not add b/c there is no TLO.
:type is_validate_locally: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" (str),
"objects" (list),
"relationships" (list)
"""
results = {}
obj = tlo
if id_ == None:
id_ = ""
if obj == None:
obj = class_from_id(type_, id_)
from crits.indicators.handlers import validate_indicator_value
if value is not None:
(value, error) = validate_indicator_value(value, object_type)
if error:
return {"success": False, "message": error}
if is_validate_locally: # no obj provided
results['success'] = True
return results
if not obj:
results['message'] = "TLO could not be found"
results['success'] = False
return results
try:
cur_len = len(obj.obj)
if file_:
data = file_.read()
filename = file_.name
md5sum = md5(data).hexdigest()
value = md5sum
reference = filename
obj.add_object(object_type, value, source, method, reference, user)
if is_validate_only == False:
obj.save(username=user)
new_len = len(obj.obj)
if new_len > cur_len:
if not is_validate_only:
results['message'] = "Object added successfully!"
results['success'] = True
if file_:
# do we have a pcap?
if data[:4] in ('\xa1\xb2\xc3\xd4', '\xd4\xc3\xb2\xa1',
'\x0a\x0d\x0d\x0a'):
handle_pcap_file(filename,
data,
source,
user=user,
related_id=id_,
related_type=type_)
else:
#XXX: MongoEngine provides no direct GridFS access so we
# need to use pymongo directly.
col = settings.COL_OBJECTS
grid = mongo_connector("%s.files" % col)
if grid.find({'md5': md5sum}).count() == 0:
put_file(filename, data, collection=col)
if add_indicator and is_validate_only == False:
from crits.indicators.handlers import handle_indicator_ind
campaign = obj.campaign if hasattr(obj, 'campaign') else None
ind_res = handle_indicator_ind(value,
source,
object_type,
IndicatorThreatTypes.UNKNOWN,
IndicatorAttackTypes.UNKNOWN,
user,
method,
reference,
add_domain=True,
campaign=campaign,
cache=cache)
if ind_res['success']:
ind = ind_res['object']
forge_relationship(class_=obj,
right_class=ind,
rel_type=RelationshipTypes.RELATED_TO,
user=user,
get_rels=is_sort_relationships)
else:
results['message'] = "Object was added, but failed to add Indicator." \
"<br>Error: " + ind_res.get('message')
if is_sort_relationships == True:
if file_ or add_indicator:
# does this line need to be here?
# obj.reload()
results['relationships'] = obj.sort_relationships(
user, meta=True)
else:
results['relationships'] = obj.sort_relationships(
user, meta=True)
else:
results[
'message'] = "Object already exists! [Type: " + object_type + "][Value: " + value + "] "
results['success'] = False
if (get_objects):
results['objects'] = obj.sort_objects()
results['id'] = str(obj.id)
return results
except ValidationError, e:
return {'success': False, 'message': str(e)}
def add_object(type_,
oid,
object_type,
name,
source,
method,
reference,
analyst,
value=None,
file_=None,
add_indicator=False,
get_objects=True,
obj=None,
is_sort_relationships=False,
is_validate_only=False,
is_validate_locally=False,
cache={}):
"""
Add an object to the database.
:param type_: The top-level object type.
:type type_: str
:param oid: The ObjectId of the top-level object.
:type oid: str
:param object_type: The type of the ObjectType being added.
:type object_type: str
:param name: The name of the ObjectType being added.
:type name: str
:param source: The name of the source adding this object.
:type source: str
:param method: The method for this object.
:type method: str
:param reference: The reference for this object.
:type reference: str
:param analyst: The user adding this object.
:type analyst: str
:param value: The value of the object.
:type value: str
:param file_: The file if the object is a file upload.
:type file_: file handle.
:param add_indicator: Also add an indicator for this object.
:type add_indicator: bool
:param get_objects: Return the formatted list of objects when completed.
:type get_object: bool
:param obj: The CRITs top-level object we are adding objects to.
This is an optional parameter used mainly for performance
reasons (by not querying mongo if we already have the
top level-object).
:type obj: :class:`crits.core.crits_mongoengine.CritsBaseAttributes`
:param is_validate_only: Only validate, do not add.
:type is_validate_only: bool
:param is_validate_locally: Only validate, do not add.
:type is_validate_locally: bool
:param cache: Cached data, typically for performance enhancements
during bulk operations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" (str),
"objects" (list),
"relationships" (list)
"""
results = {}
if oid == None:
oid = ""
if obj == None:
obj = class_from_id(type_, oid)
if not obj:
if is_validate_locally == True:
# TODO: Perform some form of validation
results['success'] = True
return results
else:
results['message'] = "Could not find item to add object to."
results['success'] = False
return results
if name == "URL" and "://" not in value.split('.')[0]:
return {
"success":
False,
"message":
"URI - URL must contain protocol prefix (e.g. http://, https://, ftp://)"
}
elif object_type == "Address":
if "ipv4" in name:
try:
validate_ipv4_address(value)
except DjangoValidationError:
return {"success": False, "message": "Invalid IPv4 address. "}
elif "ipv6" in name:
try:
validate_ipv6_address(value)
except DjangoValidationError:
return {"success": False, "message": "Invalid IPv6 address. "}
elif "cidr" in name:
try:
if '/' not in value:
raise ValidationError("")
cidr_parts = value.split('/')
if int(cidr_parts[1]) < 0 or int(cidr_parts[1]) > 128:
raise ValidationError("")
if ':' not in cidr_parts[0] and int(cidr_parts[1]) > 32:
raise ValidationError("")
validate_ipv46_address(cidr_parts[0])
except (ValidationError, ValueError) as cidr_error:
return {"success": False, "message": "Invalid CIDR address. "}
try:
cur_len = len(obj.obj)
if file_:
data = file_.read()
filename = file_.name
md5sum = md5(data).hexdigest()
value = md5sum
reference = filename
obj.add_object(object_type, name, value, source, method, reference,
analyst)
if is_validate_only == False:
obj.save(username=analyst)
new_len = len(obj.obj)
if new_len > cur_len:
results['message'] = "Object added successfully!"
results['success'] = True
if file_:
# do we have a pcap?
if data[:4] in ('\xa1\xb2\xc3\xd4', '\xd4\xc3\xb2\xa1',
'\x0a\x0d\x0d\x0a'):
handle_pcap_file(filename,
data,
source,
user=analyst,
related_id=oid,
related_type=type_)
else:
#XXX: MongoEngine provides no direct GridFS access so we
# need to use pymongo directly.
col = settings.COL_OBJECTS
grid = mongo_connector("%s.files" % col)
if grid.find({'md5': md5sum}).count() == 0:
put_file(filename, data, collection=col)
if add_indicator and is_validate_only == False:
from crits.indicators.handlers import handle_indicator_ind
if object_type != name:
object_type = "%s - %s" % (object_type, name)
campaign = obj.campaign if hasattr(obj, 'campaign') else None
ind_res = handle_indicator_ind(value,
source,
reference,
object_type,
analyst,
method,
add_domain=True,
campaign=campaign,
cache=cache)
if ind_res['success']:
ind = ind_res['object']
forge_relationship(left_class=obj,
right_class=ind,
rel_type="Related_To",
analyst=analyst,
get_rels=is_sort_relationships)
else:
results['message'] = "Object was added, but failed to add Indicator." \
"<br>Error: " + ind_res.get('message')
if is_sort_relationships == True:
if file_ or add_indicator:
# does this line need to be here?
# obj.reload()
results['relationships'] = obj.sort_relationships(
analyst, meta=True)
else:
results['relationships'] = obj.sort_relationships(
analyst, meta=True)
else:
results[
'message'] = "Object already exists! [Type: " + object_type + "][Value: " + value + "] "
results['success'] = False
if (get_objects):
results['objects'] = obj.sort_objects()
results['id'] = str(obj.id)
return results
except ValidationError, e:
return {'success': False, 'message': str(e)}
